From 00d77c33faba47a8824865370197ba269eba0966 Mon Sep 17 00:00:00 2001 From: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:51:46 +0100 Subject: [PATCH 001/332] Add development branch to workflow triggers and update Python versions --- .github/workflows/python-package.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..f321dd8 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "main", "development" ] + pull_request: + branches: [ "main", "development" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From 5eed594eccf9bb7889717c9d3625e0ee6ad8b3c1 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:32:46 +0100 Subject: [PATCH 002/332] removed utcnow and unused import --- onsrap/models.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index c7977b7..755dcc7 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from pathlib import Path from typing import Any, Mapping, Optional, Union @@ -26,10 +26,6 @@ def now() -> datetime: return datetime.now() -def utcnow() -> datetime: - return now() - - @dataclass class RuntimeID: id: str From 8b8d532e7895479218d756bfd120c2b63eec1a4a Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:33:04 +0100 Subject: [PATCH 003/332] Tweaked test to use temp file locations not broader repo locations --- tests/test_pipeline_architecture.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 1f5c269..901999e 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -63,7 +63,14 @@ def main(context): encoding="utf-8", ) - pipeline = Pipeline.from_files([writer_stage]) + pipeline = Pipeline.from_files( + [writer_stage], + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + ) first_run = pipeline.run() second_run = pipeline.run() From f7243405e27c86818d44a66d63328327e03752e5 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:35:48 +0100 Subject: [PATCH 004/332] Tweaked Readme wording for pipeline run readme --- examples/pipeline_1/runs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipeline_1/runs/README.md b/examples/pipeline_1/runs/README.md index 862c9bd..dc632ef 100644 --- a/examples/pipeline_1/runs/README.md +++ b/examples/pipeline_1/runs/README.md @@ -1,6 +1,6 @@ # Run Output Directory -In this directory is the outputs of the example example pipeline. This directory's contents are untracked if the contents are further directories. This means there are no example pipeline outputs! +In this directory is the outputs of the example pipeline. This directory's contents are untracked if the contents are further directories. This means there are no example pipeline outputs! However, you can run pipeline_1 to generate its outputs yourself. Simply type the following command into a powershell or cmd terminal: ```powershell From beed09fd78ed8bcaa9850658cc15bb414abf209b Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:48:13 +0100 Subject: [PATCH 005/332] Updated changelog for v0.1.1 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f519e85..855862e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be tracked here. +## [0.1.1] - 2026-06-24 + +This release adds run-scoped output handling for the example pipeline and aligns runtime timestamps with local time. + +### Added + +- Introduced per-run output directories so each pipeline execution writes into its own `runs//` tree. +- Anchored the example pipeline to its own `main.py` directory so runs stay inside `examples/pipeline_1/` instead of the repository working directory. +- Routed stage output paths through `ExecutionContext.run_dir` so stages can record artifacts in the active run directory. +- Switched runtime timestamp generation to local time and kept a compatibility alias for the previous helper. +- Added regression coverage for repeated runs and run-specific output paths. + + ## [0.1.0] - 2026-06-22 First tracked version of `onsrap`. From 0966e735bb83f0518f41f7dc6d8a33352809e9c1 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:49:32 +0100 Subject: [PATCH 006/332] Removed unused imports from Pipeline.py --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ffa75ad..d771f62 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -12,7 +12,7 @@ from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger -from .models import PipelineConfig, PipelineRun, PipelineStatus, RAPConfig, RunManifest, RuntimeID, StageResult, now +from .models import PipelineConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now from .stage import Stage From e7f24c495676dd4f0b3aa260762b68fcd6e7d480 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:52:35 +0100 Subject: [PATCH 007/332] Small tweaks to Example README to finish sentence and add minor amount of context. --- examples/pipeline_1/Example.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/pipeline_1/Example.md b/examples/pipeline_1/Example.md index 32c05ae..5105843 100644 --- a/examples/pipeline_1/Example.md +++ b/examples/pipeline_1/Example.md @@ -1,3 +1,5 @@ # Purpose -This example is designed to show the typical use case of using Callable entry points for \ No newline at end of file +This example is designed to show the typical use case of using Callable entry points for the package. + +The contents of this directory is trying to show how a Functional Programming approach can be orchestrated using the package. \ No newline at end of file From 2aca1b20d9724aa0c8cb862fd71d872a9c325fe1 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:54:03 +0100 Subject: [PATCH 008/332] Updated package setup.cfg to remove Python version 3.9 due to potential syntax issues. --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d270928..65415b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,6 @@ version = 0.1.1 author = ONSDigital platforms = win32 classifiers = - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -14,7 +13,7 @@ classifiers = [options] packages = find: -python_requires = >=3.9 +python_requires = >=3.10 zip_safe = no install_requires = pyyaml From 71874bab93deaefc5bdab2ccb85f19f3453aae9b Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 09:55:14 +0100 Subject: [PATCH 009/332] Removed unused variable in for loop. --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index d771f62..0d5bd32 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -158,7 +158,7 @@ def from_files( executor: StageExecutor | None = None, ) -> "Pipeline": stages: list[Stage] = [] - for position, file_path in enumerate(file_paths): + for file_path in file_paths: path = Path(file_path) stage_name = path.stem stage_dependencies = cls._dependencies_for_stage(stage_name, path, dependencies) From 8adc070578782a85d41bbd8ce4cd9ef10815d9b2 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 10:03:39 +0100 Subject: [PATCH 010/332] Changed logger creation to resolve name first so that a different logger is used based on different runitmes --- onsrap/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 0eafe7f..8c4a857 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -20,7 +20,8 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.log_dir = Path(self.config.log_dir) self.log_dir.mkdir(parents=True, exist_ok=True) - self._logger = logging.getLogger(self.config.logger_name) + logger_name = f"{self.config.logger_name}:{self.log_dir.resolve()}" + self._logger = logging.getLogger(logger_name) if not getattr(self._logger, "_onsrap_configured", False): self._logger.setLevel(getattr(logging, self.config.log_level.upper(), logging.INFO)) self._logger.propagate = False From 85749481a2730eeee7b62f6651c8ce134a1a3707 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 10:06:06 +0100 Subject: [PATCH 011/332] Minor doc tweak, added graceful missing dependency exception handling. --- onsrap/graph.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/onsrap/graph.py b/onsrap/graph.py index 18d2eab..c75e55a 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -5,6 +5,7 @@ from .errors import DependencyCycleError, DuplicateStageError, MissingDependencyError from .stage import Stage +from onsrap import stage @dataclass @@ -13,9 +14,7 @@ class StageGraph: @classmethod def from_stages(cls, stages: Iterable[Stage]) -> "StageGraph": - """ - This is the primary constructor for StageGraph, which performs validation and normalization of the stage list. - """ + """Primary constructor for :class:`StageGraph` that normalizes the stage list.""" return cls(list(stages)) def validate(self) -> None: @@ -80,6 +79,10 @@ def topological_order(self) -> list[Stage]: for stage in self.stages: for dependency in stage.dependencies: + if dependency not in dependents: + raise MissingDependencyError( + "Unknown stage dependency: {0} -> {1}".format(stage.name, dependency) + ) dependents[dependency].add(stage.name) original_order = [stage.name for stage in self.stages] From 40c2617b6c7670e79827c1cf7bc15829d43135c1 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 10:07:39 +0100 Subject: [PATCH 012/332] Removed empty file --- onsrap/run_pipeline.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 onsrap/run_pipeline.py diff --git a/onsrap/run_pipeline.py b/onsrap/run_pipeline.py deleted file mode 100644 index 842be5f..0000000 --- a/onsrap/run_pipeline.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from .runner import main - - -if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file From 50dfe8a7aab77f77c33ef822f5938bfa691ee511 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 10:12:49 +0100 Subject: [PATCH 013/332] Changed test to record logs in tmp_path location not in CWD location. --- tests/test_pipeline_architecture.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 901999e..bd4e540 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -36,6 +36,11 @@ def main(context): pipeline = Pipeline.from_files( [first_stage, second_stage], dependencies={"second_stage": ("first_stage",)}, + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, ) run = pipeline.run() @@ -90,7 +95,15 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - pipeline = Pipeline.from_files([script_stage], name="script-pipeline") + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + ) run = pipeline.run() assert run.stage_results[0].outputs.strip() == "script fallback works" From 3e6201dc5062b1958017656189738d56a97a4df4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 24 Jun 2026 16:27:03 +0100 Subject: [PATCH 014/332] Added docstrings to stage.py --- onsrap/stage.py | 171 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/onsrap/stage.py b/onsrap/stage.py index 92da700..b14b3ad 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -13,6 +13,23 @@ def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: + """ + Standardise the names of any stages dependant on other stages/processes. + + Removes trailing or leading white space from the name of any stage/process dependant + on another and turns it into a tuple of strings. + + Parameters + ---------- + dependencies : Iterable[str] | str | None + Dependency names to standardise. ``None`` returns an empty tuple. A + string is treated as a single entry in the tuple. Any iterable is converted + into a sequence of names. + Returns + ------- + normalized : tuple + A tuple of cleaned dependency names. + """ if dependencies is None: return () @@ -32,6 +49,35 @@ def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[s @dataclass class Stage: + """ + Represents a single unit of work within a pipeline. + + Can be defined by a data source process or a Python script/callable item. + Stages may be dependant on other stages and can hold metadata for themselves. + + Parameters + ---------- + ``name`` : str + The name of the Stage being run. + ``source`` : Path, Callable, or None + Item being implemented in this Stage. E.g. a file path to a Python script + or a function being executed directly. The full file path is gathered if + a path is used. + ``dependencies`` : tuple of strings + Names of stages that must be completed before this stage is attempted. These + are cleaned post initialisation to remove leading/trailing whitespace. + ``metadata`` : dictionary with string:Any key/value pairs + Location to store any summary information about the stage being run. + ``entrypoint``: str, optional + Name of the starting script to the pipeline. + ``backend`` : str, default = "python" + The name of the system that the code runs on. + + Raises + ------ + ``StageConfigurationError`` + If the stage ``name`` is empty or if the source is not a supported type. + """ name: str source: Union[Path, Callable[..., Any], None] = None dependencies: tuple[str, ...] = field(default_factory=tuple) @@ -66,6 +112,37 @@ def from_file( entrypoint: str | None = None, backend: str = "python", ) -> "Stage": + """ + Class method that checks and cleans the file path for the ``Stage``. + + Expands file path to its full name and checks whether it exists. The method + also cleans other parameters in the Stage class in the return line. + + Parameters + ---------- + ``file_path`` : str or Path + The name or file path for the script that the ``Stage`` will be running. + ``name`` : str + The name of the ``Stage`` + ``dependencies`` : Iterable[str], str, or None + The Stage/s that need to be complete before the ``Stage`` currently attempted. + ``metadata`` : Mapping[str, Any], or None + Any supporting information for the ``Stage`` being run. + ``entrypoint`` : str or None + The name of the first script for the Stage. + ``backend``: str, default = "python" + The system that the ``Stage`` is run on. + + Raises + ------ + StageConfigurationError + If the file path does not exist + + Returns + ------- + Stage + Stage class instance with cleaned/checked file path, dependencies, and metadata + """ path = Path(file_path).expanduser() if not path.exists(): raise StageConfigurationError(f"Stage source file does not exist: {path}") @@ -89,6 +166,30 @@ def from_callable( metadata: Mapping[str, Any] | None = None, backend: str = "python", ) -> "Stage": + """ + Class method that retrieves the name of the Stage from a Callable item. + + Parameters + ---------- + ``callable_object`` : Callable with any number of arguments of any type + The name or file path for the script that the Stage will be running. + ``name`` : str + The name of the Stage + ``dependencies`` : Iterable[str], str, or None + The Stage/s that need to be complete before the Stage currently attempted. + ``metadata`` : Mapping[str, Any], or None + Any supporting information for the Stage being run. + ``entrypoint`` : str or None + The name of the first script for the Stage. + ``backend``: str, default = "python" + The system that the stage is run on. + + Returns + ------- + ``Stage`` + ``Stage class`` instance with collected Stage ``name``, normalised ``dependencies`` + and ``metadata``, and defined the source as the callable_object. + """ stage_name = name or getattr(callable_object, "__name__", "stage") return cls( name=stage_name, @@ -100,6 +201,27 @@ def from_callable( @classmethod def from_dict(cls, data: Mapping[str, Any]) -> "Stage": + """ + Class method that converts a dictionary stage into a ``Stage`` class instance. + + Extracts the values from the key/value pairs in the stage and holds them as attributes. + + Parameters + ---------- + ``data`` : any number of key/value pairs of strings + The information to convert into a Stage class. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a suitable type (callable or Path). + + Returns + ------- + ``Stage`` + ``Stage`` class instance with collected ``Stage`` attributes based on the type of ``source`` + provided. + """ payload = dict(data) source = payload.pop("source", payload.pop("path", None)) @@ -141,12 +263,33 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Stage": raise StageConfigurationError("Stage dictionary must define a source, path, or callable.") def with_dependencies(self, *dependencies: str) -> "Stage": + """ + Method that normalises and adds ``dependencies`` to the ``Stage`` class attributes. + + Parameters + ---------- + ``*dependencies`` : str + Information on which scripts need to run before other scripts for this ``Stage``. + + Returns + ------- + ``Stage`` + ``Stage`` class instance with normalised ``dependencies`` attribute. + """ return replace( self, dependencies=self.dependencies + _normalize_dependencies(dependencies), ) def validate(self) -> None: + """ + Error checking on source attribute. + + Raises + ------- + ``StageConfigurationError`` + If ``source`` attribute does not define a source or does not exist. + """ if self.source is None: raise StageConfigurationError(f"Stage '{self.name}' does not define a source.") @@ -155,12 +298,26 @@ def validate(self) -> None: @property def source_path(self) -> Optional[Path]: + """ + Sets a property for the ``Stage`` class if the ``source`` is a path. + + Returns + ------- + ``source_path`` attribute to the ``Stage`` class if the ``source`` is a path. + """ if isinstance(self.source, Path): return self.source return None @property def source_label(self) -> Optional[str]: + """ + Sets a property for the ``Stage`` class with a human-readable name for the ``source``. + + Returns + ------- + ``source_label`` attribute to the ``Stage`` class if ``source`` is a callable or Path. + """ if callable(self.source): return f"{getattr(self.source, '__module__', '')}.{getattr(self.source, '__name__', self.name)}" @@ -170,5 +327,19 @@ def source_label(self) -> Optional[str]: return None def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageResult": + """ + Checks that the ``source`` is valid and then runs the ``source`` + + Properties + ---------- + context : set value "ExecutionContext" + Uses ``ExecutionContext`` class information to provide required metadata on running ``source``. + executor : set value "StageExecutor" + Uses ``StageExecutor`` class to extract the ``.execute`` method to actually run the ``source``. + + Returns + ------- + ``execute`` method of the ``StageExecutor`` class stored in the ``StageResult`` class. + """ self.validate() return executor.execute(self, context) \ No newline at end of file From 5cd4a5ebdb06b078e7d7ae9e2094edb135e7cb99 Mon Sep 17 00:00:00 2001 From: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:11:29 +0100 Subject: [PATCH 015/332] Update onsrap/graph.py to use fstring not .format --- onsrap/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/graph.py b/onsrap/graph.py index c75e55a..a9c1b5f 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -81,7 +81,7 @@ def topological_order(self) -> list[Stage]: for dependency in stage.dependencies: if dependency not in dependents: raise MissingDependencyError( - "Unknown stage dependency: {0} -> {1}".format(stage.name, dependency) + f"Unknown stage dependency: {stage.name} -> {dependency}" ) dependents[dependency].add(stage.name) From ae5fa2bc728dbb320ac15f4ce726320429b08220 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 24 Jun 2026 17:13:13 +0100 Subject: [PATCH 016/332] Removed useless stage import --- onsrap/graph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onsrap/graph.py b/onsrap/graph.py index a9c1b5f..c5da9f2 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -5,7 +5,6 @@ from .errors import DependencyCycleError, DuplicateStageError, MissingDependencyError from .stage import Stage -from onsrap import stage @dataclass From 30572c7315893c21eaadea79c96b426d808926c3 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 26 Jun 2026 11:29:03 +0100 Subject: [PATCH 017/332] Docstrings for errors.py and execution.py --- onsrap/errors.py | 35 +++++-- onsrap/execution.py | 226 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 7 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index 372d6df..8302969 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -6,27 +6,45 @@ class OnsrapError(Exception): class PipelineValidationError(OnsrapError): - """Raised when the pipeline definition is invalid.""" + """ + Raised when the pipeline definition is invalid. + Child class with ``OnsrapError`` as the parent class. + """ class StageConfigurationError(PipelineValidationError): - """Raised when a stage definition is malformed.""" + """ + Raised when a stage definition is malformed. + Child class with ``PipelineValidationError`` as the parent class. + """ class DuplicateStageError(PipelineValidationError): - """Raised when two stages share the same name.""" + """ + Raised when two stages share the same name. + Child class with ``PipelineValidationError`` as the parent class. + """ class MissingDependencyError(PipelineValidationError): - """Raised when a stage depends on an unknown stage.""" + """ + Raised when a stage depends on an unknown stage. + Child class with ``PipelineValidationError`` as the parent class. + """ class DependencyCycleError(PipelineValidationError): - """Raised when the stage graph contains a cycle.""" + """ + Raised when the stage graph contains a cycle. + Child class with ``PipelineValidationError`` as the parent class. + """ class StageExecutionError(OnsrapError): - """Raised when a stage fails during execution.""" + """ + Raised when a stage fails during execution. + Child class with ``OnsrapError`` as the parent class. + """ def __init__( self, @@ -44,4 +62,7 @@ def __init__( class StageLoadError(StageExecutionError): - """Raised when a file-backed stage cannot be loaded.""" + """ + Raised when a file-backed stage cannot be loaded. + Child class with ``StageExecutionError`` as the parent class. + """ diff --git a/onsrap/execution.py b/onsrap/execution.py index 7c52c6e..4dff349 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -19,6 +19,30 @@ @dataclass class ExecutionContext: + """ + Holds information needed to run the pipeline. + + Parameters + ---------- + ``pipeline_name`` : str + The name of the pipeline. + ``run_id`` : str + The unique identifier for the current run of the pipeline. + ``config`` : ``PipelineConfig`` class instance + The configuration required for the pipeline. + ``logger`` : ``Logger`` class instance + The logger used for this pipeline run. + ``run_dir``: Path + The directory that the run saved to. + ``started_at`` : datetime, default = current time + The time that the pipeline run started. + ``working_directory`` : Path, default = current working directory + The directory that the work is taking place in. + ``stage_results`` : dict[str, StageResult], default = dict + Stores the logs for the stage run. + ``variables`` : dict[str, Any], default = dict + Stores relevant variables regarding the stage run and their results. + """ pipeline_name: str run_id: str config: PipelineConfig @@ -30,28 +54,105 @@ class ExecutionContext: variables: dict[str, Any] = field(default_factory=dict) def record(self, result: StageResult) -> StageResult: + """ + Extracts key information from ``StageResult``. + + Saves all information on the results of the Stage to the ``stage_results`` attribute + and exclusively metadata outputs regarding the run to the ``variables`` attribute. + + Parameters + ---------- + ``result`` : ``StageResult`` + An instance of a ``StageResult`` class + + Returns + ------ + ``result`` + An unchanged ``StageResult`` instance. + """ self.stage_results[result.name] = result self.variables[result.name] = result.outputs return result def result_for(self, stage_name: str) -> StageResult | None: + """ + Getter function that returns the stage_results for a specific ``Stage``. + + Parameters + ---------- + ``stage_name`` : str + The name of the ``Stage`` that you are calling the results for. + + Returns + ------- + ``stage_results`` + Attribute for the specific `Stage` named. + """ return self.stage_results.get(stage_name) @property def stage_outputs(self) -> dict[str, Any]: + """ + Creates a ``stage_outputs`` attribute for the ``ExecutionContext`` class. + + Extracts the ```outputs`` attribute from the ``stage_results`` class for each + ``Stage`` name. + + Returns + ------- + ``stage_outputs`` + Dictionary containing the name of the stage and the associated outputs of + the run. + """ return {name: result.outputs for name, result in self.stage_results.items()} class StageExecutor(Protocol): + """ + Child class of ``Protocol`` + Implementation required + """ def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + """ + Method to run ``Stage`` however implementation required + """ ... class PythonStageExecutor: + """ + Class to run Python `Stage`. + + Contains methods that allow automatic running of individual `Stage` processes for + a pipeline. + """ def __init__(self, preferred_entrypoints: tuple[str, ...] = PREFERRED_ENTRYPOINTS): self.preferred_entrypoints = preferred_entrypoints def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + """ + Main function to select how ``Stage`` is run. + + Identifies the type of ``source`` within the ``Stage`` and runs the relevant + function for that type. + + Parameters + ---------- + ``stage`` : ``Stage`` class + The ``Stage`` that is attempting to be run. + + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``StageResult`` instance. + + Raise + ----- + ``StageExecutionError`` + If the ``source`` is not a Path or a callable object. + """ if callable(stage.source): return self._execute_callable(stage, context, stage.source, stage.source_label) @@ -71,6 +172,36 @@ def _execute_callable( callable_object: Any, source_label: str | None, ) -> StageResult: + """ + Attempt to run a callable object. + + Creates a logging instance and attempts to run the callable parsed. If + the callable cannot be run, an error is flagged and the ``StageResult`` + instance created shows a failure. If it can be run, the callable is run + and the ``StageResult`` instance shows a success. Metadata is kept for + the attempt including duration, ``name``, ``outputs``, ``source``, ``mode`` + attempted, and ``errors``. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + ``callable_object`` : Any + The callable attempting to be run. + ``source_label`` : str or None + The type of ``source`` for the ``Stage``. + + Return + ------ + ``StageResult`` class instance + + Raise + ----- + ``StageExecutionError`` + If the callable object cannot be run + """ started_at = now() context.logger.event( "Stage started", @@ -118,6 +249,33 @@ def _execute_callable( return result def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: + """ + Attempt to run a file. + + Creates a logging instance and attempts to run the file parsed based on an entrypoint. + If there is no entrypoint or the entrypoint is not a callable object, an error will be + raised. ``_execute_subprocess()`` method called if no entrypoint is found. A + ``StageResult`` instance will be created to log the results of the ``Stage``run regardless + of success or failure. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``StageResult`` class instance + + Raise + ----- + ``StageLoadError`` + If the entrypoint in the stage is unable to be run. + ``StageExecutionError`` + If the entrypoint is not found. + """ path = stage.source assert isinstance(path, Path) @@ -168,6 +326,30 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul return self._execute_subprocess(stage, context) def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> StageResult: + """ + Run the entire Python file for the ``Stage`` from the top. + + If the ``Stage`` source is a file but does not have a callable entrypoint, this method + will run the entire script top to bottom. The results of the ``Stage`` are recorded + as a ``StageResult`` instance and logging processes are complete. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``result`` + A ``StageResult`` instance holding information on the ``Stage``. + + Raise + ----- + ``StageExecutionError`` + If the ``Stage`` script was unable to be run successfully. + """ path = stage.source assert isinstance(path, Path) @@ -228,6 +410,26 @@ def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> Stag def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionContext) -> Any: + """ + Assigns appropriate parameters for a callable and runs it. + + Searches for parameter terms that likely refer to context or stage. If none of these are found, + assigns ``context`` as the first parameter and ``stage`` as the second. + + Parameters + ---------- + ``callable_object`` : Any + The callable item that is going to be run. + ``stage`` : ``Stage`` class + The ``Stage`` class instance to be a parameter for the ``callable_object``. + ``context`` : ``ExecutionContext`` class + The ``ExecutionContext`` class instance to be a parameter for the ``callable_object``. + + Returns + ------- + ``callable_object`` + An invocation of the ``callable_object`` with appropriately assigned parameters. + """ signature = inspect.signature(callable_object) parameters = list(signature.parameters.values()) @@ -282,6 +484,30 @@ def _build_success_result( *, source: str | None = None, ) -> StageResult: + """ + Create a ``StageResult`` instance showing a successful stage run. + + If the output of a ``Stage`` run is a ``StageResult`` class, set missing attributes to relevant + information from the ``Stage``. + + Parameters + ---------- + ``stage`` : ``Stage`` class + The ``Stage`` class instance being run. + ``started_at`` : datetime + The time and date that the run started. + ``finished_at`` : datetime + The time and date that the run ended. + ``output`` : Any + The output produced from the stage run. + ``source`` : str or None + The file/callable being run in the stage. + + Return + ------ + ``StageResult`` instance + Containing metadata for the stage run and showing that the run was a success. + """ if isinstance(output, StageResult): if output.name != stage.name: output.name = stage.name From eb1dc71418e12f29792626b1286e92a475100762 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 29 Jun 2026 11:16:14 +0100 Subject: [PATCH 018/332] documenting: graph.py and loader.py docstrings updated --- onsrap/graph.py | 37 +++++++++++++++++++++++++++++--- onsrap/loader.py | 56 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/onsrap/graph.py b/onsrap/graph.py index 18d2eab..c5d8f58 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -9,18 +9,42 @@ @dataclass class StageGraph: + """ + Represents an order to run stages. + + Holds an order that stages need to run in based on dependencies and logic. + + Parameters + ---------- + ``stages`` : list of ``Stage`` class items + """ stages: list[Stage] = field(default_factory=list) @classmethod def from_stages(cls, stages: Iterable[Stage]) -> "StageGraph": """ This is the primary constructor for StageGraph, which performs validation and normalization of the stage list. + + Parameters + ---------- + ``stages`` : Iterable of ``Stage`` class instances + + Returns + ------- + The ``stages`` parameter as a list. """ return cls(list(stages)) def validate(self) -> None: """ Validate the stage graph for issues such as duplicate stage names, missing dependencies, and cycles. + + Raises + ------ + ``DuplicateStageError`` + If the stage name appears multiple times in the stage list. + ``MissingDependencyError`` + If there are unknown dependencies. """ # Check for duplicate stages @@ -70,9 +94,16 @@ def topological_order(self) -> list[Stage]: stages that depend on it. That may free up more stages, which are then added to the ready list. - If the algorithm cannot place every stage, the graph contains either a - cycle or a dependency that could not be resolved. In that case a - ``DependencyCycleError`` is raised. + Returns + ------- + A list of stages ordered in the way that they need to be run through the + pipeline. + + Raises + ------ + ``DependencyCycleError`` + If the algorithm cannot place every stage, the graph contains either a + cycle or a dependency that could not be resolved. """ stage_by_name = {stage.name: stage for stage in self.stages} incoming = {stage.name: set(stage.dependencies) for stage in self.stages} diff --git a/onsrap/loader.py b/onsrap/loader.py index 21d8277..b988bfa 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -23,9 +23,23 @@ def discover_python_entrypoint(path: Path) -> str | None: the module. That keeps discovery fast and avoids running stage code just to learn how it should be invoked. - Returns ``None`` when the file exists but does not define a preferred + Parameters + ---------- + ``path`` : Path + File path for the stage being run. + + Returns + ------- + String item containing the name of the ``PREFERRED_ENTRYPOINTS`` item relevant + for the stages. + ``None`` when the file exists but does not define a preferred callable, which signals to the executor that it should treat the file as a script-style stage instead. + + Raises + ------ + ``StageConfigurationError`` + If the file path requested for the ``Stage`` does not exist. """ file_path = Path(path) @@ -61,9 +75,23 @@ def load_python_callable(path: Path, entrypoint: str): receive the execution context. It is kept separate from module loading so the executor can reuse the same import path for multiple runtime strategies. - A ``StageConfigurationError`` is raised if the chosen entrypoint does not - exist or is not callable, because that means the stage definition and the - executable surface no longer agree. + Parameters + ---------- + ``path`` : Path + The file path for the stage being run. + ``entrypoint`` : str + The name of the entrypoint function defined in the stage script. + + Raises + ------ + ``StageConfigurationError`` + If the chosen entrypoint does not exist or is not callable, because that means + the stage definition and the executable surface no longer agree. + + Returns + ------- + ``target`` + The ``entrypoint`` attribute of the module called to run the stage. """ module = load_python_module(path) target = getattr(module, entrypoint, None) @@ -87,9 +115,23 @@ def load_python_module(path: Path) -> ModuleType: The generated name is derived from the file path so repeated loads of the same stage remain stable during a run, while still avoiding collisions with - other Python modules. Import failures are converted into ``StageLoadError`` - so callers can report a stage-specific problem rather than a raw import - exception. + other Python modules. + + Parameters + ---------- + ``path`` : Path + The path for the stage. + + Returns + ------- + ``module`` + The set of code being run for the stage. + + Raises + ------ + ``StageLoadError`` + If the file is unable to be imported so callers can report a stage-specific + problem rather than a raw import exception. """ file_path = Path(path) if not file_path.exists(): From 5c651f3d8b495e95cd948d19fa3f328ea101ea93 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:31:50 +0100 Subject: [PATCH 019/332] Update onsrap/execution.py Adding suggested changes from review stage Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 4dff349..ea0d621 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -63,7 +63,7 @@ def record(self, result: StageResult) -> StageResult: Parameters ---------- ``result`` : ``StageResult`` - An instance of a ``StageResult`` class + An instance of a ``StageResult`` class which is created from the Executor classes (StageExecutor, PythonStageExecutor). Returns ------ From 95004376675c2e454cf09fbef60180c3fb922ba4 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:36:45 +0100 Subject: [PATCH 020/332] Apply suggestion from @pikes-ons --- onsrap/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index ea0d621..ffbf2d5 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -179,7 +179,7 @@ def _execute_callable( the callable cannot be run, an error is flagged and the ``StageResult`` instance created shows a failure. If it can be run, the callable is run and the ``StageResult`` instance shows a success. Metadata is kept for - the attempt including duration, ``name``, ``outputs``, ``source``, ``mode`` + the attempt including ``duration``, ``name``, ``outputs``, ``source``, ``mode`` attempted, and ``errors``. Parameters From 2a51cc7d8588f860fd2981ca3f611450f9e316f8 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:47:15 +0100 Subject: [PATCH 021/332] Apply suggestions from code review Co-authored-by: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> --- onsrap/execution.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index ffbf2d5..d5504f5 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -175,12 +175,12 @@ def _execute_callable( """ Attempt to run a callable object. - Creates a logging instance and attempts to run the callable parsed. If - the callable cannot be run, an error is flagged and the ``StageResult`` - instance created shows a failure. If it can be run, the callable is run - and the ``StageResult`` instance shows a success. Metadata is kept for - the attempt including ``duration``, ``name``, ``outputs``, ``source``, ``mode`` - attempted, and ``errors``. + Calls the logger.event() method to record an event and attempts to + run the callable parsed. If the callable cannot be run, an error is flagged + and the ``StageResult`` instance created shows a failure. If it can be run, + the callable is run and the ``StageResult`` instance shows a success. + Metadata is kept for the attempt including ``duration``, ``name``, ``outputs``, + ``source``, ``mode`` attempted, and ``errors``. Parameters ---------- @@ -252,7 +252,15 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul """ Attempt to run a file. - Creates a logging instance and attempts to run the file parsed based on an entrypoint. + """ + Attempt to run a callable object. + + Calls the logger.event() method to record an event and attempts to run + the callable parsed. If the callable cannot be run, an error is flagged and + the ``StageResult`` instance created shows a failure. If it can be run, the + callable is run and the ``StageResult`` instance shows a success. Metadata + is kept for the attempt including ``duration``, ``name``, ``outputs``, ``source``, + ``mode`` attempted, and ``errors``. If there is no entrypoint or the entrypoint is not a callable object, an error will be raised. ``_execute_subprocess()`` method called if no entrypoint is found. A ``StageResult`` instance will be created to log the results of the ``Stage``run regardless @@ -328,6 +336,9 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> StageResult: """ Run the entire Python file for the ``Stage`` from the top. + + Not desired method. Uses black-box design and obfuscates Pipeline running. Please refer + to Wiki documentation on how to implement callable solutions instead. If the ``Stage`` source is a file but does not have a callable entrypoint, this method will run the entire script top to bottom. The results of the ``Stage`` are recorded From d15516bbb87dba8d421114f23777db88ab1f0e92 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 1 Jul 2026 16:39:24 +0100 Subject: [PATCH 022/332] Completed docstrings for pipeline.py, runner.py, run_pipeline.py, logger.py, and models.py. Few points remain to pick up later --- onsrap/logger.py | 46 ++++++++ onsrap/models.py | 231 +++++++++++++++++++++++++++++++++++++++++ onsrap/pipeline.py | 172 +++++++++++++++++++++++++++++- onsrap/run_pipeline.py | 6 +- onsrap/runner.py | 52 ++++++++++ 5 files changed, 505 insertions(+), 2 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 0eafe7f..60c5bcb 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -9,12 +9,40 @@ @dataclass class LogConfig: + """ + Data class which holds information regarding how the logs are set up. + + Parameters + ---------- + ``log_dir`` : str, default = "logs/" + The directory where all logs are stored for the Pipeline. + ``log_level`` : str, default = "INFO" + Denotes how severe the log message is. + ``logger_name`` : str, default = "onsrap" + The name of the logging system. + """ log_dir: str = "logs/" log_level: str = "INFO" logger_name: str = "onsrap" class Logger: + """ + Creates a logging system. + + This system creates a logging directory and enables writing the log messages + to both console and the logging files. It allows configurable logging levels + to adjust for severity and avoids duplicating logging messages or handlers. + If the logger is unable to write to a file, the logging continues using only + the console handler. + + Parameters + ---------- + ``log_dir`` : str or Path, default = "logs/" + The directory where you'd like your logs stored. + ``log_level`` : str, default = "INFO" + The severity of the log. + """ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.config = LogConfig(log_dir=str(log_dir), log_level=log_level) self.log_dir = Path(self.config.log_dir) @@ -39,6 +67,14 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): setattr(self._logger, "_onsrap_configured", True) def __call__(self, *args: Any, **kwargs: Any) -> None: + """ + Converts Logger instances to be callable, enabling easier implementation + of logging. + + Positional arguemnts are converted to strings and joined with spaces. + Keyword arguments are serialised as JSON and appended as structured + context. + """ message = " ".join(str(arg) for arg in args) if kwargs: context = json.dumps(kwargs, default=str, sort_keys=True) @@ -46,6 +82,16 @@ def __call__(self, *args: Any, **kwargs: Any) -> None: self._logger.info(message) def event(self, message: str, **kwargs: Any) -> None: + """ + Logs a named event with optional structured context. + + Parameters + ---------- + ``message`` : str + The main description of the event to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. + """ if kwargs: self._logger.info("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) else: diff --git a/onsrap/models.py b/onsrap/models.py index c7977b7..7b9d17d 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -8,6 +8,9 @@ class StageStatus(str, Enum): + """ + Class to hold information on how the Stage has run. + """ PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -16,6 +19,9 @@ class StageStatus(str, Enum): class PipelineStatus(str, Enum): + """ + Class to hold information on how the Pipeline has run. + """ PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -23,40 +29,110 @@ class PipelineStatus(str, Enum): def now() -> datetime: + """ + Function to extract the current time in a datetime format. + """ return datetime.now() def utcnow() -> datetime: + """ + Function to extract the current time in UTC in a datetime format. + """ return now() @dataclass class RuntimeID: + """ + Holds information regarding individual runs. + + Parameters + ---------- + ``id`` : str + The id number for the run. + ``timestamp`` : datetime + The time that the run started. + ``hash`` : str + A hashed identifier created with the combined ID and + timestamp to create a unique identifier for the run. + ``short_hash`` : str + A shortened version of the ``hash`` attribute to be used + in file names for the runs. + """ id: str timestamp: datetime hash: str short_hash: str def get_id(self) -> str: + """ + Getter function to extract the ``id`` attribute. + """ return self.id def get_timestamp(self) -> datetime: + """ + Getter function to extract the ``timestamp`` attribute. + """ return self.timestamp def get_hash(self) -> str: + """ + Getter function to extract the ``hash`` attribute. + """ return self.hash def get_short_hash(self) -> str: + """ + Getter function to extract the ``short_hash`` attribute. + """ return self.short_hash @dataclass class RAPConfig: + """ + Holds information on how the Reproducible Analytical Pipeline is + configured. + + Parameters + ---------- + ``contents`` : dict[str, Any] + Contains a dictionary of string keys to Any value pairs containing + information needed to run the Pipeline. + """ contents: dict[str, Any] = field(default_factory=dict) @dataclass class PipelineConfig: + """ + Holds information required to run the whole pipeline. + + Parameters + ---------- + ``name`` : str, optional + The name of the pipeline. + ``backend`` : str, default = "python" + The system that the pipeline is run on. + ``work_dir`` : Path + The directory to run the Pipeline in. + ``project_root`` : Path + The top level directory for the whole project. + ``log_dir`` : Path + The directory to store the logs in. + ``data_dir`` : Path + The directory where the data is stored. + ``allow_subprocess_fallback`` : bool + Indicates whether the subprocess system (running the whole file + rather than an entrypoint function) should be allowed. + ``python_executable`` : str, optional + The name of the executable function for the entrypoint of the + pipeline. + ``metadata`` : dict[str, Any] + Any additional information on the pipeline. + """ name: Optional[str] = None backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) @@ -72,6 +148,21 @@ def from_any( cls, value: Union["PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None], ) -> "PipelineConfig": + """ + Converts one of several datatypes into a PipelineConfig class instance. + + Parameters + ---------- + ``value`` : PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None + The object holding metadata on how the Pipeline should run to be converted + into a PipelineConfig class instance. + + Raises + ------ + ``TypeError`` + If the datatype for the object holding information on how the pipeline is run + is not a datatype that can be converted to a PipelineConfig. + """ if value is None: return cls() @@ -91,6 +182,19 @@ def from_any( @classmethod def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": + """ + Extracts information from a mapping datatype and returns a PipelineConfig + instance. + + Parameters + ---------- + ``data`` : Mapping[str, Any] + The information to be converted into a ``PipelineConfig`` instance. + + Returns + ------- + ``PipelineConfig`` class instance + """ payload = dict(data) metadata = payload.pop("metadata", {}) @@ -125,6 +229,30 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": @classmethod def from_file(cls, path: Path) -> "PipelineConfig": + """ + Extracts a mapping item from a file containing information about how the + pipeline should run. + + Then calls the from_mapping() method to extract the information. + + Parameters + ---------- + ``path`` : Path + The file path containing information to be converted into a PipelineConfig + instance. + + Returns + ------- + ``PipelineConfig`` class instance. + + Raises + ------ + ``FileNotFoundError`` + If the file path does not exist. + ``TypeError`` + If the file containing information about how the Pipeline runs does not + contain a mapping type. + """ config_path = Path(path).expanduser() if not config_path.exists(): raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) @@ -141,6 +269,10 @@ def from_file(cls, path: Path) -> "PipelineConfig": return cls.from_mapping(raw_config) def to_dict(self) -> dict[str, Any]: + """ + Converts attributes regarding how the pipeline runs into a dictionary and holds it in + the ``metadata`` attribute of the ``PipelineConfig`` class. + """ data = { "name": self.name, "backend": self.backend, @@ -157,6 +289,36 @@ def to_dict(self) -> dict[str, Any]: @dataclass class RunManifest: + """ + Holds metadata information about the run. + + Parameters + ---------- + ``rap_name`` : str, default = "" + The name of the Pipeline. + ``run_id`` : str, default = "" + The unique ID of the run. + ``git_commit`` : str, default = None + The git commit number for the run, indicating the exact state of the code. + ``stages_run`` : list[str] + List of the names of stages that were included in this run. + ``parameters`` : dict[str, Any] + + ``inputs`` : dict[str, Any] + + ``outputs`` : dict[str, Any] + + ``backend`` : str, default = "python" + The system that the Pipeline will run in. + ``package_versions``: list[str] or str + The package versions that are used in this run. + ``timestamp`` : str, default = "" + The time that this run started. + ``reason`` : str, optional, default = None + The reason that this run took place. + ``user`` : str, optional, default = None + The person running this specific run. + """ rap_name: str = "" run_id: str = "" git_commit: Optional[str] = None @@ -185,6 +347,35 @@ class Catalog: @dataclass class StageResult: + """ + Holds information about how the stage ran. + + Parameters + ---------- + ``name`` : str + The name of the Stage run. + ``status`` : StageStatus + The status of the run at completion. + ``started_at`` : datetime + The date and time that the Stage started. + ``finished_at`` : datetime + The date and time that the Stage finished. + ``outputs`` : Any, default = None + Captures outputs of the stage being run. + ``stdout`` : str, default = "" + Captures outputs of the stage being run. + ``stderr`` : str, default = "" + Captures any errors produced during the run. + ``return_code`` : int, optional, default = None + Indicates whether the stage has run successfully or if there + was an error. + ``metadata``: dict[str, Any] + Holds information about the Stage such as file directories. + ``error`` : str, optional, default = None + Any errors produced during the run. + ``source`` : str, optional, default = None + The name/location of the code for that Stage run. + """ name: str status: StageStatus started_at: datetime @@ -199,15 +390,42 @@ class StageResult: @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``StageResult`` class called ``succeeded`` that + contains a boolean value indicating if the run was a success or not. + Updates the ``status`` attribute to record that the Stage ran successfully. + """ return self.status == StageStatus.SUCCEEDED @property def duration_seconds(self) -> float: + """ + Creates a new attribute in the ``StageResult`` class called ``duration_seconds`` + that holds the exact duration of the stage in seconds. + """ return max((self.finished_at - self.started_at).total_seconds(), 0.0) @dataclass class PipelineRun: + """ + Holds information about how the whole Pipeline ran. + + Parameters + ---------- + ``manifest`` : RunManifest class instance + Metadata on how the specific run has gone. + ``status`` : PipelineStatus class instance + Whether the Pipeline ran successfully or if there were errors. + ``started_at`` : datetime + The date and time the Pipeline started. + ``completed_at`` : datetime + The date and time the Pipeline ended. + ``stage_results`` : list[StageResult] + Holds the results for every stage run as part of the Pipeline. + ``stage_outputs`` : dict[str, Any] + Holds the outputs from all stages run as part of the Pipeline. + """ manifest: RunManifest status: PipelineStatus started_at: datetime @@ -216,6 +434,14 @@ class PipelineRun: stage_outputs: dict[str, Any] = field(default_factory=dict) def result_for(self, stage_name: str) -> Optional[StageResult]: + """ + Extracts the results for a specific stage. + + Parameters + ---------- + ``stage_name`` : str + The name of the Stage that you are requesting the results for. + """ for result in self.stage_results: if result.name == stage_name: return result @@ -223,4 +449,9 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``PipelineRun`` class called ``succeeded`` that + contains a boolean value indicating if the Pipeline was a success or not. + Updates the ``status`` attribute to record that the Pipeline ran successfully. + """ return self.status == PipelineStatus.SUCCEEDED diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ffa75ad..7cb7c3c 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -16,7 +16,32 @@ from .stage import Stage + class Pipeline: + """ + Represents an end-to-end code run. This class brings together class instances + from other modules within the package to establish what the Pipeline is. + + Sets up the metadata, configurations, logging, and executors required to run the + Pipeline. Assigns multiple attributes including those not initialised such as, + ``id``, ``graph``, ``manifest``, and ``last_run``. These take the forms of other + classes defined in other modules within this package. + + Parameters + ---------- + ``name`` : str or None + What the pipeline is called. + ``backend`` : str, default = "python" + The system used to run the pipeline. + ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + The instance containing the required information on running the Pipeline. + ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. + The required steps within the Pipeline. + ``logger`` : Logger or None + The system that is used to track the progress of the Pipeline. + ``executor`` : StageExecutor or None + The way that the Pipeline is actively run. + """ def __init__( self, name: str | None = None, @@ -52,6 +77,28 @@ def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], ) -> Stage: + """ + Extracts the ``Stage`` information from the provided stages in the Pipeline. + + Enables mappings, strings, paths, or callables to be parsed and converted into + a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return + itself. + + Parameters + ---------- + ``stage`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The information attempting to be converted into a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the information parsed is not in a suitable format to be converted into + a ``Stage`` class instance. + + Returns + ------- + ``Stage`` class instance for the stage being run. + """ if isinstance(stage, Stage): return stage @@ -67,18 +114,42 @@ def _coerce_stage( raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") def _rebuild_graph(self) -> None: + """ + Updates the ``graph`` attribute with the latest stage information. + """ self.graph = StageGraph.from_stages(self.stages) def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: + """ + Adds a step to the Pipeline. + + Creates a list called ``added_stages`` that runs the _coerce_stage() method + to extract the information from the given ``stages`` parameter. It then appends + this list to the ``stages`` attribute of the ``Pipeline`` class and updates the + StageGraph using the _rebuild_graph() method. A log instance is created to + reflect the changes. + + Parameters + ---------- + ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The new steps being added to the Pipeline. + """ added_stages = [self._coerce_stage(stage) for stage in stages] self.stages.extend(added_stages) self._rebuild_graph() self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) def ordered_stages(self) -> list[Stage]: + """ + Runs the topological_order() method on the ``graph`` attribute to extract the + correct order for the ``stages`` to be run in. + """ return self.graph.topological_order() def validate(self) -> "Pipeline": + """ + Confirms that the source files for the stage exist. + """ self.logger.event("Validating pipeline", name=self.name) for stage in self.stages: stage.validate() @@ -86,11 +157,24 @@ def validate(self) -> "Pipeline": return self def run(self) -> PipelineRun: + """ + Returns an instance of ``PipelineRunner`` which actually runs the pipeline. + """ from .runner import PipelineRunner - + return PipelineRunner(logger=self.logger).run(self) def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: + """ + Creates a ``RunManifest`` instance that contains the information about + the run of the Pipeline. + + Parameters + ---------- + ``runtime_id`` : RuntimeID + Contains information about the run to be extracted and placed into + the ``RunManifest`` instance. + """ return RunManifest( rap_name=self.name, run_id=runtime_id.get_id(), @@ -107,6 +191,12 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: ) def _create_runtime_id(self) -> RuntimeID: + """ + Creates a RuntimeID instance for the specific run. + + Establishes attributes for this specific run and returns + as a ``RuntimeID`` instance. + """ current_time = now() digest = hashlib.sha256(f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8")).hexdigest() short_hash = digest[:8] @@ -118,6 +208,18 @@ def _create_runtime_id(self) -> RuntimeID: ) def _discover_git_commit(self) -> str | None: + """ + Finds the specific version of the repository used for this run. + + Attempts to run a Git command to establish the current git commit hash + to be held in the ``RunManifest`` instance for this run. + + Returns + ------- + ``OSError`` + If Git is unable to be loaded or the Git command cannot be run for + another reason. + """ try: completed = subprocess.run( ["git", "rev-parse", "HEAD"], @@ -132,6 +234,14 @@ def _discover_git_commit(self) -> str | None: return commit or None def _package_versions(self) -> list[str]: + """ + Creates a list of packages and their versions used in this run. + + Raises + ------ + ``importlib_metadata.PackageNotFoundError`` + If the package used cannot be found in the library. + """ versions = [f"python={sys.version.split()[0]}"] try: versions.append(f"pyyaml={importlib_metadata.version('PyYAML')}") @@ -140,6 +250,10 @@ def _package_versions(self) -> list[str]: return versions def _current_user(self) -> str | None: + """ + Extracts the username for the individual completing the run. Returns a blank + value if the username cannot be extracted. + """ try: return getpass.getuser() except Exception: @@ -157,6 +271,32 @@ def from_files( logger: Logger | None = None, executor: StageExecutor | None = None, ) -> "Pipeline": + """ + Extracts the information from files regarding exactly what is being run in the pipeline and + allows for configuration of how the Pipeline is run. + + Parameters + ---------- + ``file_paths`` : Iterable[str or Path] + The files that contain the code for each stage in the pipeline. These are what + the Pipeline will run. + ``name`` : str + The name of the pipeline. + ``backend`` : str, default = "python" + The system that the pipeline is written in. + ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + The high level information required to run this specific pipeline. + ``dependencies`` : Mapping[str, Sequence[str]] or None + An object containing which stages are required to be run before other stages. + ``logger`` : Logger class or None + The logging sysem used for this Pipeline run. + ``executor`` : StageExecutor class or None + The information on exactly how to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ stages: list[Stage] = [] for position, file_path in enumerate(file_paths): path = Path(file_path) @@ -182,6 +322,19 @@ def from_files( @classmethod def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": + """ + Extracts information from a dictionary to configure a Pipeline instance as + well as what the Pipeline runs. + + Parameters + ---------- + ``cfg`` : Mapping[str, Any] + The Mapping item that contains the information needed to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ payload = dict(cfg) name = payload.pop("name", None) @@ -209,6 +362,23 @@ def _dependencies_for_stage( path: Path, dependencies: Mapping[str, Sequence[str]] | None, ) -> tuple[str, ...]: + """ + Extracts a tuple of ``dependencies`` for the requested stage. + + Will return a blank tuple if there are no ``dependencies`` for the requested + stage. Allows for ``dependencies`` to be found regardless of how the stage is + referenced in the ``dependencies`` mapping. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage that you are extracting the ``dependencies`` for. + ``path`` : Path + The filepath for the stage source. + ``dependencies`` : Mapping[str, Sequence[str]] or None + Mapping of the stage source name to their relevant ``dependencies`` (stages + required to run before the ``stage_name`` Stage). + """ if not dependencies: return () diff --git a/onsrap/run_pipeline.py b/onsrap/run_pipeline.py index 842be5f..f26acff 100644 --- a/onsrap/run_pipeline.py +++ b/onsrap/run_pipeline.py @@ -2,6 +2,10 @@ from .runner import main - +""" +This line of code establishes the function that must be called +for the Pipeline to run. main() is parsed to SystemExit as once +the main() function is run, this will then exit the system. +""" if __name__ == "__main__": raise SystemExit(main()) \ No newline at end of file diff --git a/onsrap/runner.py b/onsrap/runner.py index bbbb244..fa5c876 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -14,10 +14,39 @@ class PipelineRunner: + """ + Represents the information required to run the Pipeline. + + Parameters + ---------- + ``logger`` : Logger class type + Information used to log progress throughout the Pipeline. + """ def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() def run(self, pipeline: "Pipeline") -> PipelineRun: + """ + Method that runs a ``Pipeline`` instance. + + This method validates the source information, establishes the directories and + the context to run the pipeline within, sets out the manifest for the run, attempts + to run the stages in the order outlined by the ``StageGraph`` instance and logs all + progress alongside relevant statuses. + + It returns a PipelineRun instance containing metadata and logging information for the + specific run of the whole Pipeline. + + Parameters + ---------- + ``pipeline`` : Pipeline + A Pipeline instance that this method will run. + + Raises + ------ + ``StageExecutionError`` + If the stage is unable to be run. Logs will be created to show a failed stage. + """ pipeline.validate() runtime_id = pipeline._create_runtime_id() @@ -108,6 +137,11 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: def build_parser() -> argparse.ArgumentParser: + """ + Determines what arguments are needed when running a Pipeline from the command line. + + Enables stages to be input, followed by a name if provided. + """ parser = argparse.ArgumentParser(description="Run an onsrap pipeline from Python files.") parser.add_argument("stages", nargs="+", help="One or more Python stage files to run.") parser.add_argument("--name", default=None, help="Optional pipeline name.") @@ -115,6 +149,24 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: + """ + Entrypoint to the pipeline. + + This function can be called from the command line. It builds a parser which enables + the arguments to be held before using those arguments to build a Pipeline instance. + The pipeline.run() method is then run which runs the entire pipeline. If this runs + successfully, a 0 is returned which is the success code. + + Parameters + ---------- + ``argv`` : list[str] or None + Command line arguments to parse. + + Returns + ------- + int + Success code for completion of the run. + """ from .pipeline import Pipeline args = build_parser().parse_args(argv) From 37c4e2ad8b1fd2951bd51c861a1a07a20f94598c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 3 Jul 2026 11:05:06 +0100 Subject: [PATCH 023/332] Script for testing stage.py and minor change to documentation in execution.py --- onsrap/execution.py | 2 - tests/test_stage.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 tests/test_stage.py diff --git a/onsrap/execution.py b/onsrap/execution.py index d5504f5..a0a7396 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -251,8 +251,6 @@ def _execute_callable( def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: """ Attempt to run a file. - - """ Attempt to run a callable object. Calls the logger.event() method to record an event and attempts to run diff --git a/tests/test_stage.py b/tests/test_stage.py new file mode 100644 index 0000000..5213945 --- /dev/null +++ b/tests/test_stage.py @@ -0,0 +1,182 @@ +import pytest +from onsrap.stage import _normalize_dependencies, Stage, StageConfigurationError +from pathlib import Path +from textwrap import dedent + +def test_normalize_dependencies_none() -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + +def test_normalize_dependencies_str() -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py"," Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py","Stage_2.py", "Stage_3.py") + +@pytest.fixture +def example_function(): + """ + Test function to pass as a callable stage for stage testing. + """ + print("This is a test function") + +@pytest.fixture +def stage_test() -> Stage: + """ + Stage object for testing Stage class methods and construction. + """ + return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) + +def test_stage_creation_callable(stage_test) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info":"example"} + assert stage_test.entrypoint == None + assert stage_test.backend == "python" + +def test_stage_name_error(example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("",example_function,["stage_1"],{"info":"example"}) + +def test_stage_source_type() -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + +def test_stage_backend(example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + """ + stage_diff = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "java") + stage = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "") + stage_white_space = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "python ") + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + +def test_stage_from_files_error(tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + stage = Stage.from_file(source_file) + +def test_stage_from_callable_name() -> None: + """ + Tests that a stage name is extracted from a callable object stage. + """ + def example_function(): + pass + test = Stage.from_callable(example_function) + assert test.name == "example_function" + +@pytest.mark.skip +def test_from_dict_norm() -> None: + """ + REVIEW WITH ALEX + """ + def example_function(): + pass + data = {"name":"test_Stage", + "callable_source" : example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + +def test_with_dependencies_list(stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. + + REVIEW WITH ALEX + This test works and passes however it doesn't behave how I was expecting it to. + Was expecting the list/dictionary to be broken down so you have one tuple rather + than a tuple of dict/lists. Is this a problem or just my understanding? + """ + new_deps = ["stage2","stage3"] + new_dep_dict = {"stage1":"stage0"} + new_deps_blank = [] + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_dict = stage_test.with_dependencies(new_dep_dict) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1","['stage2', 'stage3']") + assert stage_test_dict.dependencies == ("stage_1","{'stage1': 'stage0'}") + assert stage_test_blank.dependencies == ("stage_1", '[]') + +@pytest.mark.skip +def test_validate(stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + REVIEW WITH ALEX + Does not raise a StageConfigurationError is the source is a blank string. Is + this a concern? Do we want this validate to be able to do other error checks + like if it is an int? + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + +def test_source_path(stage_test, tmp_path) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source types. + """ + stage_test.source = tmp_path/"fake_file.py" + assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = 11 + assert stage_test.source_path == None + stage_test.source = "not a file path" + assert stage_test.source_path == None + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_path == None + +def test_source_label(stage_test, tmp_path) -> None: + """ + Tests that source_label is created if the source is a Path or a callable and is None if it is + another type. + """ + stage_test.source = tmp_path/"fake_file.py" + temp_path_str = str(tmp_path/"fake_file.py") + assert stage_test.source_label == temp_path_str + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label == None + +""" +TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST +""" \ No newline at end of file From c1ad8f57cb97ff269f9404cc815aa5185a35bda0 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 11:08:51 +0100 Subject: [PATCH 024/332] Partial testing established for execution.py --- tests/test_execution.py | 121 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_execution.py diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 0000000..143f55b --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,121 @@ +from onsrap.execution import ExecutionContext +from onsrap.models import PipelineConfig, StageResult, StageStatus +from onsrap.logger import Logger +from pathlib import Path +import pytest + +@pytest.fixture +def logger() -> Logger: + """ + Logger instance for testing + """ + return Logger() + +@pytest.fixture +def config(tmp_path) -> PipelineConfig: + """ + Return a PipelineConfig object for testing. + """ + work_dir = tmp_path/"work" + project_root = tmp_path/"project" + log_dir = tmp_path/"log" + data_dir = tmp_path/"data" + return PipelineConfig( + "test_pipeline", + "python", + work_dir, + project_root, + log_dir, + data_dir, + True, + None, + {} + ) + +@pytest.fixture +def execution(config, logger, tmp_path) -> ExecutionContext: + """ + Create an ExecutionContext object for testing. + """ + run_dir = tmp_path/"run" + work_dir = tmp_path/"work_dir" + + return ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + run_dir, + '2024-05-06 15:45:30', + work_dir, + {}, + {} + ) + +@pytest.fixture +def stageresult() -> StageResult: + """ + Test StageResult instance for running ExecutionContext tests. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = "example output" + ) + + +def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: + """ + Test that the ExecutionContext creates the right attributes. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == tmp_path/"run" + assert execution.started_at == '2024-05-06 15:45:30' + assert execution.working_directory == tmp_path/"work_dir" + assert execution.stage_results == {} + assert execution.variables == {} + +def test_record(stageresult, execution) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + """ + execution.record(stageresult) + assert execution.stage_results == {'stage_test':StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None)} + assert execution.variables == {'stage_test':"example output"} + +def test_result_for(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.result_for("stage_test") == StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None) + +def test_stage_outputs(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test":"example output"} + +"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file From 8bf21b8241eeb5ed84701853e5dea1e3d29b1e53 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 17:15:52 +0100 Subject: [PATCH 025/332] Add class methods to ExecutionContext that allow file paths to be created/extracted for I/O --- .../pipeline_1/scripts/0_data_validation.py | 18 +--- .../pipeline_1/scripts/1_preprocessing.py | 32 +------- examples/pipeline_1/scripts/2_reporting.py | 33 ++------ onsrap/execution.py | 82 +++++++++++++++++++ tests/test_execution.py | 2 +- 5 files changed, 94 insertions(+), 73 deletions(-) diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index eb331cc..4a586ee 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -17,20 +17,6 @@ ) -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -99,8 +85,8 @@ def write_report(report_path: Path, report: dict[str, object]) -> None: def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index 0d8ad1c..a55180c 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_raw_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - validation_result = context.result_for("0_data_validation") - if validation_result is not None: - raw_path = validation_result.outputs.get("raw_path") - if raw_path: - return Path(raw_path) - - return data_root / "orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -112,9 +87,10 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - raw_path = resolve_raw_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + raw_path = context.resolve_given_path("0_data_validation", "raw_path", + "orders.csv", data_root) clean_path = output_root / "interim" / "1_clean_orders.csv" rows = load_orders(raw_path) diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index 337c2e5..c3f50a8 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_clean_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - preprocessing_result = context.result_for("1_preprocessing") - if preprocessing_result is not None: - clean_path = preprocessing_result.outputs.get("clean_path") - if clean_path: - return Path(clean_path) - - return data_root / "interim" / "1_clean_orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -94,9 +69,11 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - clean_path = resolve_clean_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + clean_path = context.resolve_given_path("1_preprocessing", "clean_path", + "1_clean_orders.csv", output_root, + "interim") summary_path = output_root / "processed" / "2_sales_summary.json" region_breakdown_path = output_root / "processed" / "2_revenue_by_region.csv" diff --git a/onsrap/execution.py b/onsrap/execution.py index a0a7396..fca75d0 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -105,6 +105,88 @@ def stage_outputs(self) -> dict[str, Any]: the run. """ return {name: result.outputs for name, result in self.stage_results.items()} + + def resolve_data_root(self, config: PipelineConfig | None) -> Path: + """ + Establishes the filepath that the data is held in. + + Parameters + ---------- + ``config`` : PipelineConfig + The configuration of the Pipeline being run. This holds the location filepath + for the pipeline as defined by the user in the main.py file. + + Returns + ------- + Path + The file path for the location of the data being used in the pipeline. + """ + if config is not None: + return Path(config.data_dir) + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_output_root(self, run_dir: Path | None) -> Path: + """ + Establishes the filepath that the outputs are going to be saved to. + + Parameters + ---------- + ``run_dir`` : Path + The file directory that the run results are saved to. + + Returns + ------- + Path + The file path for the outputs of the run to be saved to. + """ + if run_dir is not None: + return Path(run_dir) / "data" + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_given_path(self, stage_name: str, + path_name: str, + file_name:str, + root: Path, + add_folder: str | None = None) -> Path: + """ + Returns a file path for a requested item. + + This investigates the result of a previous stage to extract a selected path. + If the path is not available, it creates a path using a root previously derived + in main.py, the chosen directory within the root (optional), and the file path. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage where the path was outputted. + ``path_name`` : str + The name for the path within the stage results. This will be the key from the + key/value pair within the output of the previous stage. + ``root`` : Path + The file path for the root of the directory. This should be denoted through + other methods. + ``add_folder`` : str | None, default = None + Additional folder name to add into the returned file path. Additional functionality + should be added to allow for multiple folders to be added to the path. + + Returns + ------- + Path + The file path where data has previously been saved to to allow for extraction of + that data throughout the pipeline. + """ + result = self.result_for(stage_name) + if result is not None: + selected_path = result.outputs.get(path_name) + if selected_path: + return Path(selected_path) + if add_folder is not None: + #Would like to add functionality here for multiple additional folders + return root / add_folder / file_name + + return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 143f55b..7fc87b0 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -118,4 +118,4 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file +"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file From bb7d83bd21ab414c046a1ff831bdb6742eb805a6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 12:31:38 +0100 Subject: [PATCH 026/332] Minor testing implemented --- onsrap/execution.py | 12 ++++++----- tests/test_execution.py | 46 ++++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index fca75d0..050fc0d 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -149,7 +149,7 @@ def resolve_given_path(self, stage_name: str, path_name: str, file_name:str, root: Path, - add_folder: str | None = None) -> Path: + add_folder: list[str] | str | None = None) -> Path: """ Returns a file path for a requested item. @@ -167,9 +167,8 @@ def resolve_given_path(self, stage_name: str, ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. - ``add_folder`` : str | None, default = None - Additional folder name to add into the returned file path. Additional functionality - should be added to allow for multiple folders to be added to the path. + ``add_folder`` : list[str] | str | None, default = None + Additional folder name/s to add into the returned file path. Returns ------- @@ -183,7 +182,10 @@ def resolve_given_path(self, stage_name: str, if selected_path: return Path(selected_path) if add_folder is not None: - #Would like to add functionality here for multiple additional folders + if add_folder is list: + add_folder = add_folder.append(file_name) + new_path = root.joinpath(*add_folder) + return new_path return root / add_folder / file_name return root / file_name diff --git a/tests/test_execution.py b/tests/test_execution.py index 7fc87b0..756a3e2 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -12,14 +12,14 @@ def logger() -> Logger: return Logger() @pytest.fixture -def config(tmp_path) -> PipelineConfig: +def config() -> PipelineConfig: """ Return a PipelineConfig object for testing. """ - work_dir = tmp_path/"work" - project_root = tmp_path/"project" - log_dir = tmp_path/"log" - data_dir = tmp_path/"data" + work_dir = Path('tmp/work_dir') + project_root = Path('tmp/project') + log_dir = Path('tmp/log') + data_dir = "tmp/config_data" return PipelineConfig( "test_pipeline", "python", @@ -33,12 +33,12 @@ def config(tmp_path) -> PipelineConfig: ) @pytest.fixture -def execution(config, logger, tmp_path) -> ExecutionContext: +def execution(config, logger) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ - run_dir = tmp_path/"run" - work_dir = tmp_path/"work_dir" + run_dir = Path("tmp/run") + work_dir = Path('tmp/work_dir') return ExecutionContext( "test_pipeline", @@ -67,7 +67,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -75,9 +75,9 @@ def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: assert execution.run_id == "run_id_1234" assert execution.config == config assert execution.logger == logger - assert execution.run_dir == tmp_path/"run" + assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' - assert execution.working_directory == tmp_path/"work_dir" + assert execution.working_directory == Path('tmp/work_dir') assert execution.stage_results == {} assert execution.variables == {} @@ -101,6 +101,9 @@ def test_record(stageresult, execution) -> None: assert execution.variables == {'stage_test':"example output"} def test_result_for(execution, stageresult) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + """ execution.record(stageresult) assert execution.result_for("stage_test") == StageResult(name='stage_test', status='pending', @@ -115,7 +118,26 @@ def test_result_for(execution, stageresult) -> None: source=None) def test_stage_outputs(execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + """ execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file +def test_resolve_data_root(execution) -> None: + """ + Tests that resolve_data_root method extracts the path from the execution context + or, if the context is None, returns the file path for the module itself and the + data directory within that. + """ + assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") + + import onsrap.execution as execution_module + result = execution.resolve_data_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + From 799d9d1cb17d2ba334312b38b453d4dfb7ca6ef9 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 29 Jun 2026 11:16:14 +0100 Subject: [PATCH 027/332] documenting: graph.py and loader.py docstrings updated --- onsrap/graph.py | 41 +++++++++++++++++++++++++++++++---- onsrap/loader.py | 56 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/onsrap/graph.py b/onsrap/graph.py index c5da9f2..b7fb618 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -9,16 +9,42 @@ @dataclass class StageGraph: + """ + Represents an order to run stages. + + Holds an order that stages need to run in based on dependencies and logic. + + Parameters + ---------- + ``stages`` : list of ``Stage`` class items + """ stages: list[Stage] = field(default_factory=list) @classmethod def from_stages(cls, stages: Iterable[Stage]) -> "StageGraph": - """Primary constructor for :class:`StageGraph` that normalizes the stage list.""" + """ + This is the primary constructor for StageGraph, which performs validation and normalization of the stage list. + + Parameters + ---------- + ``stages`` : Iterable of ``Stage`` class instances + + Returns + ------- + The ``stages`` parameter as a list. + """ return cls(list(stages)) def validate(self) -> None: """ Validate the stage graph for issues such as duplicate stage names, missing dependencies, and cycles. + + Raises + ------ + ``DuplicateStageError`` + If the stage name appears multiple times in the stage list. + ``MissingDependencyError`` + If there are unknown dependencies. """ # Check for duplicate stages @@ -68,9 +94,16 @@ def topological_order(self) -> list[Stage]: stages that depend on it. That may free up more stages, which are then added to the ready list. - If the algorithm cannot place every stage, the graph contains either a - cycle or a dependency that could not be resolved. In that case a - ``DependencyCycleError`` is raised. + Returns + ------- + A list of stages ordered in the way that they need to be run through the + pipeline. + + Raises + ------ + ``DependencyCycleError`` + If the algorithm cannot place every stage, the graph contains either a + cycle or a dependency that could not be resolved. """ stage_by_name = {stage.name: stage for stage in self.stages} incoming = {stage.name: set(stage.dependencies) for stage in self.stages} diff --git a/onsrap/loader.py b/onsrap/loader.py index 21d8277..b988bfa 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -23,9 +23,23 @@ def discover_python_entrypoint(path: Path) -> str | None: the module. That keeps discovery fast and avoids running stage code just to learn how it should be invoked. - Returns ``None`` when the file exists but does not define a preferred + Parameters + ---------- + ``path`` : Path + File path for the stage being run. + + Returns + ------- + String item containing the name of the ``PREFERRED_ENTRYPOINTS`` item relevant + for the stages. + ``None`` when the file exists but does not define a preferred callable, which signals to the executor that it should treat the file as a script-style stage instead. + + Raises + ------ + ``StageConfigurationError`` + If the file path requested for the ``Stage`` does not exist. """ file_path = Path(path) @@ -61,9 +75,23 @@ def load_python_callable(path: Path, entrypoint: str): receive the execution context. It is kept separate from module loading so the executor can reuse the same import path for multiple runtime strategies. - A ``StageConfigurationError`` is raised if the chosen entrypoint does not - exist or is not callable, because that means the stage definition and the - executable surface no longer agree. + Parameters + ---------- + ``path`` : Path + The file path for the stage being run. + ``entrypoint`` : str + The name of the entrypoint function defined in the stage script. + + Raises + ------ + ``StageConfigurationError`` + If the chosen entrypoint does not exist or is not callable, because that means + the stage definition and the executable surface no longer agree. + + Returns + ------- + ``target`` + The ``entrypoint`` attribute of the module called to run the stage. """ module = load_python_module(path) target = getattr(module, entrypoint, None) @@ -87,9 +115,23 @@ def load_python_module(path: Path) -> ModuleType: The generated name is derived from the file path so repeated loads of the same stage remain stable during a run, while still avoiding collisions with - other Python modules. Import failures are converted into ``StageLoadError`` - so callers can report a stage-specific problem rather than a raw import - exception. + other Python modules. + + Parameters + ---------- + ``path`` : Path + The path for the stage. + + Returns + ------- + ``module`` + The set of code being run for the stage. + + Raises + ------ + ``StageLoadError`` + If the file is unable to be imported so callers can report a stage-specific + problem rather than a raw import exception. """ file_path = Path(path) if not file_path.exists(): From e805aa75638251dfa2f65d929ed09294e6b1ef89 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 1 Jul 2026 16:39:24 +0100 Subject: [PATCH 028/332] Completed docstrings for pipeline.py, runner.py, run_pipeline.py, logger.py, and models.py. Few points remain to pick up later --- onsrap/logger.py | 46 ++++++++ onsrap/models.py | 235 +++++++++++++++++++++++++++++++++++++++++ onsrap/pipeline.py | 172 +++++++++++++++++++++++++++++- onsrap/run_pipeline.py | 11 ++ onsrap/runner.py | 52 +++++++++ 5 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 onsrap/run_pipeline.py diff --git a/onsrap/logger.py b/onsrap/logger.py index 8c4a857..fd97dee 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -9,12 +9,40 @@ @dataclass class LogConfig: + """ + Data class which holds information regarding how the logs are set up. + + Parameters + ---------- + ``log_dir`` : str, default = "logs/" + The directory where all logs are stored for the Pipeline. + ``log_level`` : str, default = "INFO" + Denotes how severe the log message is. + ``logger_name`` : str, default = "onsrap" + The name of the logging system. + """ log_dir: str = "logs/" log_level: str = "INFO" logger_name: str = "onsrap" class Logger: + """ + Creates a logging system. + + This system creates a logging directory and enables writing the log messages + to both console and the logging files. It allows configurable logging levels + to adjust for severity and avoids duplicating logging messages or handlers. + If the logger is unable to write to a file, the logging continues using only + the console handler. + + Parameters + ---------- + ``log_dir`` : str or Path, default = "logs/" + The directory where you'd like your logs stored. + ``log_level`` : str, default = "INFO" + The severity of the log. + """ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.config = LogConfig(log_dir=str(log_dir), log_level=log_level) self.log_dir = Path(self.config.log_dir) @@ -40,6 +68,14 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): setattr(self._logger, "_onsrap_configured", True) def __call__(self, *args: Any, **kwargs: Any) -> None: + """ + Converts Logger instances to be callable, enabling easier implementation + of logging. + + Positional arguemnts are converted to strings and joined with spaces. + Keyword arguments are serialised as JSON and appended as structured + context. + """ message = " ".join(str(arg) for arg in args) if kwargs: context = json.dumps(kwargs, default=str, sort_keys=True) @@ -47,6 +83,16 @@ def __call__(self, *args: Any, **kwargs: Any) -> None: self._logger.info(message) def event(self, message: str, **kwargs: Any) -> None: + """ + Logs a named event with optional structured context. + + Parameters + ---------- + ``message`` : str + The main description of the event to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. + """ if kwargs: self._logger.info("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) else: diff --git a/onsrap/models.py b/onsrap/models.py index 755dcc7..92b1aec 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -8,6 +8,9 @@ class StageStatus(str, Enum): + """ + Class to hold information on how the Stage has run. + """ PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -16,6 +19,9 @@ class StageStatus(str, Enum): class PipelineStatus(str, Enum): + """ + Class to hold information on how the Pipeline has run. + """ PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -23,36 +29,110 @@ class PipelineStatus(str, Enum): def now() -> datetime: + """ + Function to extract the current time in a datetime format. + """ return datetime.now() +def utcnow() -> datetime: + """ + Function to extract the current time in UTC in a datetime format. + """ + return now() + + @dataclass class RuntimeID: + """ + Holds information regarding individual runs. + + Parameters + ---------- + ``id`` : str + The id number for the run. + ``timestamp`` : datetime + The time that the run started. + ``hash`` : str + A hashed identifier created with the combined ID and + timestamp to create a unique identifier for the run. + ``short_hash`` : str + A shortened version of the ``hash`` attribute to be used + in file names for the runs. + """ id: str timestamp: datetime hash: str short_hash: str def get_id(self) -> str: + """ + Getter function to extract the ``id`` attribute. + """ return self.id def get_timestamp(self) -> datetime: + """ + Getter function to extract the ``timestamp`` attribute. + """ return self.timestamp def get_hash(self) -> str: + """ + Getter function to extract the ``hash`` attribute. + """ return self.hash def get_short_hash(self) -> str: + """ + Getter function to extract the ``short_hash`` attribute. + """ return self.short_hash @dataclass class RAPConfig: + """ + Holds information on how the Reproducible Analytical Pipeline is + configured. + + Parameters + ---------- + ``contents`` : dict[str, Any] + Contains a dictionary of string keys to Any value pairs containing + information needed to run the Pipeline. + """ contents: dict[str, Any] = field(default_factory=dict) @dataclass class PipelineConfig: + """ + Holds information required to run the whole pipeline. + + Parameters + ---------- + ``name`` : str, optional + The name of the pipeline. + ``backend`` : str, default = "python" + The system that the pipeline is run on. + ``work_dir`` : Path + The directory to run the Pipeline in. + ``project_root`` : Path + The top level directory for the whole project. + ``log_dir`` : Path + The directory to store the logs in. + ``data_dir`` : Path + The directory where the data is stored. + ``allow_subprocess_fallback`` : bool + Indicates whether the subprocess system (running the whole file + rather than an entrypoint function) should be allowed. + ``python_executable`` : str, optional + The name of the executable function for the entrypoint of the + pipeline. + ``metadata`` : dict[str, Any] + Any additional information on the pipeline. + """ name: Optional[str] = None backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) @@ -68,6 +148,21 @@ def from_any( cls, value: Union["PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None], ) -> "PipelineConfig": + """ + Converts one of several datatypes into a PipelineConfig class instance. + + Parameters + ---------- + ``value`` : PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None + The object holding metadata on how the Pipeline should run to be converted + into a PipelineConfig class instance. + + Raises + ------ + ``TypeError`` + If the datatype for the object holding information on how the pipeline is run + is not a datatype that can be converted to a PipelineConfig. + """ if value is None: return cls() @@ -87,6 +182,19 @@ def from_any( @classmethod def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": + """ + Extracts information from a mapping datatype and returns a PipelineConfig + instance. + + Parameters + ---------- + ``data`` : Mapping[str, Any] + The information to be converted into a ``PipelineConfig`` instance. + + Returns + ------- + ``PipelineConfig`` class instance + """ payload = dict(data) metadata = payload.pop("metadata", {}) @@ -121,6 +229,30 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": @classmethod def from_file(cls, path: Path) -> "PipelineConfig": + """ + Extracts a mapping item from a file containing information about how the + pipeline should run. + + Then calls the from_mapping() method to extract the information. + + Parameters + ---------- + ``path`` : Path + The file path containing information to be converted into a PipelineConfig + instance. + + Returns + ------- + ``PipelineConfig`` class instance. + + Raises + ------ + ``FileNotFoundError`` + If the file path does not exist. + ``TypeError`` + If the file containing information about how the Pipeline runs does not + contain a mapping type. + """ config_path = Path(path).expanduser() if not config_path.exists(): raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) @@ -137,6 +269,10 @@ def from_file(cls, path: Path) -> "PipelineConfig": return cls.from_mapping(raw_config) def to_dict(self) -> dict[str, Any]: + """ + Converts attributes regarding how the pipeline runs into a dictionary and holds it in + the ``metadata`` attribute of the ``PipelineConfig`` class. + """ data = { "name": self.name, "backend": self.backend, @@ -153,6 +289,36 @@ def to_dict(self) -> dict[str, Any]: @dataclass class RunManifest: + """ + Holds metadata information about the run. + + Parameters + ---------- + ``rap_name`` : str, default = "" + The name of the Pipeline. + ``run_id`` : str, default = "" + The unique ID of the run. + ``git_commit`` : str, default = None + The git commit number for the run, indicating the exact state of the code. + ``stages_run`` : list[str] + List of the names of stages that were included in this run. + ``parameters`` : dict[str, Any] + + ``inputs`` : dict[str, Any] + + ``outputs`` : dict[str, Any] + + ``backend`` : str, default = "python" + The system that the Pipeline will run in. + ``package_versions``: list[str] or str + The package versions that are used in this run. + ``timestamp`` : str, default = "" + The time that this run started. + ``reason`` : str, optional, default = None + The reason that this run took place. + ``user`` : str, optional, default = None + The person running this specific run. + """ rap_name: str = "" run_id: str = "" git_commit: Optional[str] = None @@ -181,6 +347,35 @@ class Catalog: @dataclass class StageResult: + """ + Holds information about how the stage ran. + + Parameters + ---------- + ``name`` : str + The name of the Stage run. + ``status`` : StageStatus + The status of the run at completion. + ``started_at`` : datetime + The date and time that the Stage started. + ``finished_at`` : datetime + The date and time that the Stage finished. + ``outputs`` : Any, default = None + Captures outputs of the stage being run. + ``stdout`` : str, default = "" + Captures outputs of the stage being run. + ``stderr`` : str, default = "" + Captures any errors produced during the run. + ``return_code`` : int, optional, default = None + Indicates whether the stage has run successfully or if there + was an error. + ``metadata``: dict[str, Any] + Holds information about the Stage such as file directories. + ``error`` : str, optional, default = None + Any errors produced during the run. + ``source`` : str, optional, default = None + The name/location of the code for that Stage run. + """ name: str status: StageStatus started_at: datetime @@ -195,15 +390,42 @@ class StageResult: @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``StageResult`` class called ``succeeded`` that + contains a boolean value indicating if the run was a success or not. + Updates the ``status`` attribute to record that the Stage ran successfully. + """ return self.status == StageStatus.SUCCEEDED @property def duration_seconds(self) -> float: + """ + Creates a new attribute in the ``StageResult`` class called ``duration_seconds`` + that holds the exact duration of the stage in seconds. + """ return max((self.finished_at - self.started_at).total_seconds(), 0.0) @dataclass class PipelineRun: + """ + Holds information about how the whole Pipeline ran. + + Parameters + ---------- + ``manifest`` : RunManifest class instance + Metadata on how the specific run has gone. + ``status`` : PipelineStatus class instance + Whether the Pipeline ran successfully or if there were errors. + ``started_at`` : datetime + The date and time the Pipeline started. + ``completed_at`` : datetime + The date and time the Pipeline ended. + ``stage_results`` : list[StageResult] + Holds the results for every stage run as part of the Pipeline. + ``stage_outputs`` : dict[str, Any] + Holds the outputs from all stages run as part of the Pipeline. + """ manifest: RunManifest status: PipelineStatus started_at: datetime @@ -212,6 +434,14 @@ class PipelineRun: stage_outputs: dict[str, Any] = field(default_factory=dict) def result_for(self, stage_name: str) -> Optional[StageResult]: + """ + Extracts the results for a specific stage. + + Parameters + ---------- + ``stage_name`` : str + The name of the Stage that you are requesting the results for. + """ for result in self.stage_results: if result.name == stage_name: return result @@ -219,4 +449,9 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``PipelineRun`` class called ``succeeded`` that + contains a boolean value indicating if the Pipeline was a success or not. + Updates the ``status`` attribute to record that the Pipeline ran successfully. + """ return self.status == PipelineStatus.SUCCEEDED diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0d5bd32..14daeaf 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -16,7 +16,32 @@ from .stage import Stage + class Pipeline: + """ + Represents an end-to-end code run. This class brings together class instances + from other modules within the package to establish what the Pipeline is. + + Sets up the metadata, configurations, logging, and executors required to run the + Pipeline. Assigns multiple attributes including those not initialised such as, + ``id``, ``graph``, ``manifest``, and ``last_run``. These take the forms of other + classes defined in other modules within this package. + + Parameters + ---------- + ``name`` : str or None + What the pipeline is called. + ``backend`` : str, default = "python" + The system used to run the pipeline. + ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + The instance containing the required information on running the Pipeline. + ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. + The required steps within the Pipeline. + ``logger`` : Logger or None + The system that is used to track the progress of the Pipeline. + ``executor`` : StageExecutor or None + The way that the Pipeline is actively run. + """ def __init__( self, name: str | None = None, @@ -52,6 +77,28 @@ def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], ) -> Stage: + """ + Extracts the ``Stage`` information from the provided stages in the Pipeline. + + Enables mappings, strings, paths, or callables to be parsed and converted into + a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return + itself. + + Parameters + ---------- + ``stage`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The information attempting to be converted into a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the information parsed is not in a suitable format to be converted into + a ``Stage`` class instance. + + Returns + ------- + ``Stage`` class instance for the stage being run. + """ if isinstance(stage, Stage): return stage @@ -67,18 +114,42 @@ def _coerce_stage( raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") def _rebuild_graph(self) -> None: + """ + Updates the ``graph`` attribute with the latest stage information. + """ self.graph = StageGraph.from_stages(self.stages) def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: + """ + Adds a step to the Pipeline. + + Creates a list called ``added_stages`` that runs the _coerce_stage() method + to extract the information from the given ``stages`` parameter. It then appends + this list to the ``stages`` attribute of the ``Pipeline`` class and updates the + StageGraph using the _rebuild_graph() method. A log instance is created to + reflect the changes. + + Parameters + ---------- + ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The new steps being added to the Pipeline. + """ added_stages = [self._coerce_stage(stage) for stage in stages] self.stages.extend(added_stages) self._rebuild_graph() self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) def ordered_stages(self) -> list[Stage]: + """ + Runs the topological_order() method on the ``graph`` attribute to extract the + correct order for the ``stages`` to be run in. + """ return self.graph.topological_order() def validate(self) -> "Pipeline": + """ + Confirms that the source files for the stage exist. + """ self.logger.event("Validating pipeline", name=self.name) for stage in self.stages: stage.validate() @@ -86,11 +157,24 @@ def validate(self) -> "Pipeline": return self def run(self) -> PipelineRun: + """ + Returns an instance of ``PipelineRunner`` which actually runs the pipeline. + """ from .runner import PipelineRunner - + return PipelineRunner(logger=self.logger).run(self) def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: + """ + Creates a ``RunManifest`` instance that contains the information about + the run of the Pipeline. + + Parameters + ---------- + ``runtime_id`` : RuntimeID + Contains information about the run to be extracted and placed into + the ``RunManifest`` instance. + """ return RunManifest( rap_name=self.name, run_id=runtime_id.get_id(), @@ -107,6 +191,12 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: ) def _create_runtime_id(self) -> RuntimeID: + """ + Creates a RuntimeID instance for the specific run. + + Establishes attributes for this specific run and returns + as a ``RuntimeID`` instance. + """ current_time = now() digest = hashlib.sha256(f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8")).hexdigest() short_hash = digest[:8] @@ -118,6 +208,18 @@ def _create_runtime_id(self) -> RuntimeID: ) def _discover_git_commit(self) -> str | None: + """ + Finds the specific version of the repository used for this run. + + Attempts to run a Git command to establish the current git commit hash + to be held in the ``RunManifest`` instance for this run. + + Returns + ------- + ``OSError`` + If Git is unable to be loaded or the Git command cannot be run for + another reason. + """ try: completed = subprocess.run( ["git", "rev-parse", "HEAD"], @@ -132,6 +234,14 @@ def _discover_git_commit(self) -> str | None: return commit or None def _package_versions(self) -> list[str]: + """ + Creates a list of packages and their versions used in this run. + + Raises + ------ + ``importlib_metadata.PackageNotFoundError`` + If the package used cannot be found in the library. + """ versions = [f"python={sys.version.split()[0]}"] try: versions.append(f"pyyaml={importlib_metadata.version('PyYAML')}") @@ -140,6 +250,10 @@ def _package_versions(self) -> list[str]: return versions def _current_user(self) -> str | None: + """ + Extracts the username for the individual completing the run. Returns a blank + value if the username cannot be extracted. + """ try: return getpass.getuser() except Exception: @@ -157,6 +271,32 @@ def from_files( logger: Logger | None = None, executor: StageExecutor | None = None, ) -> "Pipeline": + """ + Extracts the information from files regarding exactly what is being run in the pipeline and + allows for configuration of how the Pipeline is run. + + Parameters + ---------- + ``file_paths`` : Iterable[str or Path] + The files that contain the code for each stage in the pipeline. These are what + the Pipeline will run. + ``name`` : str + The name of the pipeline. + ``backend`` : str, default = "python" + The system that the pipeline is written in. + ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + The high level information required to run this specific pipeline. + ``dependencies`` : Mapping[str, Sequence[str]] or None + An object containing which stages are required to be run before other stages. + ``logger`` : Logger class or None + The logging sysem used for this Pipeline run. + ``executor`` : StageExecutor class or None + The information on exactly how to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ stages: list[Stage] = [] for file_path in file_paths: path = Path(file_path) @@ -182,6 +322,19 @@ def from_files( @classmethod def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": + """ + Extracts information from a dictionary to configure a Pipeline instance as + well as what the Pipeline runs. + + Parameters + ---------- + ``cfg`` : Mapping[str, Any] + The Mapping item that contains the information needed to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ payload = dict(cfg) name = payload.pop("name", None) @@ -209,6 +362,23 @@ def _dependencies_for_stage( path: Path, dependencies: Mapping[str, Sequence[str]] | None, ) -> tuple[str, ...]: + """ + Extracts a tuple of ``dependencies`` for the requested stage. + + Will return a blank tuple if there are no ``dependencies`` for the requested + stage. Allows for ``dependencies`` to be found regardless of how the stage is + referenced in the ``dependencies`` mapping. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage that you are extracting the ``dependencies`` for. + ``path`` : Path + The filepath for the stage source. + ``dependencies`` : Mapping[str, Sequence[str]] or None + Mapping of the stage source name to their relevant ``dependencies`` (stages + required to run before the ``stage_name`` Stage). + """ if not dependencies: return () diff --git a/onsrap/run_pipeline.py b/onsrap/run_pipeline.py new file mode 100644 index 0000000..f26acff --- /dev/null +++ b/onsrap/run_pipeline.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from .runner import main + +""" +This line of code establishes the function that must be called +for the Pipeline to run. main() is parsed to SystemExit as once +the main() function is run, this will then exit the system. +""" +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/onsrap/runner.py b/onsrap/runner.py index bbbb244..fa5c876 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -14,10 +14,39 @@ class PipelineRunner: + """ + Represents the information required to run the Pipeline. + + Parameters + ---------- + ``logger`` : Logger class type + Information used to log progress throughout the Pipeline. + """ def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() def run(self, pipeline: "Pipeline") -> PipelineRun: + """ + Method that runs a ``Pipeline`` instance. + + This method validates the source information, establishes the directories and + the context to run the pipeline within, sets out the manifest for the run, attempts + to run the stages in the order outlined by the ``StageGraph`` instance and logs all + progress alongside relevant statuses. + + It returns a PipelineRun instance containing metadata and logging information for the + specific run of the whole Pipeline. + + Parameters + ---------- + ``pipeline`` : Pipeline + A Pipeline instance that this method will run. + + Raises + ------ + ``StageExecutionError`` + If the stage is unable to be run. Logs will be created to show a failed stage. + """ pipeline.validate() runtime_id = pipeline._create_runtime_id() @@ -108,6 +137,11 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: def build_parser() -> argparse.ArgumentParser: + """ + Determines what arguments are needed when running a Pipeline from the command line. + + Enables stages to be input, followed by a name if provided. + """ parser = argparse.ArgumentParser(description="Run an onsrap pipeline from Python files.") parser.add_argument("stages", nargs="+", help="One or more Python stage files to run.") parser.add_argument("--name", default=None, help="Optional pipeline name.") @@ -115,6 +149,24 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: + """ + Entrypoint to the pipeline. + + This function can be called from the command line. It builds a parser which enables + the arguments to be held before using those arguments to build a Pipeline instance. + The pipeline.run() method is then run which runs the entire pipeline. If this runs + successfully, a 0 is returned which is the success code. + + Parameters + ---------- + ``argv`` : list[str] or None + Command line arguments to parse. + + Returns + ------- + int + Success code for completion of the run. + """ from .pipeline import Pipeline args = build_parser().parse_args(argv) From e8779ea69930f5de77fcd72646fd2d15621c8582 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 16:27:40 +0100 Subject: [PATCH 029/332] Add testing for resolve_paths class functions --- onsrap/execution.py | 28 +++++++----- tests/test_execution.py | 99 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 050fc0d..ea4a378 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -145,9 +145,9 @@ def resolve_output_root(self, run_dir: Path | None) -> Path: return Path(__file__).resolve().parents[1] / "data" - def resolve_given_path(self, stage_name: str, - path_name: str, - file_name:str, + def resolve_given_path(self, stage_name: str | None, + path_name: str | None, + file_name:str | None, root: Path, add_folder: list[str] | str | None = None) -> Path: """ @@ -164,6 +164,8 @@ def resolve_given_path(self, stage_name: str, ``path_name`` : str The name for the path within the stage results. This will be the key from the key/value pair within the output of the previous stage. + ``file_name`` : str + The name of the file that you are trying to access the Path for. ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. @@ -177,18 +179,24 @@ def resolve_given_path(self, stage_name: str, that data throughout the pipeline. """ result = self.result_for(stage_name) - if result is not None: + if result is not None and path_name is not None: selected_path = result.outputs.get(path_name) if selected_path: return Path(selected_path) - if add_folder is not None: - if add_folder is list: - add_folder = add_folder.append(file_name) - new_path = root.joinpath(*add_folder) + if isinstance(add_folder, list): + if file_name is not None: + new_path = root.joinpath(*add_folder, file_name) return new_path - return root / add_folder / file_name + new_path = root.joinpath(*add_folder) + return new_path + if isinstance(add_folder, str): + if file_name is not None: + return root/ add_folder/ file_name + return root / add_folder + if file_name is not None: + return root / file_name + return root - return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 756a3e2..722449e 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -3,6 +3,7 @@ from onsrap.logger import Logger from pathlib import Path import pytest +import onsrap.execution as execution_module @pytest.fixture def logger() -> Logger: @@ -33,7 +34,7 @@ def config() -> PipelineConfig: ) @pytest.fixture -def execution(config, logger) -> ExecutionContext: +def execution(config, logger, stageresult) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ @@ -48,7 +49,7 @@ def execution(config, logger) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {}, + {"stage_test":stageresult}, {} ) @@ -67,7 +68,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config, stageresult) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -78,7 +79,7 @@ def test_executioncontext_creation(execution, logger, config) -> None: assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' assert execution.working_directory == Path('tmp/work_dir') - assert execution.stage_results == {} + assert execution.stage_results == {"stage_test":stageresult} assert execution.variables == {} def test_record(stageresult, execution) -> None: @@ -133,7 +134,7 @@ def test_resolve_data_root(execution) -> None: """ assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") - import onsrap.execution as execution_module + result = execution.resolve_data_root(None) expected = ( Path(execution_module.__file__).resolve().parents[1] @@ -141,3 +142,91 @@ def test_resolve_data_root(execution) -> None: ) assert result == expected +def test_resolve_output_root(execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, returns the file path for the module itself + and the data directory within that. + """ + run_dir = Path("tmp/run") + assert execution.resolve_output_root(run_dir) == Path("tmp/run/data") + + result = execution.resolve_output_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + +@pytest.mark.parametrize( + "add_folder,file_name,expected", + [ + ( + ["interim","testing_files"], + "clean.py", + Path("tmp/data/interim/testing_files/clean.py") + ), + ( + "interim", + "clean.py", + Path("tmp/data/interim/clean.py") + ), + ( + None, + "clean.py", + Path("tmp/data/clean.py") + ), + ( + ["interim","testing_files"], + None, + Path("tmp/data/interim/testing_files") + ), + ( + "interim", + None, + Path("tmp/data/interim") + ), + ( + None, + None, + Path("tmp/data") + ) + ], +) + + +def test_resolve_given_path_add_folders(execution, add_folder, file_name, expected) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid string + or None type. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(None, + path_name, + file_name, + root, + add_folder) == expected + +def test_resolve_given_path_norm(execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + """ + execution.record(StageResult("stage_test2", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = {"data_path":"clean.py"} )) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(stage_name, + path_name, + None, + root, + None) == Path("clean.py") \ No newline at end of file From 176443e422d6aebfb4048e2489728bd1fe5ae3c4 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 8 Jul 2026 10:53:28 +0100 Subject: [PATCH 030/332] Fixed persistent docstring issuel. --- onsrap/execution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index d5504f5..2c97fa6 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -252,7 +252,6 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul """ Attempt to run a file. - """ Attempt to run a callable object. Calls the logger.event() method to record an event and attempts to run From 1be30d2fb5b7ee409d803a62e85d6a57a9dcaae6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 3 Jul 2026 11:05:06 +0100 Subject: [PATCH 031/332] Script for testing stage.py and minor change to documentation in execution.py --- onsrap/execution.py | 1 - tests/test_stage.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/test_stage.py diff --git a/onsrap/execution.py b/onsrap/execution.py index 2c97fa6..a0a7396 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -251,7 +251,6 @@ def _execute_callable( def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: """ Attempt to run a file. - Attempt to run a callable object. Calls the logger.event() method to record an event and attempts to run diff --git a/tests/test_stage.py b/tests/test_stage.py new file mode 100644 index 0000000..5213945 --- /dev/null +++ b/tests/test_stage.py @@ -0,0 +1,182 @@ +import pytest +from onsrap.stage import _normalize_dependencies, Stage, StageConfigurationError +from pathlib import Path +from textwrap import dedent + +def test_normalize_dependencies_none() -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + +def test_normalize_dependencies_str() -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py"," Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py","Stage_2.py", "Stage_3.py") + +@pytest.fixture +def example_function(): + """ + Test function to pass as a callable stage for stage testing. + """ + print("This is a test function") + +@pytest.fixture +def stage_test() -> Stage: + """ + Stage object for testing Stage class methods and construction. + """ + return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) + +def test_stage_creation_callable(stage_test) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info":"example"} + assert stage_test.entrypoint == None + assert stage_test.backend == "python" + +def test_stage_name_error(example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("",example_function,["stage_1"],{"info":"example"}) + +def test_stage_source_type() -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + +def test_stage_backend(example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + """ + stage_diff = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "java") + stage = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "") + stage_white_space = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "python ") + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + +def test_stage_from_files_error(tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + stage = Stage.from_file(source_file) + +def test_stage_from_callable_name() -> None: + """ + Tests that a stage name is extracted from a callable object stage. + """ + def example_function(): + pass + test = Stage.from_callable(example_function) + assert test.name == "example_function" + +@pytest.mark.skip +def test_from_dict_norm() -> None: + """ + REVIEW WITH ALEX + """ + def example_function(): + pass + data = {"name":"test_Stage", + "callable_source" : example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + +def test_with_dependencies_list(stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. + + REVIEW WITH ALEX + This test works and passes however it doesn't behave how I was expecting it to. + Was expecting the list/dictionary to be broken down so you have one tuple rather + than a tuple of dict/lists. Is this a problem or just my understanding? + """ + new_deps = ["stage2","stage3"] + new_dep_dict = {"stage1":"stage0"} + new_deps_blank = [] + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_dict = stage_test.with_dependencies(new_dep_dict) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1","['stage2', 'stage3']") + assert stage_test_dict.dependencies == ("stage_1","{'stage1': 'stage0'}") + assert stage_test_blank.dependencies == ("stage_1", '[]') + +@pytest.mark.skip +def test_validate(stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + REVIEW WITH ALEX + Does not raise a StageConfigurationError is the source is a blank string. Is + this a concern? Do we want this validate to be able to do other error checks + like if it is an int? + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + +def test_source_path(stage_test, tmp_path) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source types. + """ + stage_test.source = tmp_path/"fake_file.py" + assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = 11 + assert stage_test.source_path == None + stage_test.source = "not a file path" + assert stage_test.source_path == None + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_path == None + +def test_source_label(stage_test, tmp_path) -> None: + """ + Tests that source_label is created if the source is a Path or a callable and is None if it is + another type. + """ + stage_test.source = tmp_path/"fake_file.py" + temp_path_str = str(tmp_path/"fake_file.py") + assert stage_test.source_label == temp_path_str + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label == None + +""" +TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST +""" \ No newline at end of file From b0f222485ae96ad3251e8e64969e04add4252337 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 11:08:51 +0100 Subject: [PATCH 032/332] Partial testing established for execution.py --- tests/test_execution.py | 121 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_execution.py diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 0000000..143f55b --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,121 @@ +from onsrap.execution import ExecutionContext +from onsrap.models import PipelineConfig, StageResult, StageStatus +from onsrap.logger import Logger +from pathlib import Path +import pytest + +@pytest.fixture +def logger() -> Logger: + """ + Logger instance for testing + """ + return Logger() + +@pytest.fixture +def config(tmp_path) -> PipelineConfig: + """ + Return a PipelineConfig object for testing. + """ + work_dir = tmp_path/"work" + project_root = tmp_path/"project" + log_dir = tmp_path/"log" + data_dir = tmp_path/"data" + return PipelineConfig( + "test_pipeline", + "python", + work_dir, + project_root, + log_dir, + data_dir, + True, + None, + {} + ) + +@pytest.fixture +def execution(config, logger, tmp_path) -> ExecutionContext: + """ + Create an ExecutionContext object for testing. + """ + run_dir = tmp_path/"run" + work_dir = tmp_path/"work_dir" + + return ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + run_dir, + '2024-05-06 15:45:30', + work_dir, + {}, + {} + ) + +@pytest.fixture +def stageresult() -> StageResult: + """ + Test StageResult instance for running ExecutionContext tests. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = "example output" + ) + + +def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: + """ + Test that the ExecutionContext creates the right attributes. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == tmp_path/"run" + assert execution.started_at == '2024-05-06 15:45:30' + assert execution.working_directory == tmp_path/"work_dir" + assert execution.stage_results == {} + assert execution.variables == {} + +def test_record(stageresult, execution) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + """ + execution.record(stageresult) + assert execution.stage_results == {'stage_test':StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None)} + assert execution.variables == {'stage_test':"example output"} + +def test_result_for(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.result_for("stage_test") == StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None) + +def test_stage_outputs(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test":"example output"} + +"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file From e38df80217303c33f388225c6110043f4d2b56fa Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 17:15:52 +0100 Subject: [PATCH 033/332] Add class methods to ExecutionContext that allow file paths to be created/extracted for I/O --- .../pipeline_1/scripts/0_data_validation.py | 18 +--- .../pipeline_1/scripts/1_preprocessing.py | 32 +------- examples/pipeline_1/scripts/2_reporting.py | 33 ++------ onsrap/execution.py | 82 +++++++++++++++++++ tests/test_execution.py | 2 +- 5 files changed, 94 insertions(+), 73 deletions(-) diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index eb331cc..4a586ee 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -17,20 +17,6 @@ ) -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -99,8 +85,8 @@ def write_report(report_path: Path, report: dict[str, object]) -> None: def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index 0d8ad1c..a55180c 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_raw_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - validation_result = context.result_for("0_data_validation") - if validation_result is not None: - raw_path = validation_result.outputs.get("raw_path") - if raw_path: - return Path(raw_path) - - return data_root / "orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -112,9 +87,10 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - raw_path = resolve_raw_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + raw_path = context.resolve_given_path("0_data_validation", "raw_path", + "orders.csv", data_root) clean_path = output_root / "interim" / "1_clean_orders.csv" rows = load_orders(raw_path) diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index 337c2e5..c3f50a8 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_clean_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - preprocessing_result = context.result_for("1_preprocessing") - if preprocessing_result is not None: - clean_path = preprocessing_result.outputs.get("clean_path") - if clean_path: - return Path(clean_path) - - return data_root / "interim" / "1_clean_orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -94,9 +69,11 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - clean_path = resolve_clean_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + clean_path = context.resolve_given_path("1_preprocessing", "clean_path", + "1_clean_orders.csv", output_root, + "interim") summary_path = output_root / "processed" / "2_sales_summary.json" region_breakdown_path = output_root / "processed" / "2_revenue_by_region.csv" diff --git a/onsrap/execution.py b/onsrap/execution.py index a0a7396..fca75d0 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -105,6 +105,88 @@ def stage_outputs(self) -> dict[str, Any]: the run. """ return {name: result.outputs for name, result in self.stage_results.items()} + + def resolve_data_root(self, config: PipelineConfig | None) -> Path: + """ + Establishes the filepath that the data is held in. + + Parameters + ---------- + ``config`` : PipelineConfig + The configuration of the Pipeline being run. This holds the location filepath + for the pipeline as defined by the user in the main.py file. + + Returns + ------- + Path + The file path for the location of the data being used in the pipeline. + """ + if config is not None: + return Path(config.data_dir) + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_output_root(self, run_dir: Path | None) -> Path: + """ + Establishes the filepath that the outputs are going to be saved to. + + Parameters + ---------- + ``run_dir`` : Path + The file directory that the run results are saved to. + + Returns + ------- + Path + The file path for the outputs of the run to be saved to. + """ + if run_dir is not None: + return Path(run_dir) / "data" + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_given_path(self, stage_name: str, + path_name: str, + file_name:str, + root: Path, + add_folder: str | None = None) -> Path: + """ + Returns a file path for a requested item. + + This investigates the result of a previous stage to extract a selected path. + If the path is not available, it creates a path using a root previously derived + in main.py, the chosen directory within the root (optional), and the file path. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage where the path was outputted. + ``path_name`` : str + The name for the path within the stage results. This will be the key from the + key/value pair within the output of the previous stage. + ``root`` : Path + The file path for the root of the directory. This should be denoted through + other methods. + ``add_folder`` : str | None, default = None + Additional folder name to add into the returned file path. Additional functionality + should be added to allow for multiple folders to be added to the path. + + Returns + ------- + Path + The file path where data has previously been saved to to allow for extraction of + that data throughout the pipeline. + """ + result = self.result_for(stage_name) + if result is not None: + selected_path = result.outputs.get(path_name) + if selected_path: + return Path(selected_path) + if add_folder is not None: + #Would like to add functionality here for multiple additional folders + return root / add_folder / file_name + + return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 143f55b..7fc87b0 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -118,4 +118,4 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file +"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file From 104f72bafca450f190a1ff4fca9b989508fa5e3f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 12:31:38 +0100 Subject: [PATCH 034/332] Minor testing implemented --- onsrap/execution.py | 12 ++++++----- tests/test_execution.py | 46 ++++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index fca75d0..050fc0d 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -149,7 +149,7 @@ def resolve_given_path(self, stage_name: str, path_name: str, file_name:str, root: Path, - add_folder: str | None = None) -> Path: + add_folder: list[str] | str | None = None) -> Path: """ Returns a file path for a requested item. @@ -167,9 +167,8 @@ def resolve_given_path(self, stage_name: str, ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. - ``add_folder`` : str | None, default = None - Additional folder name to add into the returned file path. Additional functionality - should be added to allow for multiple folders to be added to the path. + ``add_folder`` : list[str] | str | None, default = None + Additional folder name/s to add into the returned file path. Returns ------- @@ -183,7 +182,10 @@ def resolve_given_path(self, stage_name: str, if selected_path: return Path(selected_path) if add_folder is not None: - #Would like to add functionality here for multiple additional folders + if add_folder is list: + add_folder = add_folder.append(file_name) + new_path = root.joinpath(*add_folder) + return new_path return root / add_folder / file_name return root / file_name diff --git a/tests/test_execution.py b/tests/test_execution.py index 7fc87b0..756a3e2 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -12,14 +12,14 @@ def logger() -> Logger: return Logger() @pytest.fixture -def config(tmp_path) -> PipelineConfig: +def config() -> PipelineConfig: """ Return a PipelineConfig object for testing. """ - work_dir = tmp_path/"work" - project_root = tmp_path/"project" - log_dir = tmp_path/"log" - data_dir = tmp_path/"data" + work_dir = Path('tmp/work_dir') + project_root = Path('tmp/project') + log_dir = Path('tmp/log') + data_dir = "tmp/config_data" return PipelineConfig( "test_pipeline", "python", @@ -33,12 +33,12 @@ def config(tmp_path) -> PipelineConfig: ) @pytest.fixture -def execution(config, logger, tmp_path) -> ExecutionContext: +def execution(config, logger) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ - run_dir = tmp_path/"run" - work_dir = tmp_path/"work_dir" + run_dir = Path("tmp/run") + work_dir = Path('tmp/work_dir') return ExecutionContext( "test_pipeline", @@ -67,7 +67,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -75,9 +75,9 @@ def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: assert execution.run_id == "run_id_1234" assert execution.config == config assert execution.logger == logger - assert execution.run_dir == tmp_path/"run" + assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' - assert execution.working_directory == tmp_path/"work_dir" + assert execution.working_directory == Path('tmp/work_dir') assert execution.stage_results == {} assert execution.variables == {} @@ -101,6 +101,9 @@ def test_record(stageresult, execution) -> None: assert execution.variables == {'stage_test':"example output"} def test_result_for(execution, stageresult) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + """ execution.record(stageresult) assert execution.result_for("stage_test") == StageResult(name='stage_test', status='pending', @@ -115,7 +118,26 @@ def test_result_for(execution, stageresult) -> None: source=None) def test_stage_outputs(execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + """ execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file +def test_resolve_data_root(execution) -> None: + """ + Tests that resolve_data_root method extracts the path from the execution context + or, if the context is None, returns the file path for the module itself and the + data directory within that. + """ + assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") + + import onsrap.execution as execution_module + result = execution.resolve_data_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + From 25f7106990da668431b60a5ae8cf86e97fa47cf0 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 16:27:40 +0100 Subject: [PATCH 035/332] Add testing for resolve_paths class functions --- onsrap/execution.py | 28 +++++++----- tests/test_execution.py | 99 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 050fc0d..ea4a378 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -145,9 +145,9 @@ def resolve_output_root(self, run_dir: Path | None) -> Path: return Path(__file__).resolve().parents[1] / "data" - def resolve_given_path(self, stage_name: str, - path_name: str, - file_name:str, + def resolve_given_path(self, stage_name: str | None, + path_name: str | None, + file_name:str | None, root: Path, add_folder: list[str] | str | None = None) -> Path: """ @@ -164,6 +164,8 @@ def resolve_given_path(self, stage_name: str, ``path_name`` : str The name for the path within the stage results. This will be the key from the key/value pair within the output of the previous stage. + ``file_name`` : str + The name of the file that you are trying to access the Path for. ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. @@ -177,18 +179,24 @@ def resolve_given_path(self, stage_name: str, that data throughout the pipeline. """ result = self.result_for(stage_name) - if result is not None: + if result is not None and path_name is not None: selected_path = result.outputs.get(path_name) if selected_path: return Path(selected_path) - if add_folder is not None: - if add_folder is list: - add_folder = add_folder.append(file_name) - new_path = root.joinpath(*add_folder) + if isinstance(add_folder, list): + if file_name is not None: + new_path = root.joinpath(*add_folder, file_name) return new_path - return root / add_folder / file_name + new_path = root.joinpath(*add_folder) + return new_path + if isinstance(add_folder, str): + if file_name is not None: + return root/ add_folder/ file_name + return root / add_folder + if file_name is not None: + return root / file_name + return root - return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 756a3e2..722449e 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -3,6 +3,7 @@ from onsrap.logger import Logger from pathlib import Path import pytest +import onsrap.execution as execution_module @pytest.fixture def logger() -> Logger: @@ -33,7 +34,7 @@ def config() -> PipelineConfig: ) @pytest.fixture -def execution(config, logger) -> ExecutionContext: +def execution(config, logger, stageresult) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ @@ -48,7 +49,7 @@ def execution(config, logger) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {}, + {"stage_test":stageresult}, {} ) @@ -67,7 +68,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config, stageresult) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -78,7 +79,7 @@ def test_executioncontext_creation(execution, logger, config) -> None: assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' assert execution.working_directory == Path('tmp/work_dir') - assert execution.stage_results == {} + assert execution.stage_results == {"stage_test":stageresult} assert execution.variables == {} def test_record(stageresult, execution) -> None: @@ -133,7 +134,7 @@ def test_resolve_data_root(execution) -> None: """ assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") - import onsrap.execution as execution_module + result = execution.resolve_data_root(None) expected = ( Path(execution_module.__file__).resolve().parents[1] @@ -141,3 +142,91 @@ def test_resolve_data_root(execution) -> None: ) assert result == expected +def test_resolve_output_root(execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, returns the file path for the module itself + and the data directory within that. + """ + run_dir = Path("tmp/run") + assert execution.resolve_output_root(run_dir) == Path("tmp/run/data") + + result = execution.resolve_output_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + +@pytest.mark.parametrize( + "add_folder,file_name,expected", + [ + ( + ["interim","testing_files"], + "clean.py", + Path("tmp/data/interim/testing_files/clean.py") + ), + ( + "interim", + "clean.py", + Path("tmp/data/interim/clean.py") + ), + ( + None, + "clean.py", + Path("tmp/data/clean.py") + ), + ( + ["interim","testing_files"], + None, + Path("tmp/data/interim/testing_files") + ), + ( + "interim", + None, + Path("tmp/data/interim") + ), + ( + None, + None, + Path("tmp/data") + ) + ], +) + + +def test_resolve_given_path_add_folders(execution, add_folder, file_name, expected) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid string + or None type. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(None, + path_name, + file_name, + root, + add_folder) == expected + +def test_resolve_given_path_norm(execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + """ + execution.record(StageResult("stage_test2", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = {"data_path":"clean.py"} )) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(stage_name, + path_name, + None, + root, + None) == Path("clean.py") \ No newline at end of file From 690d1b2ee5c407d1b0803fae67d0c383e5735e72 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 8 Jul 2026 10:57:02 +0100 Subject: [PATCH 036/332] Commiting before rebase --- tests/test_execution.py | 19 ++++++- tests/test_models.py | 119 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 tests/test_models.py diff --git a/tests/test_execution.py b/tests/test_execution.py index 722449e..bbb9d06 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,4 +1,4 @@ -from onsrap.execution import ExecutionContext +from onsrap.execution import ExecutionContext, PythonStageExecutor from onsrap.models import PipelineConfig, StageResult, StageStatus from onsrap.logger import Logger from pathlib import Path @@ -158,6 +158,10 @@ def test_resolve_output_root(execution) -> None: ) assert result == expected +""" +Parameters for testing multiple add_folder options in +test_resolve_given_path_add_folders function. +""" @pytest.mark.parametrize( "add_folder,file_name,expected", [ @@ -229,4 +233,15 @@ def test_resolve_given_path_norm(execution) -> None: path_name, None, root, - None) == Path("clean.py") \ No newline at end of file + None) == Path("clean.py") + +"""TEST NOT RUN FOR StageExecutor AS COVERED UNDER PythonStageExecutor""" + +@pytest.fixture +def pythonstageexecutor() -> PythonStageExecutor: + return PythonStageExecutor(("main.py","run.py")) + +def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: + assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") + +"""CONTINUE FROM EXECUTE CLASS METHOD""" \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..e120bc2 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,119 @@ +from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, StageResult, PipelineRun, PipelineConfig +import pytest +import datetime +from pathlib import Path +from textwrap import dedent + +def test_stagestatus() -> None: + assert StageStatus.PENDING == "pending" + assert StageStatus.RUNNING == "running" + assert StageStatus.SUCCEEDED == "succeeded" + assert StageStatus.FAILED == "failed" + assert StageStatus.SKIPPED == "skipped" + +def test_pipeline_status() -> None: + assert PipelineStatus.PENDING == "pending" + assert PipelineStatus.RUNNING == "running" + assert PipelineStatus.SUCCEEDED == "succeeded" + assert PipelineStatus.FAILED == "failed" + +@pytest.fixture +def runtimeID() -> RuntimeID: + return RuntimeID(id = "abc123", + timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), + hash = "fnruw9574893ghkwq234h5kg", + short_hash = "4h5kg") + +def test_runtimeID_creation(runtimeID) -> None: + assert runtimeID.id == "abc123" + assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" + assert runtimeID.short_hash == "4h5kg" + +def test_getter_functions_runtimeID(runtimeID) -> None: + assert runtimeID.get_id() == "abc123" + assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" + assert runtimeID.get_short_hash() == "4h5kg" + +@pytest.fixture +def rapconfig() -> RAPConfig: + return RAPConfig(contents = {"name":"test_rap", + "backend":"python", + "work_dir":Path("tmp/work"), + "project_root":Path("project"), + "log_dir":Path("tmp/logs"), + "data_dir":Path("tmp/data"), + "allow_subprocess_fallback":True, + "python_executable":None, + "metadata":{"variables":["name","age"], + "num_stages":6}}) + +@pytest.fixture +def blankpipelineconfig() -> PipelineConfig: + return PipelineConfig() + +@pytest.fixture +def pipelineconfig() -> PipelineConfig: + return PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + +@pytest.fixture +def mapping() -> dict: + return {"name":"test_rap", + "backend":"python", + "work_dir":Path("tmp/work"), + "project_root":Path("project"), + "log_dir":Path("tmp/logs"), + "data_dir":Path("tmp/data"), + "allow_subprocess_fallback":True, + "python_executable":None, + "metadata":{"variables":["name","age"], + "num_stages":6}} + + +def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> None: + assert blankpipelineconfig.from_any(None) == PipelineConfig() + assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + assert blankpipelineconfig.from_any(rapconfig) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + assert blankpipelineconfig.from_any(mapping) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + + with pytest.raises(TypeError): + blankpipelineconfig.from_any(11) + +"""NOT SURE HOW TO TEST FROM_FILE()""" \ No newline at end of file From 48e7831d305af7339af530c82212f81c3aa5b394 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 9 Jul 2026 15:57:48 +0100 Subject: [PATCH 037/332] Tests for models.py and bug fixes within models.py --- onsrap/errors.py | 6 ++ tests/test_models.py | 142 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index 8302969..c22dbd2 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -66,3 +66,9 @@ class StageLoadError(StageExecutionError): Raised when a file-backed stage cannot be loaded. Child class with ``StageExecutionError`` as the parent class. """ + +class StageDependencyError(OnsrapError): + """ + Raised when incorrect inputs are provided to the dependency + attribute of a Stage. + """ \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py index e120bc2..3ab6cbe 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,10 +1,14 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, StageResult, PipelineRun, PipelineConfig +from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, PipelineRun, PipelineConfig import pytest import datetime from pathlib import Path from textwrap import dedent +from tests.test_execution import stageresult def test_stagestatus() -> None: + """ + Test that stagestatus outputs the correct values. + """ assert StageStatus.PENDING == "pending" assert StageStatus.RUNNING == "running" assert StageStatus.SUCCEEDED == "succeeded" @@ -12,6 +16,9 @@ def test_stagestatus() -> None: assert StageStatus.SKIPPED == "skipped" def test_pipeline_status() -> None: + """ + Test that pipeline status outputs the correct values. + """ assert PipelineStatus.PENDING == "pending" assert PipelineStatus.RUNNING == "running" assert PipelineStatus.SUCCEEDED == "succeeded" @@ -19,18 +26,27 @@ def test_pipeline_status() -> None: @pytest.fixture def runtimeID() -> RuntimeID: + """ + Example RuntimeID instance for testing of other methods. + """ return RuntimeID(id = "abc123", timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), hash = "fnruw9574893ghkwq234h5kg", short_hash = "4h5kg") def test_runtimeID_creation(runtimeID) -> None: + """ + Test that a RuntimeID is correctly created. + """ assert runtimeID.id == "abc123" assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" assert runtimeID.short_hash == "4h5kg" def test_getter_functions_runtimeID(runtimeID) -> None: + """ + Tests all the getter functions for the RuntimeID instance. + """ assert runtimeID.get_id() == "abc123" assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" @@ -38,6 +54,9 @@ def test_getter_functions_runtimeID(runtimeID) -> None: @pytest.fixture def rapconfig() -> RAPConfig: + """ + Example RAPConfig class instance for testing of class methods. + """ return RAPConfig(contents = {"name":"test_rap", "backend":"python", "work_dir":Path("tmp/work"), @@ -51,10 +70,16 @@ def rapconfig() -> RAPConfig: @pytest.fixture def blankpipelineconfig() -> PipelineConfig: + """ + Blank PipelineConfig instance for class method testing. + """ return PipelineConfig() @pytest.fixture def pipelineconfig() -> PipelineConfig: + """ + Example PipelineConfig completed class instance for method testing. + """ return PipelineConfig(name = "test_rap", backend = "python", work_dir = Path("tmp/work"), @@ -68,6 +93,9 @@ def pipelineconfig() -> PipelineConfig: @pytest.fixture def mapping() -> dict: + """ + Example mapping dictionary for use in testing from_mapping() method. + """ return {"name":"test_rap", "backend":"python", "work_dir":Path("tmp/work"), @@ -81,6 +109,11 @@ def mapping() -> dict: def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> None: + """ + Test derivation for a PipelineConfig instance using the from_any() method. This test + checks all methods EXCEPT from_file as this will be covered in another test due to + creation of a mock file being required. + """ assert blankpipelineconfig.from_any(None) == PipelineConfig() assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", backend = "python", @@ -116,4 +149,109 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> No with pytest.raises(TypeError): blankpipelineconfig.from_any(11) -"""NOT SURE HOW TO TEST FROM_FILE()""" \ No newline at end of file +"""NOT SURE HOW TO TEST FROM_FILE()""" + +def test_to_dict(pipelineconfig) -> None: + """ + Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values + as a dictionary. + """ + + assert pipelineconfig.to_dict() == {"name":"test_rap", + "backend":"python", + "work_dir":"tmp\\work", + "project_root":"project", + "log_dir":"tmp\\logs", + "data_dir":"tmp\\data", + "allow_subprocess_fallback":True, + "python_executable":None, + "variables":["name","age"], + "num_stages":6} + + +@pytest.fixture +def runmanifest() -> RunManifest: + """ + Example RunManifest class instance for testing of class method. + """ + return RunManifest("pipeline", + "1", + None, + ["stage1","stage2"], + {"uniqueID":"example"}, + {"input_path":"input/data/example.csv"}, + {"output_path":"output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None) + +def test_stage_result(stageresult) -> None: + """ + Uses a StageResult instance created in test_execution to ensure that + the class instance is created suitably with required defaults. + """ + assert stageresult.name == "stage_test" + assert stageresult.status == "pending" + assert stageresult.started_at == '2024-05-06 15:45:30' + assert stageresult.finished_at == '2024-05-07 15:45:30' + assert stageresult.outputs == "example output" + assert stageresult.stdout == "" + assert stageresult.stderr == "" + assert stageresult.return_code == None + assert stageresult.metadata == {} + assert stageresult.error == None + assert stageresult.source == None + +@pytest.mark.parametrize("status_stage,expected_stage", + [(StageStatus.PENDING, False), + (StageStatus.RUNNING, False), + (StageStatus.FAILED, False), + (StageStatus.SUCCEEDED, True), + (StageStatus.SKIPPED, False)]) + +def test_succeeded(stageresult, status_stage, expected_stage) -> None: + """ + Tests succeeded() method for StageResult which outputs True or False depending on + the status of the StageResult. + """ + stageresult.status = status_stage + assert stageresult.succeeded == expected_stage + +def test_duration_seconds(stageresult) -> None: + stageresult.started_at = datetime.datetime(2024,5,6,15,45,30) + stageresult.finished_at = datetime.datetime(2024,5,7,15,45,30) + seconds_value = (datetime.datetime(2024,5,7,15,45,30) - datetime.datetime(2024,5,6,15,45,30)).total_seconds() + assert stageresult.duration_seconds == seconds_value + +@pytest.fixture +def pipelinerun(stageresult, runmanifest) -> PipelineRun: + return PipelineRun(runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + [stageresult], + {"stage_test":"example output"}) + +def test_pipelinerun_configuration(pipelinerun, runmanifest, stageresult) -> None: + assert pipelinerun.manifest == runmanifest + assert pipelinerun.status == PipelineStatus.SUCCEEDED + assert pipelinerun.started_at == datetime.datetime(2024,5,6,15,45,30) + assert pipelinerun.completed_at == datetime.datetime(2024,5,7,15,45,30) + assert pipelinerun.stage_results == [stageresult] + assert pipelinerun.stage_outputs == {"stage_test":"example output"} + +def test_result_for(pipelinerun, stageresult) -> None: + assert pipelinerun.result_for("stage_test") == stageresult + assert pipelinerun.result_for("not_a_stage") == None + +@pytest.mark.parametrize("status,expected", + [(PipelineStatus.PENDING, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.FAILED, False), + (PipelineStatus.SUCCEEDED, True)]) + +def test_succeeded_pipeline(pipelinerun, status, expected) -> None: + pipelinerun.status = status + assert pipelinerun.succeeded == expected \ No newline at end of file From 81459d8f6bb07c6b969bb6a219fa014524bbd266 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 9 Jul 2026 15:58:18 +0100 Subject: [PATCH 038/332] tests for stage.py and bug fixes for stage.py incl. minor changes to models.py --- onsrap/models.py | 4 ++-- onsrap/stage.py | 35 ++++++++++++++++++++++++++++------- tests/test_stage.py | 26 +++++++------------------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 92b1aec..b37a6d5 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -270,8 +270,8 @@ def from_file(cls, path: Path) -> "PipelineConfig": def to_dict(self) -> dict[str, Any]: """ - Converts attributes regarding how the pipeline runs into a dictionary and holds it in - the ``metadata`` attribute of the ``PipelineConfig`` class. + Returns a prescriptive expression of the attributes within the PipelineConfig instance + that allows for easier processing by the user. """ data = { "name": self.name, diff --git a/onsrap/stage.py b/onsrap/stage.py index b14b3ad..1340b2d 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -5,14 +5,14 @@ from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union -from .errors import StageConfigurationError +from .errors import StageConfigurationError, StageDependencyError if TYPE_CHECKING: from .execution import ExecutionContext, StageExecutor from .models import StageResult -def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: +def _normalize_dependencies(dependencies: list[str] | str | None) -> tuple[str, ...]: """ Standardise the names of any stages dependant on other stages/processes. @@ -32,13 +32,16 @@ def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[s """ if dependencies is None: return () - + if isinstance(dependencies, list) and dependencies == []: + return () if isinstance(dependencies, str): candidate_items = [dependencies] + else: - candidate_items = list(dependencies) + candidate_items = dependencies normalized: list[str] = [] + for dependency in candidate_items: dependency_name = str(dependency).strip() if dependency_name and dependency_name not in normalized: @@ -276,9 +279,22 @@ def with_dependencies(self, *dependencies: str) -> "Stage": ``Stage`` ``Stage`` class instance with normalised ``dependencies`` attribute. """ + unpacked_deps: list = [] + for a in dependencies: + if isinstance(a,list): + unpacked_deps = unpacked_deps + a + else: + unpacked_deps.append(a) + + for i in unpacked_deps: + if isinstance(i, list): + raise StageDependencyError("Nested lists are not valid arguments for this method! " \ + "Please provided single list or individual string values") + + print(unpacked_deps) return replace( self, - dependencies=self.dependencies + _normalize_dependencies(dependencies), + dependencies=self.dependencies + _normalize_dependencies(unpacked_deps), ) def validate(self) -> None: @@ -290,8 +306,13 @@ def validate(self) -> None: ``StageConfigurationError`` If ``source`` attribute does not define a source or does not exist. """ - if self.source is None: - raise StageConfigurationError(f"Stage '{self.name}' does not define a source.") + if not (isinstance(self.source, Path) or callable(self.source)): + raise StageConfigurationError(f"Stage '{self.name}' must have a Path or Callable source.") + + if self.source is None or self.source == "": + raise StageConfigurationError( + f"Stage '{self.name}' does not define a source. Source provided: {self.source}" + ) if isinstance(self.source, Path) and not self.source.is_file(): raise StageConfigurationError(f"Stage source does not exist: {self.source}") diff --git a/tests/test_stage.py b/tests/test_stage.py index 5213945..7d6c922 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -91,15 +91,14 @@ def example_function(): test = Stage.from_callable(example_function) assert test.name == "example_function" -@pytest.mark.skip def test_from_dict_norm() -> None: """ - REVIEW WITH ALEX + Tests that a stage instance is created from a dictionary item. """ def example_function(): pass data = {"name":"test_Stage", - "callable_source" : example_function} + "callable" : example_function} stage = Stage.from_dict(data) assert stage.source == example_function @@ -107,31 +106,20 @@ def test_with_dependencies_list(stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is a list. - - REVIEW WITH ALEX - This test works and passes however it doesn't behave how I was expecting it to. - Was expecting the list/dictionary to be broken down so you have one tuple rather - than a tuple of dict/lists. Is this a problem or just my understanding? """ new_deps = ["stage2","stage3"] - new_dep_dict = {"stage1":"stage0"} new_deps_blank = [] stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_dict = stage_test.with_dependencies(new_dep_dict) stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1","['stage2', 'stage3']") - assert stage_test_dict.dependencies == ("stage_1","{'stage1': 'stage0'}") - assert stage_test_blank.dependencies == ("stage_1", '[]') + assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_blank.dependencies == ("stage_1", ) + stage_test = stage_test.with_dependencies("stage2","stage3") + assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + -@pytest.mark.skip def test_validate(stage_test, tmp_path) -> None: """ Tests whether an error is raised if the source file isn't suitable. - - REVIEW WITH ALEX - Does not raise a StageConfigurationError is the source is a blank string. Is - this a concern? Do we want this validate to be able to do other error checks - like if it is an int? """ stage_test.source = None with pytest.raises(StageConfigurationError): From f6e58cb307c97ca087cf0b73fd8c46bca30b7068 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 10 Jul 2026 11:50:48 +0100 Subject: [PATCH 039/332] Fix: Fix bug #22 and implements test to ensure this works --- onsrap/pipeline.py | 6 +++++- tests/test_pipeline.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/test_pipeline.py diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 14daeaf..d264fca 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -51,9 +51,13 @@ def __init__( logger: Logger | None = None, executor: StageExecutor | None = None, ): - self.name = name or "pipeline" self.backend = backend or "python" self.config = PipelineConfig.from_any(config) + if (self.config.name is not None) and (name == None): + self.name = self.config.name + else: + self.name = name or "pipeline" + if self.config.name is None: self.config.name = self.name self.config.backend = self.backend diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..aec62fb --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,22 @@ +from onsrap.pipeline import Pipeline, PipelineConfig +import pytest + +@pytest.fixture +def pipelineconfig() -> PipelineConfig: + return PipelineConfig(name = "test_pipeline_config") + +def test_pipeline_name(pipelineconfig): + """ + Test to confirm that Pipeline instance uses either defined name from + instance creation (shown in pipeline_named), utilises name from PipelineConfig + if no name was given (shown in pipeline_config), or defaults to "pipeline" if + no name is provided through Pipeline instance creation or through the + PipelineConfig (shown through pipeline_no_name) + """ + pipeline_named = Pipeline(name = "test_pipeline_name") + assert pipeline_named.name == "test_pipeline_name" + pipeline_config = Pipeline(name = None, config = pipelineconfig) + assert pipeline_config.name == "test_pipeline_config" + pipeline_no_name = Pipeline() + assert pipeline_no_name.name == "pipeline" + \ No newline at end of file From 3da46a2076de26cae60432e905ba97e9e658fc66 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 13:55:49 +0100 Subject: [PATCH 040/332] Fix: Applied fix for bug #20 and implemented a test for fix. --- onsrap/errors.py | 6 ++++++ onsrap/pipeline.py | 33 ++++++++++++++++++++++++++------- tests/test_pipeline.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index c22dbd2..1691b4d 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -71,4 +71,10 @@ class StageDependencyError(OnsrapError): """ Raised when incorrect inputs are provided to the dependency attribute of a Stage. + """ + +class PipelineInitialisationError(OnsrapError): + """ + Raised when there is an error in definition of the Pipeline + instance """ \ No newline at end of file diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index d264fca..3396fd3 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -6,14 +6,14 @@ import sys from importlib import metadata as importlib_metadata from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Sequence +from typing import Any, Callable, Iterable, Mapping, Sequence, Union -from .errors import StageConfigurationError +from .errors import StageConfigurationError, PipelineInitialisationError from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger from .models import PipelineConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now -from .stage import Stage +from .stage import Stage, _normalize_dependencies @@ -48,6 +48,7 @@ def __init__( backend: str = "python", config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, + dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, ): @@ -65,6 +66,13 @@ def __init__( self.logger = logger or Logger(log_dir=self.config.log_dir) self.executor = executor or PythonStageExecutor() self.stages = [self._coerce_stage(stage) for stage in (stages or [])] + self.dependencies = dependencies + if dependencies is not None and stages is None: + raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " + "for those stages. Try the from_files() method, or create your Stage objects and " \ + "parse them to the Pipeline Constructor.") + if dependencies is not None: + self._assign_dependencies(dependencies,self.stages) self.graph = StageGraph.from_stages(self.stages) self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -76,6 +84,14 @@ def __init__( backend=self.backend, stages=[stage.name for stage in self.stages], ) + def _assign_dependencies(self, + dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, + stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: + for stage in stages: + new_dependencies = self._dependencies_for_stage(stage.name,stage.source,dependencies) + stage.dependencies = _normalize_dependencies(new_dependencies) + + return stages def _coerce_stage( self, @@ -363,8 +379,8 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": @staticmethod def _dependencies_for_stage( stage_name: str, - path: Path, - dependencies: Mapping[str, Sequence[str]] | None, + path: Union[Path, Callable[..., Any], None] = None, + dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: """ Extracts a tuple of ``dependencies`` for the requested stage. @@ -385,8 +401,11 @@ def _dependencies_for_stage( """ if not dependencies: return () - - candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + if isinstance(path, Path): + candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + else: + candidates = (stage_name, str(path.__name__)) + print(candidates) for candidate in candidates: if candidate in dependencies: return tuple(str(dependency) for dependency in dependencies[candidate]) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index aec62fb..9f7e798 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,4 +1,7 @@ from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.errors import PipelineInitialisationError +from onsrap.stage import Stage +from pathlib import Path import pytest @pytest.fixture @@ -19,4 +22,39 @@ def test_pipeline_name(pipelineconfig): assert pipeline_config.name == "test_pipeline_config" pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" + + +def test_assign_dependencies(tmp_path): + def example_function(): + pass + dependencies_single = {"Stage_2":("Stage_1",)} + dependencies_multiple = {"Stage_1":["Stage_0", "Stage_0.5"], + "Stage_2":("Stage_1",)} + dependencies_non_stage_name = {"Stage_1.py":("Stage_0",), + "example_function":("Stage_1.py",)} + + with pytest.raises(PipelineInitialisationError): + Pipeline(stages = None, + dependencies = dependencies_single) + + path = tmp_path/"Stage_1.py" + pipeline_1 = Pipeline(name = "pipeline_1", + stages = [Stage("Stage_1", path, None,{}), + Stage("Stage_2", example_function, None,{})], + dependencies = dependencies_multiple) + + assert pipeline_1.stages[0].dependencies == ("Stage_0","Stage_0.5",) + assert pipeline_1.stages[1].dependencies == ("Stage_1",) + + pipeline_2 = Pipeline(name = "pipeline_2", + stages = [Stage("Stage_1", path, None,{}), + Stage("Stage_2", example_function, None,{})], + dependencies = dependencies_non_stage_name) + + assert pipeline_2.stages[0].dependencies == ("Stage_0",) + assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + + + + \ No newline at end of file From 974022914160bbfcfc207f7c37fb815189f97344 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 15:11:53 +0100 Subject: [PATCH 041/332] Completed testing suite for models.py --- tests/test_models.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 3ab6cbe..dd38580 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -151,6 +151,55 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> No """NOT SURE HOW TO TEST FROM_FILE()""" +def test_from_file(tmp_path,) -> PipelineConfig: + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + + fake_file = "path_not_real" + with pytest.raises(FileNotFoundError): + PipelineConfig.from_file(fake_file) + with pytest.raises(TypeError): + PipelineConfig.from_file(no_map_pipeline_config) + def test_to_dict(pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values From ed8ae5171b96680e74c66b5c5be1e7bd284d6ca3 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 15:34:30 +0100 Subject: [PATCH 042/332] docs/test: Add docstrings and finish testing suite for stage.py. --- tests/test_pipeline.py | 15 +++++++++------ tests/test_stage.py | 28 +++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9f7e798..e0cd641 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,11 +4,7 @@ from pathlib import Path import pytest -@pytest.fixture -def pipelineconfig() -> PipelineConfig: - return PipelineConfig(name = "test_pipeline_config") - -def test_pipeline_name(pipelineconfig): +def test_pipeline_name(): """ Test to confirm that Pipeline instance uses either defined name from instance creation (shown in pipeline_named), utilises name from PipelineConfig @@ -16,15 +12,22 @@ def test_pipeline_name(pipelineconfig): no name is provided through Pipeline instance creation or through the PipelineConfig (shown through pipeline_no_name) """ + pipeline_config = PipelineConfig(name = "test_pipeline_config") pipeline_named = Pipeline(name = "test_pipeline_name") assert pipeline_named.name == "test_pipeline_name" - pipeline_config = Pipeline(name = None, config = pipelineconfig) + pipeline_config = Pipeline(name = None, config = pipeline_config) assert pipeline_config.name == "test_pipeline_config" pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" def test_assign_dependencies(tmp_path): + """ + Test to ensure that different formats of dependencies can be parsed to the + Pipeline creation and appropriately assigned to each stage within the + Pipeline. Will also check for error raise if the dependencies are defined + but there are no defined stages. + """ def example_function(): pass dependencies_single = {"Stage_2":("Stage_1",)} diff --git a/tests/test_stage.py b/tests/test_stage.py index 7d6c922..61a92c1 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -167,4 +167,30 @@ def example_function(): """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST -""" \ No newline at end of file +""" + +def test_stage_instance_from_file(tmp_path) -> None: + """ + Tests that a Stage instance is created from a filepath. + """ + test_stage = tmp_path / "test_stage.py" + test_stage.write_text( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + assert Stage.from_file(test_stage, + entrypoint = "main") == Stage("test_stage", + test_stage.resolve(), + (), + {}, + "main", + "python") + From 2805b773afd6afd3182b592ae5478b6eb87e3715 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 17:26:56 +0100 Subject: [PATCH 043/332] docs: Updated README to cover issue #16 --- README.md | 96 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index db12f9c..8b063a8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,20 @@ located. ``` ## What is `onsrap`? -Add a summary of your project here. +Reproducible Analytical Pipelines (RAPs) are a cornerstone of high quality statistics. Reproducible refers to the concept that if code is run multiple times with the same inputs, it will produce the same outputs. A pipeline is a series of stages (small chunks of work) which are run in a specified order to produce desired outputs. Pipelines are crucial to reproducible work as they ensure that the code is run consistently. This increases the quality of the outputs by ensuring as little manual input as possible. + +ONSRap is a Python package that automatically orchestrates and runs these RAPs. The goal is to standardise how pipelines are run to reduce developer time required to convert existing code into RAP standards. As well as reducing developer time, this package also supports achieving RAP standards through items 4, 6, and 10. These are accomplished through this package by: + + - Item 4: Document everything that is needed to write and run the code + - This package includes inbuilt logging that records when the pipeline was run as well as the Pipeline configuration used. + +- Item 6: Code modules should run end-to-end without manual intervention + - This package is designed whereby once the configuration has been provided by the user and the main.py file is run, no further human input is required. + +- Item 10: Don't reinvent the wheel + - Multiple pipelines exist within the ONS and each have the potential to be orchestrated in different ways. This package aims to standardise the orchestration, ensuring consistency across pipelines. This consistency means that developers are more easily able to move between pipelines as they will all be structured in a similar way. + +For more information on the ONS Rap Minimum Standards, please see the full [standards documentation][standards]. ## Getting started @@ -18,11 +31,13 @@ requirements. It's suggested that you install this package and its requirements within a virtual environment. +Stages must be written in functional programming. A ``stage`` can be a file or a callable item, such as a function. If the ``stage`` is a file, it must have an entrypoint function (a function that, when called, runs the entirety of the stage). The ``stage`` file can run without an entrypoint, however the package has less control over the implementation and therefore best practice is inclusion of an entrypoint. + +There should be a parent file that sets out configuration, required directories and file paths, and builds the ``Pipeline`` instance. It is recommended that this is named something similar to ``main.py`` so that it is easy for users to see where the ``Pipeline`` starts. This file will be what is run through the terminal to run the entire pipeline. + ## Requirements -- Python 3.9+ installed -- a `.secrets` file with the required [secrets and credentials](#required-secrets-and-credentials) -- to have [loaded environment variables][docs-loading-environment-variables] from `.env` +- Python 3.14.6 installed Contributors have some additional requirements - please see our [contributing guidance][contributing]. @@ -53,7 +68,18 @@ Remember to update the setup and requirement files inline with any changes to yo package. ## Running the pipeline (Python only) +### Running Your Own Pipeline +To run your own Pipeline, you will need to build a ``Pipeline`` instance. This can be built using the from_files() method which requires a list of strings or Paths for your individual ``stages``. These are then compiled into a ``Pipeline`` instance. A ``Pipeline`` instance can also be created using the from_dict() method which takes a dictionary containing each attribute of the intented ``Pipeline`` instance and converts it. + +You will also need to define your ``PipelineConfig`` instance. This contains information regarding the working directory, project root, data directory, and log directory required to run the ``Pipeline`` as well as any metadata that you feel needs to be logged. + +Lastly, you need to define any ``dependencies`` required for the ``stages``. These are whether any stage needs to be run before another stage. These should be structured as a dictionary with the name of the stage as the key and the value is the stage/s that need to run before it as a tuple. +Both ``PipelineConfig`` and ``dependencies`` should be parsed into the ``Pipeline`` instance. + +Once you have your ``Pipeline`` instance, you can run the ``Pipeline.run()`` method which will run the entire ``Pipeline`` instance that has been created. + +### Example Pipeline The main runnable example now lives in `examples/pipeline_1/main.py`. It builds a three-stage pipeline from numbered scripts under `examples/pipeline_1/scripts/`. To run the example, use: @@ -66,21 +92,12 @@ Alternatively, most Python IDEs allow you to run the code directly using a `run` ## Required secrets and credentials -To run this project, you need a `.secrets` file with [secrets/credentials as -environmental variables][docs-loading-environment-variables-secrets]. The -secrets/credentials should have the following environment variable name(s): - -| Secret/credential | Environment variable name | Description | -|-------------------|---------------------------|--------------------------------------------| -| Secret 1 | `SECRET_VARIABLE_1` | Plain English description of Secret 1. | -| Credential 1 | `CREDENTIAL_VARIABLE_1` | Plain English description of Credential 1. | +No secrets or credentials are required for running this package. -Once you've added them, [load these environment variables][docs-loading-environment-variables] using -`.env`. ## Project structure layout -The cookiecutter template generated for each project will follow this folder structure: +The ONSRap repository has the following structure: ```shell . @@ -89,14 +106,46 @@ The cookiecutter template generated for each project will follow this folder str │ │ ├── raw/ │ │ ├── interim/ │ │ └── processed/ -│ └── onsrap/ -│ ├── example_modules/ -│ │ ├── __init__.py -│ │ └── example_module.py -│ ├── __init__.py -│ ├── example_config.yml -│ └── run_pipeline.py -└── ... +│ ├── onsrap/ +│ │ ├── example_modules/ +│ │ │ ├── __init__.py +│ │ │ └── example_module.py +│ │ ├── __init__.py +│ │ ├── errors.py +│ │ ├── execution.py +│ │ ├── graph.py +│ │ ├── loader.py +│ │ ├── models.py +│ │ ├── pipeline.py +│ │ ├── run_pipeline.py +│ │ ├── runner.py +│ │ └── stage.py +│ ├── examples/ +│ │ ├── pipeline_1/ +│ │ │ ├── data/ +│ │ │ │ └── orders.csv +│ │ │ ├── logs/ +│ │ │ │ └── onsrap.log +│ │ │ ├── runs/ +│ │ │ │ └── README.md +│ │ │ └── scripts/ +│ │ │ │ ├── 0_data_validation.py +│ │ │ │ ├── 1_preprocessing.py +│ │ │ │ └── 2_reporting.py +│ │ │ ├── Example.md +│ │ │ └── main.py +│ │ ├── pipeline_2/ +│ │ └── pipeline_3/ +│ ├── tests/ +│ │ ├── __init__.py +│ │ ├── repo_tests_README.md +│ │ ├── test_execution.py +│ │ ├── test_models.py +│ │ ├── test_pipeline_architecture.py +│ │ ├── test_pipeline.py +│ │ └── test_stage.py +│ └── +└── ``` ## Licence @@ -118,3 +167,4 @@ This project structure is based on the [`govcookiecutter` template project][govc [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter [docs-loading-environment-variables]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md [docs-loading-environment-variables-secrets]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md#storing-secrets-and-credentials +[standards]: https://best-practice-and-impact.github.io/ONS_minimum_RAP/ \ No newline at end of file From 45de124e8ca71725db408cdd1e1f434d626abc9c Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:31:45 +0100 Subject: [PATCH 044/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index ea4a378..59f8001 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -147,7 +147,7 @@ def resolve_output_root(self, run_dir: Path | None) -> Path: def resolve_given_path(self, stage_name: str | None, path_name: str | None, - file_name:str | None, + file_name: str | None, root: Path, add_folder: list[str] | str | None = None) -> Path: """ From 2ea3e2ac7469ae95de0e912fd6a5733ab2a1bdce Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:33:50 +0100 Subject: [PATCH 045/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 1340b2d..63dc53b 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -280,11 +280,11 @@ def with_dependencies(self, *dependencies: str) -> "Stage": ``Stage`` class instance with normalised ``dependencies`` attribute. """ unpacked_deps: list = [] - for a in dependencies: - if isinstance(a,list): - unpacked_deps = unpacked_deps + a + for dependency in dependencies: + if isinstance(dependency, list): + unpacked_deps = unpacked_deps + dependency else: - unpacked_deps.append(a) + unpacked_deps.append(dependency) for i in unpacked_deps: if isinstance(i, list): From af42b938ec4735e591e783c9326748edf2e2763c Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:34:14 +0100 Subject: [PATCH 046/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 63dc53b..56c4b9b 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -286,8 +286,8 @@ def with_dependencies(self, *dependencies: str) -> "Stage": else: unpacked_deps.append(dependency) - for i in unpacked_deps: - if isinstance(i, list): + for dependency in unpacked_deps: + if isinstance(dependency, list): raise StageDependencyError("Nested lists are not valid arguments for this method! " \ "Please provided single list or individual string values") From b6898597265324b4085b2398a7122d5bd19f59ae Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:35:29 +0100 Subject: [PATCH 047/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 56c4b9b..c725c25 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -291,7 +291,6 @@ def with_dependencies(self, *dependencies: str) -> "Stage": raise StageDependencyError("Nested lists are not valid arguments for this method! " \ "Please provided single list or individual string values") - print(unpacked_deps) return replace( self, dependencies=self.dependencies + _normalize_dependencies(unpacked_deps), From 3ee1ec557c1f51b2eb45092b134f82cecd95e3ab Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:35:43 +0100 Subject: [PATCH 048/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index d264fca..94ef373 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -53,7 +53,7 @@ def __init__( ): self.backend = backend or "python" self.config = PipelineConfig.from_any(config) - if (self.config.name is not None) and (name == None): + if (self.config.name is not None) and (name is None): self.name = self.config.name else: self.name = name or "pipeline" From 55897f4ae11ce45b22e6c2c0257e67db5dd07592 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:36:15 +0100 Subject: [PATCH 049/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 59f8001..cad1e7a 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -149,7 +149,8 @@ def resolve_given_path(self, stage_name: str | None, path_name: str | None, file_name: str | None, root: Path, - add_folder: list[str] | str | None = None) -> Path: + add_folder: list[str] | str | None = None + ) -> Path: """ Returns a file path for a requested item. From 7546afa908026bea7c864dba0c093e6383242da4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 15 Jul 2026 10:34:15 +0100 Subject: [PATCH 050/332] Fix: Actioned comments from PR #21 --- .../pipeline_1/scripts/0_data_validation.py | 4 +- .../pipeline_1/scripts/1_preprocessing.py | 4 +- examples/pipeline_1/scripts/2_reporting.py | 4 +- onsrap/errors.py | 6 +++ onsrap/execution.py | 31 ++++------- tests/test_execution.py | 52 ++++++++++++------- 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index 4a586ee..e54e2bb 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -85,8 +85,8 @@ def write_report(report_path: Path, report: dict[str, object]) -> None: def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index a55180c..1a2514c 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -87,8 +87,8 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() raw_path = context.resolve_given_path("0_data_validation", "raw_path", "orders.csv", data_root) clean_path = output_root / "interim" / "1_clean_orders.csv" diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index c3f50a8..bbbd1c2 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -69,8 +69,8 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() clean_path = context.resolve_given_path("1_preprocessing", "clean_path", "1_clean_orders.csv", output_root, "interim") diff --git a/onsrap/errors.py b/onsrap/errors.py index 1691b4d..bd29f9f 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -77,4 +77,10 @@ class PipelineInitialisationError(OnsrapError): """ Raised when there is an error in definition of the Pipeline instance + """ + +class PipelineConfigurationError(OnsrapError): + """ + Raised when there has been an issue with the PipelineConfig + instance. """ \ No newline at end of file diff --git a/onsrap/execution.py b/onsrap/execution.py index cad1e7a..3624b94 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Protocol, TYPE_CHECKING -from .errors import StageExecutionError, StageLoadError +from .errors import StageExecutionError, StageLoadError, PipelineConfigurationError from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable from .logger import Logger from .models import PipelineConfig, StageResult, StageStatus, now @@ -106,44 +106,35 @@ def stage_outputs(self) -> dict[str, Any]: """ return {name: result.outputs for name, result in self.stage_results.items()} - def resolve_data_root(self, config: PipelineConfig | None) -> Path: + def get_data_dir(self) -> Path: """ Establishes the filepath that the data is held in. - - Parameters - ---------- - ``config`` : PipelineConfig - The configuration of the Pipeline being run. This holds the location filepath - for the pipeline as defined by the user in the main.py file. Returns ------- Path The file path for the location of the data being used in the pipeline. """ - if config is not None: - return Path(config.data_dir) + if self.config is not None: + return Path(self.config.data_dir) - return Path(__file__).resolve().parents[1] / "data" + raise PipelineConfigurationError("Please parse a PipelineConfig instance to " \ + "the ExecutionContext.") - def resolve_output_root(self, run_dir: Path | None) -> Path: + def resolve_output_root(self) -> Path: """ Establishes the filepath that the outputs are going to be saved to. - Parameters - ---------- - ``run_dir`` : Path - The file directory that the run results are saved to. - Returns ------- Path The file path for the outputs of the run to be saved to. """ - if run_dir is not None: - return Path(run_dir) / "data" + if self.run_dir is not None: + return Path(self.run_dir) / "data" - return Path(__file__).resolve().parents[1] / "data" + raise PipelineConfigurationError("Please parse a run directory to " \ + "the ExecutionContext.") def resolve_given_path(self, stage_name: str | None, path_name: str | None, diff --git a/tests/test_execution.py b/tests/test_execution.py index 67ee35c..c5cafab 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest import onsrap.execution as execution_module +from onsrap.errors import PipelineConfigurationError @pytest.fixture def logger() -> Logger: @@ -129,34 +130,47 @@ def test_stage_outputs(execution, stageresult) -> None: def test_resolve_data_root(execution) -> None: """ Tests that resolve_data_root method extracts the path from the execution context - or, if the context is None, returns the file path for the module itself and the - data directory within that. + or, if the context is None, returns an error to indicate that additional input is + required. """ - assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") + assert execution.get_data_dir() == Path("tmp/config_data") + run_dir = Path("tmp/run") + work_dir = Path('tmp/work_dir') + execution_blank_config = ExecutionContext("test_pipeline", + "run_id_1234", + None, + Logger(), + run_dir, + '2024-05-06 15:45:30', + work_dir, + {"stage_test":stageresult}, + {} ) - result = execution.resolve_data_root(None) - expected = ( - Path(execution_module.__file__).resolve().parents[1] - / "data" - ) - assert result == expected + with pytest.raises(PipelineConfigurationError): + execution_blank_config.get_data_dir() def test_resolve_output_root(execution) -> None: """ Tests that resolve_output_root method extracts the path from the given run - directory or, if None are given, returns the file path for the module itself - and the data directory within that. + directory or, if None are given, raises an error to indicate additional input + is required.. """ - run_dir = Path("tmp/run") - assert execution.resolve_output_root(run_dir) == Path("tmp/run/data") + work_dir = Path('tmp/work_dir') + assert execution.resolve_output_root() == Path("tmp/run/data") - result = execution.resolve_output_root(None) - expected = ( - Path(execution_module.__file__).resolve().parents[1] - / "data" - ) - assert result == expected + execution_blank_config = ExecutionContext("test_pipeline", + "run_id_1234", + None, + Logger(), + None, + '2024-05-06 15:45:30', + work_dir, + {"stage_test":stageresult}, + {} ) + + with pytest.raises(PipelineConfigurationError): + execution_blank_config.resolve_output_root() """ Parameters for testing multiple add_folder options in From 7e6c8cb37377c5b246ac067540096899bf316a14 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 14:01:48 +0100 Subject: [PATCH 051/332] instantiated example structure --- examples/pipeline_2/conf.yaml | 33 +++++++++++++++++++ examples/pipeline_2/data/orders.csv | 7 ++++ examples/pipeline_2/main.py | 0 .../pipeline_2/scripts/0_data_validation.py | 0 .../pipeline_2/scripts/1_preprocessing.py | 0 examples/pipeline_2/scripts/2_reporting.py | 0 6 files changed, 40 insertions(+) create mode 100644 examples/pipeline_2/conf.yaml create mode 100644 examples/pipeline_2/data/orders.csv create mode 100644 examples/pipeline_2/main.py create mode 100644 examples/pipeline_2/scripts/0_data_validation.py create mode 100644 examples/pipeline_2/scripts/1_preprocessing.py create mode 100644 examples/pipeline_2/scripts/2_reporting.py diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml new file mode 100644 index 0000000..af04312 --- /dev/null +++ b/examples/pipeline_2/conf.yaml @@ -0,0 +1,33 @@ +pipeline_variables: + name: "Pipeline 2" + backend: python + stages: + - 0_data_validation: + # TODO: Put name/alias of stage as attribute here, not as key for stage. + location: "" + run: true + dependencies: [] + - 1_preprocessing: + location: "" + run: true + dependencies: + - 0_data_validation + - 2_reporting: + location: "" + run: true + dependencies: + - 1_preprocessing + working_dir: "examples/pipeline_2" + data_dir: "examples/pipeline_2/data" + output_dir: "examples/pipeline_2/output" + log_dir: "examples/pipeline_2/logs" + metadata: + example: retail-orders-using-configuration + description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." + + +stage_configuration: + 0_data_validation: + years_to_run: 2017 + time_col: "time" + target_variable: "classification" \ No newline at end of file diff --git a/examples/pipeline_2/data/orders.csv b/examples/pipeline_2/data/orders.csv new file mode 100644 index 0000000..3ffb04b --- /dev/null +++ b/examples/pipeline_2/data/orders.csv @@ -0,0 +1,7 @@ +order_id,customer_name,region,product,quantity,unit_price,order_date +1001,Alice Johnson,North,Notebook,2,3.50,2026-06-01 +1002,Ben Carter,South,Pen,5,1.20,2026-06-01 +1003,Chloe Nguyen,North,Notebook,1,3.50,2026-06-02 +1004,Daniel Patel,West,Folder,3,2.75,2026-06-03 +1005,Emma Stone,East,Pen,4,1.20,2026-06-03 +1006,Frank White, north , notebook ,2,3.50,2026-06-04 \ No newline at end of file diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/0_data_validation.py b/examples/pipeline_2/scripts/0_data_validation.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/1_preprocessing.py b/examples/pipeline_2/scripts/1_preprocessing.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py new file mode 100644 index 0000000..e69de29 From 9f83d5017f660635a9ac5ddd9ee2da14e2db7202 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 14:02:03 +0100 Subject: [PATCH 052/332] removal of one incorrect forward reference in Pipeline --- onsrap/pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 14daeaf..529e8e7 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -270,7 +270,7 @@ def from_files( dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, - ) -> "Pipeline": + ) -> Pipeline: """ Extracts the information from files regarding exactly what is being run in the pipeline and allows for configuration of how the Pipeline is run. @@ -337,6 +337,7 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": """ payload = dict(cfg) + # pipeline_variables contains pipeline information name = payload.pop("name", None) backend = payload.pop("backend", "python") config = payload.pop("config", None) @@ -356,6 +357,11 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": stages=stages, ) + def from_config(cls, config: dict) -> Pipeline: + # Wrapper for from_dict but expecting config Path/str or yaml object + # Extract pipeline_variables aka do not parse stage_configuration section of config.yaml + pass + @staticmethod def _dependencies_for_stage( stage_name: str, From 246f506b0977c2fdf68449c7d9da7ceaa56ad1ea Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 15 Jul 2026 15:29:51 +0100 Subject: [PATCH 053/332] Feat: add StageConfig class with getter method for _variables attribute --- onsrap/stage.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index b14b3ad..8beb238 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union +from datetime import datetime from .errors import StageConfigurationError @@ -342,4 +343,56 @@ def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageR ``execute`` method of the ``StageExecutor`` class stored in the ``StageResult`` class. """ self.validate() - return executor.execute(self, context) \ No newline at end of file + return executor.execute(self, context) + +@dataclass +class StageConfig: + """ + Class that holds information regarding the Stage including key information required to run the stage. + + Parameters + ---------- + ``name`` : str + The name of the stage. This should be the same as the ``Stage`` class instance. + ``_variables`` : Mapping[str, Any] | None, default = None + A mapping of variables and their basic definition. This would define standard variables + such as a sex variable alongside how it is named specifically within the data. This attribute + should not be directly interacted with. Instead, it should be defined through a yaml file or + through the set_config() method. + ``datasets`` : Mapping[str, Any] | None = None + The name of the dataset that is used within the stage alongside any useful information + regarding the data, for example the file location. + + """ + name: str + _variables: Mapping[str, Any] | None = None + datasets: Mapping[str, dict] | None = None + + def get_variables(self, variable: Iterable[str] | str | None = None) -> dict: + """ + Class method that outputs the _variables attribute. + + This method allows for the entire attribute to be extracted as well as single items + or multiple items in a list. These will be output as a dictionary of the values for + the keys requested. + + Parameters + ---------- + ``variable`` : Iterable[str] | str | None = None + """ + if variable is not None: + if isinstance(variable,Iterable): + requested_vars = {} + for requested in variable: + item = self._variables.get(requested) + requested_vars[requested] = item + if requested_vars is not None: + return requested_vars + raise StageConfigurationError("The variable/s you have requested does/do not exist") + if isinstance(variable, str): + item = self._variables.get(requested) + if item is not None: + return item + raise StageConfigurationError("The variable/s you have requested does/do not exist") + return self._variables + \ No newline at end of file From d567940f69001b39814b5d8cf0aedfd384b1faae Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 14:27:20 +0100 Subject: [PATCH 054/332] Added Logger.warning() support for Pipeline runtime warning info logging --- onsrap/logger.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index fd97dee..0fd1a31 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -98,3 +98,18 @@ def event(self, message: str, **kwargs: Any) -> None: else: self._logger.info(message) + def warning(self, message: str, **kwargs: Any) -> None: + """ + Logs a warning message with optional structured context. + + Parameters + ---------- + ``message`` : str + The main description of the warning to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. + """ + if kwargs: + self._logger.warning("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) + else: + self._logger.warning(message) \ No newline at end of file From 817ccbbf47efd829c6c32528527c7c0314f9c25e Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 15:43:22 +0100 Subject: [PATCH 055/332] Added warnings module --- onsrap/warnings.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 onsrap/warnings.py diff --git a/onsrap/warnings.py b/onsrap/warnings.py new file mode 100644 index 0000000..7fdd22b --- /dev/null +++ b/onsrap/warnings.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +class OnsrapWarning(Warning): + """Base warning for onsrap.""" + +class StageConfigurationWarning(OnsrapWarning): + """ + Raised when the pipeline configuration is not optimal. + Child class with ``OnsrapWarning`` as the parent class. + """ \ No newline at end of file From 99f038abf3d72d063cfaeabd172c9a3078af83d1 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 15:43:43 +0100 Subject: [PATCH 056/332] Initiali sketch of StageConfig use in Pipeline class. --- onsrap/pipeline.py | 49 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 529e8e7..c59989d 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -3,16 +3,18 @@ import getpass import hashlib import subprocess +import warnings import sys from importlib import metadata as importlib_metadata from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence from .errors import StageConfigurationError +from .warnings import StageConfigurationWarning from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger -from .models import PipelineConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now +from .models import PipelineConfig, StageConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now from .stage import Stage @@ -53,7 +55,7 @@ def __init__( ): self.name = name or "pipeline" self.backend = backend or "python" - self.config = PipelineConfig.from_any(config) + self.config = self._resolve_config(config) #PipelineConfig.from_any(config) if self.config.name is None: self.config.name = self.name self.config.backend = self.backend @@ -155,6 +157,9 @@ def validate(self) -> "Pipeline": stage.validate() self.graph.validate() return self + + def create_stage_config(self, s_config: str | Path) -> StageConfig: + pass def run(self) -> PipelineRun: """ @@ -259,6 +264,46 @@ def _current_user(self) -> str | None: except Exception: return None + def _resolve_config(self, config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None) -> list[PipelineConfig, StageConfig] | PipelineConfig | StageConfig: + if config is None: + return PipelineConfig.from_any(config) + if isinstance(config, str): + if config.endswith(".yaml") or config.endswith(".yml"): + return PipelineConfig.from_yaml(config) + else: + raise StageConfigurationError(f"Unsupported config file format parsed as Stage Configuration: {config!r}.") + if isinstance(config, Path): + if config.suffix in (".yaml", ".yml"): + return PipelineConfig.from_yaml(config) + else: + raise StageConfigurationError(f"Unsupported config file format parsed as Stage Configuration: {config!r}.") + if isinstance(config, PipelineConfig): + if "stage_config" in config.metadata: + warnings.warn( + "Stage Configuration found in PipelineConfig metadata. This should be moved to a separate location for StageConfiguration instantiation.", + StageConfigurationWarning + ) + self.logger.warning( + "Stage Configuration found in PipelineConfig metadata. This should be moved to a separate location for StageConfiguration instantiation." + ) + else: + warnings.warn( + "No Stage Configuration found in parsed configuration. This may lead to unexpected behavior during pipeline execution.", + StageConfigurationWarning + ) + self.logger.warning( + "No Stage Configuration found in parsed configuration. This may lead to unexpected behavior during pipeline execution." + ) + return config + if isinstance(config, Mapping): + # Look for Pipeline Configuration and Stage Configuration in keys + pass + + pipeline_config = self.config + stage_config = StageConfig() + return [pipeline_config, stage_config] + + @classmethod def from_files( cls, From fda7ea566fe5393c7941db033fd0c2840f3bb2f0 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:25:49 +0100 Subject: [PATCH 057/332] fix: Now allow users to define where their output location is in PipelineConfig and that works downstream in the ExecutionContext with the resolve_output_root() --- examples/pipeline_1/main.py | 1 + onsrap/execution.py | 2 +- onsrap/models.py | 4 ++++ onsrap/runner.py | 11 +++++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/pipeline_1/main.py b/examples/pipeline_1/main.py index 543fff9..4f71a70 100644 --- a/examples/pipeline_1/main.py +++ b/examples/pipeline_1/main.py @@ -28,6 +28,7 @@ def build_pipeline() -> Pipeline: backend="python", work_dir=PIPELINE_ROOT, project_root=PIPELINE_ROOT, + output_dir=PIPELINE_ROOT, data_dir=DATA_DIR, log_dir=LOG_DIR, metadata={ diff --git a/onsrap/execution.py b/onsrap/execution.py index 3624b94..6bdd95b 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -131,7 +131,7 @@ def resolve_output_root(self) -> Path: The file path for the outputs of the run to be saved to. """ if self.run_dir is not None: - return Path(self.run_dir) / "data" + return Path(self.run_dir) raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") diff --git a/onsrap/models.py b/onsrap/models.py index b37a6d5..8c2cd0e 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -137,6 +137,7 @@ class PipelineConfig: backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) project_root: Optional[Path] = None + output_dir: Optional[Path] = None log_dir: Path = field(default_factory=lambda: Path("logs")) data_dir: Path = field(default_factory=lambda: Path("data")) allow_subprocess_fallback: bool = True @@ -207,6 +208,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": backend = payload.pop("backend", "python") work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) + output_dir_value = payload.pop("output_dir", None) project_root = Path(project_root_value) if project_root_value is not None else work_dir log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) @@ -220,6 +222,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": backend=backend, work_dir=work_dir, project_root=project_root, + output_dir=output_dir_value, log_dir=log_dir, data_dir=data_dir, allow_subprocess_fallback=allow_subprocess_fallback, @@ -278,6 +281,7 @@ def to_dict(self) -> dict[str, Any]: "backend": self.backend, "work_dir": str(self.work_dir), "project_root": str(self.project_root) if self.project_root is not None else None, + "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), "allow_subprocess_fallback": self.allow_subprocess_fallback, diff --git a/onsrap/runner.py b/onsrap/runner.py index fa5c876..6d78310 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import warnings from pathlib import Path from typing import TYPE_CHECKING @@ -52,8 +53,14 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: runtime_id = pipeline._create_runtime_id() pipeline.id = runtime_id - project_root = Path(pipeline.config.project_root or pipeline.config.work_dir) - run_dir = project_root / "runs" / runtime_id.get_id() + if pipeline.config.output_dir is not None: + run_output = Path(pipeline.config.output_dir) + else: + warnings.warn( + "Output directory is not specified. Using project root or work directory as the run output." + ) # TODO: fill with warnings from Pipeline branch + run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) + run_dir = run_output / "runs" / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) started_at = now() From d834ea4f9091e9f632483524e39077cab2f30232 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:42:41 +0100 Subject: [PATCH 058/332] Tweaked testing for exection and models to reflect changes with output directory defintion --- tests/test_execution.py | 7 ++++--- tests/test_models.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index c5cafab..3760251 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -21,12 +21,13 @@ def config() -> PipelineConfig: work_dir = Path('tmp/work_dir') project_root = Path('tmp/project') log_dir = Path('tmp/log') - data_dir = "tmp/config_data" + data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", "python", work_dir, - project_root, + project_root, + None, log_dir, data_dir, True, @@ -157,7 +158,7 @@ def test_resolve_output_root(execution) -> None: is required.. """ work_dir = Path('tmp/work_dir') - assert execution.resolve_output_root() == Path("tmp/run/data") + assert execution.resolve_output_root() == Path("tmp/run") execution_blank_config = ExecutionContext("test_pipeline", "run_id_1234", diff --git a/tests/test_models.py b/tests/test_models.py index dd38580..715b5fe 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -210,6 +210,7 @@ def test_to_dict(pipelineconfig) -> None: "backend":"python", "work_dir":"tmp\\work", "project_root":"project", + "output_dir":None, "log_dir":"tmp\\logs", "data_dir":"tmp\\data", "allow_subprocess_fallback":True, From bff68520cc41c0ffec9dee7f307b006650727789 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:45:15 +0100 Subject: [PATCH 059/332] resolved union change --- onsrap/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 40aa65c..517c975 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -6,7 +6,7 @@ import sys from importlib import metadata as importlib_metadata from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Sequence, Union +from typing import Any, Callable, Iterable, Mapping, Sequence from .errors import StageConfigurationError, PipelineInitialisationError from .execution import PythonStageExecutor, StageExecutor @@ -379,7 +379,7 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": @staticmethod def _dependencies_for_stage( stage_name: str, - path: Union[Path, Callable[..., Any], None] = None, + path: Path | Callable[..., Any] | None = None, dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: """ From b6cd71114b931276e432d3624b7e63ff155686a9 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 14:01:48 +0100 Subject: [PATCH 060/332] instantiated example structure --- examples/pipeline_2/conf.yaml | 33 +++++++++++++++++++ examples/pipeline_2/data/orders.csv | 7 ++++ examples/pipeline_2/main.py | 0 .../pipeline_2/scripts/0_data_validation.py | 0 .../pipeline_2/scripts/1_preprocessing.py | 0 examples/pipeline_2/scripts/2_reporting.py | 0 6 files changed, 40 insertions(+) create mode 100644 examples/pipeline_2/conf.yaml create mode 100644 examples/pipeline_2/data/orders.csv create mode 100644 examples/pipeline_2/main.py create mode 100644 examples/pipeline_2/scripts/0_data_validation.py create mode 100644 examples/pipeline_2/scripts/1_preprocessing.py create mode 100644 examples/pipeline_2/scripts/2_reporting.py diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml new file mode 100644 index 0000000..af04312 --- /dev/null +++ b/examples/pipeline_2/conf.yaml @@ -0,0 +1,33 @@ +pipeline_variables: + name: "Pipeline 2" + backend: python + stages: + - 0_data_validation: + # TODO: Put name/alias of stage as attribute here, not as key for stage. + location: "" + run: true + dependencies: [] + - 1_preprocessing: + location: "" + run: true + dependencies: + - 0_data_validation + - 2_reporting: + location: "" + run: true + dependencies: + - 1_preprocessing + working_dir: "examples/pipeline_2" + data_dir: "examples/pipeline_2/data" + output_dir: "examples/pipeline_2/output" + log_dir: "examples/pipeline_2/logs" + metadata: + example: retail-orders-using-configuration + description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." + + +stage_configuration: + 0_data_validation: + years_to_run: 2017 + time_col: "time" + target_variable: "classification" \ No newline at end of file diff --git a/examples/pipeline_2/data/orders.csv b/examples/pipeline_2/data/orders.csv new file mode 100644 index 0000000..3ffb04b --- /dev/null +++ b/examples/pipeline_2/data/orders.csv @@ -0,0 +1,7 @@ +order_id,customer_name,region,product,quantity,unit_price,order_date +1001,Alice Johnson,North,Notebook,2,3.50,2026-06-01 +1002,Ben Carter,South,Pen,5,1.20,2026-06-01 +1003,Chloe Nguyen,North,Notebook,1,3.50,2026-06-02 +1004,Daniel Patel,West,Folder,3,2.75,2026-06-03 +1005,Emma Stone,East,Pen,4,1.20,2026-06-03 +1006,Frank White, north , notebook ,2,3.50,2026-06-04 \ No newline at end of file diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/0_data_validation.py b/examples/pipeline_2/scripts/0_data_validation.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/1_preprocessing.py b/examples/pipeline_2/scripts/1_preprocessing.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py new file mode 100644 index 0000000..e69de29 From bcca0342d76340b6dc221ffbfcbb4e587356511a Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 14:02:03 +0100 Subject: [PATCH 061/332] removal of one incorrect forward reference in Pipeline --- onsrap/pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 517c975..7168826 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -290,7 +290,7 @@ def from_files( dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, - ) -> "Pipeline": + ) -> Pipeline: """ Extracts the information from files regarding exactly what is being run in the pipeline and allows for configuration of how the Pipeline is run. @@ -357,6 +357,7 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": """ payload = dict(cfg) + # pipeline_variables contains pipeline information name = payload.pop("name", None) backend = payload.pop("backend", "python") config = payload.pop("config", None) @@ -376,6 +377,11 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": stages=stages, ) + def from_config(cls, config: dict) -> Pipeline: + # Wrapper for from_dict but expecting config Path/str or yaml object + # Extract pipeline_variables aka do not parse stage_configuration section of config.yaml + pass + @staticmethod def _dependencies_for_stage( stage_name: str, From 05f6b646ddb320b2d96f4620a6547d70d30bd011 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 15 Jul 2026 15:29:51 +0100 Subject: [PATCH 062/332] Feat: add StageConfig class with getter method for _variables attribute --- onsrap/stage.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index c725c25..3a196f0 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union +from datetime import datetime from .errors import StageConfigurationError, StageDependencyError @@ -362,4 +363,56 @@ def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageR ``execute`` method of the ``StageExecutor`` class stored in the ``StageResult`` class. """ self.validate() - return executor.execute(self, context) \ No newline at end of file + return executor.execute(self, context) + +@dataclass +class StageConfig: + """ + Class that holds information regarding the Stage including key information required to run the stage. + + Parameters + ---------- + ``name`` : str + The name of the stage. This should be the same as the ``Stage`` class instance. + ``_variables`` : Mapping[str, Any] | None, default = None + A mapping of variables and their basic definition. This would define standard variables + such as a sex variable alongside how it is named specifically within the data. This attribute + should not be directly interacted with. Instead, it should be defined through a yaml file or + through the set_config() method. + ``datasets`` : Mapping[str, Any] | None = None + The name of the dataset that is used within the stage alongside any useful information + regarding the data, for example the file location. + + """ + name: str + _variables: Mapping[str, Any] | None = None + datasets: Mapping[str, dict] | None = None + + def get_variables(self, variable: Iterable[str] | str | None = None) -> dict: + """ + Class method that outputs the _variables attribute. + + This method allows for the entire attribute to be extracted as well as single items + or multiple items in a list. These will be output as a dictionary of the values for + the keys requested. + + Parameters + ---------- + ``variable`` : Iterable[str] | str | None = None + """ + if variable is not None: + if isinstance(variable,Iterable): + requested_vars = {} + for requested in variable: + item = self._variables.get(requested) + requested_vars[requested] = item + if requested_vars is not None: + return requested_vars + raise StageConfigurationError("The variable/s you have requested does/do not exist") + if isinstance(variable, str): + item = self._variables.get(requested) + if item is not None: + return item + raise StageConfigurationError("The variable/s you have requested does/do not exist") + return self._variables + \ No newline at end of file From 6495e7f016aea2f00921d9d1e95700622c0eec5c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 16 Jul 2026 11:48:01 +0100 Subject: [PATCH 063/332] Adding a second pipeline example --- examples/pipeline_2/data/orders_cleaned.csv | 7 ++ examples/pipeline_2/data/orders_prepped.csv | 7 ++ examples/pipeline_2/scripts/0_clean_data.py | 59 ++++++++++++++ .../pipeline_2/scripts/0_data_validation.py | 0 examples/pipeline_2/scripts/1_derive_vars.py | 76 +++++++++++++++++++ .../pipeline_2/scripts/1_preprocessing.py | 0 examples/pipeline_2/scripts/2_reporting.py | 3 + 7 files changed, 152 insertions(+) create mode 100644 examples/pipeline_2/data/orders_cleaned.csv create mode 100644 examples/pipeline_2/data/orders_prepped.csv create mode 100644 examples/pipeline_2/scripts/0_clean_data.py delete mode 100644 examples/pipeline_2/scripts/0_data_validation.py create mode 100644 examples/pipeline_2/scripts/1_derive_vars.py delete mode 100644 examples/pipeline_2/scripts/1_preprocessing.py diff --git a/examples/pipeline_2/data/orders_cleaned.csv b/examples/pipeline_2/data/orders_cleaned.csv new file mode 100644 index 0000000..c3e721b --- /dev/null +++ b/examples/pipeline_2/data/orders_cleaned.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date +1001,north,notebook,2,3.5,2026-06-01 +1002,south,pen,5,1.2,2026-06-01 +1003,north,notebook,1,3.5,2026-06-02 +1004,west,folder,3,2.75,2026-06-03 +1005,east,pen,4,1.2,2026-06-03 +1006,north,notebook,2,3.5,2026-06-04 diff --git a/examples/pipeline_2/data/orders_prepped.csv b/examples/pipeline_2/data/orders_prepped.csv new file mode 100644 index 0000000..a65d404 --- /dev/null +++ b/examples/pipeline_2/data/orders_prepped.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order +1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False +1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False +1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True +1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False +1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False +1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py new file mode 100644 index 0000000..9f5c673 --- /dev/null +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -0,0 +1,59 @@ +import pandas as pd + + +def check_variables(df, expected_variables): + missing = [] + for i in expected_variables: + if i in df.columns: + pass + else: + missing.append(i) + if missing == []: + print("All variables present") + print(f"Missing the following variables: {missing}") + + +def remove_identifiable(df,identifiable_cols): + for i in identifiable_cols: + if i in df.columns: + df = df.drop(i, axis = 1) + else: + pass + return df + + +def standardise_columns(df): + df.columns = [col.lower() for col in df.columns] + df.columns = [col.capitalize() for col in df.columns] + for item in df.columns: + df[item] = df[item].apply(lambda x: x.lower() if isinstance(x,str) else x) + df[item] = df[item].apply(lambda x: x.strip() if isinstance(x,str) else x) + return df + + + +def main(): + orders = pd.read_csv("examples/pipeline_2/data/orders.csv") + + expected_variables = ["order_id", + "customer_name", + "region", + "product", + "quantity", + "unit_price", + "order_date", + "order_method"] + + identifiable_cols = ["customer_name", + "age", + "dob", + "address"] + + check_variables(orders,expected_variables) + print(orders.dtypes) + orders = remove_identifiable(orders, identifiable_cols) + orders = standardise_columns(orders) + print(orders) + orders.to_csv("examples/pipeline_2/data/orders_cleaned.csv", index = False) + +main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/0_data_validation.py b/examples/pipeline_2/scripts/0_data_validation.py deleted file mode 100644 index e69de29..0000000 diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py new file mode 100644 index 0000000..9b0540f --- /dev/null +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -0,0 +1,76 @@ +import pandas as pd +import numpy as np +from datetime import timedelta + +def correct_date_time(df): + df["Order_date"] = pd.to_datetime(df["Order_date"]) + return df + + +def estimate_delivery(df, delivery_times): + df["Estimated_delvery_date"] = df["Order_date"] + pd.to_timedelta(df["Region"].map(delivery_times), unit = "D") + df["Delivery_day"] = df["Estimated_delvery_date"].dt.day_name() + return df + +def total_cost(df): + df["Total_cost"] = df["Quantity"] * df["Unit_price"] + return df + +def order_date_values(df): + df["Order_day"] = df["Order_date"].dt.day_name() + df["Order_month"] = df["Order_date"].dt.month_name() + return(df) + +def size_order_alert(df): + df["Large_order"] = df["Quantity"] > df["Quantity"].quantile(0.75) + df["Small_order"] = df["Quantity"] < df["Quantity"].quantile(0.25) + return df + +def postage_cost(df): + df["Postage"] = np.select( + [ + df["Large_order"], + df["Small_order"] + ], + [ + 5.00, + 1.00 + ], + default = 2.50 + ) + return df + +def production_cost(df): + df["Production_cost"] = np.select( + [ + df["notebook"], + df["pen"] + ], + [ + 1.00, + 0.3, + ], + default = 2.50 + ) + return df + + + +def main(): + df = pd.read_csv("examples/pipeline_2/data/orders_cleaned.csv") + delivery_times = {"north":14, + "south":4, + "east":7, + "west":7} + + df = correct_date_time(df) + df = estimate_delivery(df, delivery_times) + df = total_cost(df) + df = order_date_values(df) + df = size_order_alert(df) + df = postage_cost(df) + df.to_csv("examples/pipeline_2/data/orders_prepped.csv", index = False) + + + +main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/1_preprocessing.py b/examples/pipeline_2/scripts/1_preprocessing.py deleted file mode 100644 index e69de29..0000000 diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index e69de29..aa56bf7 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -0,0 +1,3 @@ +import pandas + +orders = \ No newline at end of file From 953c5a0572ea08c39955601c9be19c735ff6acfa Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 3 Jul 2026 11:05:06 +0100 Subject: [PATCH 064/332] Script for testing stage.py and minor change to documentation in execution.py --- onsrap/execution.py | 1 - tests/test_stage.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/test_stage.py diff --git a/onsrap/execution.py b/onsrap/execution.py index 2c97fa6..a0a7396 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -251,7 +251,6 @@ def _execute_callable( def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: """ Attempt to run a file. - Attempt to run a callable object. Calls the logger.event() method to record an event and attempts to run diff --git a/tests/test_stage.py b/tests/test_stage.py new file mode 100644 index 0000000..5213945 --- /dev/null +++ b/tests/test_stage.py @@ -0,0 +1,182 @@ +import pytest +from onsrap.stage import _normalize_dependencies, Stage, StageConfigurationError +from pathlib import Path +from textwrap import dedent + +def test_normalize_dependencies_none() -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + +def test_normalize_dependencies_str() -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py"," Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py","Stage_2.py", "Stage_3.py") + +@pytest.fixture +def example_function(): + """ + Test function to pass as a callable stage for stage testing. + """ + print("This is a test function") + +@pytest.fixture +def stage_test() -> Stage: + """ + Stage object for testing Stage class methods and construction. + """ + return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) + +def test_stage_creation_callable(stage_test) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info":"example"} + assert stage_test.entrypoint == None + assert stage_test.backend == "python" + +def test_stage_name_error(example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("",example_function,["stage_1"],{"info":"example"}) + +def test_stage_source_type() -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + +def test_stage_backend(example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + """ + stage_diff = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "java") + stage = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "") + stage_white_space = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "python ") + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + +def test_stage_from_files_error(tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + stage = Stage.from_file(source_file) + +def test_stage_from_callable_name() -> None: + """ + Tests that a stage name is extracted from a callable object stage. + """ + def example_function(): + pass + test = Stage.from_callable(example_function) + assert test.name == "example_function" + +@pytest.mark.skip +def test_from_dict_norm() -> None: + """ + REVIEW WITH ALEX + """ + def example_function(): + pass + data = {"name":"test_Stage", + "callable_source" : example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + +def test_with_dependencies_list(stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. + + REVIEW WITH ALEX + This test works and passes however it doesn't behave how I was expecting it to. + Was expecting the list/dictionary to be broken down so you have one tuple rather + than a tuple of dict/lists. Is this a problem or just my understanding? + """ + new_deps = ["stage2","stage3"] + new_dep_dict = {"stage1":"stage0"} + new_deps_blank = [] + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_dict = stage_test.with_dependencies(new_dep_dict) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1","['stage2', 'stage3']") + assert stage_test_dict.dependencies == ("stage_1","{'stage1': 'stage0'}") + assert stage_test_blank.dependencies == ("stage_1", '[]') + +@pytest.mark.skip +def test_validate(stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + REVIEW WITH ALEX + Does not raise a StageConfigurationError is the source is a blank string. Is + this a concern? Do we want this validate to be able to do other error checks + like if it is an int? + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + +def test_source_path(stage_test, tmp_path) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source types. + """ + stage_test.source = tmp_path/"fake_file.py" + assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = 11 + assert stage_test.source_path == None + stage_test.source = "not a file path" + assert stage_test.source_path == None + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_path == None + +def test_source_label(stage_test, tmp_path) -> None: + """ + Tests that source_label is created if the source is a Path or a callable and is None if it is + another type. + """ + stage_test.source = tmp_path/"fake_file.py" + temp_path_str = str(tmp_path/"fake_file.py") + assert stage_test.source_label == temp_path_str + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label == None + +""" +TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST +""" \ No newline at end of file From 897a00e6b120a42f23350a8245ce656ee15945ed Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 11:08:51 +0100 Subject: [PATCH 065/332] Partial testing established for execution.py --- tests/test_execution.py | 121 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_execution.py diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 0000000..143f55b --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,121 @@ +from onsrap.execution import ExecutionContext +from onsrap.models import PipelineConfig, StageResult, StageStatus +from onsrap.logger import Logger +from pathlib import Path +import pytest + +@pytest.fixture +def logger() -> Logger: + """ + Logger instance for testing + """ + return Logger() + +@pytest.fixture +def config(tmp_path) -> PipelineConfig: + """ + Return a PipelineConfig object for testing. + """ + work_dir = tmp_path/"work" + project_root = tmp_path/"project" + log_dir = tmp_path/"log" + data_dir = tmp_path/"data" + return PipelineConfig( + "test_pipeline", + "python", + work_dir, + project_root, + log_dir, + data_dir, + True, + None, + {} + ) + +@pytest.fixture +def execution(config, logger, tmp_path) -> ExecutionContext: + """ + Create an ExecutionContext object for testing. + """ + run_dir = tmp_path/"run" + work_dir = tmp_path/"work_dir" + + return ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + run_dir, + '2024-05-06 15:45:30', + work_dir, + {}, + {} + ) + +@pytest.fixture +def stageresult() -> StageResult: + """ + Test StageResult instance for running ExecutionContext tests. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = "example output" + ) + + +def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: + """ + Test that the ExecutionContext creates the right attributes. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == tmp_path/"run" + assert execution.started_at == '2024-05-06 15:45:30' + assert execution.working_directory == tmp_path/"work_dir" + assert execution.stage_results == {} + assert execution.variables == {} + +def test_record(stageresult, execution) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + """ + execution.record(stageresult) + assert execution.stage_results == {'stage_test':StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None)} + assert execution.variables == {'stage_test':"example output"} + +def test_result_for(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.result_for("stage_test") == StageResult(name='stage_test', + status='pending', + started_at='2024-05-06 15:45:30', + finished_at='2024-05-07 15:45:30', + outputs="example output", + stdout='', + stderr='', + return_code=None, + metadata={}, + error=None, + source=None) + +def test_stage_outputs(execution, stageresult) -> None: + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test":"example output"} + +"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file From b7a18868b8a042a1072244a9324571c8cd78c941 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 17:15:52 +0100 Subject: [PATCH 066/332] Add class methods to ExecutionContext that allow file paths to be created/extracted for I/O --- .../pipeline_1/scripts/0_data_validation.py | 18 +--- .../pipeline_1/scripts/1_preprocessing.py | 32 +------- examples/pipeline_1/scripts/2_reporting.py | 33 ++------ onsrap/execution.py | 82 +++++++++++++++++++ tests/test_execution.py | 2 +- 5 files changed, 94 insertions(+), 73 deletions(-) diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index eb331cc..4a586ee 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -17,20 +17,6 @@ ) -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -99,8 +85,8 @@ def write_report(report_path: Path, report: dict[str, object]) -> None: def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index 0d8ad1c..a55180c 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_raw_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - validation_result = context.result_for("0_data_validation") - if validation_result is not None: - raw_path = validation_result.outputs.get("raw_path") - if raw_path: - return Path(raw_path) - - return data_root / "orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -112,9 +87,10 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - raw_path = resolve_raw_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + raw_path = context.resolve_given_path("0_data_validation", "raw_path", + "orders.csv", data_root) clean_path = output_root / "interim" / "1_clean_orders.csv" rows = load_orders(raw_path) diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index 337c2e5..c3f50a8 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -7,31 +7,6 @@ from typing import Any -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_clean_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - preprocessing_result = context.result_for("1_preprocessing") - if preprocessing_result is not None: - clean_path = preprocessing_result.outputs.get("clean_path") - if clean_path: - return Path(clean_path) - - return data_root / "interim" / "1_clean_orders.csv" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -94,9 +69,11 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - clean_path = resolve_clean_path(context, data_root) + data_root = context.resolve_data_root(config = context.config) + output_root = context.resolve_output_root(run_dir = context.run_dir) + clean_path = context.resolve_given_path("1_preprocessing", "clean_path", + "1_clean_orders.csv", output_root, + "interim") summary_path = output_root / "processed" / "2_sales_summary.json" region_breakdown_path = output_root / "processed" / "2_revenue_by_region.csv" diff --git a/onsrap/execution.py b/onsrap/execution.py index a0a7396..fca75d0 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -105,6 +105,88 @@ def stage_outputs(self) -> dict[str, Any]: the run. """ return {name: result.outputs for name, result in self.stage_results.items()} + + def resolve_data_root(self, config: PipelineConfig | None) -> Path: + """ + Establishes the filepath that the data is held in. + + Parameters + ---------- + ``config`` : PipelineConfig + The configuration of the Pipeline being run. This holds the location filepath + for the pipeline as defined by the user in the main.py file. + + Returns + ------- + Path + The file path for the location of the data being used in the pipeline. + """ + if config is not None: + return Path(config.data_dir) + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_output_root(self, run_dir: Path | None) -> Path: + """ + Establishes the filepath that the outputs are going to be saved to. + + Parameters + ---------- + ``run_dir`` : Path + The file directory that the run results are saved to. + + Returns + ------- + Path + The file path for the outputs of the run to be saved to. + """ + if run_dir is not None: + return Path(run_dir) / "data" + + return Path(__file__).resolve().parents[1] / "data" + + def resolve_given_path(self, stage_name: str, + path_name: str, + file_name:str, + root: Path, + add_folder: str | None = None) -> Path: + """ + Returns a file path for a requested item. + + This investigates the result of a previous stage to extract a selected path. + If the path is not available, it creates a path using a root previously derived + in main.py, the chosen directory within the root (optional), and the file path. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage where the path was outputted. + ``path_name`` : str + The name for the path within the stage results. This will be the key from the + key/value pair within the output of the previous stage. + ``root`` : Path + The file path for the root of the directory. This should be denoted through + other methods. + ``add_folder`` : str | None, default = None + Additional folder name to add into the returned file path. Additional functionality + should be added to allow for multiple folders to be added to the path. + + Returns + ------- + Path + The file path where data has previously been saved to to allow for extraction of + that data throughout the pipeline. + """ + result = self.result_for(stage_name) + if result is not None: + selected_path = result.outputs.get(path_name) + if selected_path: + return Path(selected_path) + if add_folder is not None: + #Would like to add functionality here for multiple additional folders + return root / add_folder / file_name + + return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 143f55b..7fc87b0 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -118,4 +118,4 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE FROM STAGEEXECUTOR CLASS""" \ No newline at end of file +"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file From 2970dc7fd1d37eae4fc468f1c2fae781ebb32d31 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 12:31:38 +0100 Subject: [PATCH 067/332] Minor testing implemented --- onsrap/execution.py | 12 ++++++----- tests/test_execution.py | 46 ++++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index fca75d0..050fc0d 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -149,7 +149,7 @@ def resolve_given_path(self, stage_name: str, path_name: str, file_name:str, root: Path, - add_folder: str | None = None) -> Path: + add_folder: list[str] | str | None = None) -> Path: """ Returns a file path for a requested item. @@ -167,9 +167,8 @@ def resolve_given_path(self, stage_name: str, ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. - ``add_folder`` : str | None, default = None - Additional folder name to add into the returned file path. Additional functionality - should be added to allow for multiple folders to be added to the path. + ``add_folder`` : list[str] | str | None, default = None + Additional folder name/s to add into the returned file path. Returns ------- @@ -183,7 +182,10 @@ def resolve_given_path(self, stage_name: str, if selected_path: return Path(selected_path) if add_folder is not None: - #Would like to add functionality here for multiple additional folders + if add_folder is list: + add_folder = add_folder.append(file_name) + new_path = root.joinpath(*add_folder) + return new_path return root / add_folder / file_name return root / file_name diff --git a/tests/test_execution.py b/tests/test_execution.py index 7fc87b0..756a3e2 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -12,14 +12,14 @@ def logger() -> Logger: return Logger() @pytest.fixture -def config(tmp_path) -> PipelineConfig: +def config() -> PipelineConfig: """ Return a PipelineConfig object for testing. """ - work_dir = tmp_path/"work" - project_root = tmp_path/"project" - log_dir = tmp_path/"log" - data_dir = tmp_path/"data" + work_dir = Path('tmp/work_dir') + project_root = Path('tmp/project') + log_dir = Path('tmp/log') + data_dir = "tmp/config_data" return PipelineConfig( "test_pipeline", "python", @@ -33,12 +33,12 @@ def config(tmp_path) -> PipelineConfig: ) @pytest.fixture -def execution(config, logger, tmp_path) -> ExecutionContext: +def execution(config, logger) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ - run_dir = tmp_path/"run" - work_dir = tmp_path/"work_dir" + run_dir = Path("tmp/run") + work_dir = Path('tmp/work_dir') return ExecutionContext( "test_pipeline", @@ -67,7 +67,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -75,9 +75,9 @@ def test_executioncontext_creation(execution, tmp_path, logger, config) -> None: assert execution.run_id == "run_id_1234" assert execution.config == config assert execution.logger == logger - assert execution.run_dir == tmp_path/"run" + assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' - assert execution.working_directory == tmp_path/"work_dir" + assert execution.working_directory == Path('tmp/work_dir') assert execution.stage_results == {} assert execution.variables == {} @@ -101,6 +101,9 @@ def test_record(stageresult, execution) -> None: assert execution.variables == {'stage_test':"example output"} def test_result_for(execution, stageresult) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + """ execution.record(stageresult) assert execution.result_for("stage_test") == StageResult(name='stage_test', status='pending', @@ -115,7 +118,26 @@ def test_result_for(execution, stageresult) -> None: source=None) def test_stage_outputs(execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + """ execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -"""TESTING TO CONTINUE RESOLVE CLASS METHODS""" \ No newline at end of file +def test_resolve_data_root(execution) -> None: + """ + Tests that resolve_data_root method extracts the path from the execution context + or, if the context is None, returns the file path for the module itself and the + data directory within that. + """ + assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") + + import onsrap.execution as execution_module + result = execution.resolve_data_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + From 187222889b34e3ab357dcc2acfa21465c5478b7b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 7 Jul 2026 16:27:40 +0100 Subject: [PATCH 068/332] Add testing for resolve_paths class functions --- onsrap/execution.py | 28 +++++++----- tests/test_execution.py | 99 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 050fc0d..ea4a378 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -145,9 +145,9 @@ def resolve_output_root(self, run_dir: Path | None) -> Path: return Path(__file__).resolve().parents[1] / "data" - def resolve_given_path(self, stage_name: str, - path_name: str, - file_name:str, + def resolve_given_path(self, stage_name: str | None, + path_name: str | None, + file_name:str | None, root: Path, add_folder: list[str] | str | None = None) -> Path: """ @@ -164,6 +164,8 @@ def resolve_given_path(self, stage_name: str, ``path_name`` : str The name for the path within the stage results. This will be the key from the key/value pair within the output of the previous stage. + ``file_name`` : str + The name of the file that you are trying to access the Path for. ``root`` : Path The file path for the root of the directory. This should be denoted through other methods. @@ -177,18 +179,24 @@ def resolve_given_path(self, stage_name: str, that data throughout the pipeline. """ result = self.result_for(stage_name) - if result is not None: + if result is not None and path_name is not None: selected_path = result.outputs.get(path_name) if selected_path: return Path(selected_path) - if add_folder is not None: - if add_folder is list: - add_folder = add_folder.append(file_name) - new_path = root.joinpath(*add_folder) + if isinstance(add_folder, list): + if file_name is not None: + new_path = root.joinpath(*add_folder, file_name) return new_path - return root / add_folder / file_name + new_path = root.joinpath(*add_folder) + return new_path + if isinstance(add_folder, str): + if file_name is not None: + return root/ add_folder/ file_name + return root / add_folder + if file_name is not None: + return root / file_name + return root - return root / file_name class StageExecutor(Protocol): diff --git a/tests/test_execution.py b/tests/test_execution.py index 756a3e2..722449e 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -3,6 +3,7 @@ from onsrap.logger import Logger from pathlib import Path import pytest +import onsrap.execution as execution_module @pytest.fixture def logger() -> Logger: @@ -33,7 +34,7 @@ def config() -> PipelineConfig: ) @pytest.fixture -def execution(config, logger) -> ExecutionContext: +def execution(config, logger, stageresult) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ @@ -48,7 +49,7 @@ def execution(config, logger) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {}, + {"stage_test":stageresult}, {} ) @@ -67,7 +68,7 @@ def stageresult() -> StageResult: ) -def test_executioncontext_creation(execution, logger, config) -> None: +def test_executioncontext_creation(execution, logger, config, stageresult) -> None: """ Test that the ExecutionContext creates the right attributes. """ @@ -78,7 +79,7 @@ def test_executioncontext_creation(execution, logger, config) -> None: assert execution.run_dir == Path("tmp/run") assert execution.started_at == '2024-05-06 15:45:30' assert execution.working_directory == Path('tmp/work_dir') - assert execution.stage_results == {} + assert execution.stage_results == {"stage_test":stageresult} assert execution.variables == {} def test_record(stageresult, execution) -> None: @@ -133,7 +134,7 @@ def test_resolve_data_root(execution) -> None: """ assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") - import onsrap.execution as execution_module + result = execution.resolve_data_root(None) expected = ( Path(execution_module.__file__).resolve().parents[1] @@ -141,3 +142,91 @@ def test_resolve_data_root(execution) -> None: ) assert result == expected +def test_resolve_output_root(execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, returns the file path for the module itself + and the data directory within that. + """ + run_dir = Path("tmp/run") + assert execution.resolve_output_root(run_dir) == Path("tmp/run/data") + + result = execution.resolve_output_root(None) + expected = ( + Path(execution_module.__file__).resolve().parents[1] + / "data" + ) + assert result == expected + +@pytest.mark.parametrize( + "add_folder,file_name,expected", + [ + ( + ["interim","testing_files"], + "clean.py", + Path("tmp/data/interim/testing_files/clean.py") + ), + ( + "interim", + "clean.py", + Path("tmp/data/interim/clean.py") + ), + ( + None, + "clean.py", + Path("tmp/data/clean.py") + ), + ( + ["interim","testing_files"], + None, + Path("tmp/data/interim/testing_files") + ), + ( + "interim", + None, + Path("tmp/data/interim") + ), + ( + None, + None, + Path("tmp/data") + ) + ], +) + + +def test_resolve_given_path_add_folders(execution, add_folder, file_name, expected) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid string + or None type. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(None, + path_name, + file_name, + root, + add_folder) == expected + +def test_resolve_given_path_norm(execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + """ + execution.record(StageResult("stage_test2", + StageStatus.PENDING, + '2024-05-06 15:45:30', + '2024-05-07 15:45:30', + metadata={}, + outputs = {"data_path":"clean.py"} )) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path(stage_name, + path_name, + None, + root, + None) == Path("clean.py") \ No newline at end of file From afbbcf67edebb3ccc671de189d7be643c6a93f79 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 8 Jul 2026 10:57:02 +0100 Subject: [PATCH 069/332] Commiting before rebase --- tests/test_execution.py | 19 ++++++- tests/test_models.py | 119 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 tests/test_models.py diff --git a/tests/test_execution.py b/tests/test_execution.py index 722449e..bbb9d06 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,4 +1,4 @@ -from onsrap.execution import ExecutionContext +from onsrap.execution import ExecutionContext, PythonStageExecutor from onsrap.models import PipelineConfig, StageResult, StageStatus from onsrap.logger import Logger from pathlib import Path @@ -158,6 +158,10 @@ def test_resolve_output_root(execution) -> None: ) assert result == expected +""" +Parameters for testing multiple add_folder options in +test_resolve_given_path_add_folders function. +""" @pytest.mark.parametrize( "add_folder,file_name,expected", [ @@ -229,4 +233,15 @@ def test_resolve_given_path_norm(execution) -> None: path_name, None, root, - None) == Path("clean.py") \ No newline at end of file + None) == Path("clean.py") + +"""TEST NOT RUN FOR StageExecutor AS COVERED UNDER PythonStageExecutor""" + +@pytest.fixture +def pythonstageexecutor() -> PythonStageExecutor: + return PythonStageExecutor(("main.py","run.py")) + +def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: + assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") + +"""CONTINUE FROM EXECUTE CLASS METHOD""" \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..e120bc2 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,119 @@ +from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, StageResult, PipelineRun, PipelineConfig +import pytest +import datetime +from pathlib import Path +from textwrap import dedent + +def test_stagestatus() -> None: + assert StageStatus.PENDING == "pending" + assert StageStatus.RUNNING == "running" + assert StageStatus.SUCCEEDED == "succeeded" + assert StageStatus.FAILED == "failed" + assert StageStatus.SKIPPED == "skipped" + +def test_pipeline_status() -> None: + assert PipelineStatus.PENDING == "pending" + assert PipelineStatus.RUNNING == "running" + assert PipelineStatus.SUCCEEDED == "succeeded" + assert PipelineStatus.FAILED == "failed" + +@pytest.fixture +def runtimeID() -> RuntimeID: + return RuntimeID(id = "abc123", + timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), + hash = "fnruw9574893ghkwq234h5kg", + short_hash = "4h5kg") + +def test_runtimeID_creation(runtimeID) -> None: + assert runtimeID.id == "abc123" + assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" + assert runtimeID.short_hash == "4h5kg" + +def test_getter_functions_runtimeID(runtimeID) -> None: + assert runtimeID.get_id() == "abc123" + assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" + assert runtimeID.get_short_hash() == "4h5kg" + +@pytest.fixture +def rapconfig() -> RAPConfig: + return RAPConfig(contents = {"name":"test_rap", + "backend":"python", + "work_dir":Path("tmp/work"), + "project_root":Path("project"), + "log_dir":Path("tmp/logs"), + "data_dir":Path("tmp/data"), + "allow_subprocess_fallback":True, + "python_executable":None, + "metadata":{"variables":["name","age"], + "num_stages":6}}) + +@pytest.fixture +def blankpipelineconfig() -> PipelineConfig: + return PipelineConfig() + +@pytest.fixture +def pipelineconfig() -> PipelineConfig: + return PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + +@pytest.fixture +def mapping() -> dict: + return {"name":"test_rap", + "backend":"python", + "work_dir":Path("tmp/work"), + "project_root":Path("project"), + "log_dir":Path("tmp/logs"), + "data_dir":Path("tmp/data"), + "allow_subprocess_fallback":True, + "python_executable":None, + "metadata":{"variables":["name","age"], + "num_stages":6}} + + +def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> None: + assert blankpipelineconfig.from_any(None) == PipelineConfig() + assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + assert blankpipelineconfig.from_any(rapconfig) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + assert blankpipelineconfig.from_any(mapping) == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + + with pytest.raises(TypeError): + blankpipelineconfig.from_any(11) + +"""NOT SURE HOW TO TEST FROM_FILE()""" \ No newline at end of file From 7f291a51b787a85a76cde7194c4fa213dd07c82c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 6 Jul 2026 11:08:51 +0100 Subject: [PATCH 070/332] Partial testing established for execution.py --- tests/test_execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index bbb9d06..67ee35c 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -244,4 +244,4 @@ def pythonstageexecutor() -> PythonStageExecutor: def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") -"""CONTINUE FROM EXECUTE CLASS METHOD""" \ No newline at end of file +"""CONTINUE FROM EXECUTE CLASS METHOD""" From a98e0d85d215ec16d56da9f5987e370a5da38075 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 9 Jul 2026 15:57:48 +0100 Subject: [PATCH 071/332] Tests for models.py and bug fixes within models.py --- onsrap/errors.py | 6 ++ tests/test_models.py | 142 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index 8302969..c22dbd2 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -66,3 +66,9 @@ class StageLoadError(StageExecutionError): Raised when a file-backed stage cannot be loaded. Child class with ``StageExecutionError`` as the parent class. """ + +class StageDependencyError(OnsrapError): + """ + Raised when incorrect inputs are provided to the dependency + attribute of a Stage. + """ \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py index e120bc2..3ab6cbe 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,10 +1,14 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, StageResult, PipelineRun, PipelineConfig +from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, PipelineRun, PipelineConfig import pytest import datetime from pathlib import Path from textwrap import dedent +from tests.test_execution import stageresult def test_stagestatus() -> None: + """ + Test that stagestatus outputs the correct values. + """ assert StageStatus.PENDING == "pending" assert StageStatus.RUNNING == "running" assert StageStatus.SUCCEEDED == "succeeded" @@ -12,6 +16,9 @@ def test_stagestatus() -> None: assert StageStatus.SKIPPED == "skipped" def test_pipeline_status() -> None: + """ + Test that pipeline status outputs the correct values. + """ assert PipelineStatus.PENDING == "pending" assert PipelineStatus.RUNNING == "running" assert PipelineStatus.SUCCEEDED == "succeeded" @@ -19,18 +26,27 @@ def test_pipeline_status() -> None: @pytest.fixture def runtimeID() -> RuntimeID: + """ + Example RuntimeID instance for testing of other methods. + """ return RuntimeID(id = "abc123", timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), hash = "fnruw9574893ghkwq234h5kg", short_hash = "4h5kg") def test_runtimeID_creation(runtimeID) -> None: + """ + Test that a RuntimeID is correctly created. + """ assert runtimeID.id == "abc123" assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" assert runtimeID.short_hash == "4h5kg" def test_getter_functions_runtimeID(runtimeID) -> None: + """ + Tests all the getter functions for the RuntimeID instance. + """ assert runtimeID.get_id() == "abc123" assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" @@ -38,6 +54,9 @@ def test_getter_functions_runtimeID(runtimeID) -> None: @pytest.fixture def rapconfig() -> RAPConfig: + """ + Example RAPConfig class instance for testing of class methods. + """ return RAPConfig(contents = {"name":"test_rap", "backend":"python", "work_dir":Path("tmp/work"), @@ -51,10 +70,16 @@ def rapconfig() -> RAPConfig: @pytest.fixture def blankpipelineconfig() -> PipelineConfig: + """ + Blank PipelineConfig instance for class method testing. + """ return PipelineConfig() @pytest.fixture def pipelineconfig() -> PipelineConfig: + """ + Example PipelineConfig completed class instance for method testing. + """ return PipelineConfig(name = "test_rap", backend = "python", work_dir = Path("tmp/work"), @@ -68,6 +93,9 @@ def pipelineconfig() -> PipelineConfig: @pytest.fixture def mapping() -> dict: + """ + Example mapping dictionary for use in testing from_mapping() method. + """ return {"name":"test_rap", "backend":"python", "work_dir":Path("tmp/work"), @@ -81,6 +109,11 @@ def mapping() -> dict: def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> None: + """ + Test derivation for a PipelineConfig instance using the from_any() method. This test + checks all methods EXCEPT from_file as this will be covered in another test due to + creation of a mock file being required. + """ assert blankpipelineconfig.from_any(None) == PipelineConfig() assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", backend = "python", @@ -116,4 +149,109 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> No with pytest.raises(TypeError): blankpipelineconfig.from_any(11) -"""NOT SURE HOW TO TEST FROM_FILE()""" \ No newline at end of file +"""NOT SURE HOW TO TEST FROM_FILE()""" + +def test_to_dict(pipelineconfig) -> None: + """ + Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values + as a dictionary. + """ + + assert pipelineconfig.to_dict() == {"name":"test_rap", + "backend":"python", + "work_dir":"tmp\\work", + "project_root":"project", + "log_dir":"tmp\\logs", + "data_dir":"tmp\\data", + "allow_subprocess_fallback":True, + "python_executable":None, + "variables":["name","age"], + "num_stages":6} + + +@pytest.fixture +def runmanifest() -> RunManifest: + """ + Example RunManifest class instance for testing of class method. + """ + return RunManifest("pipeline", + "1", + None, + ["stage1","stage2"], + {"uniqueID":"example"}, + {"input_path":"input/data/example.csv"}, + {"output_path":"output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None) + +def test_stage_result(stageresult) -> None: + """ + Uses a StageResult instance created in test_execution to ensure that + the class instance is created suitably with required defaults. + """ + assert stageresult.name == "stage_test" + assert stageresult.status == "pending" + assert stageresult.started_at == '2024-05-06 15:45:30' + assert stageresult.finished_at == '2024-05-07 15:45:30' + assert stageresult.outputs == "example output" + assert stageresult.stdout == "" + assert stageresult.stderr == "" + assert stageresult.return_code == None + assert stageresult.metadata == {} + assert stageresult.error == None + assert stageresult.source == None + +@pytest.mark.parametrize("status_stage,expected_stage", + [(StageStatus.PENDING, False), + (StageStatus.RUNNING, False), + (StageStatus.FAILED, False), + (StageStatus.SUCCEEDED, True), + (StageStatus.SKIPPED, False)]) + +def test_succeeded(stageresult, status_stage, expected_stage) -> None: + """ + Tests succeeded() method for StageResult which outputs True or False depending on + the status of the StageResult. + """ + stageresult.status = status_stage + assert stageresult.succeeded == expected_stage + +def test_duration_seconds(stageresult) -> None: + stageresult.started_at = datetime.datetime(2024,5,6,15,45,30) + stageresult.finished_at = datetime.datetime(2024,5,7,15,45,30) + seconds_value = (datetime.datetime(2024,5,7,15,45,30) - datetime.datetime(2024,5,6,15,45,30)).total_seconds() + assert stageresult.duration_seconds == seconds_value + +@pytest.fixture +def pipelinerun(stageresult, runmanifest) -> PipelineRun: + return PipelineRun(runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + [stageresult], + {"stage_test":"example output"}) + +def test_pipelinerun_configuration(pipelinerun, runmanifest, stageresult) -> None: + assert pipelinerun.manifest == runmanifest + assert pipelinerun.status == PipelineStatus.SUCCEEDED + assert pipelinerun.started_at == datetime.datetime(2024,5,6,15,45,30) + assert pipelinerun.completed_at == datetime.datetime(2024,5,7,15,45,30) + assert pipelinerun.stage_results == [stageresult] + assert pipelinerun.stage_outputs == {"stage_test":"example output"} + +def test_result_for(pipelinerun, stageresult) -> None: + assert pipelinerun.result_for("stage_test") == stageresult + assert pipelinerun.result_for("not_a_stage") == None + +@pytest.mark.parametrize("status,expected", + [(PipelineStatus.PENDING, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.FAILED, False), + (PipelineStatus.SUCCEEDED, True)]) + +def test_succeeded_pipeline(pipelinerun, status, expected) -> None: + pipelinerun.status = status + assert pipelinerun.succeeded == expected \ No newline at end of file From 2e0f95352099bcf655f6a1df87de282618e9c9ca Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 9 Jul 2026 15:58:18 +0100 Subject: [PATCH 072/332] tests for stage.py and bug fixes for stage.py incl. minor changes to models.py --- onsrap/models.py | 4 ++-- onsrap/stage.py | 35 ++++++++++++++++++++++++++++------- tests/test_stage.py | 26 +++++++------------------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 92b1aec..b37a6d5 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -270,8 +270,8 @@ def from_file(cls, path: Path) -> "PipelineConfig": def to_dict(self) -> dict[str, Any]: """ - Converts attributes regarding how the pipeline runs into a dictionary and holds it in - the ``metadata`` attribute of the ``PipelineConfig`` class. + Returns a prescriptive expression of the attributes within the PipelineConfig instance + that allows for easier processing by the user. """ data = { "name": self.name, diff --git a/onsrap/stage.py b/onsrap/stage.py index 8beb238..576be83 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -6,14 +6,14 @@ from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union from datetime import datetime -from .errors import StageConfigurationError +from .errors import StageConfigurationError, StageDependencyError if TYPE_CHECKING: from .execution import ExecutionContext, StageExecutor from .models import StageResult -def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: +def _normalize_dependencies(dependencies: list[str] | str | None) -> tuple[str, ...]: """ Standardise the names of any stages dependant on other stages/processes. @@ -33,13 +33,16 @@ def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[s """ if dependencies is None: return () - + if isinstance(dependencies, list) and dependencies == []: + return () if isinstance(dependencies, str): candidate_items = [dependencies] + else: - candidate_items = list(dependencies) + candidate_items = dependencies normalized: list[str] = [] + for dependency in candidate_items: dependency_name = str(dependency).strip() if dependency_name and dependency_name not in normalized: @@ -277,9 +280,22 @@ def with_dependencies(self, *dependencies: str) -> "Stage": ``Stage`` ``Stage`` class instance with normalised ``dependencies`` attribute. """ + unpacked_deps: list = [] + for a in dependencies: + if isinstance(a,list): + unpacked_deps = unpacked_deps + a + else: + unpacked_deps.append(a) + + for i in unpacked_deps: + if isinstance(i, list): + raise StageDependencyError("Nested lists are not valid arguments for this method! " \ + "Please provided single list or individual string values") + + print(unpacked_deps) return replace( self, - dependencies=self.dependencies + _normalize_dependencies(dependencies), + dependencies=self.dependencies + _normalize_dependencies(unpacked_deps), ) def validate(self) -> None: @@ -291,8 +307,13 @@ def validate(self) -> None: ``StageConfigurationError`` If ``source`` attribute does not define a source or does not exist. """ - if self.source is None: - raise StageConfigurationError(f"Stage '{self.name}' does not define a source.") + if not (isinstance(self.source, Path) or callable(self.source)): + raise StageConfigurationError(f"Stage '{self.name}' must have a Path or Callable source.") + + if self.source is None or self.source == "": + raise StageConfigurationError( + f"Stage '{self.name}' does not define a source. Source provided: {self.source}" + ) if isinstance(self.source, Path) and not self.source.is_file(): raise StageConfigurationError(f"Stage source does not exist: {self.source}") diff --git a/tests/test_stage.py b/tests/test_stage.py index 5213945..7d6c922 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -91,15 +91,14 @@ def example_function(): test = Stage.from_callable(example_function) assert test.name == "example_function" -@pytest.mark.skip def test_from_dict_norm() -> None: """ - REVIEW WITH ALEX + Tests that a stage instance is created from a dictionary item. """ def example_function(): pass data = {"name":"test_Stage", - "callable_source" : example_function} + "callable" : example_function} stage = Stage.from_dict(data) assert stage.source == example_function @@ -107,31 +106,20 @@ def test_with_dependencies_list(stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is a list. - - REVIEW WITH ALEX - This test works and passes however it doesn't behave how I was expecting it to. - Was expecting the list/dictionary to be broken down so you have one tuple rather - than a tuple of dict/lists. Is this a problem or just my understanding? """ new_deps = ["stage2","stage3"] - new_dep_dict = {"stage1":"stage0"} new_deps_blank = [] stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_dict = stage_test.with_dependencies(new_dep_dict) stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1","['stage2', 'stage3']") - assert stage_test_dict.dependencies == ("stage_1","{'stage1': 'stage0'}") - assert stage_test_blank.dependencies == ("stage_1", '[]') + assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_blank.dependencies == ("stage_1", ) + stage_test = stage_test.with_dependencies("stage2","stage3") + assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + -@pytest.mark.skip def test_validate(stage_test, tmp_path) -> None: """ Tests whether an error is raised if the source file isn't suitable. - - REVIEW WITH ALEX - Does not raise a StageConfigurationError is the source is a blank string. Is - this a concern? Do we want this validate to be able to do other error checks - like if it is an int? """ stage_test.source = None with pytest.raises(StageConfigurationError): From 423a351716bbcb0c37114ce4b36c6d3ae6041df4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 10 Jul 2026 11:50:48 +0100 Subject: [PATCH 073/332] Fix: Fix bug #22 and implements test to ensure this works --- onsrap/pipeline.py | 8 ++++++-- tests/test_pipeline.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/test_pipeline.py diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index c59989d..93617d1 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -53,9 +53,13 @@ def __init__( logger: Logger | None = None, executor: StageExecutor | None = None, ): - self.name = name or "pipeline" self.backend = backend or "python" - self.config = self._resolve_config(config) #PipelineConfig.from_any(config) + self.config = PipelineConfig.from_any(config) + if (self.config.name is not None) and (name == None): + self.name = self.config.name + else: + self.name = name or "pipeline" + if self.config.name is None: self.config.name = self.name self.config.backend = self.backend diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..aec62fb --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,22 @@ +from onsrap.pipeline import Pipeline, PipelineConfig +import pytest + +@pytest.fixture +def pipelineconfig() -> PipelineConfig: + return PipelineConfig(name = "test_pipeline_config") + +def test_pipeline_name(pipelineconfig): + """ + Test to confirm that Pipeline instance uses either defined name from + instance creation (shown in pipeline_named), utilises name from PipelineConfig + if no name was given (shown in pipeline_config), or defaults to "pipeline" if + no name is provided through Pipeline instance creation or through the + PipelineConfig (shown through pipeline_no_name) + """ + pipeline_named = Pipeline(name = "test_pipeline_name") + assert pipeline_named.name == "test_pipeline_name" + pipeline_config = Pipeline(name = None, config = pipelineconfig) + assert pipeline_config.name == "test_pipeline_config" + pipeline_no_name = Pipeline() + assert pipeline_no_name.name == "pipeline" + \ No newline at end of file From 0b8953f6c32516de59db321287fb31d05119b144 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 13:55:49 +0100 Subject: [PATCH 074/332] Fix: Applied fix for bug #20 and implemented a test for fix. --- onsrap/errors.py | 6 ++++++ onsrap/pipeline.py | 31 +++++++++++++++++++++++++------ tests/test_pipeline.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index c22dbd2..1691b4d 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -71,4 +71,10 @@ class StageDependencyError(OnsrapError): """ Raised when incorrect inputs are provided to the dependency attribute of a Stage. + """ + +class PipelineInitialisationError(OnsrapError): + """ + Raised when there is an error in definition of the Pipeline + instance """ \ No newline at end of file diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 93617d1..d7d61a6 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -7,7 +7,7 @@ import sys from importlib import metadata as importlib_metadata from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Sequence +from typing import Any, Callable, Iterable, Mapping, Sequence, Union from .errors import StageConfigurationError from .warnings import StageConfigurationWarning @@ -15,7 +15,7 @@ from .graph import StageGraph from .logger import Logger from .models import PipelineConfig, StageConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now -from .stage import Stage +from .stage import Stage, _normalize_dependencies @@ -50,6 +50,7 @@ def __init__( backend: str = "python", config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, + dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, ): @@ -67,6 +68,13 @@ def __init__( self.logger = logger or Logger(log_dir=self.config.log_dir) self.executor = executor or PythonStageExecutor() self.stages = [self._coerce_stage(stage) for stage in (stages or [])] + self.dependencies = dependencies + if dependencies is not None and stages is None: + raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " + "for those stages. Try the from_files() method, or create your Stage objects and " \ + "parse them to the Pipeline Constructor.") + if dependencies is not None: + self._assign_dependencies(dependencies,self.stages) self.graph = StageGraph.from_stages(self.stages) self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -78,6 +86,14 @@ def __init__( backend=self.backend, stages=[stage.name for stage in self.stages], ) + def _assign_dependencies(self, + dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, + stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: + for stage in stages: + new_dependencies = self._dependencies_for_stage(stage.name,stage.source,dependencies) + stage.dependencies = _normalize_dependencies(new_dependencies) + + return stages def _coerce_stage( self, @@ -414,8 +430,8 @@ def from_config(cls, config: dict) -> Pipeline: @staticmethod def _dependencies_for_stage( stage_name: str, - path: Path, - dependencies: Mapping[str, Sequence[str]] | None, + path: Union[Path, Callable[..., Any], None] = None, + dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: """ Extracts a tuple of ``dependencies`` for the requested stage. @@ -436,8 +452,11 @@ def _dependencies_for_stage( """ if not dependencies: return () - - candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + if isinstance(path, Path): + candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + else: + candidates = (stage_name, str(path.__name__)) + print(candidates) for candidate in candidates: if candidate in dependencies: return tuple(str(dependency) for dependency in dependencies[candidate]) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index aec62fb..9f7e798 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,4 +1,7 @@ from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.errors import PipelineInitialisationError +from onsrap.stage import Stage +from pathlib import Path import pytest @pytest.fixture @@ -19,4 +22,39 @@ def test_pipeline_name(pipelineconfig): assert pipeline_config.name == "test_pipeline_config" pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" + + +def test_assign_dependencies(tmp_path): + def example_function(): + pass + dependencies_single = {"Stage_2":("Stage_1",)} + dependencies_multiple = {"Stage_1":["Stage_0", "Stage_0.5"], + "Stage_2":("Stage_1",)} + dependencies_non_stage_name = {"Stage_1.py":("Stage_0",), + "example_function":("Stage_1.py",)} + + with pytest.raises(PipelineInitialisationError): + Pipeline(stages = None, + dependencies = dependencies_single) + + path = tmp_path/"Stage_1.py" + pipeline_1 = Pipeline(name = "pipeline_1", + stages = [Stage("Stage_1", path, None,{}), + Stage("Stage_2", example_function, None,{})], + dependencies = dependencies_multiple) + + assert pipeline_1.stages[0].dependencies == ("Stage_0","Stage_0.5",) + assert pipeline_1.stages[1].dependencies == ("Stage_1",) + + pipeline_2 = Pipeline(name = "pipeline_2", + stages = [Stage("Stage_1", path, None,{}), + Stage("Stage_2", example_function, None,{})], + dependencies = dependencies_non_stage_name) + + assert pipeline_2.stages[0].dependencies == ("Stage_0",) + assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + + + + \ No newline at end of file From 07b06b1b5417922f86f72fc5b7be03aa3629f09f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 15:11:53 +0100 Subject: [PATCH 075/332] Completed testing suite for models.py --- tests/test_models.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 3ab6cbe..dd38580 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -151,6 +151,55 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> No """NOT SURE HOW TO TEST FROM_FILE()""" +def test_from_file(tmp_path,) -> PipelineConfig: + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == PipelineConfig(name = "test_rap", + backend = "python", + work_dir = Path("tmp/work"), + project_root = Path("project"), + log_dir = Path("tmp/logs"), + data_dir = Path("tmp/data"), + allow_subprocess_fallback = True, + python_executable = None, + metadata = {"variables":["name","age"], + "num_stages":6}) + + fake_file = "path_not_real" + with pytest.raises(FileNotFoundError): + PipelineConfig.from_file(fake_file) + with pytest.raises(TypeError): + PipelineConfig.from_file(no_map_pipeline_config) + def test_to_dict(pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values From 29e72a40ec1a056206ce8a54d6311012b95a9570 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 15:34:30 +0100 Subject: [PATCH 076/332] docs/test: Add docstrings and finish testing suite for stage.py. --- tests/test_pipeline.py | 15 +++++++++------ tests/test_stage.py | 28 +++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9f7e798..e0cd641 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,11 +4,7 @@ from pathlib import Path import pytest -@pytest.fixture -def pipelineconfig() -> PipelineConfig: - return PipelineConfig(name = "test_pipeline_config") - -def test_pipeline_name(pipelineconfig): +def test_pipeline_name(): """ Test to confirm that Pipeline instance uses either defined name from instance creation (shown in pipeline_named), utilises name from PipelineConfig @@ -16,15 +12,22 @@ def test_pipeline_name(pipelineconfig): no name is provided through Pipeline instance creation or through the PipelineConfig (shown through pipeline_no_name) """ + pipeline_config = PipelineConfig(name = "test_pipeline_config") pipeline_named = Pipeline(name = "test_pipeline_name") assert pipeline_named.name == "test_pipeline_name" - pipeline_config = Pipeline(name = None, config = pipelineconfig) + pipeline_config = Pipeline(name = None, config = pipeline_config) assert pipeline_config.name == "test_pipeline_config" pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" def test_assign_dependencies(tmp_path): + """ + Test to ensure that different formats of dependencies can be parsed to the + Pipeline creation and appropriately assigned to each stage within the + Pipeline. Will also check for error raise if the dependencies are defined + but there are no defined stages. + """ def example_function(): pass dependencies_single = {"Stage_2":("Stage_1",)} diff --git a/tests/test_stage.py b/tests/test_stage.py index 7d6c922..61a92c1 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -167,4 +167,30 @@ def example_function(): """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST -""" \ No newline at end of file +""" + +def test_stage_instance_from_file(tmp_path) -> None: + """ + Tests that a Stage instance is created from a filepath. + """ + test_stage = tmp_path / "test_stage.py" + test_stage.write_text( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + assert Stage.from_file(test_stage, + entrypoint = "main") == Stage("test_stage", + test_stage.resolve(), + (), + {}, + "main", + "python") + From b32ee5453edab1cb9417efbffdbb96a6283c95e6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 14 Jul 2026 17:26:56 +0100 Subject: [PATCH 077/332] docs: Updated README to cover issue #16 --- README.md | 96 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index db12f9c..8b063a8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,20 @@ located. ``` ## What is `onsrap`? -Add a summary of your project here. +Reproducible Analytical Pipelines (RAPs) are a cornerstone of high quality statistics. Reproducible refers to the concept that if code is run multiple times with the same inputs, it will produce the same outputs. A pipeline is a series of stages (small chunks of work) which are run in a specified order to produce desired outputs. Pipelines are crucial to reproducible work as they ensure that the code is run consistently. This increases the quality of the outputs by ensuring as little manual input as possible. + +ONSRap is a Python package that automatically orchestrates and runs these RAPs. The goal is to standardise how pipelines are run to reduce developer time required to convert existing code into RAP standards. As well as reducing developer time, this package also supports achieving RAP standards through items 4, 6, and 10. These are accomplished through this package by: + + - Item 4: Document everything that is needed to write and run the code + - This package includes inbuilt logging that records when the pipeline was run as well as the Pipeline configuration used. + +- Item 6: Code modules should run end-to-end without manual intervention + - This package is designed whereby once the configuration has been provided by the user and the main.py file is run, no further human input is required. + +- Item 10: Don't reinvent the wheel + - Multiple pipelines exist within the ONS and each have the potential to be orchestrated in different ways. This package aims to standardise the orchestration, ensuring consistency across pipelines. This consistency means that developers are more easily able to move between pipelines as they will all be structured in a similar way. + +For more information on the ONS Rap Minimum Standards, please see the full [standards documentation][standards]. ## Getting started @@ -18,11 +31,13 @@ requirements. It's suggested that you install this package and its requirements within a virtual environment. +Stages must be written in functional programming. A ``stage`` can be a file or a callable item, such as a function. If the ``stage`` is a file, it must have an entrypoint function (a function that, when called, runs the entirety of the stage). The ``stage`` file can run without an entrypoint, however the package has less control over the implementation and therefore best practice is inclusion of an entrypoint. + +There should be a parent file that sets out configuration, required directories and file paths, and builds the ``Pipeline`` instance. It is recommended that this is named something similar to ``main.py`` so that it is easy for users to see where the ``Pipeline`` starts. This file will be what is run through the terminal to run the entire pipeline. + ## Requirements -- Python 3.9+ installed -- a `.secrets` file with the required [secrets and credentials](#required-secrets-and-credentials) -- to have [loaded environment variables][docs-loading-environment-variables] from `.env` +- Python 3.14.6 installed Contributors have some additional requirements - please see our [contributing guidance][contributing]. @@ -53,7 +68,18 @@ Remember to update the setup and requirement files inline with any changes to yo package. ## Running the pipeline (Python only) +### Running Your Own Pipeline +To run your own Pipeline, you will need to build a ``Pipeline`` instance. This can be built using the from_files() method which requires a list of strings or Paths for your individual ``stages``. These are then compiled into a ``Pipeline`` instance. A ``Pipeline`` instance can also be created using the from_dict() method which takes a dictionary containing each attribute of the intented ``Pipeline`` instance and converts it. + +You will also need to define your ``PipelineConfig`` instance. This contains information regarding the working directory, project root, data directory, and log directory required to run the ``Pipeline`` as well as any metadata that you feel needs to be logged. + +Lastly, you need to define any ``dependencies`` required for the ``stages``. These are whether any stage needs to be run before another stage. These should be structured as a dictionary with the name of the stage as the key and the value is the stage/s that need to run before it as a tuple. +Both ``PipelineConfig`` and ``dependencies`` should be parsed into the ``Pipeline`` instance. + +Once you have your ``Pipeline`` instance, you can run the ``Pipeline.run()`` method which will run the entire ``Pipeline`` instance that has been created. + +### Example Pipeline The main runnable example now lives in `examples/pipeline_1/main.py`. It builds a three-stage pipeline from numbered scripts under `examples/pipeline_1/scripts/`. To run the example, use: @@ -66,21 +92,12 @@ Alternatively, most Python IDEs allow you to run the code directly using a `run` ## Required secrets and credentials -To run this project, you need a `.secrets` file with [secrets/credentials as -environmental variables][docs-loading-environment-variables-secrets]. The -secrets/credentials should have the following environment variable name(s): - -| Secret/credential | Environment variable name | Description | -|-------------------|---------------------------|--------------------------------------------| -| Secret 1 | `SECRET_VARIABLE_1` | Plain English description of Secret 1. | -| Credential 1 | `CREDENTIAL_VARIABLE_1` | Plain English description of Credential 1. | +No secrets or credentials are required for running this package. -Once you've added them, [load these environment variables][docs-loading-environment-variables] using -`.env`. ## Project structure layout -The cookiecutter template generated for each project will follow this folder structure: +The ONSRap repository has the following structure: ```shell . @@ -89,14 +106,46 @@ The cookiecutter template generated for each project will follow this folder str │ │ ├── raw/ │ │ ├── interim/ │ │ └── processed/ -│ └── onsrap/ -│ ├── example_modules/ -│ │ ├── __init__.py -│ │ └── example_module.py -│ ├── __init__.py -│ ├── example_config.yml -│ └── run_pipeline.py -└── ... +│ ├── onsrap/ +│ │ ├── example_modules/ +│ │ │ ├── __init__.py +│ │ │ └── example_module.py +│ │ ├── __init__.py +│ │ ├── errors.py +│ │ ├── execution.py +│ │ ├── graph.py +│ │ ├── loader.py +│ │ ├── models.py +│ │ ├── pipeline.py +│ │ ├── run_pipeline.py +│ │ ├── runner.py +│ │ └── stage.py +│ ├── examples/ +│ │ ├── pipeline_1/ +│ │ │ ├── data/ +│ │ │ │ └── orders.csv +│ │ │ ├── logs/ +│ │ │ │ └── onsrap.log +│ │ │ ├── runs/ +│ │ │ │ └── README.md +│ │ │ └── scripts/ +│ │ │ │ ├── 0_data_validation.py +│ │ │ │ ├── 1_preprocessing.py +│ │ │ │ └── 2_reporting.py +│ │ │ ├── Example.md +│ │ │ └── main.py +│ │ ├── pipeline_2/ +│ │ └── pipeline_3/ +│ ├── tests/ +│ │ ├── __init__.py +│ │ ├── repo_tests_README.md +│ │ ├── test_execution.py +│ │ ├── test_models.py +│ │ ├── test_pipeline_architecture.py +│ │ ├── test_pipeline.py +│ │ └── test_stage.py +│ └── +└── ``` ## Licence @@ -118,3 +167,4 @@ This project structure is based on the [`govcookiecutter` template project][govc [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter [docs-loading-environment-variables]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md [docs-loading-environment-variables-secrets]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md#storing-secrets-and-credentials +[standards]: https://best-practice-and-impact.github.io/ONS_minimum_RAP/ \ No newline at end of file From b86b6a53f2c7c1f8d7b4cef894692c5c4f63da0c Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:31:45 +0100 Subject: [PATCH 078/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index ea4a378..59f8001 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -147,7 +147,7 @@ def resolve_output_root(self, run_dir: Path | None) -> Path: def resolve_given_path(self, stage_name: str | None, path_name: str | None, - file_name:str | None, + file_name: str | None, root: Path, add_folder: list[str] | str | None = None) -> Path: """ From 3f5c317147015e0e8c42b1a16945a71e401a97e0 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:33:50 +0100 Subject: [PATCH 079/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 576be83..e6f9a16 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -281,11 +281,11 @@ def with_dependencies(self, *dependencies: str) -> "Stage": ``Stage`` class instance with normalised ``dependencies`` attribute. """ unpacked_deps: list = [] - for a in dependencies: - if isinstance(a,list): - unpacked_deps = unpacked_deps + a + for dependency in dependencies: + if isinstance(dependency, list): + unpacked_deps = unpacked_deps + dependency else: - unpacked_deps.append(a) + unpacked_deps.append(dependency) for i in unpacked_deps: if isinstance(i, list): From 4e0cbee96c8a9e56c27bed84f3a1db2a8a8c0ad3 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:34:14 +0100 Subject: [PATCH 080/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index e6f9a16..4fede0a 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -287,8 +287,8 @@ def with_dependencies(self, *dependencies: str) -> "Stage": else: unpacked_deps.append(dependency) - for i in unpacked_deps: - if isinstance(i, list): + for dependency in unpacked_deps: + if isinstance(dependency, list): raise StageDependencyError("Nested lists are not valid arguments for this method! " \ "Please provided single list or individual string values") From 6e38e67e2449a1c4ff83dd9e13952a9d19189138 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:35:29 +0100 Subject: [PATCH 081/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 4fede0a..3a196f0 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -292,7 +292,6 @@ def with_dependencies(self, *dependencies: str) -> "Stage": raise StageDependencyError("Nested lists are not valid arguments for this method! " \ "Please provided single list or individual string values") - print(unpacked_deps) return replace( self, dependencies=self.dependencies + _normalize_dependencies(unpacked_deps), From 81db0afa0cf7ae23985b1ca8a0227182489dfa21 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:35:43 +0100 Subject: [PATCH 082/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index d7d61a6..eb4c565 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -56,7 +56,7 @@ def __init__( ): self.backend = backend or "python" self.config = PipelineConfig.from_any(config) - if (self.config.name is not None) and (name == None): + if (self.config.name is not None) and (name is None): self.name = self.config.name else: self.name = name or "pipeline" From b3798cd77d91773e07f387737bf03cafba2b5307 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:36:15 +0100 Subject: [PATCH 083/332] Apply suggestion from @BelowBayesline Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 59f8001..cad1e7a 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -149,7 +149,8 @@ def resolve_given_path(self, stage_name: str | None, path_name: str | None, file_name: str | None, root: Path, - add_folder: list[str] | str | None = None) -> Path: + add_folder: list[str] | str | None = None + ) -> Path: """ Returns a file path for a requested item. From 2e4b4f33c4e16037e0a913c537e46945e926c000 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 15 Jul 2026 10:34:15 +0100 Subject: [PATCH 084/332] Fix: Actioned comments from PR #21 --- .../pipeline_1/scripts/0_data_validation.py | 4 +- .../pipeline_1/scripts/1_preprocessing.py | 4 +- examples/pipeline_1/scripts/2_reporting.py | 4 +- onsrap/errors.py | 6 +++ onsrap/execution.py | 31 ++++------- tests/test_execution.py | 52 ++++++++++++------- 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index 4a586ee..e54e2bb 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -85,8 +85,8 @@ def write_report(report_path: Path, report: dict[str, object]) -> None: def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index a55180c..1a2514c 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -87,8 +87,8 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() raw_path = context.resolve_given_path("0_data_validation", "raw_path", "orders.csv", data_root) clean_path = output_root / "interim" / "1_clean_orders.csv" diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index c3f50a8..bbbd1c2 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -69,8 +69,8 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = context.resolve_data_root(config = context.config) - output_root = context.resolve_output_root(run_dir = context.run_dir) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() clean_path = context.resolve_given_path("1_preprocessing", "clean_path", "1_clean_orders.csv", output_root, "interim") diff --git a/onsrap/errors.py b/onsrap/errors.py index 1691b4d..bd29f9f 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -77,4 +77,10 @@ class PipelineInitialisationError(OnsrapError): """ Raised when there is an error in definition of the Pipeline instance + """ + +class PipelineConfigurationError(OnsrapError): + """ + Raised when there has been an issue with the PipelineConfig + instance. """ \ No newline at end of file diff --git a/onsrap/execution.py b/onsrap/execution.py index cad1e7a..3624b94 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Protocol, TYPE_CHECKING -from .errors import StageExecutionError, StageLoadError +from .errors import StageExecutionError, StageLoadError, PipelineConfigurationError from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable from .logger import Logger from .models import PipelineConfig, StageResult, StageStatus, now @@ -106,44 +106,35 @@ def stage_outputs(self) -> dict[str, Any]: """ return {name: result.outputs for name, result in self.stage_results.items()} - def resolve_data_root(self, config: PipelineConfig | None) -> Path: + def get_data_dir(self) -> Path: """ Establishes the filepath that the data is held in. - - Parameters - ---------- - ``config`` : PipelineConfig - The configuration of the Pipeline being run. This holds the location filepath - for the pipeline as defined by the user in the main.py file. Returns ------- Path The file path for the location of the data being used in the pipeline. """ - if config is not None: - return Path(config.data_dir) + if self.config is not None: + return Path(self.config.data_dir) - return Path(__file__).resolve().parents[1] / "data" + raise PipelineConfigurationError("Please parse a PipelineConfig instance to " \ + "the ExecutionContext.") - def resolve_output_root(self, run_dir: Path | None) -> Path: + def resolve_output_root(self) -> Path: """ Establishes the filepath that the outputs are going to be saved to. - Parameters - ---------- - ``run_dir`` : Path - The file directory that the run results are saved to. - Returns ------- Path The file path for the outputs of the run to be saved to. """ - if run_dir is not None: - return Path(run_dir) / "data" + if self.run_dir is not None: + return Path(self.run_dir) / "data" - return Path(__file__).resolve().parents[1] / "data" + raise PipelineConfigurationError("Please parse a run directory to " \ + "the ExecutionContext.") def resolve_given_path(self, stage_name: str | None, path_name: str | None, diff --git a/tests/test_execution.py b/tests/test_execution.py index 67ee35c..c5cafab 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest import onsrap.execution as execution_module +from onsrap.errors import PipelineConfigurationError @pytest.fixture def logger() -> Logger: @@ -129,34 +130,47 @@ def test_stage_outputs(execution, stageresult) -> None: def test_resolve_data_root(execution) -> None: """ Tests that resolve_data_root method extracts the path from the execution context - or, if the context is None, returns the file path for the module itself and the - data directory within that. + or, if the context is None, returns an error to indicate that additional input is + required. """ - assert execution.resolve_data_root(execution.config) == Path("tmp/config_data") + assert execution.get_data_dir() == Path("tmp/config_data") + run_dir = Path("tmp/run") + work_dir = Path('tmp/work_dir') + execution_blank_config = ExecutionContext("test_pipeline", + "run_id_1234", + None, + Logger(), + run_dir, + '2024-05-06 15:45:30', + work_dir, + {"stage_test":stageresult}, + {} ) - result = execution.resolve_data_root(None) - expected = ( - Path(execution_module.__file__).resolve().parents[1] - / "data" - ) - assert result == expected + with pytest.raises(PipelineConfigurationError): + execution_blank_config.get_data_dir() def test_resolve_output_root(execution) -> None: """ Tests that resolve_output_root method extracts the path from the given run - directory or, if None are given, returns the file path for the module itself - and the data directory within that. + directory or, if None are given, raises an error to indicate additional input + is required.. """ - run_dir = Path("tmp/run") - assert execution.resolve_output_root(run_dir) == Path("tmp/run/data") + work_dir = Path('tmp/work_dir') + assert execution.resolve_output_root() == Path("tmp/run/data") - result = execution.resolve_output_root(None) - expected = ( - Path(execution_module.__file__).resolve().parents[1] - / "data" - ) - assert result == expected + execution_blank_config = ExecutionContext("test_pipeline", + "run_id_1234", + None, + Logger(), + None, + '2024-05-06 15:45:30', + work_dir, + {"stage_test":stageresult}, + {} ) + + with pytest.raises(PipelineConfigurationError): + execution_blank_config.resolve_output_root() """ Parameters for testing multiple add_folder options in From 7b55430fdec33be85d75524e83aa1fde261861df Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:25:49 +0100 Subject: [PATCH 085/332] fix: Now allow users to define where their output location is in PipelineConfig and that works downstream in the ExecutionContext with the resolve_output_root() --- examples/pipeline_1/main.py | 1 + onsrap/execution.py | 2 +- onsrap/models.py | 4 ++++ onsrap/runner.py | 11 +++++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/pipeline_1/main.py b/examples/pipeline_1/main.py index 543fff9..4f71a70 100644 --- a/examples/pipeline_1/main.py +++ b/examples/pipeline_1/main.py @@ -28,6 +28,7 @@ def build_pipeline() -> Pipeline: backend="python", work_dir=PIPELINE_ROOT, project_root=PIPELINE_ROOT, + output_dir=PIPELINE_ROOT, data_dir=DATA_DIR, log_dir=LOG_DIR, metadata={ diff --git a/onsrap/execution.py b/onsrap/execution.py index 3624b94..6bdd95b 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -131,7 +131,7 @@ def resolve_output_root(self) -> Path: The file path for the outputs of the run to be saved to. """ if self.run_dir is not None: - return Path(self.run_dir) / "data" + return Path(self.run_dir) raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") diff --git a/onsrap/models.py b/onsrap/models.py index b37a6d5..8c2cd0e 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -137,6 +137,7 @@ class PipelineConfig: backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) project_root: Optional[Path] = None + output_dir: Optional[Path] = None log_dir: Path = field(default_factory=lambda: Path("logs")) data_dir: Path = field(default_factory=lambda: Path("data")) allow_subprocess_fallback: bool = True @@ -207,6 +208,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": backend = payload.pop("backend", "python") work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) + output_dir_value = payload.pop("output_dir", None) project_root = Path(project_root_value) if project_root_value is not None else work_dir log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) @@ -220,6 +222,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": backend=backend, work_dir=work_dir, project_root=project_root, + output_dir=output_dir_value, log_dir=log_dir, data_dir=data_dir, allow_subprocess_fallback=allow_subprocess_fallback, @@ -278,6 +281,7 @@ def to_dict(self) -> dict[str, Any]: "backend": self.backend, "work_dir": str(self.work_dir), "project_root": str(self.project_root) if self.project_root is not None else None, + "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), "allow_subprocess_fallback": self.allow_subprocess_fallback, diff --git a/onsrap/runner.py b/onsrap/runner.py index fa5c876..6d78310 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import warnings from pathlib import Path from typing import TYPE_CHECKING @@ -52,8 +53,14 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: runtime_id = pipeline._create_runtime_id() pipeline.id = runtime_id - project_root = Path(pipeline.config.project_root or pipeline.config.work_dir) - run_dir = project_root / "runs" / runtime_id.get_id() + if pipeline.config.output_dir is not None: + run_output = Path(pipeline.config.output_dir) + else: + warnings.warn( + "Output directory is not specified. Using project root or work directory as the run output." + ) # TODO: fill with warnings from Pipeline branch + run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) + run_dir = run_output / "runs" / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) started_at = now() From 1024408a2746c8102dccac523f270ba4e78d0d84 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:42:41 +0100 Subject: [PATCH 086/332] Tweaked testing for exection and models to reflect changes with output directory defintion --- tests/test_execution.py | 7 ++++--- tests/test_models.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index c5cafab..3760251 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -21,12 +21,13 @@ def config() -> PipelineConfig: work_dir = Path('tmp/work_dir') project_root = Path('tmp/project') log_dir = Path('tmp/log') - data_dir = "tmp/config_data" + data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", "python", work_dir, - project_root, + project_root, + None, log_dir, data_dir, True, @@ -157,7 +158,7 @@ def test_resolve_output_root(execution) -> None: is required.. """ work_dir = Path('tmp/work_dir') - assert execution.resolve_output_root() == Path("tmp/run/data") + assert execution.resolve_output_root() == Path("tmp/run") execution_blank_config = ExecutionContext("test_pipeline", "run_id_1234", diff --git a/tests/test_models.py b/tests/test_models.py index dd38580..715b5fe 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -210,6 +210,7 @@ def test_to_dict(pipelineconfig) -> None: "backend":"python", "work_dir":"tmp\\work", "project_root":"project", + "output_dir":None, "log_dir":"tmp\\logs", "data_dir":"tmp\\data", "allow_subprocess_fallback":True, From 123baf0dd53c27ee39d86ab704ed6d28b0330594 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 11:45:15 +0100 Subject: [PATCH 087/332] resolved union change --- onsrap/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index eb4c565..7ceccee 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -7,7 +7,7 @@ import sys from importlib import metadata as importlib_metadata from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Sequence, Union +from typing import Any, Callable, Iterable, Mapping, Sequence from .errors import StageConfigurationError from .warnings import StageConfigurationWarning @@ -430,7 +430,7 @@ def from_config(cls, config: dict) -> Pipeline: @staticmethod def _dependencies_for_stage( stage_name: str, - path: Union[Path, Callable[..., Any], None] = None, + path: Path | Callable[..., Any] | None = None, dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: """ From e1546d3dabf459d2e9f3ebc8d622cabf3c14ba53 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 15 Jul 2026 15:43:43 +0100 Subject: [PATCH 088/332] Initiali sketch of StageConfig use in Pipeline class. --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7ceccee..2ef6561 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -55,7 +55,7 @@ def __init__( executor: StageExecutor | None = None, ): self.backend = backend or "python" - self.config = PipelineConfig.from_any(config) + self.config = self._resolve_config(config) if (self.config.name is not None) and (name is None): self.name = self.config.name else: From c04c05adc2dddd3a115bb46eeac7d8876dc0f584 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:30:20 +0100 Subject: [PATCH 089/332] Removed StageConfig from stage.py and embellished it in models.py --- onsrap/models.py | 149 ++++++++++++++++++++++++++++++++++++++++++++++- onsrap/stage.py | 53 +---------------- 2 files changed, 149 insertions(+), 53 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8c2cd0e..89a38a3 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,10 +1,13 @@ from __future__ import annotations +import warnings from dataclasses import dataclass, field from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Mapping, Optional, Union +from typing import Any, Iterable, Mapping, Optional, Union + +from .errors import StageConfigurationError class StageStatus(str, Enum): @@ -124,6 +127,9 @@ class PipelineConfig: The directory to store the logs in. ``data_dir`` : Path The directory where the data is stored. + ``output_dir`` : Path, optional + The directory where pipeline outputs should be written. Not used internally + by the runner; exposed for stage code to read via ``context.config.output_dir``. ``allow_subprocess_fallback`` : bool Indicates whether the subprocess system (running the whole file rather than an entrypoint function) should be allowed. @@ -140,6 +146,7 @@ class PipelineConfig: output_dir: Optional[Path] = None log_dir: Path = field(default_factory=lambda: Path("logs")) data_dir: Path = field(default_factory=lambda: Path("data")) + output_dir: Optional[Path] = None allow_subprocess_fallback: bool = True python_executable: Optional[str] = None metadata: dict[str, Any] = field(default_factory=dict) @@ -212,7 +219,19 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": project_root = Path(project_root_value) if project_root_value is not None else work_dir log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) - allow_subprocess_fallback = bool(payload.pop("allow_subprocess_fallback", True)) + output_dir_value = payload.pop("output_dir", None) + output_dir = Path(output_dir_value) if output_dir_value is not None else None + raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) + if isinstance(raw_subprocess_fallback, str): + warnings.warn( + "allow_subprocess_fallback should be a boolean, not a string. " + f"Received {raw_subprocess_fallback!r}. Use an unquoted YAML boolean.", + UserWarning, + stacklevel=2, + ) + allow_subprocess_fallback = raw_subprocess_fallback.strip().lower() not in ("false", "0", "no", "off") + else: + allow_subprocess_fallback = bool(raw_subprocess_fallback) python_executable = payload.pop("python_executable", None) metadata.update(payload) @@ -225,6 +244,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": output_dir=output_dir_value, log_dir=log_dir, data_dir=data_dir, + output_dir=output_dir, allow_subprocess_fallback=allow_subprocess_fallback, python_executable=python_executable, metadata=metadata, @@ -284,6 +304,7 @@ def to_dict(self) -> dict[str, Any]: "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), + "output_dir": str(self.output_dir) if self.output_dir is not None else None, "allow_subprocess_fallback": self.allow_subprocess_fallback, "python_executable": self.python_executable, } @@ -291,6 +312,130 @@ def to_dict(self) -> dict[str, Any]: return data +@dataclass +class StageConfig: + """ + Holds configuration that should be exposed to an individual stage at runtime. + + Parameters + ---------- + ``name`` : str + The name of the stage that this configuration applies to. + ``_variables`` : dict[str, Any] + Arbitrary stage-scoped variables. + ``datasets`` : dict[str, Any] + Optional dataset-related metadata for the stage. + ``metadata`` : dict[str, Any] + Additional supporting metadata for the stage configuration. + """ + name: str + _variables: dict[str, Any] = field(default_factory=dict) + datasets: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> "StageConfig": + """ + Build a ``StageConfig`` from a mapping loaded from code or configuration files. + + The ``datasets`` and ``metadata`` keys are extracted into their dedicated + attributes. All remaining keys are treated as stage variables that should be + exposed to the stage at runtime. + + Parameters + ---------- + ``name`` : str + Stage name that this configuration applies to. + ``data`` : Mapping[str, Any] or None + Raw configuration payload for that stage. + + Returns + ------- + ``StageConfig`` + A normalized stage configuration object. + """ + payload = dict(data or {}) + + datasets = payload.pop("datasets", {}) + if isinstance(datasets, Mapping): + datasets = dict(datasets) + else: + raise StageConfigurationError("Stage datasets must be provided as a mapping.") + + metadata = payload.pop("metadata", {}) + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} + + return cls( + name=str(name).strip(), + _variables=payload, + datasets=datasets, + metadata=metadata, + ) + + @property + def variables(self) -> dict[str, Any]: + """ + Return a copy of the stage variables without datasets or metadata. + """ + return dict(self._variables) + + def get(self, variable: str, default: Any = None) -> Any: + """ + Return a configured variable if present, otherwise return ``default``. + """ + return self._variables.get(variable, default) + + def require(self, variable: str) -> Any: + """ + Return a configured variable and raise if the stage does not define it. + """ + if variable not in self._variables: + raise StageConfigurationError( + f"Stage configuration '{self.name}' does not define '{variable}'." + ) + return self._variables[variable] + + def get_variables(self, variable: Iterable[str] | str | None = None) -> Any: + """ + Return all configured variables, one configured variable, or a selected subset. + """ + if variable is None: + return dict(self._variables) + + if isinstance(variable, str): + return self.require(variable) + + requested_variables: dict[str, Any] = {} + missing_variables: list[str] = [] + for requested_name in variable: + if requested_name in self._variables: + requested_variables[requested_name] = self._variables[requested_name] + else: + missing_variables.append(requested_name) + + if missing_variables: + missing = ", ".join(sorted(missing_variables)) + raise StageConfigurationError( + f"Stage configuration '{self.name}' does not define: {missing}." + ) + + return requested_variables + + def to_dict(self) -> dict[str, Any]: + """ + Serialize the stage configuration back to a mapping suitable for manifests. + """ + data = dict(self._variables) + if self.datasets: + data["datasets"] = dict(self.datasets) + if self.metadata: + data["metadata"] = dict(self.metadata) + return data + + @dataclass class RunManifest: """ diff --git a/onsrap/stage.py b/onsrap/stage.py index 3a196f0..57616e0 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -355,6 +355,8 @@ def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageR ---------- context : set value "ExecutionContext" Uses ``ExecutionContext`` class information to provide required metadata on running ``source``. + Any stage-specific configuration resolved by the ``Pipeline`` is available through + ``context.stage_config`` while this stage is running. executor : set value "StageExecutor" Uses ``StageExecutor`` class to extract the ``.execute`` method to actually run the ``source``. @@ -364,55 +366,4 @@ def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageR """ self.validate() return executor.execute(self, context) - -@dataclass -class StageConfig: - """ - Class that holds information regarding the Stage including key information required to run the stage. - - Parameters - ---------- - ``name`` : str - The name of the stage. This should be the same as the ``Stage`` class instance. - ``_variables`` : Mapping[str, Any] | None, default = None - A mapping of variables and their basic definition. This would define standard variables - such as a sex variable alongside how it is named specifically within the data. This attribute - should not be directly interacted with. Instead, it should be defined through a yaml file or - through the set_config() method. - ``datasets`` : Mapping[str, Any] | None = None - The name of the dataset that is used within the stage alongside any useful information - regarding the data, for example the file location. - - """ - name: str - _variables: Mapping[str, Any] | None = None - datasets: Mapping[str, dict] | None = None - - def get_variables(self, variable: Iterable[str] | str | None = None) -> dict: - """ - Class method that outputs the _variables attribute. - - This method allows for the entire attribute to be extracted as well as single items - or multiple items in a list. These will be output as a dictionary of the values for - the keys requested. - - Parameters - ---------- - ``variable`` : Iterable[str] | str | None = None - """ - if variable is not None: - if isinstance(variable,Iterable): - requested_vars = {} - for requested in variable: - item = self._variables.get(requested) - requested_vars[requested] = item - if requested_vars is not None: - return requested_vars - raise StageConfigurationError("The variable/s you have requested does/do not exist") - if isinstance(variable, str): - item = self._variables.get(requested) - if item is not None: - return item - raise StageConfigurationError("The variable/s you have requested does/do not exist") - return self._variables \ No newline at end of file From 1a2f24766a588aa833c5136018452fcf2622f30d Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:30:58 +0100 Subject: [PATCH 090/332] Modified ExecutionContext to account for StageConfig objects --- onsrap/execution.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 6bdd95b..2f213d5 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -11,7 +11,7 @@ from .errors import StageExecutionError, StageLoadError, PipelineConfigurationError from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable from .logger import Logger -from .models import PipelineConfig, StageResult, StageStatus, now +from .models import PipelineConfig, StageConfig, StageResult, StageStatus, now if TYPE_CHECKING: from .stage import Stage @@ -40,8 +40,12 @@ class ExecutionContext: The directory that the work is taking place in. ``stage_results`` : dict[str, StageResult], default = dict Stores the logs for the stage run. + ``stage_configs`` : dict[str, StageConfig], default = dict + Stage-name keyed configuration mapping resolved by the ``Pipeline``. ``variables`` : dict[str, Any], default = dict Stores relevant variables regarding the stage run and their results. + ``active_stage_name`` : str or None, default = None + Name of the stage currently being executed. Used to expose ``stage_config``. """ pipeline_name: str run_id: str @@ -51,7 +55,9 @@ class ExecutionContext: started_at: datetime = field(default_factory=now) working_directory: Path = field(default_factory=Path.cwd) stage_results: dict[str, StageResult] = field(default_factory=dict) + stage_configs: dict[str, StageConfig] = field(default_factory=dict) variables: dict[str, Any] = field(default_factory=dict) + active_stage_name: str | None = None def record(self, result: StageResult) -> StageResult: """ @@ -90,6 +96,29 @@ def result_for(self, stage_name: str) -> StageResult | None: """ return self.stage_results.get(stage_name) + def set_active_stage(self, stage_name: str | None) -> None: + """ + Mark the stage currently being executed so ``stage_config`` resolves correctly. + """ + self.active_stage_name = stage_name + + def stage_config_for(self, stage_name: str) -> StageConfig | None: + """ + Return the configuration bound to a specific stage name, if one exists. + """ + return self.stage_configs.get(stage_name) + + @property + def stage_config(self) -> StageConfig | None: + """ + Return the configuration for the stage currently being executed. + + This property is ``None`` outside an active stage run. + """ + if self.active_stage_name is None: + return None + return self.stage_config_for(self.active_stage_name) + @property def stage_outputs(self) -> dict[str, Any]: """ From e10c3ae7963706cabc2123c6b6a0e90ca5bb33c3 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:31:31 +0100 Subject: [PATCH 091/332] Modified PipelineRunner to account for StageConfig & changes to ExecutionContext --- onsrap/runner.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 6d78310..ed91955 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -33,7 +33,9 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: This method validates the source information, establishes the directories and the context to run the pipeline within, sets out the manifest for the run, attempts to run the stages in the order outlined by the ``StageGraph`` instance and logs all - progress alongside relevant statuses. + progress alongside relevant statuses. Before each stage executes, the runner binds + the current stage name onto the ``ExecutionContext`` so ``context.stage_config`` + resolves to the correct stage-specific configuration. It returns a PipelineRun instance containing metadata and logging information for the specific run of the whole Pipeline. @@ -72,6 +74,7 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: run_dir=run_dir, started_at=started_at, working_directory=pipeline.config.work_dir, + stage_configs=dict(pipeline.stage_configs), ) ordered_stages = pipeline.ordered_stages() @@ -91,7 +94,11 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: try: for stage in ordered_stages: self.logger.event("Executing stage", name=stage.name, source=stage.source_label) - result = stage.run(context, pipeline.executor) + context.set_active_stage(stage.name) + try: + result = stage.run(context, pipeline.executor) + finally: + context.set_active_stage(None) context.record(result) stage_results.append(result) manifest.stages_run.append(result.name) From 4e68efb551afae57b591b3064681d896024aa950 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:31:44 +0100 Subject: [PATCH 092/332] Added StageConfig to __init__ export --- onsrap/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onsrap/__init__.py b/onsrap/__init__.py index 908f017..8329c36 100644 --- a/onsrap/__init__.py +++ b/onsrap/__init__.py @@ -20,6 +20,7 @@ RAPDataset, RunManifest, RuntimeID, + StageConfig, StageResult, StageStatus, ) @@ -47,6 +48,7 @@ "RAPDataset", "RunManifest", "RuntimeID", + "StageConfig", "Stage", "StageConfigurationError", "StageExecutionError", From 0a44117eabf57ca082e96efe345e896b1ab488cb Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:32:44 +0100 Subject: [PATCH 093/332] feat: Added StageConfig and configuration handling to Pipeline architecture. Implemented through numerous new methods and concepts, with documentation. --- onsrap/pipeline.py | 439 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 393 insertions(+), 46 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 2ef6561..487a556 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence -from .errors import StageConfigurationError +from .errors import StageConfigurationError, PipelineInitialisationError from .warnings import StageConfigurationWarning from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph @@ -54,20 +54,24 @@ def __init__( logger: Logger | None = None, executor: StageExecutor | None = None, ): - self.backend = backend or "python" - self.config = self._resolve_config(config) - if (self.config.name is not None) and (name is None): - self.name = self.config.name - else: - self.name = name or "pipeline" + resolved_config, resolved_stage_configs, configured_stages = self._resolve_config(config) + + self.name = name or resolved_config.name or "pipeline" + self.backend = backend or resolved_config.backend or "python" + if backend == "python" and resolved_config.backend != "python": + self.backend = resolved_config.backend + self.config = resolved_config if self.config.name is None: self.config.name = self.name self.config.backend = self.backend self.logger = logger or Logger(log_dir=self.config.log_dir) self.executor = executor or PythonStageExecutor() - self.stages = [self._coerce_stage(stage) for stage in (stages or [])] + if stages is None: + self.stages = configured_stages + else: + self.stages = [self._coerce_stage(stage) for stage in stages] self.dependencies = dependencies if dependencies is not None and stages is None: raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " @@ -75,6 +79,8 @@ def __init__( "parse them to the Pipeline Constructor.") if dependencies is not None: self._assign_dependencies(dependencies,self.stages) + self.stage_configs = dict(resolved_stage_configs) + self._sync_stage_configs() self.graph = StageGraph.from_stages(self.stages) self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -158,6 +164,7 @@ def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[. """ added_stages = [self._coerce_stage(stage) for stage in stages] self.stages.extend(added_stages) + self._sync_stage_configs() self._rebuild_graph() self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) @@ -173,13 +180,75 @@ def validate(self) -> "Pipeline": Confirms that the source files for the stage exist. """ self.logger.event("Validating pipeline", name=self.name) + self._validate_stage_configs() for stage in self.stages: stage.validate() self.graph.validate() return self - def create_stage_config(self, s_config: str | Path) -> StageConfig: - pass + def create_stage_config( + self, + s_config: Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> StageConfig: + """ + Create a ``StageConfig`` from direct data, a stage-name keyed mapping, or a config file. + + Parameters + ---------- + ``s_config`` : Mapping[str, Any] | str | Path + Either a single stage payload, a mapping keyed by stage name, or a config file. + Config files may contain a top-level ``stage_configuration`` section or may consist + solely of stage-name keyed configuration entries. + ``name`` : str or None, keyword-only + Stage name to extract when the input contains more than one stage configuration. + + Returns + ------- + ``StageConfig`` + The normalized stage configuration for the requested stage. + + Raises + ------ + ``StageConfigurationError`` + If the input cannot be resolved to exactly one stage configuration. + """ + if isinstance(s_config, Mapping): + if "pipeline_variables" in s_config or "stage_configuration" in s_config or "stage_config" in s_config: + _, stage_config_payload = self._split_config_sections(s_config) + stage_configs = self._build_stage_configs(stage_config_payload) + elif name is not None and name in s_config and isinstance(s_config[name], Mapping): + stage_configs = self._build_stage_configs(s_config) + else: + if name is None: + if len(s_config) != 1: + raise StageConfigurationError( + "A stage configuration mapping must include exactly one stage when no name is provided." + ) + name, stage_payload = next(iter(s_config.items())) + else: + stage_payload = s_config + + if not isinstance(stage_payload, Mapping): + raise StageConfigurationError("Stage configuration values must be provided as a mapping.") + + return StageConfig.from_mapping(str(name), stage_payload) + + return self._select_stage_config(stage_configs, name=name) + + raw_payload = self._load_config_mapping(s_config) + if "pipeline_variables" in raw_payload or "stage_configuration" in raw_payload or "stage_config" in raw_payload: + _, stage_config_payload = self._split_config_sections(raw_payload) + stage_configs = self._build_stage_configs(stage_config_payload) + elif all(isinstance(value, Mapping) for value in raw_payload.values()): + stage_configs = self._build_stage_configs(raw_payload) + else: + raise StageConfigurationError( + "Config files passed to create_stage_config must define a stage-configuration section or a mapping of stage names to configuration mappings." + ) + + return self._select_stage_config(stage_configs, name=name) def run(self) -> PipelineRun: """ @@ -205,7 +274,7 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: run_id=runtime_id.get_id(), git_commit=self._discover_git_commit(), stages_run=[], - parameters=self.config.to_dict(), + parameters=self._manifest_parameters(), inputs={stage.name: list(stage.dependencies) for stage in self.stages}, outputs={}, backend=self.backend, @@ -284,44 +353,300 @@ def _current_user(self) -> str | None: except Exception: return None - def _resolve_config(self, config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None) -> list[PipelineConfig, StageConfig] | PipelineConfig | StageConfig: + def _resolve_config( + self, + config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None, + ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage]]: + """ + Resolve supported configuration inputs into pipeline config, stage config, and stages. + + This method is the main normalization step for configuration injection. It accepts + already-constructed config objects, raw mappings, and YAML files, and converts them + into the three objects the pipeline needs before execution starts. + """ if config is None: - return PipelineConfig.from_any(config) - if isinstance(config, str): - if config.endswith(".yaml") or config.endswith(".yml"): - return PipelineConfig.from_yaml(config) - else: - raise StageConfigurationError(f"Unsupported config file format parsed as Stage Configuration: {config!r}.") - if isinstance(config, Path): - if config.suffix in (".yaml", ".yml"): - return PipelineConfig.from_yaml(config) - else: - raise StageConfigurationError(f"Unsupported config file format parsed as Stage Configuration: {config!r}.") + return PipelineConfig.from_any(config), {}, [] + if isinstance(config, PipelineConfig): - if "stage_config" in config.metadata: + stage_configuration = config.metadata.get("stage_configuration") + if stage_configuration is None: + stage_configuration = config.metadata.get("stage_config") + + if stage_configuration is not None: warnings.warn( - "Stage Configuration found in PipelineConfig metadata. This should be moved to a separate location for StageConfiguration instantiation.", - StageConfigurationWarning - ) - self.logger.warning( - "Stage Configuration found in PipelineConfig metadata. This should be moved to a separate location for StageConfiguration instantiation." + "Stage configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", + StageConfigurationWarning, ) + return config, self._build_stage_configs(stage_configuration), [] + + raw_config = self._load_config_mapping(config) + pipeline_payload, stage_config_payload = self._split_config_sections(raw_config) + normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) + stage_definitions = normalized_pipeline_payload.pop("stages", ()) + + pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) + stage_configs = self._build_stage_configs(stage_config_payload) + configured_stages = self._build_stages_from_config( + stage_definitions, + backend=pipeline_config.backend, + work_dir=pipeline_config.work_dir, + ) + return pipeline_config, stage_configs, configured_stages + + def _sync_stage_configs(self) -> None: + """ + Ensure every known stage has a ``StageConfig`` entry, even if it is empty. + """ + for stage in self.stages: + self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) + + def _validate_stage_configs(self) -> None: + """ + Confirm that every configured stage name matches a stage present in the pipeline. + """ + self._sync_stage_configs() + stage_names = {stage.name for stage in self.stages} + unknown_stage_configs = sorted(name for name in self.stage_configs if name not in stage_names) + if unknown_stage_configs and stage_names: + missing = ", ".join(unknown_stage_configs) + raise StageConfigurationError( + f"Stage configuration was provided for unknown stages: {missing}." + ) + + def _manifest_parameters(self) -> dict[str, Any]: + """ + Build the manifest parameter payload, including per-stage configuration. + """ + parameters = self.config.to_dict() + if self.stage_configs: + parameters["stage_configuration"] = { + name: stage_config.to_dict() + for name, stage_config in self.stage_configs.items() + } + return parameters + + @staticmethod + def _select_stage_config( + stage_configs: Mapping[str, StageConfig], + *, + name: str | None, + ) -> StageConfig: + """ + Select one stage configuration from a stage-name keyed mapping. + + When ``name`` is omitted, exactly one stage configuration must be present. + """ + if name is not None: + if name not in stage_configs: + raise StageConfigurationError(f"Stage configuration '{name}' was not found.") + return stage_configs[name] + + if len(stage_configs) != 1: + raise StageConfigurationError( + "The provided input resolves to multiple stage configurations; specify a stage name." + ) + + return next(iter(stage_configs.values())) + + @staticmethod + def _load_config_mapping( + config: RAPConfig | Mapping[str, Any] | str | Path, + ) -> dict[str, Any]: + """ + Load raw configuration data from a RAP config object, mapping, or YAML file. + """ + if isinstance(config, RAPConfig): + return dict(config.contents) + + if isinstance(config, Mapping): + return dict(config) + + config_path = Path(config).expanduser() + if config_path.suffix.lower() not in (".yaml", ".yml"): + raise StageConfigurationError( + f"Unsupported config file format parsed as Stage Configuration: {config!r}." + ) + if not config_path.exists(): + raise FileNotFoundError(f"Config file does not exist: {config_path}") + + import yaml + + raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if raw_config is None: + return {} + if not isinstance(raw_config, Mapping): + raise TypeError("Pipeline config file must contain a mapping at the top level.") + return dict(raw_config) + + @staticmethod + def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any], Mapping[str, Any] | None]: + """ + Split a raw config payload into pipeline-level and stage-level sections. + + Composite config payloads may use top-level ``pipeline_variables`` and + ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless + a stage-configuration key is present. + """ + if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: + pipeline_payload = raw_config.get("pipeline_variables", {}) + if not isinstance(pipeline_payload, Mapping): + raise StageConfigurationError("The 'pipeline_variables' section must be a mapping.") + stage_payload = raw_config.get("stage_configuration", raw_config.get("stage_config")) + return dict(pipeline_payload), stage_payload + + pipeline_payload = dict(raw_config) + stage_payload = pipeline_payload.pop("stage_configuration", pipeline_payload.pop("stage_config", None)) + return pipeline_payload, stage_payload + + @staticmethod + def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str, Any]: + """ + Normalize supported aliases in the pipeline section before model construction. + + Recognized aliases: + + - ``working_dir`` → ``work_dir`` (only when ``work_dir`` is absent). + + If both ``working_dir`` and ``work_dir`` are present at the same time, a + ``UserWarning`` is emitted and ``working_dir`` is left in the payload where + it will be silently absorbed into ``PipelineConfig.metadata``. + """ + normalized_payload = dict(pipeline_payload) + if "working_dir" in normalized_payload: + if "work_dir" not in normalized_payload: + normalized_payload["work_dir"] = normalized_payload.pop("working_dir") else: warnings.warn( - "No Stage Configuration found in parsed configuration. This may lead to unexpected behavior during pipeline execution.", - StageConfigurationWarning + "Both 'working_dir' and 'work_dir' were found in the pipeline configuration. " + "'work_dir' will be used and 'working_dir' will be ignored.", + UserWarning, + stacklevel=2, ) - self.logger.warning( - "No Stage Configuration found in parsed configuration. This may lead to unexpected behavior during pipeline execution." - ) - return config - if isinstance(config, Mapping): - # Look for Pipeline Configuration and Stage Configuration in keys - pass - - pipeline_config = self.config - stage_config = StageConfig() - return [pipeline_config, stage_config] + return normalized_payload + + @staticmethod + def _build_stage_configs(stage_configuration: Mapping[str, Any] | None) -> dict[str, StageConfig]: + """ + Build a stage-name keyed configuration mapping for any number of configured stages. + + The returned mapping scales linearly with the provided stage entries and is used as + the canonical runtime lookup structure for stage configuration. + """ + if stage_configuration is None: + return {} + if not isinstance(stage_configuration, Mapping): + raise StageConfigurationError("Stage configuration must be a mapping keyed by stage name.") + + return { + str(stage_name): StageConfig.from_mapping(str(stage_name), stage_payload) + for stage_name, stage_payload in stage_configuration.items() + } + + def _build_stages_from_config( + self, + stage_definitions: Sequence[Any] | None, + *, + backend: str, + work_dir: Path, + ) -> list[Stage]: + """ + Convert configured stage definitions into ``Stage`` instances. + + Each entry is resolved independently, so the method can process any number of + stage definitions supplied in the pipeline configuration. + """ + if not stage_definitions: + return [] + if not isinstance(stage_definitions, Sequence) or isinstance(stage_definitions, (str, bytes)): + raise StageConfigurationError("Configured stages must be provided as a sequence.") + + configured_stages: list[Stage] = [] + for stage_definition in stage_definitions: + stage = self._stage_from_config_definition(stage_definition, backend=backend, work_dir=work_dir) + if stage is not None: + configured_stages.append(stage) + return configured_stages + + def _stage_from_config_definition( + self, + stage_definition: Any, + *, + backend: str, + work_dir: Path, + ) -> Stage | None: + """ + Resolve one configured stage entry into a ``Stage`` instance. + + Supported forms include ready-made ``Stage`` objects, paths, callables, full stage + dictionaries, and compact ``{stage_name: {...}}`` definitions from YAML config files. + Entries with ``run: false`` are skipped. + """ + if isinstance(stage_definition, Stage): + return stage_definition + + if isinstance(stage_definition, (str, Path)) or callable(stage_definition): + return self._coerce_stage(stage_definition) + + if not isinstance(stage_definition, Mapping): + raise StageConfigurationError("Configured stage entries must be mappings, paths, or callables.") + + payload = dict(stage_definition) + if any(key in payload for key in ("name", "source", "path", "callable")): + return Stage.from_dict(payload) + + if len(payload) != 1: + raise StageConfigurationError( + "Configured stage mappings must define exactly one stage name." + ) + + stage_name, stage_payload = next(iter(payload.items())) + if not isinstance(stage_payload, Mapping): + raise StageConfigurationError("Configured stage details must be provided as a mapping.") + + stage_options = dict(stage_payload) + if not bool(stage_options.pop("run", True)): + return None + + location = stage_options.pop("location", stage_options.pop("source", stage_options.pop("path", None))) + dependencies = stage_options.pop("dependencies", ()) + entrypoint = stage_options.pop("entrypoint", None) + metadata = stage_options.pop("metadata", {}) + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} + metadata.update(stage_options) + + source = self._resolve_stage_source(stage_name=str(stage_name), location=location, work_dir=work_dir) + return Stage.from_file( + source, + name=str(stage_name), + dependencies=dependencies, + metadata=metadata, + entrypoint=entrypoint, + backend=backend, + ) + + @staticmethod + def _resolve_stage_source(stage_name: str, location: Any, work_dir: Path) -> Path: + """ + Resolve the source path for a configured stage. + + Empty locations default to ``work_dir / "scripts" / ".py"``. Relative + paths are first interpreted as given and then relative to ``work_dir``. + """ + if location in (None, ""): + return work_dir / "scripts" / f"{stage_name}.py" + + candidate = Path(location).expanduser() + if candidate.is_absolute() or candidate.exists(): + return candidate + + work_dir_candidate = work_dir / candidate + if work_dir_candidate.exists(): + return work_dir_candidate + + return candidate @classmethod @@ -400,6 +725,9 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": ------- A ``Pipeline`` class instance. """ + if "pipeline_variables" in cfg or "stage_configuration" in cfg or "stage_config" in cfg: + return cls(config=cfg) + payload = dict(cfg) # pipeline_variables contains pipeline information @@ -422,10 +750,29 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": stages=stages, ) - def from_config(cls, config: dict) -> Pipeline: - # Wrapper for from_dict but expecting config Path/str or yaml object - # Extract pipeline_variables aka do not parse stage_configuration section of config.yaml - pass + @classmethod + def from_config( + cls, + config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + backend: str = "python", + logger: Logger | None = None, + executor: StageExecutor | None = None, + ) -> "Pipeline": + """ + Construct a pipeline directly from a composite configuration payload or file. + + This is the preferred entrypoint when configuration defines both pipeline-level + settings and the stage-level configuration that should be injected at runtime. + """ + return cls( + name=name, + backend=backend, + config=config, + logger=logger, + executor=executor, + ) @staticmethod def _dependencies_for_stage( From 7eedb388495366fd5566563ca3e3673c82b2ab17 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:33:11 +0100 Subject: [PATCH 094/332] tests: added StageConfig/config-parsing related tests --- tests/test_pipeline_architecture.py | 269 +++++++++++++++++++++++++++- 1 file changed, 268 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index bd4e540..79fe757 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -3,6 +3,10 @@ from pathlib import Path from textwrap import dedent +import pytest +import yaml + +from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph from onsrap.pipeline import Pipeline from onsrap.stage import Stage @@ -121,4 +125,267 @@ def test_stage_graph_detects_cycles() -> None: except Exception as exc: # noqa: BLE001 assert exc.__class__.__name__ == "DependencyCycleError" else: - raise AssertionError("Expected a dependency cycle error") \ No newline at end of file + raise AssertionError("Expected a dependency cycle error") + + +def test_pipeline_from_config_builds_stages_and_injects_stage_config(tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_file = scripts_dir / "0_data_validation.py" + stage_file.write_text( + dedent( + """ + def run(context): + return { + "stage_name": context.stage_config.name, + "years_to_run": context.stage_config.get("years_to_run"), + "target_variable": context.stage_config.require("target_variable"), + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + dedent( + f""" + pipeline_variables: + name: "configured-pipeline" + backend: python + working_dir: "{tmp_path.as_posix()}" + project_root: "{tmp_path.as_posix()}" + log_dir: "{(tmp_path / 'logs').as_posix()}" + stages: + - 0_data_validation: + location: "" + run: true + dependencies: [] + + stage_configuration: + 0_data_validation: + years_to_run: 2017 + target_variable: "classification" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + pipeline = Pipeline.from_config(config_file) + + assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] + assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 + + run = pipeline.run() + + assert run.stage_outputs["0_data_validation"] == { + "stage_name": "0_data_validation", + "years_to_run": 2017, + "target_variable": "classification", + } + + +def test_pipeline_rejects_unknown_stage_configuration(tmp_path: Path) -> None: + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return "ok" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + pipeline = Pipeline.from_files( + [stage_file], + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + "stage_configuration": { + "missing_stage": {"years_to_run": 2017}, + }, + }, + ) + + with pytest.raises(StageConfigurationError, match="unknown stages"): + pipeline.validate() + + +def test_pipeline_from_config_parses_stage_configuration_payloads(tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + for stage_name in ("0_extract", "1_transform"): + (scripts_dir / f"{stage_name}.py").write_text( + dedent( + """ + def run(context): + return context.stage_config.to_dict() + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_payload = { + "pipeline_variables": { + "name": "parse-test", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "data_dir": (tmp_path / "data").as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "metadata": { + "description": "configuration parsing test", + }, + "stages": [ + { + "0_extract": { + "location": "", + "run": True, + "dependencies": [], + "owner": "analytics", + } + }, + { + "1_transform": { + "location": "", + "run": True, + "dependencies": ["0_extract"], + } + }, + ], + }, + "stage_configuration": { + "0_extract": { + "years_to_run": 2017, + "datasets": { + "orders": { + "path": "data/orders.csv", + } + }, + "metadata": { + "purpose": "extract", + }, + }, + "1_transform": { + "target_variable": "classification", + "metadata": { + "purpose": "transform", + }, + }, + }, + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + + pipeline = Pipeline.from_config(config_file) + + assert pipeline.name == "parse-test" + assert pipeline.config.work_dir == tmp_path + assert pipeline.config.project_root == tmp_path + assert pipeline.config.log_dir == tmp_path / "logs" + assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] + assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + assert pipeline.stages[0].metadata["owner"] == "analytics" + assert pipeline.stages[1].dependencies == ("0_extract",) + assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017} + assert pipeline.stage_configs["0_extract"].datasets == {"orders": {"path": "data/orders.csv"}} + assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} + assert pipeline.create_stage_config(config_file, name="1_transform").require("target_variable") == "classification" + + +def test_pipeline_from_config_scales_stage_configuration_to_many_stages(tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_count = 6 + stage_names = [f"{index}_stage" for index in range(stage_count)] + + for index, stage_name in enumerate(stage_names): + stage_file = scripts_dir / f"{stage_name}.py" + previous_stage_name = stage_names[index - 1] if index > 0 else None + stage_file.write_text( + dedent( + f""" + def run(context): + previous_ordinal = None + if {index} > 0: + previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] + return {{ + "stage_name": context.stage_config.name, + "ordinal": context.stage_config.require("ordinal"), + "label": context.stage_config.require("label"), + "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), + "known_stage_configs": sorted(context.stage_configs), + "previous_ordinal": previous_ordinal, + }} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + stage_definitions = [] + stage_configuration = {} + for index, stage_name in enumerate(stage_names): + dependencies = [stage_names[index - 1]] if index > 0 else [] + stage_definitions.append( + { + stage_name: { + "location": "", + "run": True, + "dependencies": dependencies, + } + } + ) + stage_configuration[stage_name] = { + "ordinal": index, + "label": f"label-{index}", + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + yaml.safe_dump( + { + "pipeline_variables": { + "name": "many-stage-pipeline", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "stages": stage_definitions, + }, + "stage_configuration": stage_configuration, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + pipeline = Pipeline.from_config(config_file) + + assert [stage.name for stage in pipeline.stages] == stage_names + assert sorted(pipeline.stage_configs) == stage_names + + run = pipeline.run() + + assert run.manifest.stages_run == stage_names + assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names + + for index, stage_name in enumerate(stage_names): + output = run.stage_outputs[stage_name] + assert output["stage_name"] == stage_name + assert output["ordinal"] == index + assert output["label"] == f"label-{index}" + assert output["first_stage_ordinal"] == 0 + assert output["known_stage_configs"] == stage_names + expected_previous = None if index == 0 else index - 1 + assert output["previous_ordinal"] == expected_previous \ No newline at end of file From 7972f1243c1e96fbbe7648854b35f003b1cbff56 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:33:28 +0100 Subject: [PATCH 095/332] tmp: Added a configuration.md file to summarise configuration changes in this current state --- configuration.md | 492 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 492 insertions(+) create mode 100644 configuration.md diff --git a/configuration.md b/configuration.md new file mode 100644 index 0000000..790ae9a --- /dev/null +++ b/configuration.md @@ -0,0 +1,492 @@ +# Configuration + +This document describes how configuration flows through the onsrap pipeline +architecture, from the initial input accepted at construction time through to the +point where individual stage scripts read their own variables at execution time. + +--- + +## Overview + +onsrap uses two distinct levels of configuration. + +| Level | Object | Scope | +|---|---|---| +| Pipeline | `PipelineConfig` | Execution environment, directories, backend, run metadata | +| Stage | `StageConfig` | Per-stage variables injected at execution time | + +Both objects are constructed during `Pipeline` initialisation and are immutable +for the duration of a run. They are kept separate so that pipeline orchestration +concerns (where to write logs, which Python interpreter to use) never bleed into +the domain logic a stage script contains. + +--- + +## Configuration Objects + +### `PipelineConfig` + +Defined in `onsrap/models.py`. Holds every setting that controls how the runner +behaves. + +| Field | Type | Default | Description | +|---|---|---|---| +| `name` | `str \| None` | `None` | Pipeline name. Back-filled from the `Pipeline.name` argument when absent. | +| `backend` | `str` | `"python"` | Execution backend. Currently only `"python"` is implemented. | +| `work_dir` | `Path` | `Path.cwd()` | Working directory used for stage file discovery and subprocess execution. | +| `project_root` | `Path \| None` | `None` → falls back to `work_dir` | Root used to construct `runs/` output directories. | +| `log_dir` | `Path` | `Path("logs")` | Directory where `onsrap.log` is written. | +| `data_dir` | `Path` | `Path("data")` | Conventional location for input data. Not enforced by the runner; available to stages via `context.config.data_dir`. | +| `output_dir` | `Path \| None` | `None` | Conventional location for pipeline outputs. Not enforced by the runner; available to stages via `context.config.output_dir`. | +| `allow_subprocess_fallback` | `bool` | `True` | When `True`, stage files without a recognised entrypoint function (`run`, `main`, `execute`) are executed as plain scripts via subprocess. Set to `False` to require entrypoints everywhere. | +| `python_executable` | `str \| None` | `None` → `sys.executable` | Python interpreter used in subprocess fallback mode. | +| `metadata` | `dict[str, Any]` | `{}` | Arbitrary additional values. Any unrecognised key from a raw config mapping is absorbed here rather than raising an error. | + +`PipelineConfig` can be constructed directly in code, loaded with +`PipelineConfig.from_any()`, or produced automatically by `Pipeline._resolve_config()` +from a raw mapping or YAML file. + +### `StageConfig` + +Defined in `onsrap/models.py`. Holds every setting that should be visible to one +specific stage script. + +| Attribute | Access | Description | +|---|---|---| +| `name` | `stage_config.name` | The stage name this config belongs to. | +| `_variables` | `.get(key)`, `.require(key)`, `.variables`, `.get_variables(...)` | Arbitrary key/value pairs — the main carrier for stage parameters. All YAML keys not named `datasets` or `metadata` end up here. | +| `datasets` | `stage_config.datasets` | Mapping of dataset identifiers to their properties (e.g. file path, format). | +| `metadata` | `stage_config.metadata` | Supporting metadata about the stage configuration itself (e.g. purpose, owner). | + +Accessing variables from stage code: + +```python +# Optional: returns default if the key is absent +value = context.stage_config.get("years_to_run") +value = context.stage_config.get("years_to_run", default=2020) + +# Mandatory: raises StageConfigurationError if the key is absent +value = context.stage_config.require("target_variable") + +# All variables at once +all_vars = context.stage_config.variables # returns a copy + +# Selected subset (raises if any are missing) +subset = context.stage_config.get_variables(["years_to_run", "target_variable"]) +``` + +--- + +## Entry Points + +`Pipeline` is the single public entry point for all configuration. Four class +methods accept configuration in different ways. + +``` +Pipeline(config=...) — direct constructor; most flexible +Pipeline.from_config(path) — preferred when a composite config file defines everything +Pipeline.from_files([...], config=...) — explicit stage file list with optional config +Pipeline.from_dict({...}) — construct from an in-memory mapping +``` + +All four funnel into `Pipeline.__init__`, which calls `_resolve_config()` as its +first action. That single call produces the three objects the pipeline needs before +execution can start: `PipelineConfig`, `dict[str, StageConfig]`, and +`list[Stage]`. + +--- + +## Supported Config Input Types + +`Pipeline._resolve_config()` and `Pipeline._load_config_mapping()` together +handle the following types for the `config` parameter. + +| Type | Behaviour | +|---|---| +| `None` | Constructs a fully-defaulted `PipelineConfig`. No stage configs. | +| `PipelineConfig` instance | Used directly. Stage configuration may optionally be embedded in `config.metadata["stage_configuration"]` (backwards-compatibility path; emits a `StageConfigurationWarning`). | +| `RAPConfig` instance | Delegates to `PipelineConfig.from_mapping(config.contents)`. | +| `Mapping[str, Any]` | Parsed as a composite or flat config payload (see below). | +| `str` / `Path` | Read and YAML-parsed; the resulting mapping is treated as above. Only `.yaml` / `.yml` files are accepted. | + +--- + +## Config Payload Formats + +A raw mapping (or YAML file loaded into a mapping) is classified by +`_split_config_sections()` into one of two shapes. + +### Composite format (recommended) + +Used when any of the keys `pipeline_variables`, `stage_configuration`, or +`stage_config` are present at the top level. This is the format used by +`examples/pipeline_2/conf.yaml`. + +```yaml +pipeline_variables: + name: "My Pipeline" + backend: python + working_dir: "path/to/pipeline" # alias for work_dir + project_root: "path/to/pipeline" + log_dir: "path/to/pipeline/logs" + data_dir: "path/to/pipeline/data" + output_dir: "path/to/pipeline/output" + stages: + - 0_data_validation: + location: "" # empty → scripts/0_data_validation.py + run: true + dependencies: [] + - 1_preprocessing: + location: "" + run: true + dependencies: + - 0_data_validation + metadata: + description: "Example pipeline" + +stage_configuration: + 0_data_validation: + years_to_run: 2017 + time_col: "order_date" + target_variable: "classification" + datasets: + orders: + path: "data/orders.csv" + metadata: + purpose: "validate raw inputs" + 1_preprocessing: + drop_columns: ["id", "notes"] +``` + +In this format: +- Everything under `pipeline_variables` becomes `PipelineConfig`. +- Everything under `stage_configuration` becomes the `stage_configs` mapping. +- Any key at the top level outside these two sections is **ignored**. + +### Flat format + +Used when none of the composite section markers are present. The entire mapping +is treated as pipeline config; an optional nested `stage_configuration` or +`stage_config` key within it carries the stage-level config. + +```python +Pipeline.from_files( + ["scripts/0_data_validation.py", "scripts/1_preprocessing.py"], + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + "stage_configuration": { + "0_data_validation": { + "years_to_run": 2017, + "target_variable": "classification", + } + }, + }, +) +``` + +--- + +## Pipeline-Level Parsing Flow + +``` +config input (any supported type) + │ + ▼ +Pipeline._resolve_config() + │ + ├─ None ─────────────────────────────► PipelineConfig() (defaults) + │ + ├─ PipelineConfig instance ──────────► used as-is + │ (stage config read from .metadata if present) + │ + └─ everything else + │ + ▼ + Pipeline._load_config_mapping() + Normalises input to dict[str, Any]: + RAPConfig → config.contents + Mapping → dict(mapping) + str / Path → yaml.safe_load(file) + │ + ▼ + Pipeline._split_config_sections() + Detects composite vs flat shape. + Returns (pipeline_payload, stage_config_payload). + │ + ▼ + Pipeline._normalize_pipeline_payload() + Maps recognised field aliases: + working_dir → work_dir (only when work_dir absent) + Pops the stages list before handing payload on. + │ + ▼ + PipelineConfig.from_mapping(pipeline_payload) + Extracts recognised fields; remaining keys + are absorbed into PipelineConfig.metadata. +``` + +### Recognised `pipeline_variables` keys + +`name`, `backend`, `work_dir` / `working_dir`, `project_root`, `log_dir`, +`data_dir`, `output_dir`, `allow_subprocess_fallback`, `python_executable`, +`metadata`, `stages`. + +Any other key is silently absorbed into `PipelineConfig.metadata`. This is +intentional — it allows pipelines to carry arbitrary project metadata — but it +also means a typo in a recognised key name will not raise an error. + +--- + +## Stage Configuration Parsing Flow + +``` +stage_configuration section (Mapping[str, Any]) + │ + ▼ +Pipeline._build_stage_configs() +Iterates every stage name in the mapping. +For each stage: + │ + ▼ +StageConfig.from_mapping(stage_name, stage_payload) +Splits the stage payload into three buckets: + datasets → StageConfig.datasets + metadata → StageConfig.metadata + everything else → StageConfig._variables + │ + ▼ +Pipeline.stage_configs (dict[str, StageConfig]) +One entry per configured stage. Stages without an +explicit entry get a default empty StageConfig via +Pipeline._sync_stage_configs(). +``` + +--- + +## Stage Definition Parsing Flow + +When stages are listed under `pipeline_variables.stages` in a composite YAML, +`Pipeline._build_stages_from_config()` converts each entry into a `Stage` object. + +``` +stages: list (each entry one of several forms) + │ + ▼ +Pipeline._stage_from_config_definition() +Dispatches on the type of the entry: + + Stage instance ─────────────────────────────► returned as-is + str / Path / callable ──────────────────────► Pipeline._coerce_stage() + Mapping with name/source/path/callable keys ► Stage.from_dict() + {stage_name: {...}} single-key mapping ─────► stage_name-keyed form (most common in YAML) + │ + ▼ (for single-key YAML form) +Extracts from the inner mapping: + run → if false, stage is skipped entirely + location → source file path (alias: source, path) + dependencies → list of prerequisite stage names + entrypoint → explicit function name (optional) + metadata → stage metadata dict + all others → merged into metadata + │ + ▼ +Pipeline._resolve_stage_source(stage_name, location, work_dir) + location is None or "" ─► work_dir / "scripts" / ".py" + absolute path ──────────► used as-is + relative, exists ───────► used as-is + relative, absent ───────► work_dir / location tried first + neither exists ─────────► raw candidate returned (Stage.validate() will fail later) + │ + ▼ +Stage.from_file(source, name, dependencies, metadata, entrypoint, backend) +Expands the path, verifies it exists, stores resolved absolute path. +``` + +--- + +## Config Consumption at Execution Time + +`PipelineRunner.run()` is the bridge between the `Pipeline` object (which holds +all parsed configuration) and the actual execution of stage scripts. + +``` +Pipeline.stage_configs (dict[str, StageConfig]) + │ + │ copied at run start + ▼ +PipelineRunner.run() + Creates ExecutionContext with: + config = pipeline.config (PipelineConfig) + stage_configs = dict(pipeline.stage_configs) + │ + │ for each stage in topological order + ▼ +context.set_active_stage(stage.name) + Sets ExecutionContext.active_stage_name + │ + ▼ +stage.run(context, executor) + → PythonStageExecutor.execute(stage, context) + → loads stage module; calls entrypoint(context) + +Within the stage function: + context.stage_config # StageConfig for THIS stage + context.stage_config.get(key) # optional variable lookup + context.stage_config.require(key) # mandatory variable lookup + context.stage_config_for(name) # any other stage's StageConfig + context.config # PipelineConfig (directories etc.) + context.run_dir # project_root/runs/ + context.result_for(name) # StageResult from an earlier stage + │ + ▼ +context.set_active_stage(None) # reset after stage finishes +``` + +### What stages can read + +| `context` attribute | Type | Contains | +|---|---|---| +| `context.config` | `PipelineConfig` | Directories, backend, subprocess settings | +| `context.stage_config` | `StageConfig \| None` | This stage's variables, datasets, metadata | +| `context.stage_config_for(name)` | `StageConfig \| None` | Any stage's config by name | +| `context.stage_configs` | `dict[str, StageConfig]` | All stage configs | +| `context.run_id` | `str` | Unique run identifier | +| `context.run_dir` | `Path` | `project_root / "runs" / run_id` | +| `context.result_for(name)` | `StageResult \| None` | Output of a previously run stage | +| `context.stage_outputs` | `dict[str, Any]` | All outputs from stages run so far | + +--- + +## Validation Model + +Configuration parsing and execution validation are intentionally separate phases. + +| Phase | When | What is checked | +|---|---|---| +| **Parse** | `Pipeline.__init__` | Type validity; YAML syntax; that stage source files exist (`Stage.from_file` calls `path.exists()`); `StageConfig` is built for each configured stage name | +| **Validate** | `Pipeline.validate()`, called automatically at the start of every `Pipeline.run()` | Stage source files still exist; dependency graph is acyclic and complete; every name in `stage_configuration` matches a real stage | + +Calling `pipeline.validate()` explicitly before `pipeline.run()` is safe and +useful in test suites or CI pipelines. + +--- + +## Manifest Serialisation + +When a pipeline run completes, `Pipeline._manifest_parameters()` serialises the +full configuration state into `RunManifest.parameters`. This includes the +`PipelineConfig` fields and a nested `stage_configuration` block containing every +`StageConfig.to_dict()`. This means the exact parameters used for any given run +are reproducible from the manifest alone. + +--- + +## Known Pitfalls + +### 1 — Relative paths are resolved against the process working directory + +`work_dir`, `log_dir`, `data_dir`, and `output_dir` are stored as `Path` objects +constructed directly from whatever string is in the config. A value like +`"examples/pipeline_2"` is therefore resolved relative to wherever Python is +running when the pipeline is constructed, not relative to the YAML file's +location. + +**Mitigation**: Use absolute paths in YAML configs, or construct `PipelineConfig` +in Python code where you can use `Path(__file__).parent` to anchor paths relative +to the config module. + +### 2 — Unrecognised `pipeline_variables` keys are silently absorbed into metadata + +`PipelineConfig.from_mapping()` pops every recognised field and then calls +`metadata.update(remaining_payload)`. A typo such as `log_dirs:` instead of +`log_dir:` will not raise; instead the default `Path("logs")` will be used and +the misspelled key will appear in `pipeline.config.metadata`. + +**Mitigation**: When debugging unexpected defaults, inspect +`pipeline.config.metadata` to see which keys were not recognised. + +### 3 — Stage configs are not cross-checked until `validate()` runs + +`Pipeline.__init__` does not validate that every stage named in +`stage_configuration` has a matching stage in the `stages` list. That check runs +in `_validate_stage_configs()`, which is called inside `validate()`, which is +called at the start of `run()`. A name mismatch (e.g., a renamed stage script) +will therefore only surface when the pipeline is actually executed. + +**Mitigation**: Call `pipeline.validate()` explicitly after construction in +environments where you want early failure. + +### 4 — Keys outside `pipeline_variables` in a composite YAML are ignored + +When the composite format is detected, `_split_config_sections()` reads only +`pipeline_variables` and `stage_configuration` / `stage_config`. Any other top-level +key in the YAML file is silently discarded. + +```yaml +pipeline_variables: + name: "my-pipeline" +work_dir: "/path/that/will/be/ignored" # ← this key is outside pipeline_variables +stage_configuration: ... +``` + +### 5 — `allow_subprocess_fallback` as a quoted YAML string + +YAML `false` (unquoted) parses to Python `False`. The quoted string `"false"` +parses to Python `"false"`. Because `PipelineConfig` now explicitly checks for +string values and maps common representations to booleans, a quoted `"false"` +will emit a `UserWarning` and be interpreted as `False`. However, to avoid any +ambiguity, use unquoted YAML booleans: + +```yaml +allow_subprocess_fallback: false # correct — unquoted YAML boolean +allow_subprocess_fallback: "false" # warns — will be treated as False +``` + +### 6 — Passing a composite YAML to `PipelineConfig.from_file()` directly + +`PipelineConfig.from_file()` (and `PipelineConfig.from_any()` when given a path) +does not understand the `pipeline_variables` / `stage_configuration` structure. +If a composite YAML is loaded this way, `pipeline_variables` and +`stage_configuration` are treated as unknown keys and absorbed into +`PipelineConfig.metadata`. The resulting `PipelineConfig` will have default +values for all fields. + +**Mitigation**: Always pass composite YAML files to `Pipeline.from_config()` or +as the `config=` argument to `Pipeline(...)`. Reserve `PipelineConfig.from_file()` +for flat, pipeline-only YAML files. + +### 7 — Stage source resolution for relative non-existent paths + +`_resolve_stage_source()` tries the literal path, then `work_dir / path`. If +neither exists it returns the raw candidate. `Stage.from_file()` then calls +`path.exists()` and raises `StageConfigurationError`. This means a typo in a +`location` field is caught at construction time, not silently deferred, but the +error message will point to the stage file rather than the config key. + +### 8 — Unknown stage definition keys are absorbed into `Stage.metadata` + +In `_stage_from_config_definition()`, any key under a stage entry that is not +`run`, `location` / `source` / `path`, `dependencies`, `entrypoint`, or +`metadata` is merged into `Stage.metadata`. This is intentional — it lets you +attach arbitrary properties to a stage definition (e.g. `owner: analytics`) — +but it also means a misspelled reserved key such as `dependancies` will silently +appear in metadata rather than being recognised as a dependency list. + +--- + +## Extensibility + +Stage configuration scales linearly: `_build_stage_configs()` iterates every +key in the `stage_configuration` mapping and constructs one `StageConfig` per +entry. Adding a new stage to the pipeline requires: + +1. Adding the stage entry to `pipeline_variables.stages` in the YAML (or passing + it to `from_files`). +2. Adding the corresponding entry to `stage_configuration` in the YAML (or + passing it in the flat config mapping). + +No other changes are needed. `Pipeline._sync_stage_configs()` ensures that every +stage that does not have an explicit entry still receives an empty `StageConfig` +so that `context.stage_config` is never `None` during execution. From 2e17323fdb9763065e43ac0c875c9c60e0b048ce Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:39:04 +0100 Subject: [PATCH 096/332] fix: Removed RAPConfig and touchpoints - deprecated and misdirective --- configuration.md | 2 -- onsrap/__init__.py | 2 -- onsrap/models.py | 22 ++-------------------- onsrap/pipeline.py | 22 +++++++++------------- 4 files changed, 11 insertions(+), 37 deletions(-) diff --git a/configuration.md b/configuration.md index 790ae9a..e71cf6f 100644 --- a/configuration.md +++ b/configuration.md @@ -105,7 +105,6 @@ handle the following types for the `config` parameter. |---|---| | `None` | Constructs a fully-defaulted `PipelineConfig`. No stage configs. | | `PipelineConfig` instance | Used directly. Stage configuration may optionally be embedded in `config.metadata["stage_configuration"]` (backwards-compatibility path; emits a `StageConfigurationWarning`). | -| `RAPConfig` instance | Delegates to `PipelineConfig.from_mapping(config.contents)`. | | `Mapping[str, Any]` | Parsed as a composite or flat config payload (see below). | | `str` / `Path` | Read and YAML-parsed; the resulting mapping is treated as above. Only `.yaml` / `.yml` files are accepted. | @@ -206,7 +205,6 @@ Pipeline._resolve_config() ▼ Pipeline._load_config_mapping() Normalises input to dict[str, Any]: - RAPConfig → config.contents Mapping → dict(mapping) str / Path → yaml.safe_load(file) │ diff --git a/onsrap/__init__.py b/onsrap/__init__.py index 8329c36..0be47a6 100644 --- a/onsrap/__init__.py +++ b/onsrap/__init__.py @@ -16,7 +16,6 @@ PipelineConfig, PipelineRun, PipelineStatus, - RAPConfig, RAPDataset, RunManifest, RuntimeID, @@ -44,7 +43,6 @@ "PipelineStatus", "PipelineValidationError", "PythonStageExecutor", - "RAPConfig", "RAPDataset", "RunManifest", "RuntimeID", diff --git a/onsrap/models.py b/onsrap/models.py index 89a38a3..adb5522 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -93,21 +93,6 @@ def get_short_hash(self) -> str: return self.short_hash -@dataclass -class RAPConfig: - """ - Holds information on how the Reproducible Analytical Pipeline is - configured. - - Parameters - ---------- - ``contents`` : dict[str, Any] - Contains a dictionary of string keys to Any value pairs containing - information needed to run the Pipeline. - """ - contents: dict[str, Any] = field(default_factory=dict) - - @dataclass class PipelineConfig: """ @@ -154,14 +139,14 @@ class PipelineConfig: @classmethod def from_any( cls, - value: Union["PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None], + value: Union["PipelineConfig", Mapping[str, Any], str, Path, None], ) -> "PipelineConfig": """ Converts one of several datatypes into a PipelineConfig class instance. Parameters ---------- - ``value`` : PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None + ``value`` : PipelineConfig, Mapping[str, Any], str, Path, or None The object holding metadata on how the Pipeline should run to be converted into a PipelineConfig class instance. @@ -177,9 +162,6 @@ def from_any( if isinstance(value, cls): return value - if isinstance(value, RAPConfig): - return cls.from_mapping(value.contents) - if isinstance(value, Mapping): return cls.from_mapping(dict(value)) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 487a556..5552c5d 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -14,11 +14,10 @@ from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger -from .models import PipelineConfig, StageConfig, PipelineRun, RAPConfig, RunManifest, RuntimeID, now +from .models import PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now from .stage import Stage, _normalize_dependencies - class Pipeline: """ Represents an end-to-end code run. This class brings together class instances @@ -35,7 +34,7 @@ class Pipeline: What the pipeline is called. ``backend`` : str, default = "python" The system used to run the pipeline. - ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None The instance containing the required information on running the Pipeline. ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. The required steps within the Pipeline. @@ -48,7 +47,7 @@ def __init__( self, name: str | None = None, backend: str = "python", - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, logger: Logger | None = None, @@ -355,7 +354,7 @@ def _current_user(self) -> str | None: def _resolve_config( self, - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None, ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage]]: """ Resolve supported configuration inputs into pipeline config, stage config, and stages. @@ -450,14 +449,11 @@ def _select_stage_config( @staticmethod def _load_config_mapping( - config: RAPConfig | Mapping[str, Any] | str | Path, + config: Mapping[str, Any] | str | Path, ) -> dict[str, Any]: """ - Load raw configuration data from a RAP config object, mapping, or YAML file. + Load raw configuration data from a mapping or YAML file. """ - if isinstance(config, RAPConfig): - return dict(config.contents) - if isinstance(config, Mapping): return dict(config) @@ -656,7 +652,7 @@ def from_files( *, name: str | None = None, backend: str = "python", - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, @@ -674,7 +670,7 @@ def from_files( The name of the pipeline. ``backend`` : str, default = "python" The system that the pipeline is written in. - ``config`` : PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None The high level information required to run this specific pipeline. ``dependencies`` : Mapping[str, Sequence[str]] or None An object containing which stages are required to be run before other stages. @@ -753,7 +749,7 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": @classmethod def from_config( cls, - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path, + config: PipelineConfig | Mapping[str, Any] | str | Path, *, name: str | None = None, backend: str = "python", From e9b6384f498b32ba094378b54816563d8abbc01e Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 09:59:49 +0100 Subject: [PATCH 097/332] doc: Standardised future references/typing across codebase --- onsrap/execution.py | 14 +++++++------- onsrap/graph.py | 2 +- onsrap/loader.py | 3 ++- onsrap/models.py | 10 +++++----- onsrap/pipeline.py | 6 +++--- onsrap/runner.py | 2 +- onsrap/stage.py | 10 +++++----- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 2f213d5..16500d8 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -225,7 +225,7 @@ class StageExecutor(Protocol): Child class of ``Protocol`` Implementation required """ - def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Method to run ``Stage`` however implementation required """ @@ -242,7 +242,7 @@ class PythonStageExecutor: def __init__(self, preferred_entrypoints: tuple[str, ...] = PREFERRED_ENTRYPOINTS): self.preferred_entrypoints = preferred_entrypoints - def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Main function to select how ``Stage`` is run. @@ -280,7 +280,7 @@ def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: def _execute_callable( self, - stage: "Stage", + stage: Stage, context: ExecutionContext, callable_object: Any, source_label: str | None, @@ -361,7 +361,7 @@ def _execute_callable( ) return result - def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def _execute_file(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Attempt to run a file. Attempt to run a callable object. @@ -444,7 +444,7 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul return self._execute_subprocess(stage, context) - def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Run the entire Python file for the ``Stage`` from the top. @@ -531,7 +531,7 @@ def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> Stag return result -def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionContext) -> Any: +def _invoke_callable(callable_object: Any, stage: Stage, context: ExecutionContext) -> Any: """ Assigns appropriate parameters for a callable and runs it. @@ -599,7 +599,7 @@ def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionCon def _build_success_result( - stage: "Stage", + stage: Stage, started_at: datetime, finished_at: datetime, output: Any, diff --git a/onsrap/graph.py b/onsrap/graph.py index b7fb618..6da5931 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -21,7 +21,7 @@ class StageGraph: stages: list[Stage] = field(default_factory=list) @classmethod - def from_stages(cls, stages: Iterable[Stage]) -> "StageGraph": + def from_stages(cls, stages: Iterable[Stage]) -> StageGraph: """ This is the primary constructor for StageGraph, which performs validation and normalization of the stage list. diff --git a/onsrap/loader.py b/onsrap/loader.py index b988bfa..80c01ab 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -6,6 +6,7 @@ import sys from pathlib import Path from types import ModuleType +from typing import Any from .errors import StageConfigurationError, StageLoadError @@ -65,7 +66,7 @@ def discover_python_entrypoint(path: Path) -> str | None: return None -def load_python_callable(path: Path, entrypoint: str): +def load_python_callable(path: Path, entrypoint: str) -> Any: """ Import a stage module and return the named callable from it. diff --git a/onsrap/models.py b/onsrap/models.py index adb5522..140e807 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -139,8 +139,8 @@ class PipelineConfig: @classmethod def from_any( cls, - value: Union["PipelineConfig", Mapping[str, Any], str, Path, None], - ) -> "PipelineConfig": + value: Union[PipelineConfig, Mapping[str, Any], str, Path, None], + ) -> PipelineConfig: """ Converts one of several datatypes into a PipelineConfig class instance. @@ -171,7 +171,7 @@ def from_any( raise TypeError("Unsupported pipeline config type: {0!r}".format(type(value))) @classmethod - def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": + def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: """ Extracts information from a mapping datatype and returns a PipelineConfig instance. @@ -233,7 +233,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": ) @classmethod - def from_file(cls, path: Path) -> "PipelineConfig": + def from_file(cls, path: Path) -> PipelineConfig: """ Extracts a mapping item from a file containing information about how the pipeline should run. @@ -316,7 +316,7 @@ class StageConfig: metadata: dict[str, Any] = field(default_factory=dict) @classmethod - def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> "StageConfig": + def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> StageConfig: """ Build a ``StageConfig`` from a mapping loaded from code or configuration files. diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 5552c5d..454c591 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -174,7 +174,7 @@ def ordered_stages(self) -> list[Stage]: """ return self.graph.topological_order() - def validate(self) -> "Pipeline": + def validate(self) -> Pipeline: """ Confirms that the source files for the stage exist. """ @@ -707,7 +707,7 @@ def from_files( ) @classmethod - def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": + def from_dict(cls, cfg: Mapping[str, Any]) -> Pipeline: """ Extracts information from a dictionary to configure a Pipeline instance as well as what the Pipeline runs. @@ -755,7 +755,7 @@ def from_config( backend: str = "python", logger: Logger | None = None, executor: StageExecutor | None = None, - ) -> "Pipeline": + ) -> Pipeline: """ Construct a pipeline directly from a composite configuration payload or file. diff --git a/onsrap/runner.py b/onsrap/runner.py index ed91955..720eadd 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -26,7 +26,7 @@ class PipelineRunner: def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() - def run(self, pipeline: "Pipeline") -> PipelineRun: + def run(self, pipeline: Pipeline) -> PipelineRun: """ Method that runs a ``Pipeline`` instance. diff --git a/onsrap/stage.py b/onsrap/stage.py index 57616e0..4e61e39 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -115,7 +115,7 @@ def from_file( metadata: Mapping[str, Any] | None = None, entrypoint: str | None = None, backend: str = "python", - ) -> "Stage": + ) -> Stage: """ Class method that checks and cleans the file path for the ``Stage``. @@ -169,7 +169,7 @@ def from_callable( dependencies: Iterable[str] | str | None = None, metadata: Mapping[str, Any] | None = None, backend: str = "python", - ) -> "Stage": + ) -> Stage: """ Class method that retrieves the name of the Stage from a Callable item. @@ -204,7 +204,7 @@ def from_callable( ) @classmethod - def from_dict(cls, data: Mapping[str, Any]) -> "Stage": + def from_dict(cls, data: Mapping[str, Any]) -> Stage: """ Class method that converts a dictionary stage into a ``Stage`` class instance. @@ -266,7 +266,7 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Stage": raise StageConfigurationError("Stage dictionary must define a source, path, or callable.") - def with_dependencies(self, *dependencies: str) -> "Stage": + def with_dependencies(self, *dependencies: str) -> Stage: """ Method that normalises and adds ``dependencies`` to the ``Stage`` class attributes. @@ -347,7 +347,7 @@ def source_label(self) -> Optional[str]: return None - def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageResult": + def run(self, context: ExecutionContext, executor: StageExecutor) -> StageResult: """ Checks that the ``source`` is valid and then runs the ``source`` From dccb168b845fe42319885434835fde341fa167e3 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 10:04:06 +0100 Subject: [PATCH 098/332] doc: Documented PipelineRunner.run() method to try to provide a little more context for developers --- onsrap/runner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/onsrap/runner.py b/onsrap/runner.py index 720eadd..0f0f1a4 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -50,6 +50,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ``StageExecutionError`` If the stage is unable to be run. Logs will be created to show a failed stage. """ + # Initial Pipeline steps - validate, create run ID and any relevant directories. pipeline.validate() runtime_id = pipeline._create_runtime_id() @@ -65,6 +66,8 @@ def run(self, pipeline: Pipeline) -> PipelineRun: run_dir = run_output / "runs" / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) + # Initialise the ExecutionContext which will be passed to each stage as it runs. This + # context will hold the configuration for the pipeline and for each stage. started_at = now() context = ExecutionContext( pipeline_name=pipeline.name, @@ -77,6 +80,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: stage_configs=dict(pipeline.stage_configs), ) + # Ensure the stages are in order and create a manifest that explains the run. ordered_stages = pipeline.ordered_stages() manifest = pipeline._construct_manifest(runtime_id=runtime_id) manifest.stages_run = [] @@ -90,6 +94,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: stages=[stage.name for stage in ordered_stages], ) + # Execution of the stages in the dependency-driven order. stage_results = [] try: for stage in ordered_stages: @@ -103,13 +108,16 @@ def run(self, pipeline: Pipeline) -> PipelineRun: stage_results.append(result) manifest.stages_run.append(result.name) manifest.outputs[result.name] = result.outputs + except StageExecutionError as exc: + # Handle recording of execution errors. if exc.result is not None and context.result_for(exc.result.name) is None: context.record(exc.result) stage_results.append(exc.result) manifest.stages_run.append(exc.result.name) manifest.outputs[exc.result.name] = exc.result.outputs + # Log the failure and raise the exception to indicate the pipeline has failed. completed_at = now() run = PipelineRun( manifest=manifest, @@ -129,6 +137,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ) raise + # If the pipeline has completed successfully, record the completion and return the run information. completed_at = now() run = PipelineRun( manifest=manifest, From 2ceccd137c877785675cd4e707f1f19a94c48d2c Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 16 Jul 2026 12:08:23 +0100 Subject: [PATCH 099/332] fix: removed RAPConfig references after branch merges --- onsrap/models.py | 3 --- tests/test_models.py | 30 ++---------------------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 140e807..fcfec6f 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -201,8 +201,6 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: project_root = Path(project_root_value) if project_root_value is not None else work_dir log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) - output_dir_value = payload.pop("output_dir", None) - output_dir = Path(output_dir_value) if output_dir_value is not None else None raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) if isinstance(raw_subprocess_fallback, str): warnings.warn( @@ -226,7 +224,6 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: output_dir=output_dir_value, log_dir=log_dir, data_dir=data_dir, - output_dir=output_dir, allow_subprocess_fallback=allow_subprocess_fallback, python_executable=python_executable, metadata=metadata, diff --git a/tests/test_models.py b/tests/test_models.py index 715b5fe..c0f44ee 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,4 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RAPConfig, RunManifest, PipelineRun, PipelineConfig +from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig import pytest import datetime from pathlib import Path @@ -52,22 +52,6 @@ def test_getter_functions_runtimeID(runtimeID) -> None: assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" assert runtimeID.get_short_hash() == "4h5kg" -@pytest.fixture -def rapconfig() -> RAPConfig: - """ - Example RAPConfig class instance for testing of class methods. - """ - return RAPConfig(contents = {"name":"test_rap", - "backend":"python", - "work_dir":Path("tmp/work"), - "project_root":Path("project"), - "log_dir":Path("tmp/logs"), - "data_dir":Path("tmp/data"), - "allow_subprocess_fallback":True, - "python_executable":None, - "metadata":{"variables":["name","age"], - "num_stages":6}}) - @pytest.fixture def blankpipelineconfig() -> PipelineConfig: """ @@ -108,7 +92,7 @@ def mapping() -> dict: "num_stages":6}} -def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> None: +def test_from_any(mapping, pipelineconfig, blankpipelineconfig) -> None: """ Test derivation for a PipelineConfig instance using the from_any() method. This test checks all methods EXCEPT from_file as this will be covered in another test due to @@ -125,16 +109,6 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig, rapconfig) -> No python_executable = None, metadata = {"variables":["name","age"], "num_stages":6}) - assert blankpipelineconfig.from_any(rapconfig) == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) assert blankpipelineconfig.from_any(mapping) == PipelineConfig(name = "test_rap", backend = "python", work_dir = Path("tmp/work"), From dd00eb8cb01a654f36e34cf57c72d5b5c23100b0 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 16 Jul 2026 15:47:53 +0100 Subject: [PATCH 100/332] Feat: Filled out stages for example pipeline 2 --- .../{data => input_data}/orders.csv | 0 .../orders_cleaned.csv | 0 .../orders_prepped.csv | 14 ++-- examples/pipeline_2/scripts/0_clean_data.py | 5 +- .../pipeline_2/scripts/0_data_validation.py | 0 examples/pipeline_2/scripts/1_derive_vars.py | 26 ++++-- .../pipeline_2/scripts/1_preprocessing.py | 0 examples/pipeline_2/scripts/2_reporting.py | 80 ++++++++++++++++++- 8 files changed, 105 insertions(+), 20 deletions(-) rename examples/pipeline_2/{data => input_data}/orders.csv (100%) rename examples/pipeline_2/{data => processed_data}/orders_cleaned.csv (100%) rename examples/pipeline_2/{data => processed_data}/orders_prepped.csv (65%) delete mode 100644 examples/pipeline_2/scripts/0_data_validation.py delete mode 100644 examples/pipeline_2/scripts/1_preprocessing.py diff --git a/examples/pipeline_2/data/orders.csv b/examples/pipeline_2/input_data/orders.csv similarity index 100% rename from examples/pipeline_2/data/orders.csv rename to examples/pipeline_2/input_data/orders.csv diff --git a/examples/pipeline_2/data/orders_cleaned.csv b/examples/pipeline_2/processed_data/orders_cleaned.csv similarity index 100% rename from examples/pipeline_2/data/orders_cleaned.csv rename to examples/pipeline_2/processed_data/orders_cleaned.csv diff --git a/examples/pipeline_2/data/orders_prepped.csv b/examples/pipeline_2/processed_data/orders_prepped.csv similarity index 65% rename from examples/pipeline_2/data/orders_prepped.csv rename to examples/pipeline_2/processed_data/orders_prepped.csv index a65d404..b6c5d70 100644 --- a/examples/pipeline_2/data/orders_prepped.csv +++ b/examples/pipeline_2/processed_data/orders_prepped.csv @@ -1,7 +1,7 @@ -Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order -1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False -1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False -1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True -1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False -1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False -1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False +Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order,Postage,Total_production_cost,Order_profit +1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False,2.5,2.0,2.5 +1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False,5.0,1.5,-0.5 +1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True,1.0,1.0,1.5 +1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False,2.5,2.25,3.5 +1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False,5.0,1.2,-1.4000000000000004 +1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False,2.5,2.0,2.5 diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py index 9f5c673..4c2f234 100644 --- a/examples/pipeline_2/scripts/0_clean_data.py +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -33,7 +33,7 @@ def standardise_columns(df): def main(): - orders = pd.read_csv("examples/pipeline_2/data/orders.csv") + orders = pd.read_csv("examples/pipeline_2/input_data/orders.csv") expected_variables = ["order_id", "customer_name", @@ -53,7 +53,6 @@ def main(): print(orders.dtypes) orders = remove_identifiable(orders, identifiable_cols) orders = standardise_columns(orders) - print(orders) - orders.to_csv("examples/pipeline_2/data/orders_cleaned.csv", index = False) + orders.to_csv("examples/pipeline_2/processed_data/orders_cleaned.csv", index = False) main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/0_data_validation.py b/examples/pipeline_2/scripts/0_data_validation.py deleted file mode 100644 index e69de29..0000000 diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py index 9b0540f..aeb44e4 100644 --- a/examples/pipeline_2/scripts/1_derive_vars.py +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -41,23 +41,31 @@ def postage_cost(df): return df def production_cost(df): - df["Production_cost"] = np.select( + df["Total_production_cost"] = np.select( [ - df["notebook"], - df["pen"] + df["Product"] == "notebook", + df["Product"] == "pen", + df["Product"] == "folder" ], [ - 1.00, - 0.3, + (1.00*df["Quantity"]), + (0.3*df["Quantity"]), + (0.75*df["Quantity"]) + ], - default = 2.50 + default = 0 ) return df +def profit_per_order(df): + df["Order_profit"] = df["Total_cost"]-df["Total_production_cost"]-df["Postage"] + return df + + def main(): - df = pd.read_csv("examples/pipeline_2/data/orders_cleaned.csv") + df = pd.read_csv("examples/pipeline_2/processed_data/orders_cleaned.csv") delivery_times = {"north":14, "south":4, "east":7, @@ -69,7 +77,9 @@ def main(): df = order_date_values(df) df = size_order_alert(df) df = postage_cost(df) - df.to_csv("examples/pipeline_2/data/orders_prepped.csv", index = False) + df = production_cost(df) + df = profit_per_order(df) + df.to_csv("examples/pipeline_2/processed_data/orders_prepped.csv", index = False) diff --git a/examples/pipeline_2/scripts/1_preprocessing.py b/examples/pipeline_2/scripts/1_preprocessing.py deleted file mode 100644 index e69de29..0000000 diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index aa56bf7..02d1569 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -1,3 +1,79 @@ -import pandas +import pandas as pd +import tabulate +from pathlib import Path -orders = \ No newline at end of file +orders = pd.read_csv("examples/pipeline_2/processed_data/orders_prepped.csv") + +report = [] + +num_format = "{:.2f}" +##PROFIT PER REGION## +profit_per_region = orders.groupby("Region")["Order_profit"].sum() + +highest_prof_region = profit_per_region.idxmax().capitalize() +highest_prof_value = num_format.format((profit_per_region.max())) + +lowest_prof_region = profit_per_region.idxmin().capitalize() +lowest_prof_region = num_format.format((profit_per_region.min())) + +##QUANTITY PER REGION## +quantity_per_region = orders.groupby("Region")["Quantity"].sum() + +highest_quant_region = quantity_per_region.idxmax().capitalize() +highest_quant_value = quantity_per_region.max() + +lowest_quant_region = quantity_per_region.idxmin().capitalize() +lowest_quant_value = quantity_per_region.min() + +##ORDER DAY POP## +delivery_day_frequency = orders["Order_day"].value_counts() +highest_delivery_day = delivery_day_frequency.idxmin().capitalize() + +##ORDER COUNTS## +large_order_num = (orders["Large_order"] == True).sum() + +small_order_num = (orders["Small_order"] == True).sum() + +##ORDERS PER REGION## +orders_per_region = orders["Region"].value_counts() +highest_order_region = quantity_per_region.idxmax().capitalize() + +##TOTALS## +total_count = orders.count() +total_profit = num_format.format(orders["Order_profit"].sum()) + +##PROFIT PER PRODUCT## +profit_per_product = orders.groupby("Product")["Order_profit"].sum().sort_values(ascending=False) + +highest_profit_product = profit_per_product.idxmax().capitalize() +highest_profit_value = num_format.format((profit_per_product.max())) + +lowest_profit_product = profit_per_product.idxmin().capitalize() +lowest_profit_value = num_format.format((profit_per_product.min())) + +##CURATE REPORT## +report.append("# June 2026 Order Summary") +report.append("") +report.append("## Summary") +report.append("") +report.append(f"Total Orders: {total_count}") +report.append("") +report.append(f"Total Profit: {total_profit}") +report.append("") +report.append("## Region Analysis") +report.append(f"The region that produced the highest number of orders: {highest_order_region}") +report.append(f"The region that produced the highest profit: {highest_prof_region} at £{highest_prof_value}") +report.append(f"The region that produced the lowest profit: {lowest_prof_region} at £{lowest_profit_value}") +report.append(f"The region that had the highest quantity of items ordered: {highest_quant_region} at {highest_quant_value}") +report.append(f"The region that had the highest quantity of items ordered: {lowest_quant_region} at {lowest_quant_value}") +report.append("") +report.append("## Product Analysis") +report.append(f"The product that had the highest profit: {highest_profit_product} at £{highest_profit_value}") +report.append(f"The product that had the lowest profit: {lowest_profit_product} at £{lowest_profit_value}") +report.append("") +report.append("## Order Analysis") +report.append(f"The most orders occured on a {highest_delivery_day}.") +report.append(f"{large_order_num} order/s were Large (greater than 75% of the rest).") +report.append(f"{small_order_num} order/s were Small (less than 25% of the rest).") + +output_path = Path("") \ No newline at end of file From 88dced98a35a4ae8e3f68c1c85e5938b703c09ab Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 16 Jul 2026 16:27:00 +0100 Subject: [PATCH 101/332] Feat: Finalise stage 2 of example Pipeline_2 --- examples/pipeline_2/outputs/order_analysis.md | 33 ++++ examples/pipeline_2/scripts/2_reporting.py | 146 +++++++++++------- 2 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 examples/pipeline_2/outputs/order_analysis.md diff --git a/examples/pipeline_2/outputs/order_analysis.md b/examples/pipeline_2/outputs/order_analysis.md new file mode 100644 index 0000000..d1ea234 --- /dev/null +++ b/examples/pipeline_2/outputs/order_analysis.md @@ -0,0 +1,33 @@ +# Order Summary + +This is an automatically generated report showing key information on orders of products. + +## Summary + +Total Orders: **6** + +Total Profit: **£8.10** + +## Region Analysis +Highest number of orders: **North** + +Highest profit: **North** at **£6.50** + +Lowest profit: **East** at **£-1.90** + +Highest quantity of items ordered: **North** at **5** + +Lowest quantity of items ordered: **West** at **3** + +## Product Analysis +Highest profit: **Notebook** at **£6.50** + +Lowest profit: **Pen** at **£-1.90** + +## Order Analysis + +The most orders occured on a **Tuesday**. + +**2** order/s were Large (greater than 75% of orders for the period). + +**1** order/s were Small (less than 25% of orders for the period). \ No newline at end of file diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index 02d1569..5b85e38 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -1,79 +1,117 @@ import pandas as pd -import tabulate from pathlib import Path -orders = pd.read_csv("examples/pipeline_2/processed_data/orders_prepped.csv") - -report = [] - -num_format = "{:.2f}" ##PROFIT PER REGION## -profit_per_region = orders.groupby("Region")["Order_profit"].sum() +def per_region_profits(orders, values, num_format): + profit_per_region = orders.groupby("Region")["Order_profit"].sum() -highest_prof_region = profit_per_region.idxmax().capitalize() -highest_prof_value = num_format.format((profit_per_region.max())) + values["highest_prof_region"] = profit_per_region.idxmax().capitalize() + values["highest_prof_value"] = num_format.format((profit_per_region.max())) -lowest_prof_region = profit_per_region.idxmin().capitalize() -lowest_prof_region = num_format.format((profit_per_region.min())) + values["lowest_prof_region"] = profit_per_region.idxmin().capitalize() + values["lowest_profit_value"] = num_format.format((profit_per_region.min())) ##QUANTITY PER REGION## -quantity_per_region = orders.groupby("Region")["Quantity"].sum() +def per_region_quantity(orders, values): + quantity_per_region = orders.groupby("Region")["Quantity"].sum() -highest_quant_region = quantity_per_region.idxmax().capitalize() -highest_quant_value = quantity_per_region.max() + values["highest_quant_region"] = quantity_per_region.idxmax().capitalize() + values["highest_quant_value"] = quantity_per_region.max() -lowest_quant_region = quantity_per_region.idxmin().capitalize() -lowest_quant_value = quantity_per_region.min() + values["lowest_quant_region"] = quantity_per_region.idxmin().capitalize() + values["lowest_quant_value"] = quantity_per_region.min() ##ORDER DAY POP## -delivery_day_frequency = orders["Order_day"].value_counts() -highest_delivery_day = delivery_day_frequency.idxmin().capitalize() +def orders_per_day(orders, values): + delivery_day_frequency = orders["Order_day"].value_counts() + values["highest_delivery_day"] = delivery_day_frequency.idxmin().capitalize() ##ORDER COUNTS## -large_order_num = (orders["Large_order"] == True).sum() +def order_quantity(orders, values): + values["large_order_num"] = (orders["Large_order"] == True).sum() -small_order_num = (orders["Small_order"] == True).sum() + values["small_order_num"] = (orders["Small_order"] == True).sum() ##ORDERS PER REGION## -orders_per_region = orders["Region"].value_counts() -highest_order_region = quantity_per_region.idxmax().capitalize() +def per_region_orders(orders, values): + orders_per_region = orders["Region"].value_counts() + values["highest_order_region"] = orders_per_region.idxmax().capitalize() ##TOTALS## -total_count = orders.count() -total_profit = num_format.format(orders["Order_profit"].sum()) +def total_summaries(orders,values,num_format): + values["total_count"] = orders["Order_id"].count() + values["total_profit"] = num_format.format(orders["Order_profit"].sum()) ##PROFIT PER PRODUCT## -profit_per_product = orders.groupby("Product")["Order_profit"].sum().sort_values(ascending=False) +def profit_per_product(orders,values, num_format): + df = orders.groupby("Product")["Order_profit"].sum().sort_values(ascending=False) -highest_profit_product = profit_per_product.idxmax().capitalize() -highest_profit_value = num_format.format((profit_per_product.max())) + values["highest_profit_product"] = df.idxmax().capitalize() + values["highest_profit_value"] = num_format.format((df.max())) -lowest_profit_product = profit_per_product.idxmin().capitalize() -lowest_profit_value = num_format.format((profit_per_product.min())) + values["lowest_profit_product"] = df.idxmin().capitalize() + values["lowest_profit_value"] = num_format.format((df.min())) ##CURATE REPORT## -report.append("# June 2026 Order Summary") -report.append("") -report.append("## Summary") -report.append("") -report.append(f"Total Orders: {total_count}") -report.append("") -report.append(f"Total Profit: {total_profit}") -report.append("") -report.append("## Region Analysis") -report.append(f"The region that produced the highest number of orders: {highest_order_region}") -report.append(f"The region that produced the highest profit: {highest_prof_region} at £{highest_prof_value}") -report.append(f"The region that produced the lowest profit: {lowest_prof_region} at £{lowest_profit_value}") -report.append(f"The region that had the highest quantity of items ordered: {highest_quant_region} at {highest_quant_value}") -report.append(f"The region that had the highest quantity of items ordered: {lowest_quant_region} at {lowest_quant_value}") -report.append("") -report.append("## Product Analysis") -report.append(f"The product that had the highest profit: {highest_profit_product} at £{highest_profit_value}") -report.append(f"The product that had the lowest profit: {lowest_profit_product} at £{lowest_profit_value}") -report.append("") -report.append("## Order Analysis") -report.append(f"The most orders occured on a {highest_delivery_day}.") -report.append(f"{large_order_num} order/s were Large (greater than 75% of the rest).") -report.append(f"{small_order_num} order/s were Small (less than 25% of the rest).") - -output_path = Path("") \ No newline at end of file +def curate_report(report, values): + report.append("# Order Summary") + report.append("") + report.append("This is an automatically generated report showing key information on orders of products.") + report.append("") + report.append("## Summary") + report.append("") + report.append(f"Total Orders: **{values["total_count"]}**") + report.append("") + report.append(f"Total Profit: **£{values["total_profit"]}**") + report.append("") + report.append("## Region Analysis") + report.append(f"Highest number of orders: **{values["highest_order_region"]}**") + report.append("") + report.append(f"Highest profit: **{values["highest_prof_region"]}** at **£{values["highest_prof_value"]}**") + report.append("") + report.append(f"Lowest profit: **{values["lowest_prof_region"]}** at **£{values["lowest_profit_value"]}**") + report.append("") + report.append(f"Highest quantity of items ordered: **{values["highest_quant_region"]}** at **{values["highest_quant_value"]}**") + report.append("") + report.append(f"Lowest quantity of items ordered: **{values["lowest_quant_region"]}** at **{values["lowest_quant_value"]}**") + report.append("") + report.append("## Product Analysis") + report.append(f"Highest profit: **{values["highest_profit_product"]}** at **£{values["highest_profit_value"]}**") + report.append("") + report.append(f"Lowest profit: **{values["lowest_profit_product"]}** at **£{values["lowest_profit_value"]}**") + report.append("") + report.append("## Order Analysis") + report.append("") + report.append(f"The most orders occured on a **{values["highest_delivery_day"]}**.") + report.append("") + report.append(f"**{values["large_order_num"]}** order/s were Large (greater than 75% of orders for the period).") + report.append("") + report.append(f"**{values["small_order_num"]}** order/s were Small (less than 25% of orders for the period).") + +def write_report(report): + report_file = Path("examples/pipeline_2/outputs/order_analysis.md") + + report_file.write_text( + "\n".join(report), + encoding="utf-8" + ) + +def main(): + orders = pd.read_csv("examples/pipeline_2/processed_data/orders_prepped.csv") + + report = [] + values = {} + + num_format = "{:.2f}" + + per_region_profits(orders, values, num_format) + per_region_quantity(orders, values) + orders_per_day(orders, values) + order_quantity(orders, values) + per_region_orders(orders, values) + total_summaries(orders,values,num_format) + profit_per_product(orders,values, num_format) + curate_report(report, values) + write_report(report) + +main() \ No newline at end of file From 9085f8dd240aa11a406b6dbc474517aa11897583 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 16 Jul 2026 16:49:18 +0100 Subject: [PATCH 102/332] Feat: Stage configs written for example pipeline 2 --- examples/pipeline_2/conf.yaml | 83 ++++++++++++++++--- .../processed_data/orders_cleaned.csv | 7 -- .../processed_data/orders_prepped.csv | 7 -- examples/pipeline_2/scripts/0_clean_data.py | 4 +- examples/pipeline_2/scripts/1_derive_vars.py | 4 +- 5 files changed, 74 insertions(+), 31 deletions(-) delete mode 100644 examples/pipeline_2/processed_data/orders_cleaned.csv delete mode 100644 examples/pipeline_2/processed_data/orders_prepped.csv diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index af04312..311bbb3 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -2,24 +2,24 @@ pipeline_variables: name: "Pipeline 2" backend: python stages: - - 0_data_validation: - # TODO: Put name/alias of stage as attribute here, not as key for stage. - location: "" + - 0_clean_data: + name: "0_clean_data" + location: "examples/pipeline_2/scripts/0_clean_data.py" run: true dependencies: [] - - 1_preprocessing: - location: "" + - 1_derive_vars: + location: "examples/pipeline_2/scripts/1_derive_vars.py" run: true dependencies: - - 0_data_validation + - 0_clean_data - 2_reporting: - location: "" + location: "examples/pipeline_2/scripts/2_reporting.py" run: true dependencies: - - 1_preprocessing + - 1_derive_vars working_dir: "examples/pipeline_2" data_dir: "examples/pipeline_2/data" - output_dir: "examples/pipeline_2/output" + output_dir: "examples/pipeline_2/outputs" log_dir: "examples/pipeline_2/logs" metadata: example: retail-orders-using-configuration @@ -27,7 +27,64 @@ pipeline_variables: stage_configuration: - 0_data_validation: - years_to_run: 2017 - time_col: "time" - target_variable: "classification" \ No newline at end of file + 0_clean_data: + expected_variables: + - "order_id" + - "customer_name" + - "region" + - "product" + - "quantity" + - "unit_price" + - "order_date" + - "order_method + identifiable_cols: + - "customer_name" + - "age" + - "dob" + - "address + output_location: "examples/pipeline_2/data/orders_cleaned.csv" + input_location: "examples/pipeline_2/data/orders.csv" + 1_derive_vars: + input_location: "examples/pipeline_2/data/orders_cleaned.csv" + output_location: "examples/pipeline_2/data/orders_prepped.csv" + delivery_times: + north: 14 + south: 4 + east: 7 + west: 7 + order_date: "Order_date" + estimated_delivery_date: "Estimated_delivery_date" + delivery_day: "Delivery_day" + region: "Region" + total_cost: "Total_cost" + quantity: "Quantity" + unit_price: "Unit_price" + order_day: "Order_day" + order_month: "Order_month" + large_order: "Large_order" + small_order: "Small_order" + postage: "Postage" + postage_values: + large_order: 5.00 + small_order: 1.00 + default: 2.50 + product: "Product" + product_costs: + notebook: 1.00 + pen: 0.3 + folder: 0.75 + total_production_cost: "Total_production_cost" + order_profit: "Order_profit" + 2_reporting: + input_location: "examples/pipeline_2/processed_data/orders_prepped.csv" + report_location: "examples/pipeline_2/outputs/order_analysis.md" + region: "Region" + order_profit: "Order_profit" + quantity: "Quantity" + order_day: "Order_day" + large_order: "Large_order" + small_order: "Small_order" + order_id: "Order_id" + order_profit: "Order_profit" + product: "Product" + num_format: "{:.2f}" diff --git a/examples/pipeline_2/processed_data/orders_cleaned.csv b/examples/pipeline_2/processed_data/orders_cleaned.csv deleted file mode 100644 index c3e721b..0000000 --- a/examples/pipeline_2/processed_data/orders_cleaned.csv +++ /dev/null @@ -1,7 +0,0 @@ -Order_id,Region,Product,Quantity,Unit_price,Order_date -1001,north,notebook,2,3.5,2026-06-01 -1002,south,pen,5,1.2,2026-06-01 -1003,north,notebook,1,3.5,2026-06-02 -1004,west,folder,3,2.75,2026-06-03 -1005,east,pen,4,1.2,2026-06-03 -1006,north,notebook,2,3.5,2026-06-04 diff --git a/examples/pipeline_2/processed_data/orders_prepped.csv b/examples/pipeline_2/processed_data/orders_prepped.csv deleted file mode 100644 index b6c5d70..0000000 --- a/examples/pipeline_2/processed_data/orders_prepped.csv +++ /dev/null @@ -1,7 +0,0 @@ -Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order,Postage,Total_production_cost,Order_profit -1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False,2.5,2.0,2.5 -1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False,5.0,1.5,-0.5 -1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True,1.0,1.0,1.5 -1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False,2.5,2.25,3.5 -1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False,5.0,1.2,-1.4000000000000004 -1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False,2.5,2.0,2.5 diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py index 4c2f234..15997df 100644 --- a/examples/pipeline_2/scripts/0_clean_data.py +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -33,7 +33,7 @@ def standardise_columns(df): def main(): - orders = pd.read_csv("examples/pipeline_2/input_data/orders.csv") + orders = pd.read_csv("examples/pipeline_2/data/orders.csv") expected_variables = ["order_id", "customer_name", @@ -53,6 +53,6 @@ def main(): print(orders.dtypes) orders = remove_identifiable(orders, identifiable_cols) orders = standardise_columns(orders) - orders.to_csv("examples/pipeline_2/processed_data/orders_cleaned.csv", index = False) + orders.to_csv("examples/pipeline_2/data/orders_cleaned.csv", index = False) main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py index aeb44e4..9673afe 100644 --- a/examples/pipeline_2/scripts/1_derive_vars.py +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -65,7 +65,7 @@ def profit_per_order(df): def main(): - df = pd.read_csv("examples/pipeline_2/processed_data/orders_cleaned.csv") + df = pd.read_csv("examples/pipeline_2/data/orders_cleaned.csv") delivery_times = {"north":14, "south":4, "east":7, @@ -79,7 +79,7 @@ def main(): df = postage_cost(df) df = production_cost(df) df = profit_per_order(df) - df.to_csv("examples/pipeline_2/processed_data/orders_prepped.csv", index = False) + df.to_csv("examples/pipeline_2/data/orders_prepped.csv", index = False) From f1e8e4ef0c7d7bc4264e6088a38f7399858cd354 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 17 Jul 2026 09:45:29 +0100 Subject: [PATCH 103/332] Tweaked file paths for example_2 pipeline --- examples/pipeline_2/{input_data => data}/orders.csv | 0 examples/pipeline_2/data/orders_cleaned.csv | 7 +++++++ examples/pipeline_2/data/orders_prepped.csv | 7 +++++++ examples/pipeline_2/scripts/2_reporting.py | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) rename examples/pipeline_2/{input_data => data}/orders.csv (100%) create mode 100644 examples/pipeline_2/data/orders_cleaned.csv create mode 100644 examples/pipeline_2/data/orders_prepped.csv diff --git a/examples/pipeline_2/input_data/orders.csv b/examples/pipeline_2/data/orders.csv similarity index 100% rename from examples/pipeline_2/input_data/orders.csv rename to examples/pipeline_2/data/orders.csv diff --git a/examples/pipeline_2/data/orders_cleaned.csv b/examples/pipeline_2/data/orders_cleaned.csv new file mode 100644 index 0000000..c3e721b --- /dev/null +++ b/examples/pipeline_2/data/orders_cleaned.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date +1001,north,notebook,2,3.5,2026-06-01 +1002,south,pen,5,1.2,2026-06-01 +1003,north,notebook,1,3.5,2026-06-02 +1004,west,folder,3,2.75,2026-06-03 +1005,east,pen,4,1.2,2026-06-03 +1006,north,notebook,2,3.5,2026-06-04 diff --git a/examples/pipeline_2/data/orders_prepped.csv b/examples/pipeline_2/data/orders_prepped.csv new file mode 100644 index 0000000..b6c5d70 --- /dev/null +++ b/examples/pipeline_2/data/orders_prepped.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order,Postage,Total_production_cost,Order_profit +1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False,2.5,2.0,2.5 +1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False,5.0,1.5,-0.5 +1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True,1.0,1.0,1.5 +1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False,2.5,2.25,3.5 +1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False,5.0,1.2,-1.4000000000000004 +1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False,2.5,2.0,2.5 diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index 5b85e38..5f8f6de 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -97,7 +97,7 @@ def write_report(report): ) def main(): - orders = pd.read_csv("examples/pipeline_2/processed_data/orders_prepped.csv") + orders = pd.read_csv("examples/pipeline_2/data/orders_prepped.csv") report = [] values = {} From ffb69cee7d69ca3f1882259189db7a2d9e62106f Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 17 Jul 2026 09:49:45 +0100 Subject: [PATCH 104/332] fix: added get_stage_config() method to return the dict(self._variables) or StageConfig if optional argument is parsed. Preferred access point over stage_config @property method --- onsrap/execution.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/onsrap/execution.py b/onsrap/execution.py index 16500d8..6c85d55 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -113,6 +113,10 @@ def stage_config(self) -> StageConfig | None: """ Return the configuration for the stage currently being executed. + The preferred access method for this is ``get_stage_config()`` which allows + for optional arguments to return the full ``StageConfig`` instance or just + the variables dictionary. + This property is ``None`` outside an active stage run. """ if self.active_stage_name is None: @@ -164,6 +168,30 @@ def resolve_output_root(self) -> Path: raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") + + def get_stage_config(self, vars_only: bool = True) -> dict | StageConfig: + """ + Returns the configuration for the stage currently being executed, with optional arguments. + + Optional argument ``vars_only`` can be set to ``False`` to return the full ``StageConfig`` instance, + rather than just the variables dictionary. + + If you want to access ``metadata`` or ``dataframes`` from the ``StageConfig``, you must set ``vars_only`` to False. + + Parameters + ---------- + ``vars_only`` : bool, default = True + If True, returns only the variables dictionary from the ``StageConfig``. If False, returns the full ``StageConfig`` instance. + + Returns + ------- + dict or StageConfig + The parameters contained within the configuration for the currently active stage. + If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. + """ + if vars_only: + return self.stage_config_for(self.active_stage_name).variables() + return self.stage_config_for(self.active_stage_name) or {} def resolve_given_path(self, stage_name: str | None, path_name: str | None, From b8d746d361c3c57df67ef4bd25a4ffe0dcdc3883 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 17 Jul 2026 10:47:34 +0100 Subject: [PATCH 105/332] Added custom warning --- onsrap/runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 0f0f1a4..878eceb 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from .errors import StageExecutionError +from .warnings import StageConfigurationWarning from .execution import ExecutionContext from .logger import Logger from .models import PipelineRun, PipelineStatus, now @@ -60,7 +61,8 @@ def run(self, pipeline: Pipeline) -> PipelineRun: run_output = Path(pipeline.config.output_dir) else: warnings.warn( - "Output directory is not specified. Using project root or work directory as the run output." + "Output directory is not specified. Using project root or work directory as the run output.", + StageConfigurationWarning ) # TODO: fill with warnings from Pipeline branch run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) run_dir = run_output / "runs" / runtime_id.get_id() From e88b2a11772617d1d8762d5932a70decaee2f78a Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 17 Jul 2026 10:47:45 +0100 Subject: [PATCH 106/332] Added whitespace and todo --- onsrap/pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 454c591..a7289d6 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -63,14 +63,16 @@ def __init__( self.config = resolved_config if self.config.name is None: self.config.name = self.name + self.config.backend = self.backend - self.logger = logger or Logger(log_dir=self.config.log_dir) self.executor = executor or PythonStageExecutor() + if stages is None: self.stages = configured_stages else: self.stages = [self._coerce_stage(stage) for stage in stages] + self.dependencies = dependencies if dependencies is not None and stages is None: raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " @@ -78,6 +80,7 @@ def __init__( "parse them to the Pipeline Constructor.") if dependencies is not None: self._assign_dependencies(dependencies,self.stages) + self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() self.graph = StageGraph.from_stages(self.stages) @@ -483,6 +486,7 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless a stage-configuration key is present. """ + #TODO: Enforce this behaviour using Errors if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: pipeline_payload = raw_config.get("pipeline_variables", {}) if not isinstance(pipeline_payload, Mapping): From c5c1b6b523e515925dbc29924c0db5c2662bdb7c Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 30 Jun 2026 09:26:30 +0100 Subject: [PATCH 107/332] Created main2.py in example 1 to show order of stages provided is less relevant than dependencies. --- examples/pipeline_1/main2.py | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 examples/pipeline_1/main2.py diff --git a/examples/pipeline_1/main2.py b/examples/pipeline_1/main2.py new file mode 100644 index 0000000..f4bf7fb --- /dev/null +++ b/examples/pipeline_1/main2.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from pathlib import Path + +from onsrap import Pipeline, PipelineConfig + + +PIPELINE_ROOT = Path(__file__).resolve().parent +SCRIPTS_DIR = PIPELINE_ROOT / "scripts" +DATA_DIR = PIPELINE_ROOT / "data" +LOG_DIR = PIPELINE_ROOT / "logs" + +""" +In this version of the main pipeline script, the order of the stage files has been altered. +The preprocessing stage is now listed before the data validation stage in the `stage_files` list. +However, the dependencies remain unchanged, meaning that the pipeline will still enforce that data +validation must be completed before preprocessing can run. + +This means that changing the order of the stage files in pipeline does not affect the execution order +of the stages, which is defined by the dependencies. +""" + + +def build_pipeline() -> Pipeline: + stage_files = [ # Altered Script Order + SCRIPTS_DIR / "1_preprocessing.py", + SCRIPTS_DIR / "0_data_validation.py", + SCRIPTS_DIR / "2_reporting.py", + ] + + dependencies = { + "1_preprocessing": ("0_data_validation",), + "2_reporting": ("1_preprocessing",), + } + + config = PipelineConfig( + name="pipeline_1", + backend="python", + work_dir=PIPELINE_ROOT, + project_root=PIPELINE_ROOT, + data_dir=DATA_DIR, + log_dir=LOG_DIR, + metadata={ + "example": "retail-orders", + "description": "Validate, clean, and summarize a small orders dataset.", + }, + ) + + return Pipeline.from_files( + stage_files, + name="pipeline_1", + backend="python", + config=config, + dependencies=dependencies, + ) + + +def main() -> None: + run = build_pipeline().run() + report = run.manifest.outputs["2_reporting"] + + print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + print(f"Summary report written to: {report['report_path']}") + print(f"Cleaned data written to: {report['clean_path']}") + + +if __name__ == "__main__": + main() From f339a7989df5c4247e14f08bf001ff3d954576d0 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 30 Jun 2026 09:26:45 +0100 Subject: [PATCH 108/332] Added TODOs, tweaked pipeline construction logic --- onsrap/pipeline.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7168826..acaff89 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -64,7 +64,7 @@ def __init__( self.config.backend = self.backend self.logger = logger or Logger(log_dir=self.config.log_dir) - self.executor = executor or PythonStageExecutor() + self.executor = PythonStageExecutor() if executor is None else executor self.stages = [self._coerce_stage(stage) for stage in (stages or [])] self.dependencies = dependencies if dependencies is not None and stages is None: @@ -73,7 +73,8 @@ def __init__( "parse them to the Pipeline Constructor.") if dependencies is not None: self._assign_dependencies(dependencies,self.stages) - self.graph = StageGraph.from_stages(self.stages) + self.graph = StageGraph.from_stages(self.stages) if len(self.stages) > 0 else StageGraph() + # TODO: assess this graph and Executor default value instantiation behaviour self.id: RuntimeID | None = None self.manifest: RunManifest | None = None self.last_run: PipelineRun | None = None @@ -93,6 +94,8 @@ def _assign_dependencies(self, return stages + # TODO: Add a method to add dependencies to the pipeline after initialization + # TODO: Re-order methods to be more logical/readable in order (public, private, classmethods, staticmethods) def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], From 716cbce3158ef2557a937419e91b70fc86704c12 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 30 Jun 2026 09:35:55 +0100 Subject: [PATCH 109/332] tweak: Removed Union[] and replaced with | operator --- onsrap/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index 3a196f0..6b5eab1 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -83,7 +83,7 @@ class Stage: If the stage ``name`` is empty or if the source is not a supported type. """ name: str - source: Union[Path, Callable[..., Any], None] = None + source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) metadata: dict[str, Any] = field(default_factory=dict) entrypoint: Optional[str] = None From ff7d185bdcc7c38472d6bbc92ba7a3b3ecd001bd Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 17 Jul 2026 12:22:27 +0100 Subject: [PATCH 110/332] feat: Added TODOs for approaching Config, with initial config structure --- examples/pipeline_2/main.py | 13 +++++ onsrap/models.py | 12 ++++- onsrap/pipeline.py | 101 +++++++++++++++++++++++------------- onsrap/stage.py | 1 + 4 files changed, 90 insertions(+), 37 deletions(-) diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py index e69de29..d017a46 100644 --- a/examples/pipeline_2/main.py +++ b/examples/pipeline_2/main.py @@ -0,0 +1,13 @@ +import yaml +from onsrap import Pipeline, PipelineConfig, StageConfig + + + + +def main() -> None: + config = yaml.safe_load(open("conf.yaml")) + + Pipeline.from_config(config) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/onsrap/models.py b/onsrap/models.py index fcfec6f..ef4a1f5 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -102,6 +102,8 @@ class PipelineConfig: ---------- ``name`` : str, optional The name of the pipeline. + + ``backend`` : str, default = "python" The system that the pipeline is run on. ``work_dir`` : Path @@ -125,17 +127,25 @@ class PipelineConfig: Any additional information on the pipeline. """ name: Optional[str] = None + stages_to_run: Optional[dict[str, bool]] = None backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) project_root: Optional[Path] = None output_dir: Optional[Path] = None log_dir: Path = field(default_factory=lambda: Path("logs")) data_dir: Path = field(default_factory=lambda: Path("data")) - output_dir: Optional[Path] = None allow_subprocess_fallback: bool = True python_executable: Optional[str] = None metadata: dict[str, Any] = field(default_factory=dict) + def __post_init__(self) -> None: + """ + Post-initialization method to ensure that the ``work_dir`` and ``project_root`` + attributes are set correctly. + """ + if self.stages_to_run is None: + self.stages_to_run = {} + @classmethod def from_any( cls, diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index bcd9414..d56b452 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -18,6 +18,8 @@ from .stage import Stage, _normalize_dependencies +ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") + class Pipeline: """ Represents an end-to-end code run. This class brings together class instances @@ -47,29 +49,35 @@ def __init__( self, name: str | None = None, backend: str = "python", - config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, + config: PipelineConfig | Mapping[str, Any] | None = None, stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, ): + # if config is not None: resolved_config, resolved_stage_configs, configured_stages = self._resolve_config(config) self.name = name or resolved_config.name or "pipeline" self.backend = backend or resolved_config.backend or "python" if backend == "python" and resolved_config.backend != "python": + # TODO: Error or Warning here? What is preferred? self.backend = resolved_config.backend self.config = resolved_config if self.config.name is None: self.config.name = self.name - self.config.backend = self.backend self.logger = logger or Logger(log_dir=self.config.log_dir) - self.executor = executor or PythonStageExecutor() + self.executor = executor or PythonStageExecutor() # TODO: don't just default - check with config for Executor definition if stages is None: self.stages = configured_stages + # TODO: Add check to see if parsed and from-config stages are different + #elif: len(configured_stages) != 0: + # Warn or Error saying that they have parsed stages AND configured stages + # We COULD check to see if these are identical Stage objects and in order, + # but simplicity might be easier. else: self.stages = [self._coerce_stage(stage) for stage in stages] @@ -79,11 +87,15 @@ def __init__( "for those stages. Try the from_files() method, or create your Stage objects and " \ "parse them to the Pipeline Constructor.") if dependencies is not None: - self._assign_dependencies(dependencies,self.stages) + self._assign_dependencies(dependencies, self.stages) self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() + # TODO: link with comment on issue #28 - We need to integrate StageGraph and validation + # with stages_to_run which will be in PipelineConfig. This allows users to turn on and off Stages + # but we haven't accounted for what that looks like in StageGraph/Pipeline orchestration. self.graph = StageGraph.from_stages(self.stages) + self.graph.validate() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None self.last_run: PipelineRun | None = None @@ -94,11 +106,12 @@ def __init__( backend=self.backend, stages=[stage.name for stage in self.stages], ) + def _assign_dependencies(self, dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: for stage in stages: - new_dependencies = self._dependencies_for_stage(stage.name,stage.source,dependencies) + new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) stage.dependencies = _normalize_dependencies(new_dependencies) return stages @@ -359,7 +372,7 @@ def _current_user(self) -> str | None: def _resolve_config( self, - config: PipelineConfig | Mapping[str, Any] | str | Path | None, + config: PipelineConfig | Mapping[str, Any] | None, ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage]]: """ Resolve supported configuration inputs into pipeline config, stage config, and stages. @@ -372,9 +385,10 @@ def _resolve_config( return PipelineConfig.from_any(config), {}, [] if isinstance(config, PipelineConfig): - stage_configuration = config.metadata.get("stage_configuration") + # + stage_configuration = config.metadata.get("stage_configuration", None) if stage_configuration is None: - stage_configuration = config.metadata.get("stage_config") + stage_configuration = config.metadata.get("stage_config", None) if stage_configuration is not None: warnings.warn( @@ -386,6 +400,14 @@ def _resolve_config( raw_config = self._load_config_mapping(config) pipeline_payload, stage_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) + + # PipelineConfig needs to know what stages to run + # Extract run order from stages + + #run_order = self._extract_run_order(pipeline_payload) + # - Look for stages in pipeline_payload + # - Create a dict, where keys are stage names, and values are where run = true or false + # - Ensure through StageGraph at some point that dependencies are met. stage_definitions = normalized_pipeline_payload.pop("stages", ()) pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) @@ -463,7 +485,7 @@ def _load_config_mapping( return dict(config) config_path = Path(config).expanduser() - if config_path.suffix.lower() not in (".yaml", ".yml"): + if config_path.suffix.lower() not in ACCEPTED_CONFIG_TYPES: raise StageConfigurationError( f"Unsupported config file format parsed as Stage Configuration: {config!r}." ) @@ -474,8 +496,10 @@ def _load_config_mapping( raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if raw_config is None: + # TODO: Warn if it fails? return {} if not isinstance(raw_config, Mapping): + # TODO: maybe this should be richer error messaging raise TypeError("Pipeline config file must contain a mapping at the top level.") return dict(raw_config) @@ -488,7 +512,9 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless a stage-configuration key is present. """ - #TODO: Enforce this behaviour using Errors + # TODO: Enforce this behaviour using Errors + # TODO: Ensure that if stage_config or other keys grabbed are None, warn or error. + # TODO: The if statement is messy and non-intuitive if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: pipeline_payload = raw_config.get("pipeline_variables", {}) if not isinstance(pipeline_payload, Mapping): @@ -497,6 +523,8 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any return dict(pipeline_payload), stage_payload pipeline_payload = dict(raw_config) + # The line below may never happen as it asks for "stage_config" but the if statement above also does this, + # and this code only actions if that if statement does not complete. stage_payload = pipeline_payload.pop("stage_configuration", pipeline_payload.pop("stage_config", None)) return pipeline_payload, stage_payload @@ -713,42 +741,38 @@ def from_files( ) @classmethod - def from_dict(cls, cfg: Mapping[str, Any]) -> Pipeline: + def from_dict( + cls, + config: PipelineConfig | Mapping[str, Any] | str | Path, + name: str | None = None, + backend: str = "python", + logger: Logger | None = None, + executor: StageExecutor | None = None, + ) -> Pipeline: """ Extracts information from a dictionary to configure a Pipeline instance as well as what the Pipeline runs. Parameters ---------- - ``cfg`` : Mapping[str, Any] - The Mapping item that contains the information needed to run the Pipeline. + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path + The object containing the information needed to run the Pipeline. Returns ------- A ``Pipeline`` class instance. """ - if "pipeline_variables" in cfg or "stage_configuration" in cfg or "stage_config" in cfg: - return cls(config=cfg) - - payload = dict(cfg) + pipe_payload, stage_payload = cls._split_config_sections(config) # pipeline_variables contains pipeline information - name = payload.pop("name", None) - backend = payload.pop("backend", "python") - config = payload.pop("config", None) - stages = payload.pop("stages", []) - - if config is None and payload: - config = payload - elif isinstance(config, Mapping) and payload: - combined_config = dict(config) - combined_config.update(payload) - config = combined_config + name = pipe_payload.pop("name", None) + backend = pipe_payload.pop("backend", "python") + stages = pipe_payload.pop("stages", []) return cls( name=name, backend=backend, - config=config, + config=pipe_payload, stages=stages, ) @@ -756,7 +780,6 @@ def from_dict(cls, cfg: Mapping[str, Any]) -> Pipeline: def from_config( cls, config: PipelineConfig | Mapping[str, Any] | str | Path, - *, name: str | None = None, backend: str = "python", logger: Logger | None = None, @@ -768,13 +791,19 @@ def from_config( This is the preferred entrypoint when configuration defines both pipeline-level settings and the stage-level configuration that should be injected at runtime. """ - return cls( - name=name, - backend=backend, + #TODO: Add str/Path behaviour handling + # UPDATE: Not needed as is handled in _resolve_config() + + # Add behaviour handling for str or Path instances for config arg parsed. + # if str, resolve Path, then parse to yaml.safe_load() to extract the mapping. + # Then just parse the mapping to from_dict() to extract the Pipeline instance. + return cls.from_dict( config=config, - logger=logger, - executor=executor, - ) + name=name, + backend=backend, + logger=logger, + executor=executor + ) @staticmethod def _dependencies_for_stage( diff --git a/onsrap/stage.py b/onsrap/stage.py index 7364319..c03b71e 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -83,6 +83,7 @@ class Stage: If the stage ``name`` is empty or if the source is not a supported type. """ name: str + run: bool source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) metadata: dict[str, Any] = field(default_factory=dict) From 41295833d0e7aebe58ef7813bb5b2b763c317505 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 17 Jul 2026 12:23:44 +0100 Subject: [PATCH 111/332] tweak: added TODO for conf.yaml in example2 pipeline to handle different bool value types if YAML doesn't already --- examples/pipeline_2/conf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index 311bbb3..1e041fa 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -5,7 +5,7 @@ pipeline_variables: - 0_clean_data: name: "0_clean_data" location: "examples/pipeline_2/scripts/0_clean_data.py" - run: true + run: true # TODO: 1/0 or y/n dependencies: [] - 1_derive_vars: location: "examples/pipeline_2/scripts/1_derive_vars.py" From 39737cbe3c7631932bf4794c4fea9f6a7f5b899d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 20 Jul 2026 11:37:52 +0100 Subject: [PATCH 112/332] Tweak: add error messages for TODOs regarding Pipeline backend, executor conflict, and stages parsed through Pipeline init and configuration file --- onsrap/pipeline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index d56b452..0f59930 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -61,23 +61,25 @@ def __init__( self.name = name or resolved_config.name or "pipeline" self.backend = backend or resolved_config.backend or "python" if backend == "python" and resolved_config.backend != "python": - # TODO: Error or Warning here? What is preferred? - self.backend = resolved_config.backend + raise PipelineInitialisationError(f"Pipeline backend {backend} does not align with PipelineConfig backend {resolved_config.backend}.") self.config = resolved_config if self.config.name is None: self.config.name = self.name self.logger = logger or Logger(log_dir=self.config.log_dir) - self.executor = executor or PythonStageExecutor() # TODO: don't just default - check with config for Executor definition + if executor is None: + if self.backend == "python": + self.executor = PythonStageExecutor() + else: + raise PipelineInitialisationError("Requested backend does not have a compatible executor. Available executors are: Python.") + else: + self.executor = executor if stages is None: self.stages = configured_stages - # TODO: Add check to see if parsed and from-config stages are different - #elif: len(configured_stages) != 0: - # Warn or Error saying that they have parsed stages AND configured stages - # We COULD check to see if these are identical Stage objects and in order, - # but simplicity might be easier. + elif len(configured_stages) != 0: + raise PipelineInitialisationError("Stages parsed through both Pipeline initialisation AND config file. Please choose one method.") else: self.stages = [self._coerce_stage(stage) for stage in stages] From a562c04a10f386d8c28ab25236e052ad86fc3488 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 20 Jul 2026 11:56:01 +0100 Subject: [PATCH 113/332] tweak: Reordered methods in Pipeline Class to suitable order --- onsrap/pipeline.py | 514 ++++++++++++++++++++++----------------------- 1 file changed, 256 insertions(+), 258 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0f59930..3ebd97a 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -108,91 +108,33 @@ def __init__( backend=self.backend, stages=[stage.name for stage in self.stages], ) - - def _assign_dependencies(self, - dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, - stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: - for stage in stages: - new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) - stage.dependencies = _normalize_dependencies(new_dependencies) - - return stages - - # TODO: Add a method to add dependencies to the pipeline after initialization - # TODO: Re-order methods to be more logical/readable in order (public, private, classmethods, staticmethods) - def _coerce_stage( - self, - stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], - ) -> Stage: - """ - Extracts the ``Stage`` information from the provided stages in the Pipeline. - - Enables mappings, strings, paths, or callables to be parsed and converted into - a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return - itself. - - Parameters - ---------- - ``stage`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] - The information attempting to be converted into a ``Stage`` class instance. - - Raises - ------ - ``StageConfigurationError`` - If the information parsed is not in a suitable format to be converted into - a ``Stage`` class instance. - - Returns - ------- - ``Stage`` class instance for the stage being run. - """ - if isinstance(stage, Stage): - return stage - - if isinstance(stage, Mapping): - return Stage.from_dict(stage) - - if callable(stage): - return Stage.from_callable(stage) - - if isinstance(stage, (str, Path)): - return Stage.from_file(stage) - - raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") - - def _rebuild_graph(self) -> None: - """ - Updates the ``graph`` attribute with the latest stage information. - """ - self.graph = StageGraph.from_stages(self.stages) - def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: - """ - Adds a step to the Pipeline. - - Creates a list called ``added_stages`` that runs the _coerce_stage() method - to extract the information from the given ``stages`` parameter. It then appends - this list to the ``stages`` attribute of the ``Pipeline`` class and updates the - StageGraph using the _rebuild_graph() method. A log instance is created to - reflect the changes. - - Parameters - ---------- - ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] - The new steps being added to the Pipeline. - """ - added_stages = [self._coerce_stage(stage) for stage in stages] - self.stages.extend(added_stages) - self._sync_stage_configs() - self._rebuild_graph() - self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) - + """ + Adds a step to the Pipeline. + + Creates a list called ``added_stages`` that runs the _coerce_stage() method + to extract the information from the given ``stages`` parameter. It then appends + this list to the ``stages`` attribute of the ``Pipeline`` class and updates the + StageGraph using the _rebuild_graph() method. A log instance is created to + reflect the changes. + + Parameters + ---------- + ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The new steps being added to the Pipeline. + """ + added_stages = [self._coerce_stage(stage) for stage in stages] + self.stages.extend(added_stages) + self._sync_stage_configs() + self._rebuild_graph() + self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) + def ordered_stages(self) -> list[Stage]: - """ - Runs the topological_order() method on the ``graph`` attribute to extract the - correct order for the ``stages`` to be run in. - """ - return self.graph.topological_order() + """ + Runs the topological_order() method on the ``graph`` attribute to extract the + correct order for the ``stages`` to be run in. + """ + return self.graph.topological_order() def validate(self) -> Pipeline: """ @@ -277,6 +219,64 @@ def run(self) -> PipelineRun: return PipelineRunner(logger=self.logger).run(self) + + def _assign_dependencies(self, + dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, + stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: + for stage in stages: + new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) + stage.dependencies = _normalize_dependencies(new_dependencies) + + return stages + + # TODO: Add a method to add dependencies to the pipeline after initialization + def _coerce_stage( + self, + stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], + ) -> Stage: + """ + Extracts the ``Stage`` information from the provided stages in the Pipeline. + + Enables mappings, strings, paths, or callables to be parsed and converted into + a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return + itself. + + Parameters + ---------- + ``stage`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The information attempting to be converted into a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the information parsed is not in a suitable format to be converted into + a ``Stage`` class instance. + + Returns + ------- + ``Stage`` class instance for the stage being run. + """ + if isinstance(stage, Stage): + return stage + + if isinstance(stage, Mapping): + return Stage.from_dict(stage) + + if callable(stage): + return Stage.from_callable(stage) + + if isinstance(stage, (str, Path)): + return Stage.from_file(stage) + + raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") + + def _rebuild_graph(self) -> None: + """ + Updates the ``graph`` attribute with the latest stage information. + """ + self.graph = StageGraph.from_stages(self.stages) + + def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: """ Creates a ``RunManifest`` instance that contains the information about @@ -403,7 +403,7 @@ def _resolve_config( pipeline_payload, stage_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) - # PipelineConfig needs to know what stages to run + # TODO: PipelineConfig needs to know what stages to run # Extract run order from stages #run_order = self._extract_run_order(pipeline_payload) @@ -452,152 +452,31 @@ def _manifest_parameters(self) -> dict[str, Any]: for name, stage_config in self.stage_configs.items() } return parameters - - @staticmethod - def _select_stage_config( - stage_configs: Mapping[str, StageConfig], - *, - name: str | None, - ) -> StageConfig: - """ - Select one stage configuration from a stage-name keyed mapping. - - When ``name`` is omitted, exactly one stage configuration must be present. - """ - if name is not None: - if name not in stage_configs: - raise StageConfigurationError(f"Stage configuration '{name}' was not found.") - return stage_configs[name] - - if len(stage_configs) != 1: - raise StageConfigurationError( - "The provided input resolves to multiple stage configurations; specify a stage name." - ) - - return next(iter(stage_configs.values())) - - @staticmethod - def _load_config_mapping( - config: Mapping[str, Any] | str | Path, - ) -> dict[str, Any]: - """ - Load raw configuration data from a mapping or YAML file. - """ - if isinstance(config, Mapping): - return dict(config) - - config_path = Path(config).expanduser() - if config_path.suffix.lower() not in ACCEPTED_CONFIG_TYPES: - raise StageConfigurationError( - f"Unsupported config file format parsed as Stage Configuration: {config!r}." - ) - if not config_path.exists(): - raise FileNotFoundError(f"Config file does not exist: {config_path}") - - import yaml - - raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) - if raw_config is None: - # TODO: Warn if it fails? - return {} - if not isinstance(raw_config, Mapping): - # TODO: maybe this should be richer error messaging - raise TypeError("Pipeline config file must contain a mapping at the top level.") - return dict(raw_config) - - @staticmethod - def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any], Mapping[str, Any] | None]: - """ - Split a raw config payload into pipeline-level and stage-level sections. - - Composite config payloads may use top-level ``pipeline_variables`` and - ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless - a stage-configuration key is present. - """ - # TODO: Enforce this behaviour using Errors - # TODO: Ensure that if stage_config or other keys grabbed are None, warn or error. - # TODO: The if statement is messy and non-intuitive - if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: - pipeline_payload = raw_config.get("pipeline_variables", {}) - if not isinstance(pipeline_payload, Mapping): - raise StageConfigurationError("The 'pipeline_variables' section must be a mapping.") - stage_payload = raw_config.get("stage_configuration", raw_config.get("stage_config")) - return dict(pipeline_payload), stage_payload - - pipeline_payload = dict(raw_config) - # The line below may never happen as it asks for "stage_config" but the if statement above also does this, - # and this code only actions if that if statement does not complete. - stage_payload = pipeline_payload.pop("stage_configuration", pipeline_payload.pop("stage_config", None)) - return pipeline_payload, stage_payload - - @staticmethod - def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str, Any]: - """ - Normalize supported aliases in the pipeline section before model construction. - - Recognized aliases: - - - ``working_dir`` → ``work_dir`` (only when ``work_dir`` is absent). - - If both ``working_dir`` and ``work_dir`` are present at the same time, a - ``UserWarning`` is emitted and ``working_dir`` is left in the payload where - it will be silently absorbed into ``PipelineConfig.metadata``. - """ - normalized_payload = dict(pipeline_payload) - if "working_dir" in normalized_payload: - if "work_dir" not in normalized_payload: - normalized_payload["work_dir"] = normalized_payload.pop("working_dir") - else: - warnings.warn( - "Both 'working_dir' and 'work_dir' were found in the pipeline configuration. " - "'work_dir' will be used and 'working_dir' will be ignored.", - UserWarning, - stacklevel=2, - ) - return normalized_payload - - @staticmethod - def _build_stage_configs(stage_configuration: Mapping[str, Any] | None) -> dict[str, StageConfig]: - """ - Build a stage-name keyed configuration mapping for any number of configured stages. - - The returned mapping scales linearly with the provided stage entries and is used as - the canonical runtime lookup structure for stage configuration. - """ - if stage_configuration is None: - return {} - if not isinstance(stage_configuration, Mapping): - raise StageConfigurationError("Stage configuration must be a mapping keyed by stage name.") - - return { - str(stage_name): StageConfig.from_mapping(str(stage_name), stage_payload) - for stage_name, stage_payload in stage_configuration.items() - } - + def _build_stages_from_config( - self, - stage_definitions: Sequence[Any] | None, - *, - backend: str, - work_dir: Path, - ) -> list[Stage]: - """ - Convert configured stage definitions into ``Stage`` instances. - - Each entry is resolved independently, so the method can process any number of - stage definitions supplied in the pipeline configuration. - """ - if not stage_definitions: - return [] - if not isinstance(stage_definitions, Sequence) or isinstance(stage_definitions, (str, bytes)): - raise StageConfigurationError("Configured stages must be provided as a sequence.") - - configured_stages: list[Stage] = [] - for stage_definition in stage_definitions: - stage = self._stage_from_config_definition(stage_definition, backend=backend, work_dir=work_dir) - if stage is not None: - configured_stages.append(stage) - return configured_stages + self, + stage_definitions: Sequence[Any] | None, + *, + backend: str, + work_dir: Path, + ) -> list[Stage]: + """ + Convert configured stage definitions into ``Stage`` instances. + + Each entry is resolved independently, so the method can process any number of + stage definitions supplied in the pipeline configuration. + """ + if not stage_definitions: + return [] + if not isinstance(stage_definitions, Sequence) or isinstance(stage_definitions, (str, bytes)): + raise StageConfigurationError("Configured stages must be provided as a sequence.") + + configured_stages: list[Stage] = [] + for stage_definition in stage_definitions: + stage = self._stage_from_config_definition(stage_definition, backend=backend, work_dir=work_dir) + if stage is not None: + configured_stages.append(stage) + return configured_stages def _stage_from_config_definition( self, @@ -658,28 +537,7 @@ def _stage_from_config_definition( entrypoint=entrypoint, backend=backend, ) - - @staticmethod - def _resolve_stage_source(stage_name: str, location: Any, work_dir: Path) -> Path: - """ - Resolve the source path for a configured stage. - - Empty locations default to ``work_dir / "scripts" / ".py"``. Relative - paths are first interpreted as given and then relative to ``work_dir``. - """ - if location in (None, ""): - return work_dir / "scripts" / f"{stage_name}.py" - - candidate = Path(location).expanduser() - if candidate.is_absolute() or candidate.exists(): - return candidate - - work_dir_candidate = work_dir / candidate - if work_dir_candidate.exists(): - return work_dir_candidate - - return candidate - + @classmethod def from_files( @@ -764,6 +622,7 @@ def from_dict( ------- A ``Pipeline`` class instance. """ + # TODO: Finish this class method to include missing variables and clarify where sourced from config. pipe_payload, stage_payload = cls._split_config_sections(config) # pipeline_variables contains pipeline information @@ -793,12 +652,6 @@ def from_config( This is the preferred entrypoint when configuration defines both pipeline-level settings and the stage-level configuration that should be injected at runtime. """ - #TODO: Add str/Path behaviour handling - # UPDATE: Not needed as is handled in _resolve_config() - - # Add behaviour handling for str or Path instances for config arg parsed. - # if str, resolve Path, then parse to yaml.safe_load() to extract the mapping. - # Then just parse the mapping to from_dict() to extract the Pipeline instance. return cls.from_dict( config=config, name=name, @@ -807,6 +660,151 @@ def from_config( executor=executor ) + + + @staticmethod + def _select_stage_config( + stage_configs: Mapping[str, StageConfig], + *, + name: str | None, + ) -> StageConfig: + """ + Select one stage configuration from a stage-name keyed mapping. + + When ``name`` is omitted, exactly one stage configuration must be present. + """ + if name is not None: + if name not in stage_configs: + raise StageConfigurationError(f"Stage configuration '{name}' was not found.") + return stage_configs[name] + + if len(stage_configs) != 1: + raise StageConfigurationError( + "The provided input resolves to multiple stage configurations; specify a stage name." + ) + + return next(iter(stage_configs.values())) + + @staticmethod + def _load_config_mapping( + config: Mapping[str, Any] | str | Path, + ) -> dict[str, Any]: + """ + Load raw configuration data from a mapping or YAML file. + """ + if isinstance(config, Mapping): + return dict(config) + + config_path = Path(config).expanduser() + if config_path.suffix.lower() not in ACCEPTED_CONFIG_TYPES: + raise StageConfigurationError( + f"Unsupported config file format parsed as Stage Configuration: {config!r}." + ) + if not config_path.exists(): + raise FileNotFoundError(f"Config file does not exist: {config_path}") + + import yaml + + raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if raw_config is None: + # TODO: Warn if it fails? + return {} + if not isinstance(raw_config, Mapping): + # TODO: maybe this should be richer error messaging + raise TypeError("Pipeline config file must contain a mapping at the top level.") + return dict(raw_config) + + @staticmethod + def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any], Mapping[str, Any] | None]: + """ + Split a raw config payload into pipeline-level and stage-level sections. + + Composite config payloads may use top-level ``pipeline_variables`` and + ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless + a stage-configuration key is present. + """ + # TODO: Enforce this behaviour using Errors + # TODO: Ensure that if stage_config or other keys grabbed are None, warn or error. + # TODO: The if statement is messy and non-intuitive + if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: + pipeline_payload = raw_config.get("pipeline_variables", {}) + if not isinstance(pipeline_payload, Mapping): + raise StageConfigurationError("The 'pipeline_variables' section must be a mapping.") + stage_payload = raw_config.get("stage_configuration", raw_config.get("stage_config")) + return dict(pipeline_payload), stage_payload + + pipeline_payload = dict(raw_config) + # The line below may never happen as it asks for "stage_config" but the if statement above also does this, + # and this code only actions if that if statement does not complete. + stage_payload = pipeline_payload.pop("stage_configuration", pipeline_payload.pop("stage_config", None)) + return pipeline_payload, stage_payload + + @staticmethod + def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str, Any]: + """ + Normalize supported aliases in the pipeline section before model construction. + + Recognized aliases: + + - ``working_dir`` → ``work_dir`` (only when ``work_dir`` is absent). + + If both ``working_dir`` and ``work_dir`` are present at the same time, a + ``UserWarning`` is emitted and ``working_dir`` is left in the payload where + it will be silently absorbed into ``PipelineConfig.metadata``. + """ + normalized_payload = dict(pipeline_payload) + if "working_dir" in normalized_payload: + if "work_dir" not in normalized_payload: + normalized_payload["work_dir"] = normalized_payload.pop("working_dir") + else: + warnings.warn( + "Both 'working_dir' and 'work_dir' were found in the pipeline configuration. " + "'work_dir' will be used and 'working_dir' will be ignored.", + UserWarning, + stacklevel=2, + ) + return normalized_payload + + @staticmethod + def _build_stage_configs(stage_configuration: Mapping[str, Any] | None) -> dict[str, StageConfig]: + """ + Build a stage-name keyed configuration mapping for any number of configured stages. + + The returned mapping scales linearly with the provided stage entries and is used as + the canonical runtime lookup structure for stage configuration. + """ + if stage_configuration is None: + return {} + if not isinstance(stage_configuration, Mapping): + raise StageConfigurationError("Stage configuration must be a mapping keyed by stage name.") + + return { + str(stage_name): StageConfig.from_mapping(str(stage_name), stage_payload) + for stage_name, stage_payload in stage_configuration.items() + } + + + @staticmethod + def _resolve_stage_source(stage_name: str, location: Any, work_dir: Path) -> Path: + """ + Resolve the source path for a configured stage. + + Empty locations default to ``work_dir / "scripts" / ".py"``. Relative + paths are first interpreted as given and then relative to ``work_dir``. + """ + if location in (None, ""): + return work_dir / "scripts" / f"{stage_name}.py" + + candidate = Path(location).expanduser() + if candidate.is_absolute() or candidate.exists(): + return candidate + + work_dir_candidate = work_dir / candidate + if work_dir_candidate.exists(): + return work_dir_candidate + + return candidate + @staticmethod def _dependencies_for_stage( stage_name: str, @@ -841,4 +839,4 @@ def _dependencies_for_stage( if candidate in dependencies: return tuple(str(dependency) for dependency in dependencies[candidate]) - return () \ No newline at end of file + return () From d31a8884a1cb40c9b8ad7868bdf4c63dfc543f36 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 20 Jul 2026 15:18:46 +0100 Subject: [PATCH 114/332] feat: add a new class method to Pipeline - add_dependencies() --- onsrap/pipeline.py | 55 +++++++++++++++++++++++++++++++++++++++++- tests/test_pipeline.py | 33 +++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 3ebd97a..4c939ef 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -219,6 +219,59 @@ def run(self) -> PipelineRun: return PipelineRunner(logger=self.logger).run(self) + def add_dependencies(self, + *dependencies: tuple[str]| dict[str, Sequence[str]]) -> None: + """ + Adds a set of ``dependencies`` for the Pipeline after the Pipeline initialisation. + + This method takes any number of positional arguments and imputes them as + ``dependencies``. It looks at each argument parsed, checks the data type against + the existing ``Pipeline`` ``dependencies`` and if they are the same data type, it + will take every stage within the ``Pipeline`` instance. It will then run the + ``_dependencies_for_stage()`` class method and normalize any ``dependencies`` before + adding them to the individual ``Stage`` instances. It will then append these + ``dependencies`` directly to the ``dependencies`` in the ``Pipeline`` instance before + rerunning the ``StageGraph`` creation to ensure the new ``dependencies`` are considered. + A logging entry will be created to track that these ``dependencies`` are added. + + Parameters + ---------- + ``*dependencies`` : tuple[str]| dict[str, Sequence[str]] + Any number of dependencies that you would like to add to the Pipeline. + + Raises + ------ + ``PipelineInitializationError`` + If the dependency you are attempting to add to the Pipeline doesn't match + the datatype for dependencies currently in the Pipeline. + """ + + for dependency in dependencies: + if self.dependencies is not None and not isinstance(dependency, type(self.dependencies)): + raise PipelineInitialisationError("Existing dependencies are not the same type as new dependencies") + + for stage in self.stages: + new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependency) + existing = stage.dependencies or () + new = existing + tuple(_normalize_dependencies(new_dependencies)) + + stage.dependencies = tuple(dict.fromkeys(new)) + + if isinstance(dependency, tuple): + existing = self.dependencies or () + self.dependencies = tuple(existing | dependency) + elif isinstance(dependency, dict): + for stage_name, deps in dependency.items(): + existing = self.dependencies.get(stage_name,[]) + combined = existing + tuple(deps) + self.dependencies[stage_name] = tuple(dict.fromkeys(combined)) + + self.graph = StageGraph.from_stages(self.stages) + self.graph.validate() + + self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + + def _assign_dependencies(self, dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, @@ -226,6 +279,7 @@ def _assign_dependencies(self, for stage in stages: new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) stage.dependencies = _normalize_dependencies(new_dependencies) + #TODO: Should this aldo return pipeline.dependencies as the normalised values? return stages @@ -834,7 +888,6 @@ def _dependencies_for_stage( candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) else: candidates = (stage_name, str(path.__name__)) - print(candidates) for candidate in candidates: if candidate in dependencies: return tuple(str(dependency) for dependency in dependencies[candidate]) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index e0cd641..64eb154 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -58,6 +58,39 @@ def example_function(): assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) +def test_add_dependencies_single_dict(tmp_path): + """ + Tests that a dictionary correctly assigns dependencies to + individual stages and the Pipeline instance. + """ + + path_1 = tmp_path/"Stage_1.py" + path_2 = tmp_path/"Stage_2.py" + path_0 = tmp_path/"Stage_0.py" + dependencies_multiple = {"Stage_0":(), + "Stage_1":(), + "Stage_2":("Stage_1",)} + dep_dict = {"Stage_1":("Stage_0",), + "Stage_2":("Stage_0","Stage_1")} + dep_tuple = ("Stage_0.25",) + stage_1 = Stage("Stage_1", source = path_1, dependencies = {}) + stage_2 = Stage("Stage_2", source = path_2, dependencies = {}) + stage_0 = Stage("Stage_0", source = path_0, dependencies = {}) + + pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], + dependencies = dependencies_multiple) + with pytest.raises(PipelineInitialisationError): + pipeline_dict.add_dependencies(dep_tuple) + + pipeline_dict.add_dependencies(dep_dict) + assert stage_1.dependencies == ("Stage_0",) + assert stage_2.dependencies == ("Stage_1","Stage_0",) + assert stage_0.dependencies == () + assert pipeline_dict.dependencies == {"Stage_0":(), + "Stage_1":("Stage_0",), + "Stage_2":("Stage_1","Stage_0",)} + + \ No newline at end of file From b51126fffe699d4fab98eb30d84de0b808d636cc Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 20 Jul 2026 16:15:46 +0100 Subject: [PATCH 115/332] tweak: removed a blank line but clearing all changes before next commit --- onsrap/pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 4c939ef..cdf6e68 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -283,7 +283,6 @@ def _assign_dependencies(self, return stages - # TODO: Add a method to add dependencies to the pipeline after initialization def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], @@ -441,7 +440,6 @@ def _resolve_config( return PipelineConfig.from_any(config), {}, [] if isinstance(config, PipelineConfig): - # stage_configuration = config.metadata.get("stage_configuration", None) if stage_configuration is None: stage_configuration = config.metadata.get("stage_config", None) From 760bedff6af5e7a86e1c43b505c641a3f4871128 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 20 Jul 2026 17:18:49 +0100 Subject: [PATCH 116/332] feat: Add methods to allow stages_to_run configuration to be parsed through PipelineConfig to Pipeline and subsequently through StageGraph for validation --- onsrap/models.py | 71 +++++++++++++++++++++++++++++++++++++++++++++- onsrap/pipeline.py | 40 +++++++++++++++++++------- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index ef4a1f5..9a5bca2 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Iterable, Mapping, Optional, Union -from .errors import StageConfigurationError +from .errors import StageConfigurationError, PipelineConfigurationError class StageStatus(str, Enum): @@ -204,7 +204,9 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: metadata = {"metadata": metadata} name = payload.pop("name", None) + backend = payload.pop("backend", "python") + stages_to_run = cls._extract_stages_run(payload) work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) output_dir_value = payload.pop("output_dir", None) @@ -228,6 +230,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: return cls( name=name, + stages_to_run = stages_to_run, backend=backend, work_dir=work_dir, project_root=project_root, @@ -238,6 +241,72 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: python_executable=python_executable, metadata=metadata, ) + + def _extract_stages_run(self, + payload: Mapping[str, Any] + ) -> dict[str, bool]: + """ + Method to extract stages_to_run configuration and convert all values to boolean values. + + Parameters + ---------- + ``payload`` : Mapping[str, Any] + The dictionary where the stages_to_run configuration is being extracted from. + + Returns + ------- + ``boolean_dict`` + A dictionary of stage_name:bool to indicate whether a stage is being run. + """ + stages_to_run = payload.pop("stages_to_run", None) + if stages_to_run is None: + pass + #TODO: Add default to add all stages here + boolean_dict = {stage_name: self._to_bool(value) for stage_name,value in stages_to_run} + return boolean_dict + + + def _to_bool(self, value): + """ + Method to convert values to boolean True/False values. + + Integers convert to boolean where 0 = False and 1 = True. A certain subset of strings are + accepted for conversion. Any other strings will. raise an error. + + Parameters + ---------- + ``value`` : bool | int | str + + Returns + ------- + ``value`` + The value input but converted to a boolean value. + + Raises + ------ + ``ValueError`` + When the value has not been able to be converted to a boolean. + """ + + if isinstance(value, bool): + return value + + if isinstance(value, int): + return bool(value) + + if isinstance(value, str): + value = value.strip().lower() + + if value in {"true", "yes", "y", "1"}: + return True + + if value in {"false", "no", "n", "0"}: + return False + + raise ValueError(f"Cannot convert {value!r} to bool") + + raise ValueError(f"Cannot convert {value!r} to bool") + @classmethod def from_file(cls, path: Path) -> PipelineConfig: diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index cdf6e68..3d4cd32 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -93,10 +93,32 @@ def __init__( self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() - # TODO: link with comment on issue #28 - We need to integrate StageGraph and validation - # with stages_to_run which will be in PipelineConfig. This allows users to turn on and off Stages - # but we haven't accounted for what that looks like in StageGraph/Pipeline orchestration. - self.graph = StageGraph.from_stages(self.stages) + + #SOPHIE'S ATTEMPT AT IMPLEMENTING STAGES_TO_RUN + + #dictionary of stage name: stage class instance + stage_lookup = {stage.name: stage + for stage in self.stages} + + #list of all stage names selected to run in config + stage_names_to_run = [stage_name + for stage_name, value in self.config.stages_to_run.items() + if value] + + #run standard StageGraph if all stages are present in stage_names_to_run + if set(stage_lookup.keys()) == set(stage_names_to_run): + self.graph = StageGraph.from_stages(self.stages) + else: + #Checks that all stages in stage_names_to_run exist in the Pipeline. + for stage_name in stage_names_to_run: + if stage_name not in list(stage_lookup.keys()): + raise PipelineInitialisationError("You're trying to run a stage that does not exist. Please add the stage to the Pipeline.") + #list of Stage instances for stages that should be run following configuration + stages_to_run = [stage_lookup[name] + for name in stage_names_to_run + if name in stage_lookup] + self.graph = StageGraph.from_stages(stages_to_run) + self.graph.validate() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -455,13 +477,6 @@ def _resolve_config( pipeline_payload, stage_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) - # TODO: PipelineConfig needs to know what stages to run - # Extract run order from stages - - #run_order = self._extract_run_order(pipeline_payload) - # - Look for stages in pipeline_payload - # - Create a dict, where keys are stage names, and values are where run = true or false - # - Ensure through StageGraph at some point that dependencies are met. stage_definitions = normalized_pipeline_payload.pop("stages", ()) pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) @@ -473,6 +488,8 @@ def _resolve_config( ) return pipeline_config, stage_configs, configured_stages + + def _sync_stage_configs(self) -> None: """ Ensure every known stage has a ``StageConfig`` entry, even if it is empty. @@ -804,6 +821,7 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str ``UserWarning`` is emitted and ``working_dir`` is left in the payload where it will be silently absorbed into ``PipelineConfig.metadata``. """ + #TODO: Add normalization for stages_to_run normalized_payload = dict(pipeline_payload) if "working_dir" in normalized_payload: if "work_dir" not in normalized_payload: From be80a132061934d3abef33c0fae910dbe9e9142b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 08:55:40 +0100 Subject: [PATCH 117/332] tweak: improve error messaging in _load_config_mapping() method --- onsrap/pipeline.py | 10 ++++++---- onsrap/warnings.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 3d4cd32..1ff8b01 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -301,7 +301,6 @@ def _assign_dependencies(self, for stage in stages: new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) stage.dependencies = _normalize_dependencies(new_dependencies) - #TODO: Should this aldo return pipeline.dependencies as the normalised values? return stages @@ -776,11 +775,14 @@ def _load_config_mapping( raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if raw_config is None: - # TODO: Warn if it fails? + warnings.warn("No configuration has loaded from the configuration file. Please check " \ + "your configurations.") return {} if not isinstance(raw_config, Mapping): - # TODO: maybe this should be richer error messaging - raise TypeError("Pipeline config file must contain a mapping at the top level.") + raise TypeError("Configuration file must contain a mapping at the top level. Please ensure" \ + "that your configuration file is structured into key:value pairs in the notation that suits" \ + "the configuration file that you are using. The top level key value pairs should reflect " \ + "the Pipeline and Stage configurations.") return dict(raw_config) @staticmethod diff --git a/onsrap/warnings.py b/onsrap/warnings.py index 7fdd22b..2d9f236 100644 --- a/onsrap/warnings.py +++ b/onsrap/warnings.py @@ -4,6 +4,12 @@ class OnsrapWarning(Warning): """Base warning for onsrap.""" class StageConfigurationWarning(OnsrapWarning): + """ + Raised when the stage configuration is not optimal. + Child class with ``OnsrapWarning`` as the parent class. + """ + +class PipelineConfigurationWarning(OnsrapWarning): """ Raised when the pipeline configuration is not optimal. Child class with ``OnsrapWarning`` as the parent class. From e615cdbeb2efac04f683222ccb43fc7a5beb21e4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 12:08:36 +0100 Subject: [PATCH 118/332] feat: improves _split_config_sections method and adds _extract_keys method to support --- onsrap/pipeline.py | 121 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 21 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 1ff8b01..c4ab70f 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -9,8 +9,8 @@ from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence -from .errors import StageConfigurationError, PipelineInitialisationError -from .warnings import StageConfigurationWarning +from .errors import StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError +from .warnings import StageConfigurationWarning, PipelineConfigurationWarning from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger @@ -786,30 +786,109 @@ def _load_config_mapping( return dict(raw_config) @staticmethod - def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[dict[str, Any], Mapping[str, Any] | None]: + def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, Any] | None, Mapping[str, Any] | None]: """ Split a raw config payload into pipeline-level and stage-level sections. - Composite config payloads may use top-level ``pipeline_variables`` and - ``stage_configuration`` keys. Flat payloads are treated as pipeline config unless - a stage-configuration key is present. - """ - # TODO: Enforce this behaviour using Errors - # TODO: Ensure that if stage_config or other keys grabbed are None, warn or error. - # TODO: The if statement is messy and non-intuitive - if "pipeline_variables" in raw_config or "stage_configuration" in raw_config or "stage_config" in raw_config: - pipeline_payload = raw_config.get("pipeline_variables", {}) - if not isinstance(pipeline_payload, Mapping): - raise StageConfigurationError("The 'pipeline_variables' section must be a mapping.") - stage_payload = raw_config.get("stage_configuration", raw_config.get("stage_config")) - return dict(pipeline_payload), stage_payload - - pipeline_payload = dict(raw_config) - # The line below may never happen as it asks for "stage_config" but the if statement above also does this, - # and this code only actions if that if statement does not complete. - stage_payload = pipeline_payload.pop("stage_configuration", pipeline_payload.pop("stage_config", None)) + Allows for configurations that have "stage_configuration", "stage_config", "pipeline_variables" + and "pipeline_config" as the key. The key is identified and used to pull the values for the + configuration from the ``raw_config``. It is then Nonetype checked and Type checked to ensure + that appropriate information is extracted and errors are produced if any of these checks fail. + + Parameters + ---------- + ``raw_config``: Mapping[str, Any] + Contents of the configuration file previously extracted. + + Returns + ------- + ``pipeline_payload``: Mapping[str, Any] | None + Contents of the pipeline configuration settings defined in the configuration file. + ``stage_payload``: Mapping[str, Any] | None + Contents of the stage configuration settings defined in the configuration file. + + Raises + ------ + ``PipelineConfigurationWarning`` + If blank values for pipeline_payload or stage_payload are detected. + If there are remaining keys in the ``raw_config`` that have not been extracted. + + ``PipelineConfigurationError`` + If the pipeline_payload or stage_payload are not mapping types. + """ + possible_stage_keys = ("stage_configuration", "stage_config") + possible_pipeline_keys = ("pipeline_variables","pipeline_config") + + stage_configuration = Pipeline._extract_keys(possible_stage_keys, raw_config) + pipeline_configuration = Pipeline._extract_keys(possible_pipeline_keys, raw_config) + + pipeline_payload = raw_config.get(pipeline_configuration,{}) + + if pipeline_payload is None: + warnings.warn("Blank pipeline configuration detected. Please check that this is correct.", PipelineConfigurationWarning) + stage_payload = raw_config.get(stage_configuration,{}) + if stage_payload is None: + warnings.warn("Blank stage configuration detected. Please check that this is correct.", StageConfigurationWarning) + + remaining_keys = set(raw_config) - [pipeline_configuration, stage_configuration] + if remaining_keys: + warnings.warn("There are remaining sections in your configuration file that have not been extracted." \ + " Please check that all your configurations are in the pipeline or stage configuration keys.", + PipelineConfigurationWarning) + + if not isinstance(pipeline_payload, Mapping): + raise PipelineConfigurationError(f"The {pipeline_configuration} section must be a mapping.") + if not isinstance(stage_payload, Mapping): + raise PipelineConfigurationError(f"The {stage_configuration} section must be a mapping.") + return pipeline_payload, stage_payload + @staticmethod + def _extract_keys(possible_keys: tuple[str, ...], + dictionary: Mapping[str, Any]) -> str: + """ + Checks whether a provided dictionary has a key that has been previously defined. + + Creates a list for all specified keys that are present in the dictionary and checks + the number of keys that match. This should only be 1 so if there are any fewer or + additional then appropriate errors are raised. + + Parameters + ---------- + ``possible_keys``: tuple[str, ...] + Set of string keys that are possibly in the dictionary provided. + ``dictionary``: Mapping[str, Any] + Dictionary that is being checked for valid keys. + + Returns + ------- + ``key``: str + String value for the key that is present in the ``dictionary`` out of the + ``possible_keys`` values. + + Raises + ------ + ``PipelineConfigurationError`` + If no keys in the ``dictionary`` are also in the ``possible_keys`` tuple. + + ``PipelineConfigurationWarning`` + If more than one key in the possible_keys is found, alerts user that it will + default to the first selected option and records the key that is selected. + """ + + matches = [key for key in possible_keys if key in dictionary] + + if len(matches) == 1: + key = matches[0] + elif len(matches) == 0: + raise PipelineConfigurationError(f"No valid keys were found in the configuration. Please ensure that your top level key is one of: {possible_keys}.") + else: + warnings.warn(f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", PipelineConfigurationWarning) + key = matches[0] + + return key + + @staticmethod def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str, Any]: """ From 732cda09c08598936853216fda97d8163fe0754a Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 14:26:03 +0100 Subject: [PATCH 119/332] tweak: added normalized aliases for stages_to_run - further may be needed at a later date --- onsrap/pipeline.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index c4ab70f..9d82b41 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -897,12 +897,13 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str Recognized aliases: - ``working_dir`` → ``work_dir`` (only when ``work_dir`` is absent). + - ``stage_to_run`` → ``stages_to_run`` (only when ``stages_to_run`` is absent). If both ``working_dir`` and ``work_dir`` are present at the same time, a ``UserWarning`` is emitted and ``working_dir`` is left in the payload where it will be silently absorbed into ``PipelineConfig.metadata``. """ - #TODO: Add normalization for stages_to_run + normalized_payload = dict(pipeline_payload) if "working_dir" in normalized_payload: if "work_dir" not in normalized_payload: @@ -914,6 +915,17 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str UserWarning, stacklevel=2, ) + if "stage_to_run" in normalized_payload: + if "stages_to_run" not in normalized_payload: + normalized_payload["stages_to_run"] = normalized_payload.pop("stage_to_run") + else: + warnings.warn( + "Both 'stage_to_run' and 'stages_to_run' were found in the pipeline configuration. " + "'stages_to_run' will be used and 'stage_to_run' will be ignored.", + UserWarning, + stacklevel=2, + ) + return normalized_payload @staticmethod From e744d00f97d771bd40ef67df1e51dc59e57e218a Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 15:12:56 +0100 Subject: [PATCH 120/332] tweak: adjusted _extract_stages_run method to return None value if no stages are set to true and subsequent process in pipeline.py init to run all stages in this instance --- onsrap/models.py | 9 ++++++--- onsrap/pipeline.py | 16 ++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 9a5bca2..69bd5e8 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -244,7 +244,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: def _extract_stages_run(self, payload: Mapping[str, Any] - ) -> dict[str, bool]: + ) -> dict[str, bool] | None: """ Method to extract stages_to_run configuration and convert all values to boolean values. @@ -257,11 +257,14 @@ def _extract_stages_run(self, ------- ``boolean_dict`` A dictionary of stage_name:bool to indicate whether a stage is being run. + + None + If stages_to_run does not exist within the configuration. """ stages_to_run = payload.pop("stages_to_run", None) if stages_to_run is None: - pass - #TODO: Add default to add all stages here + return None + boolean_dict = {stage_name: self._to_bool(value) for stage_name,value in stages_to_run} return boolean_dict diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 9d82b41..1e4f926 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -100,10 +100,15 @@ def __init__( stage_lookup = {stage.name: stage for stage in self.stages} + #sets stage_names_to_run as all stages possible if none are specified + if self.config.stages_to_run is None: + warnings.warn("No stages specified to run. All stages running by default.", PipelineConfigurationWarning) + stage_names_to_run = list(stage_lookup.keys()) + else: #list of all stage names selected to run in config - stage_names_to_run = [stage_name - for stage_name, value in self.config.stages_to_run.items() - if value] + stage_names_to_run = [stage_name + for stage_name, value in self.config.stages_to_run.items() + if value] #run standard StageGraph if all stages are present in stage_names_to_run if set(stage_lookup.keys()) == set(stage_names_to_run): @@ -115,8 +120,7 @@ def __init__( raise PipelineInitialisationError("You're trying to run a stage that does not exist. Please add the stage to the Pipeline.") #list of Stage instances for stages that should be run following configuration stages_to_run = [stage_lookup[name] - for name in stage_names_to_run - if name in stage_lookup] + for name in stage_names_to_run] self.graph = StageGraph.from_stages(stages_to_run) self.graph.validate() @@ -903,7 +907,7 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str ``UserWarning`` is emitted and ``working_dir`` is left in the payload where it will be silently absorbed into ``PipelineConfig.metadata``. """ - + normalized_payload = dict(pipeline_payload) if "working_dir" in normalized_payload: if "work_dir" not in normalized_payload: From 269d21802655d0c6d6caaf367b2ab8b00d5287c2 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 15:36:34 +0100 Subject: [PATCH 121/332] fix: correct all tests in test_execution.py to run with changes --- tests/test_execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_execution.py b/tests/test_execution.py index 3760251..613e89a 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -25,6 +25,7 @@ def config() -> PipelineConfig: return PipelineConfig( "test_pipeline", "python", + None, work_dir, project_root, None, From 928c85d0ba264797681ffcfad751184d912a5a54 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 15:53:44 +0100 Subject: [PATCH 122/332] fix: correct pytests in test_stage.py. Removed run parameter in stage class instance --- onsrap/models.py | 133 ++++++++++++++++++++++---------------------- onsrap/stage.py | 4 +- tests/test_stage.py | 2 +- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 69bd5e8..15550e4 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -206,7 +206,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: name = payload.pop("name", None) backend = payload.pop("backend", "python") - stages_to_run = cls._extract_stages_run(payload) + stages_to_run = PipelineConfig._extract_stages_run(payload) work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) output_dir_value = payload.pop("output_dir", None) @@ -242,8 +242,69 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: metadata=metadata, ) - def _extract_stages_run(self, - payload: Mapping[str, Any] + @classmethod + def from_file(cls, path: Path) -> PipelineConfig: + """ + Extracts a mapping item from a file containing information about how the + pipeline should run. + + Then calls the from_mapping() method to extract the information. + + Parameters + ---------- + ``path`` : Path + The file path containing information to be converted into a PipelineConfig + instance. + + Returns + ------- + ``PipelineConfig`` class instance. + + Raises + ------ + ``FileNotFoundError`` + If the file path does not exist. + ``TypeError`` + If the file containing information about how the Pipeline runs does not + contain a mapping type. + """ + config_path = Path(path).expanduser() + if not config_path.exists(): + raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) + + import yaml + + raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if raw_config is None: + return cls() + + if not isinstance(raw_config, Mapping): + raise TypeError("Pipeline config file must contain a mapping at the top level.") + + return cls.from_mapping(raw_config) + + def to_dict(self) -> dict[str, Any]: + """ + Returns a prescriptive expression of the attributes within the PipelineConfig instance + that allows for easier processing by the user. + """ + data = { + "name": self.name, + "backend": self.backend, + "work_dir": str(self.work_dir), + "project_root": str(self.project_root) if self.project_root is not None else None, + "output_dir": str(self.output_dir) if self.output_dir is not None else None, + "log_dir": str(self.log_dir), + "data_dir": str(self.data_dir), + "output_dir": str(self.output_dir) if self.output_dir is not None else None, + "allow_subprocess_fallback": self.allow_subprocess_fallback, + "python_executable": self.python_executable, + } + data.update(self.metadata) + return data + + @staticmethod + def _extract_stages_run(payload: Mapping[str, Any] ) -> dict[str, bool] | None: """ Method to extract stages_to_run configuration and convert all values to boolean values. @@ -265,11 +326,11 @@ def _extract_stages_run(self, if stages_to_run is None: return None - boolean_dict = {stage_name: self._to_bool(value) for stage_name,value in stages_to_run} + boolean_dict = {stage_name: PipelineConfig._to_bool(value) for stage_name,value in stages_to_run} return boolean_dict - - def _to_bool(self, value): + @staticmethod + def _to_bool(value): """ Method to convert values to boolean True/False values. @@ -311,66 +372,6 @@ def _to_bool(self, value): raise ValueError(f"Cannot convert {value!r} to bool") - @classmethod - def from_file(cls, path: Path) -> PipelineConfig: - """ - Extracts a mapping item from a file containing information about how the - pipeline should run. - - Then calls the from_mapping() method to extract the information. - - Parameters - ---------- - ``path`` : Path - The file path containing information to be converted into a PipelineConfig - instance. - - Returns - ------- - ``PipelineConfig`` class instance. - - Raises - ------ - ``FileNotFoundError`` - If the file path does not exist. - ``TypeError`` - If the file containing information about how the Pipeline runs does not - contain a mapping type. - """ - config_path = Path(path).expanduser() - if not config_path.exists(): - raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) - - import yaml - - raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) - if raw_config is None: - return cls() - - if not isinstance(raw_config, Mapping): - raise TypeError("Pipeline config file must contain a mapping at the top level.") - - return cls.from_mapping(raw_config) - - def to_dict(self) -> dict[str, Any]: - """ - Returns a prescriptive expression of the attributes within the PipelineConfig instance - that allows for easier processing by the user. - """ - data = { - "name": self.name, - "backend": self.backend, - "work_dir": str(self.work_dir), - "project_root": str(self.project_root) if self.project_root is not None else None, - "output_dir": str(self.output_dir) if self.output_dir is not None else None, - "log_dir": str(self.log_dir), - "data_dir": str(self.data_dir), - "output_dir": str(self.output_dir) if self.output_dir is not None else None, - "allow_subprocess_fallback": self.allow_subprocess_fallback, - "python_executable": self.python_executable, - } - data.update(self.metadata) - return data @dataclass diff --git a/onsrap/stage.py b/onsrap/stage.py index c03b71e..8e766fd 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -82,8 +82,8 @@ class Stage: ``StageConfigurationError`` If the stage ``name`` is empty or if the source is not a supported type. """ + #TODO: Do we need a run indicator within the stage? name: str - run: bool source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) metadata: dict[str, Any] = field(default_factory=dict) @@ -101,7 +101,7 @@ def __post_init__(self) -> None: self.source = self.source.expanduser() elif self.source is not None and not callable(self.source): raise StageConfigurationError("Stage source must be a path, callable, or None.") - + self.dependencies = _normalize_dependencies(self.dependencies) self.metadata = dict(self.metadata or {}) self.backend = str(self.backend or "python").strip() or "python" diff --git a/tests/test_stage.py b/tests/test_stage.py index 61a92c1..f3ab3ae 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -68,7 +68,7 @@ def test_stage_backend(example_function) -> None: {"info":"example"}, backend = "java") stage = Stage("callable_stage",example_function,["stage_1"], {"info":"example"}, backend = "") - stage_white_space = Stage("callable_stage",example_function,["stage_1"], + stage_white_space = Stage("callable_stage", example_function,["stage_1"], {"info":"example"}, backend = "python ") assert stage_diff.backend == "java" assert stage.backend == "python" From a4cda10449db0bfbb7a89a687eaa5736f06743a1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 21 Jul 2026 17:10:26 +0100 Subject: [PATCH 123/332] fix: fix pytests in test_pipeline.py and test_stage.py --- onsrap/pipeline.py | 5 +++-- tests/test_pipeline.py | 23 ++++++++++++++--------- tests/test_stage.py | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 1e4f926..ea1ebe2 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -101,9 +101,10 @@ def __init__( for stage in self.stages} #sets stage_names_to_run as all stages possible if none are specified - if self.config.stages_to_run is None: + if self.config.stages_to_run == {}: warnings.warn("No stages specified to run. All stages running by default.", PipelineConfigurationWarning) stage_names_to_run = list(stage_lookup.keys()) + else: #list of all stage names selected to run in config stage_names_to_run = [stage_name @@ -834,7 +835,7 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, if stage_payload is None: warnings.warn("Blank stage configuration detected. Please check that this is correct.", StageConfigurationWarning) - remaining_keys = set(raw_config) - [pipeline_configuration, stage_configuration] + remaining_keys = set(raw_config) - {pipeline_configuration, stage_configuration} if remaining_keys: warnings.warn("There are remaining sections in your configuration file that have not been extracted." \ " Please check that all your configurations are in the pipeline or stage configuration keys.", diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 64eb154..84a07f3 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -30,9 +30,13 @@ def test_assign_dependencies(tmp_path): """ def example_function(): pass + + path_1 = tmp_path/"Stage_1.py" + path_0 = tmp_path/"Stage_0.py" + dependencies_single = {"Stage_2":("Stage_1",)} - dependencies_multiple = {"Stage_1":["Stage_0", "Stage_0.5"], - "Stage_2":("Stage_1",)} + dependencies_multiple = {"Stage_1":["Stage_0"], + "Stage_2":("Stage_1", "Stage_0")} dependencies_non_stage_name = {"Stage_1.py":("Stage_0",), "example_function":("Stage_1.py",)} @@ -40,18 +44,19 @@ def example_function(): Pipeline(stages = None, dependencies = dependencies_single) - path = tmp_path/"Stage_1.py" pipeline_1 = Pipeline(name = "pipeline_1", - stages = [Stage("Stage_1", path, None,{}), - Stage("Stage_2", example_function, None,{})], + stages = [Stage("Stage_1", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], dependencies = dependencies_multiple) - assert pipeline_1.stages[0].dependencies == ("Stage_0","Stage_0.5",) - assert pipeline_1.stages[1].dependencies == ("Stage_1",) + assert pipeline_1.stages[0].dependencies == ("Stage_0",) + assert pipeline_1.stages[1].dependencies == ("Stage_1","Stage_0") pipeline_2 = Pipeline(name = "pipeline_2", - stages = [Stage("Stage_1", path, None,{}), - Stage("Stage_2", example_function, None,{})], + stages = [Stage("Stage_1.py", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], dependencies = dependencies_non_stage_name) assert pipeline_2.stages[0].dependencies == ("Stage_0",) diff --git a/tests/test_stage.py b/tests/test_stage.py index f3ab3ae..759bfc1 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -25,7 +25,7 @@ def example_function(): """ Test function to pass as a callable stage for stage testing. """ - print("This is a test function") + pass @pytest.fixture def stage_test() -> Stage: From 1da852be32348de81517b9f85d642c185a187bcb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 22 Jul 2026 09:27:04 +0100 Subject: [PATCH 124/332] fix: pytests in test_pipeline_architecture.py --- onsrap/pipeline.py | 34 +++++++++++++++++--------- onsrap/runner.py | 2 ++ tests/test_pipeline_architecture.py | 38 ++++++++++++++--------------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ea1ebe2..fab3233 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -541,6 +541,9 @@ def _build_stages_from_config( """ if not stage_definitions: return [] + + stage_definitions = list(stage_definitions) + if not isinstance(stage_definitions, Sequence) or isinstance(stage_definitions, (str, bytes)): raise StageConfigurationError("Configured stages must be provided as a sequence.") @@ -695,19 +698,22 @@ def from_dict( ------- A ``Pipeline`` class instance. """ - # TODO: Finish this class method to include missing variables and clarify where sourced from config. - pipe_payload, stage_payload = cls._split_config_sections(config) + # REMOVED AS THIS WAS RUNNING TWICE. SHOULD DISCUSS WHAT TO DO ABOUT THIS + #METHOD AND WHETHER IT IS NEEDED + + #pipe_payload, stage_payload = cls._split_config_sections(config) # pipeline_variables contains pipeline information - name = pipe_payload.pop("name", None) - backend = pipe_payload.pop("backend", "python") - stages = pipe_payload.pop("stages", []) + #name = pipe_payload.get("name", None) + #backend = pipe_payload.get("backend", "python") + #stages = pipe_payload.get("stages", []) + #print(stages) return cls( name=name, backend=backend, - config=pipe_payload, - stages=stages, + config=config, + stages=None, ) @classmethod @@ -725,8 +731,10 @@ def from_config( This is the preferred entrypoint when configuration defines both pipeline-level settings and the stage-level configuration that should be injected at runtime. """ + extracted_config = Pipeline._load_config_mapping(config) + return cls.from_dict( - config=config, + config=extracted_config, name=name, backend=backend, logger=logger, @@ -823,14 +831,17 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, """ possible_stage_keys = ("stage_configuration", "stage_config") possible_pipeline_keys = ("pipeline_variables","pipeline_config") - + print("Assigned Possible Keys") stage_configuration = Pipeline._extract_keys(possible_stage_keys, raw_config) + print("Extracted Stage Configuration name") pipeline_configuration = Pipeline._extract_keys(possible_pipeline_keys, raw_config) + print("Extracted Pipeline Configuration name") pipeline_payload = raw_config.get(pipeline_configuration,{}) - + print("Get Pipeline Configuration") if pipeline_payload is None: warnings.warn("Blank pipeline configuration detected. Please check that this is correct.", PipelineConfigurationWarning) + stage_payload = raw_config.get(stage_configuration,{}) if stage_payload is None: warnings.warn("Blank stage configuration detected. Please check that this is correct.", StageConfigurationWarning) @@ -845,7 +856,7 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, raise PipelineConfigurationError(f"The {pipeline_configuration} section must be a mapping.") if not isinstance(stage_payload, Mapping): raise PipelineConfigurationError(f"The {stage_configuration} section must be a mapping.") - + print("Completed split") return pipeline_payload, stage_payload @staticmethod @@ -882,7 +893,6 @@ def _extract_keys(possible_keys: tuple[str, ...], """ matches = [key for key in possible_keys if key in dictionary] - if len(matches) == 1: key = matches[0] elif len(matches) == 0: diff --git a/onsrap/runner.py b/onsrap/runner.py index 878eceb..593a4f5 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -83,6 +83,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ) # Ensure the stages are in order and create a manifest that explains the run. + ordered_stages = pipeline.ordered_stages() manifest = pipeline._construct_manifest(runtime_id=runtime_id) manifest.stages_run = [] @@ -158,6 +159,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: run_id=runtime_id.get_id(), stages=len(stage_results), ) + return run diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 79fe757..131ec9b 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -40,10 +40,10 @@ def main(context): pipeline = Pipeline.from_files( [first_stage, second_stage], dependencies={"second_stage": ("first_stage",)}, - config={ - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {} }, ) @@ -74,11 +74,10 @@ def main(context): pipeline = Pipeline.from_files( [writer_stage], - config={ - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", - }, + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, ) first_run = pipeline.run() @@ -102,11 +101,10 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa pipeline = Pipeline.from_files( [script_stage], name="script-pipeline", - config={ - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", - }, + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, ) run = pipeline.run() @@ -160,7 +158,7 @@ def run(context): log_dir: "{(tmp_path / 'logs').as_posix()}" stages: - 0_data_validation: - location: "" + location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" run: true dependencies: [] @@ -176,6 +174,9 @@ def run(context): pipeline = Pipeline.from_config(config_file) + #TODO: Fix above line of code. _split_config method is running twice when runs through from_config as it is called + #as part of the __init__ and part of the from_dict() that from_config() calls. Need to review how to normalise. + assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 @@ -203,10 +204,9 @@ def run(context): pipeline = Pipeline.from_files( [stage_file], - config={ - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, "stage_configuration": { "missing_stage": {"years_to_run": 2017}, }, From 6757e35ab84b1af9186eb690ea086586d4961954 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 22 Jul 2026 10:48:28 +0100 Subject: [PATCH 125/332] feat: updated example pipeline 2 to run off a config file. Further work needed to impute stage configurations into stages and extract logs --- examples/pipeline_2/conf.yaml | 46 +++++++++++++++++------------------ examples/pipeline_2/main.py | 12 +++++++-- onsrap/models.py | 2 +- onsrap/pipeline.py | 5 ---- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index 1e041fa..fb8e3a3 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -1,30 +1,31 @@ pipeline_variables: - name: "Pipeline 2" + name: "Config Example Pipeline" backend: python stages: - 0_clean_data: name: "0_clean_data" - location: "examples/pipeline_2/scripts/0_clean_data.py" - run: true # TODO: 1/0 or y/n + location: examples/pipeline_2/scripts/0_clean_data.py dependencies: [] - 1_derive_vars: - location: "examples/pipeline_2/scripts/1_derive_vars.py" - run: true + location: examples/pipeline_2/scripts/1_derive_vars.py dependencies: - - 0_clean_data + - "0_clean_data" - 2_reporting: - location: "examples/pipeline_2/scripts/2_reporting.py" - run: true + location: examples/pipeline_2/scripts/2_reporting.py dependencies: - - 1_derive_vars - working_dir: "examples/pipeline_2" - data_dir: "examples/pipeline_2/data" - output_dir: "examples/pipeline_2/outputs" - log_dir: "examples/pipeline_2/logs" + - "1_derive_vars" + working_dir: Path(__file__).resolve().parent + project_root: Path(__file__).resolve().parent + data_dir: (Path(__file__).resolve().parent)/data + output_dir: (Path(__file__).resolve().parent)/outputs + log_dir: (Path(__file__).resolve().parent)/logs metadata: example: retail-orders-using-configuration description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." - + stages_to_run: + 0_clean_data: True + 1_derive_vars: True + 2_reporting: True stage_configuration: 0_clean_data: @@ -36,17 +37,17 @@ stage_configuration: - "quantity" - "unit_price" - "order_date" - - "order_method + - "order_method" identifiable_cols: - "customer_name" - "age" - "dob" - - "address - output_location: "examples/pipeline_2/data/orders_cleaned.csv" - input_location: "examples/pipeline_2/data/orders.csv" + - "address" + output_location: (Path(__file__).resolve().parent)/data/orders_cleaned.csv + input_location: (Path(__file__).resolve().parent)/data/orders.csv 1_derive_vars: - input_location: "examples/pipeline_2/data/orders_cleaned.csv" - output_location: "examples/pipeline_2/data/orders_prepped.csv" + input_location: (Path(__file__).resolve().parent)/data/orders_cleaned.csv + output_location: (Path(__file__).resolve().parent)/data/orders_prepped.csv delivery_times: north: 14 south: 4 @@ -76,8 +77,8 @@ stage_configuration: total_production_cost: "Total_production_cost" order_profit: "Order_profit" 2_reporting: - input_location: "examples/pipeline_2/processed_data/orders_prepped.csv" - report_location: "examples/pipeline_2/outputs/order_analysis.md" + input_location: (Path(__file__).resolve().parent)/data/orders_prepped.csv + report_location: (Path(__file__).resolve().parent)/outputs/order_analysis.md region: "Region" order_profit: "Order_profit" quantity: "Quantity" @@ -85,6 +86,5 @@ stage_configuration: large_order: "Large_order" small_order: "Small_order" order_id: "Order_id" - order_profit: "Order_profit" product: "Product" num_format: "{:.2f}" diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py index d017a46..8622dbd 100644 --- a/examples/pipeline_2/main.py +++ b/examples/pipeline_2/main.py @@ -1,13 +1,21 @@ import yaml from onsrap import Pipeline, PipelineConfig, StageConfig +from pathlib import Path def main() -> None: - config = yaml.safe_load(open("conf.yaml")) + config_path = (Path(__file__).resolve().parent)/"conf.yaml" + print(config_path) - Pipeline.from_config(config) + pipeline = Pipeline.from_config(config_path) + + run = pipeline.run() + + print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + print(f"Summary report written to: {pipeline.config.output_dir}") + print(f"Cleaned data written to: {pipeline.config.data_dir}") if __name__ == "__main__": main() \ No newline at end of file diff --git a/onsrap/models.py b/onsrap/models.py index 15550e4..8925348 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -326,7 +326,7 @@ def _extract_stages_run(payload: Mapping[str, Any] if stages_to_run is None: return None - boolean_dict = {stage_name: PipelineConfig._to_bool(value) for stage_name,value in stages_to_run} + boolean_dict = {stage_name: PipelineConfig._to_bool(value) for stage_name,value in stages_to_run.items()} return boolean_dict @staticmethod diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index fab3233..5dd0370 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -831,14 +831,10 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, """ possible_stage_keys = ("stage_configuration", "stage_config") possible_pipeline_keys = ("pipeline_variables","pipeline_config") - print("Assigned Possible Keys") stage_configuration = Pipeline._extract_keys(possible_stage_keys, raw_config) - print("Extracted Stage Configuration name") pipeline_configuration = Pipeline._extract_keys(possible_pipeline_keys, raw_config) - print("Extracted Pipeline Configuration name") pipeline_payload = raw_config.get(pipeline_configuration,{}) - print("Get Pipeline Configuration") if pipeline_payload is None: warnings.warn("Blank pipeline configuration detected. Please check that this is correct.", PipelineConfigurationWarning) @@ -856,7 +852,6 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, raise PipelineConfigurationError(f"The {pipeline_configuration} section must be a mapping.") if not isinstance(stage_payload, Mapping): raise PipelineConfigurationError(f"The {stage_configuration} section must be a mapping.") - print("Completed split") return pipeline_payload, stage_payload @staticmethod From 24968904299a1cc7df1355fbc9f930f4dc874b3d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 22 Jul 2026 10:52:05 +0100 Subject: [PATCH 126/332] tweak: example pipeline 2 to enable logs to work --- examples/pipeline_1/main.py | 2 +- examples/pipeline_2/conf.yaml | 22 +++++++++++----------- examples/pipeline_2/main.py | 2 ++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/pipeline_1/main.py b/examples/pipeline_1/main.py index 4f71a70..08242e3 100644 --- a/examples/pipeline_1/main.py +++ b/examples/pipeline_1/main.py @@ -49,7 +49,7 @@ def build_pipeline() -> Pipeline: def main() -> None: run = build_pipeline().run() report = run.manifest.outputs["2_reporting"] - + print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") print(f"Summary report written to: {report['report_path']}") print(f"Cleaned data written to: {report['clean_path']}") diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index fb8e3a3..ab0244c 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -14,11 +14,11 @@ pipeline_variables: location: examples/pipeline_2/scripts/2_reporting.py dependencies: - "1_derive_vars" - working_dir: Path(__file__).resolve().parent - project_root: Path(__file__).resolve().parent - data_dir: (Path(__file__).resolve().parent)/data - output_dir: (Path(__file__).resolve().parent)/outputs - log_dir: (Path(__file__).resolve().parent)/logs + working_dir: examples/pipeline_2 + project_root: examples/pipeline_2 + data_dir: examples/pipeline_2/data + output_dir: examples/pipeline_2/outputs + log_dir: examples/pipeline_2/logs metadata: example: retail-orders-using-configuration description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." @@ -43,11 +43,11 @@ stage_configuration: - "age" - "dob" - "address" - output_location: (Path(__file__).resolve().parent)/data/orders_cleaned.csv - input_location: (Path(__file__).resolve().parent)/data/orders.csv + output_location: examples/pipeline_2/data/orders_cleaned.csv + input_location: examples/pipeline_2/data/orders.csv 1_derive_vars: - input_location: (Path(__file__).resolve().parent)/data/orders_cleaned.csv - output_location: (Path(__file__).resolve().parent)/data/orders_prepped.csv + input_location: examples/pipeline_2/data/orders_cleaned.csv + output_location: examples/pipeline_2/data/orders_prepped.csv delivery_times: north: 14 south: 4 @@ -77,8 +77,8 @@ stage_configuration: total_production_cost: "Total_production_cost" order_profit: "Order_profit" 2_reporting: - input_location: (Path(__file__).resolve().parent)/data/orders_prepped.csv - report_location: (Path(__file__).resolve().parent)/outputs/order_analysis.md + input_location: examples/pipeline_2/data/orders_prepped.csv + report_location: examples/pipeline_2/outputs/order_analysis.md region: "Region" order_profit: "Order_profit" quantity: "Quantity" diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py index 8622dbd..baaf10b 100644 --- a/examples/pipeline_2/main.py +++ b/examples/pipeline_2/main.py @@ -12,6 +12,8 @@ def main() -> None: pipeline = Pipeline.from_config(config_path) run = pipeline.run() + report = run.manifest.outputs + print(report) print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") print(f"Summary report written to: {pipeline.config.output_dir}") From c0f9139684774ce9bb25632c025891e775f7cf9b Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 12:42:53 +0100 Subject: [PATCH 127/332] Removed Union[] calls --- onsrap/models.py | 6 +++--- onsrap/stage.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8925348..9d34a3f 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -5,7 +5,7 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Iterable, Mapping, Optional, Union +from typing import Any, Iterable, Mapping, Optional from .errors import StageConfigurationError, PipelineConfigurationError @@ -149,7 +149,7 @@ def __post_init__(self) -> None: @classmethod def from_any( cls, - value: Union[PipelineConfig, Mapping[str, Any], str, Path, None], + value: PipelineConfig | Mapping[str, Any] | str | Path | None, ) -> PipelineConfig: """ Converts one of several datatypes into a PipelineConfig class instance. @@ -538,7 +538,7 @@ class RunManifest: inputs: dict[str, Any] = field(default_factory=dict) outputs: dict[str, Any] = field(default_factory=dict) backend: str = "python" - package_versions: Union[list[str], str] = field(default_factory=list) + package_versions: list[str] | str = field(default_factory=list) timestamp: str = "" reason: Optional[str] = None user: Optional[str] = None diff --git a/onsrap/stage.py b/onsrap/stage.py index 8e766fd..fccc713 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING from datetime import datetime from .errors import StageConfigurationError, StageDependencyError From be3a373e01790568697608acb5dbdb1949b3c84b Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 14:07:30 +0100 Subject: [PATCH 128/332] Changed constructor stage definition logic. Added Method to handle stages_to_run logic prior to self.graph definition. --- onsrap/pipeline.py | 80 ++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 5dd0370..42aa9d1 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -68,20 +68,23 @@ def __init__( self.config.name = self.name self.logger = logger or Logger(log_dir=self.config.log_dir) - if executor is None: + if executor is not None: + self.executor = executor + else: if self.backend == "python": self.executor = PythonStageExecutor() else: raise PipelineInitialisationError("Requested backend does not have a compatible executor. Available executors are: Python.") - else: - self.executor = executor - - if stages is None: - self.stages = configured_stages - elif len(configured_stages) != 0: - raise PipelineInitialisationError("Stages parsed through both Pipeline initialisation AND config file. Please choose one method.") - else: - self.stages = [self._coerce_stage(stage) for stage in stages] + + if stages is not None and configured_stages: + raise PipelineInitialisationError( + "Stages parsed through both Pipeline construction and configuration file. Either provide stages through the constructor or the configuration file, not both." + ) + self.stages = ( + configured_stages + if stages is None + else [self._coerce_stage(stage) for stage in stages] + ) self.dependencies = dependencies if dependencies is not None and stages is None: @@ -93,36 +96,8 @@ def __init__( self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() - - #SOPHIE'S ATTEMPT AT IMPLEMENTING STAGES_TO_RUN - #dictionary of stage name: stage class instance - stage_lookup = {stage.name: stage - for stage in self.stages} - - #sets stage_names_to_run as all stages possible if none are specified - if self.config.stages_to_run == {}: - warnings.warn("No stages specified to run. All stages running by default.", PipelineConfigurationWarning) - stage_names_to_run = list(stage_lookup.keys()) - - else: - #list of all stage names selected to run in config - stage_names_to_run = [stage_name - for stage_name, value in self.config.stages_to_run.items() - if value] - - #run standard StageGraph if all stages are present in stage_names_to_run - if set(stage_lookup.keys()) == set(stage_names_to_run): - self.graph = StageGraph.from_stages(self.stages) - else: - #Checks that all stages in stage_names_to_run exist in the Pipeline. - for stage_name in stage_names_to_run: - if stage_name not in list(stage_lookup.keys()): - raise PipelineInitialisationError("You're trying to run a stage that does not exist. Please add the stage to the Pipeline.") - #list of Stage instances for stages that should be run following configuration - stages_to_run = [stage_lookup[name] - for name in stage_names_to_run] - self.graph = StageGraph.from_stages(stages_to_run) + self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) self.graph.validate() self.id: RuntimeID | None = None @@ -614,6 +589,33 @@ def _stage_from_config_definition( backend=backend, ) + def _resolve_stages_to_run(self) -> list[Stage]: + """ + + """ + stage_lookup = {stage.name: stage + for stage in self.stages} + + if self.config.stages_to_run == {}: + warnings.warn("No stages specified to run. All stages running by default.", PipelineConfigurationWarning) + stage_names_to_run = list(stage_lookup.keys()) + + else: + stage_names_to_run = [ + stage_name + for stage_name, value in self.config.stages_to_run.items() + if value + ] + + if set(stage_lookup.keys()) == set(stage_names_to_run): + return self.stages + else: + # Check that all stages in stage_names_to_run exist in the Pipeline. + for stage_name in stage_names_to_run: + if stage_name not in list(stage_lookup.keys()): + raise PipelineInitialisationError("You're trying to run a stage that does not exist. Please add the stage to the Pipeline.") + + return [stage_lookup[name] for name in stage_names_to_run] @classmethod def from_files( From 3a1bb29608a66e87b3cebeff5ce9abefb3a763da Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 14:09:40 +0100 Subject: [PATCH 129/332] tweak: visual changes --- onsrap/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onsrap/models.py b/onsrap/models.py index 9d34a3f..90350d2 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -326,7 +326,11 @@ def _extract_stages_run(payload: Mapping[str, Any] if stages_to_run is None: return None - boolean_dict = {stage_name: PipelineConfig._to_bool(value) for stage_name,value in stages_to_run.items()} + boolean_dict = { + stage_name: PipelineConfig._to_bool(value) + for stage_name, value in stages_to_run.items() + } + return boolean_dict @staticmethod From d748fd4405d5d4f4b8df691453433212a3ab928c Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 14:57:08 +0100 Subject: [PATCH 130/332] tweak: indentation --- examples/pipeline_2/conf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index ab0244c..1386391 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -20,8 +20,8 @@ pipeline_variables: output_dir: examples/pipeline_2/outputs log_dir: examples/pipeline_2/logs metadata: - example: retail-orders-using-configuration - description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." + example: retail-orders-using-configuration + description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." stages_to_run: 0_clean_data: True 1_derive_vars: True From a1508f2e2a9590643089af66bc44423cce347abb Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 16:38:50 +0100 Subject: [PATCH 131/332] feat: Added enabled stage initial structure but with TODOs to handle stages_to_run logic from PipelineConfig --- onsrap/pipeline.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 42aa9d1..2694ab8 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -96,7 +96,7 @@ def __init__( self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() - + self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) self.graph.validate() @@ -110,6 +110,7 @@ def __init__( backend=self.backend, stages=[stage.name for stage in self.stages], ) + def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: """ Adds a step to the Pipeline. @@ -128,9 +129,41 @@ def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[. added_stages = [self._coerce_stage(stage) for stage in stages] self.stages.extend(added_stages) self._sync_stage_configs() - self._rebuild_graph() + self._rebuild_graph() # TODO: This will define the StageGraph from self.stages + # We don't want this behaviour following changes to PipelineConfig.stages_to_run logic. + # See _resolve_stages_to_run() for the sort of approach we want + + # TODO: Adding Stages needs to interface with Pipeline Config's stages to run + # TODO: CARE: adding stages_to_run when stages_to_run is an empty dict changes behaviour: + # Empty dict behaviour defaults to running the entire pipeline + # Adding a stage in stages_to_run will make that one Stage run! + # TODO: If adding a stage, need to verify/check dependencies! self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) + def enable_stage(self, *stage_name: str) -> None: + # TODO: accept lists of strings + if not set(stage_name).issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError("You're trying to enable a stage that does not exist. Please add the stage to the Pipeline.") + + for name in stage_name: + self.config.stages_to_run[name] = True + # TODO: Do something with the StageGraph! + self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) + # TODO: Look at StageGraph.validate() + # TODO: Do something with dependencies! + + def disable_stage(self, *stage_name: str) -> None: + # TODO: accept lists of strings + if not set(stage_name).issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError("You're trying to disable a stage that does not exist. Please add the stage to the Pipeline.") + + for name in stage_name: + self.config.stages_to_run[name] = False + # TODO: Do something with the StageGraph! + self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) + # TODO: Look at StageGraph.validate() + # TODO: Do something with dependencies! + def ordered_stages(self) -> list[Stage]: """ Runs the topological_order() method on the ``graph`` attribute to extract the @@ -177,6 +210,7 @@ def create_stage_config( ``StageConfigurationError`` If the input cannot be resolved to exactly one stage configuration. """ + # TODO: Comment the logical sections here for better readability if isinstance(s_config, Mapping): if "pipeline_variables" in s_config or "stage_configuration" in s_config or "stage_config" in s_config: _, stage_config_payload = self._split_config_sections(s_config) @@ -591,7 +625,7 @@ def _stage_from_config_definition( def _resolve_stages_to_run(self) -> list[Stage]: """ - + # TODO: Document """ stage_lookup = {stage.name: stage for stage in self.stages} From 5a5f2cc87f0f2697c2eac7612f809a7078888d01 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 16:39:07 +0100 Subject: [PATCH 132/332] tweak: removed TODO from Stage --- onsrap/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onsrap/stage.py b/onsrap/stage.py index fccc713..58827dd 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -82,7 +82,6 @@ class Stage: ``StageConfigurationError`` If the stage ``name`` is empty or if the source is not a supported type. """ - #TODO: Do we need a run indicator within the stage? name: str source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) From 56552caedebf23a1c7889855c8ef7f2e94d1788f Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 22 Jul 2026 16:42:54 +0100 Subject: [PATCH 133/332] tweak: resolved comments from PR on returned error message richness. --- onsrap/pipeline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 2694ab8..9632437 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -19,6 +19,7 @@ ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") +AVAILABLE_EXECUTORS = ("python",) class Pipeline: """ @@ -74,7 +75,7 @@ def __init__( if self.backend == "python": self.executor = PythonStageExecutor() else: - raise PipelineInitialisationError("Requested backend does not have a compatible executor. Available executors are: Python.") + raise PipelineInitialisationError(f"Requested backend does not have a compatible executor. Available executors are: {', '.join(AVAILABLE_EXECUTORS)}.") if stages is not None and configured_stages: raise PipelineInitialisationError( @@ -647,7 +648,7 @@ def _resolve_stages_to_run(self) -> list[Stage]: # Check that all stages in stage_names_to_run exist in the Pipeline. for stage_name in stage_names_to_run: if stage_name not in list(stage_lookup.keys()): - raise PipelineInitialisationError("You're trying to run a stage that does not exist. Please add the stage to the Pipeline.") + raise PipelineInitialisationError(f"You're trying to run a stage that does not exist: '{stage_name}'. Please add the stage to the Pipeline.") return [stage_lookup[name] for name in stage_names_to_run] From bdf200beb1116759d88ca0cbc544196e12b71c7b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 09:31:04 +0100 Subject: [PATCH 134/332] tweak: add __str__ method and __repr__ method to Stage class --- onsrap/stage.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/onsrap/stage.py b/onsrap/stage.py index 58827dd..0da31a2 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -105,6 +105,21 @@ def __post_init__(self) -> None: self.metadata = dict(self.metadata or {}) self.backend = str(self.backend or "python").strip() or "python" + def __str__(self) -> str: + return ( + f"Stage Instance Attributes\n" + f"--------------------------\n" + f"Name: {self.name}\nSource: {self.source_label} \n" + f"Dependencies: {self.dependencies}\nMetadata: {self.metadata} \n" + f"Entrypoint: {self.entrypoint} \nBackend: {self.backend} \n)" + ) + + def __repr__(self) -> str: + return ( + f"Stage(name={self.name},source={self.source_label}, " + f"dependencies={self.dependencies},metadata={self.metadata}, " + f"entrypoint={self.entrypoint}, backend={self.backend})" + ) @classmethod def from_file( cls, @@ -151,6 +166,7 @@ def from_file( if not path.exists(): raise StageConfigurationError(f"Stage source file does not exist: {path}") + return cls( name=name or path.stem, source=path.resolve(), From 86626294f7e4e37cb20bae02f33370d09d36d8a8 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 10:17:23 +0100 Subject: [PATCH 135/332] doc: add docstrings for __str__ and __repr__ for stage class --- onsrap/stage.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/onsrap/stage.py b/onsrap/stage.py index 0da31a2..d581656 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -106,6 +106,14 @@ def __post_init__(self) -> None: self.backend = str(self.backend or "python").strip() or "python" def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Stage`` class. + + Returns + ------- + str + A string representation of the ``Stage`` class with its attributes. + """ return ( f"Stage Instance Attributes\n" f"--------------------------\n" @@ -115,11 +123,22 @@ def __str__(self) -> str: ) def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``Stage`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for + debugging purposes. + + Returns + ------- + str + A string representation of the ``Stage`` class with its attributes. + """ return ( f"Stage(name={self.name},source={self.source_label}, " f"dependencies={self.dependencies},metadata={self.metadata}, " f"entrypoint={self.entrypoint}, backend={self.backend})" ) + @classmethod def from_file( cls, From 021d2b56742dcd6010fccb5b697ef1c471c6acf9 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 11:36:13 +0100 Subject: [PATCH 136/332] feat: __str__ and __repr__ functions for Pipeline class --- onsrap/pipeline.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 9632437..04605d0 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -112,6 +112,44 @@ def __init__( stages=[stage.name for stage in self.stages], ) + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Pipeline`` class. + + Returns + ------- + str + A string representation of the ``Pipeline`` class with its attributes. + """ + return ( + f"Pipeline Instance Attributes\n" + f"--------------------------\n" + f"Name: {self.name}\nBackend: {self.backend} \n" + f"Configuration: {self.config}\nStages: {self.stages} \n" + f"Dependencies: {self.dependencies} \nLogger: {self.logger} \n)" + f"Executor: {self.executor} \nGraph: {self.graph} \n" + f"ID: {self.id} \nManifest: {self.manifest} \nLast Run: {self.last_run}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``Pipeline`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for + debugging purposes. + + Returns + ------- + str + A string representation of the ``Pipeline`` class with its attributes. + """ + return ( + f"Pipeline(name={self.name},backend={self.backend}, " + f"stages={self.stages},dependencies={self.dependencies}, " + f"logger={self.logger},executor={self.executor},graph={self.graph}, " + f"id={self.id},manifest={self.manifest},last_run={self.last_run})" + ) + + def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: """ Adds a step to the Pipeline. From 2b0daa2273d155a67890ecedf20b606bfe1b74de Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 11:40:07 +0100 Subject: [PATCH 137/332] feat: __str__ and __repr__ methods for PipelineRunner class actioning issue #33 --- onsrap/pipeline.py | 4 ++-- onsrap/runner.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 04605d0..9af44a3 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -134,8 +134,8 @@ def __str__(self) -> str: def __repr__(self) -> str: """ Representation method that returns a human readable representation of the ``Pipeline`` class. - This method is structured to be more concise than the ``__str__`` method and is intended for - debugging purposes. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. Returns ------- diff --git a/onsrap/runner.py b/onsrap/runner.py index 593a4f5..8aaac45 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -27,6 +27,36 @@ class PipelineRunner: def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``PipelineRunner`` class. + + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return ( + f"PipelineRunner Instance Attributes\n" + f"--------------------------\n" + f"Logger: {self.logger} \n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``PipelineRunner`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return ( + f"PipelineRunner(logger={self.logger})" + ) + def run(self, pipeline: Pipeline) -> PipelineRun: """ Method that runs a ``Pipeline`` instance. From 320f50826ed58dfe62842fee028cf2c16af5b6a6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 13:10:31 +0100 Subject: [PATCH 138/332] feat: add __str__ and __repr__ methods to additional classes used in initialisation of Pipeline class as per issue #37 --- onsrap/logger.py | 32 ++++++++++++++++++ onsrap/models.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 0fd1a31..6f61800 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -82,6 +82,38 @@ def __call__(self, *args: Any, **kwargs: Any) -> None: message = f"{message} | {context}" if message else context self._logger.info(message) + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Logger`` + class. + + Returns + ------- + str + A string representation of the ``Logger`` class with its attributes. + """ + return ( + f"Logger Instance Attributes\n" + f"--------------------------\n" + f"Log Directory: {self.log_dir.resolve()}\n" + f"Log Level: {self.config.log_level}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the + ``Logger`` class. This method is structured to be more concise than + the ``__str__`` method and is intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``Logger`` class with its attributes. + """ + return ( + f"Logger(log_dir={self.log_dir.resolve()}, log_level={self.config.log_level})" + ) + def event(self, message: str, **kwargs: Any) -> None: """ Logs a named event with optional structured context. diff --git a/onsrap/models.py b/onsrap/models.py index 90350d2..12ee1d1 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -102,8 +102,9 @@ class PipelineConfig: ---------- ``name`` : str, optional The name of the pipeline. - - + ``stages_to_run`` : dict[str, bool], optional + A dictionary of all stage names alongside a boolean value that indicates + whether the stage should be run or not. ``backend`` : str, default = "python" The system that the pipeline is run on. ``work_dir`` : Path @@ -146,6 +147,46 @@ def __post_init__(self) -> None: if self.stages_to_run is None: self.stages_to_run = {} + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``PipelineConfig`` class. + + Returns + ------- + str + A string representation of the ``PipelineConfig`` class with its attributes. + """ + return ( + f"PipelineConfig Instance Attributes\n" + f"--------------------------\n" + f"Name: {self.name}\nStages To Run: {self.stages_to_run}\n" + f"Backend: {self.backend} \n" + f"Work Directory: {self.work_dir}\nProject Root: {self.project_root}\n" + f"Output Directory: {self.output_dir}\nLog Directory: {self.log_dir}\n" + f"Data Directory: {self.data_dir}\nAllow Subprocess Fallback: {self.allow_subprocess_fallback}\n" + f"Python Executable: {self.python_executable}\nMetadata: {self.metadata}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``PipelineConfig`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``PipelineConfig`` class with its attributes. + """ + return ( + f"PipelineConfig(name={self.name}, stages_to_run={self.stages_to_run}, " + f"backend={self.backend}, " + f"work_dir={self.work_dir},project_root={self.project_root}, " + f"output_dir={self.output_dir},log_dir={self.log_dir},data_dir={self.data_dir}, " + f"allow_subprocess_fallback={self.allow_subprocess_fallback}, " + f"python_executable={self.python_executable},metadata={self.metadata})" + ) + @classmethod def from_any( cls, @@ -547,6 +588,47 @@ class RunManifest: reason: Optional[str] = None user: Optional[str] = None + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``RunManifest`` + class. + + Returns + ------- + str + A string representation of the ``RunManifest`` class with its attributes. + """ + return ( + f"RunManifest Instance Attributes\n" + f"--------------------------\n" + f"RAP Name: {self.rap_name}\nRun ID: {self.run_id} \n" + f"Git Commit: {self.git_commit}\nStages Run: {self.stages_run} \n" + f"Parameters: {self.parameters} \nInputs: {self.inputs} \n" + f"Outputs: {self.outputs} \n Backend: {self.backend} \n" + f"Package Versions: {self.package_versions} \nTimestamp: {self.timestamp}\n" + f"Reason: {self.reason} \nUser: {self.user}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the + ``RunManifest`` class. This method is structured to be more concise than + the ``__str__`` method and is intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``RunManifest`` class with its attributes. + """ + return ( + f"RunManifest(rap_name={self.rap_name}, run_id={self.run_id}, " + f"git_commit={self.git_commit}, stages_run={self.stages_run}, " + f"parameters={self.parameters}, inputs={self.inputs}, " + f"outputs={self.outputs}, backend = {self.backend}, " + f"package_versions={self.package_versions}, " + f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" + ) + class RAPDataset: def __init__(self): From 58dd76d397c8fe07bbdfda89191864e15124e0e4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 14:44:36 +0100 Subject: [PATCH 139/332] tweak: mild adjustments to all __str__ and __repr__ methods to tidy up. Related to issues #32, #33, #34, and #37 --- onsrap/execution.py | 6 ++++++ onsrap/logger.py | 4 +--- onsrap/models.py | 35 ++++++++++++++++++++++------------- onsrap/pipeline.py | 14 ++++++++------ onsrap/stage.py | 8 +++----- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 6c85d55..aa300bd 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -270,6 +270,12 @@ class PythonStageExecutor: def __init__(self, preferred_entrypoints: tuple[str, ...] = PREFERRED_ENTRYPOINTS): self.preferred_entrypoints = preferred_entrypoints + def __str__(self) -> str: + return f"PythonStageExecutor: \n Preferred Entrypoints: {self.preferred_entrypoints})" + + def __repr__(self) -> str: + return f"PythonStageExecutor(preferred_entrypoints={self.preferred_entrypoints})" + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Main function to select how ``Stage`` is run. diff --git a/onsrap/logger.py b/onsrap/logger.py index 6f61800..ff065f1 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -93,10 +93,8 @@ def __str__(self) -> str: A string representation of the ``Logger`` class with its attributes. """ return ( - f"Logger Instance Attributes\n" - f"--------------------------\n" f"Log Directory: {self.log_dir.resolve()}\n" - f"Log Level: {self.config.log_level}\n" + f" Log Level: {self.config.log_level}" ) def __repr__(self) -> str: diff --git a/onsrap/models.py b/onsrap/models.py index 12ee1d1..4fddf5b 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,5 +1,6 @@ from __future__ import annotations +from textwrap import indent import warnings from dataclasses import dataclass, field from datetime import datetime @@ -157,14 +158,12 @@ def __str__(self) -> str: A string representation of the ``PipelineConfig`` class with its attributes. """ return ( - f"PipelineConfig Instance Attributes\n" - f"--------------------------\n" - f"Name: {self.name}\nStages To Run: {self.stages_to_run}\n" - f"Backend: {self.backend} \n" - f"Work Directory: {self.work_dir}\nProject Root: {self.project_root}\n" - f"Output Directory: {self.output_dir}\nLog Directory: {self.log_dir}\n" - f"Data Directory: {self.data_dir}\nAllow Subprocess Fallback: {self.allow_subprocess_fallback}\n" - f"Python Executable: {self.python_executable}\nMetadata: {self.metadata}\n" + f" Name: {self.name}\n Stages To Run: {_format_dict(self.stages_to_run, indent=4)}\n" + f" Backend: {self.backend} \n" + f" Work Directory: {self.work_dir}\n Project Root: {self.project_root}\n" + f" Output Directory: {self.output_dir}\n Log Directory: {self.log_dir}\n" + f" Data Directory: {self.data_dir}\n Allow Subprocess Fallback: {self.allow_subprocess_fallback}\n" + f" Python Executable: {self.python_executable}\n Metadata: \n{_format_dict(self.metadata, indent=8)}" ) def __repr__(self) -> str: @@ -598,13 +597,13 @@ def __str__(self) -> str: str A string representation of the ``RunManifest`` class with its attributes. """ + return ( - f"RunManifest Instance Attributes\n" - f"--------------------------\n" - f"RAP Name: {self.rap_name}\nRun ID: {self.run_id} \n" + f"\nRAP Name: {self.rap_name}\nRun ID: {self.run_id} \n" f"Git Commit: {self.git_commit}\nStages Run: {self.stages_run} \n" - f"Parameters: {self.parameters} \nInputs: {self.inputs} \n" - f"Outputs: {self.outputs} \n Backend: {self.backend} \n" + f"Parameters: \n{_format_dict(self.parameters, indent=4)} \n" + f"Inputs: \n{_format_dict(self.inputs, indent=4)} \n" + f"Outputs: \n{_format_dict(self.outputs, indent = 4)} \nBackend: {self.backend} \n" f"Package Versions: {self.package_versions} \nTimestamp: {self.timestamp}\n" f"Reason: {self.reason} \nUser: {self.user}\n" ) @@ -752,3 +751,13 @@ def succeeded(self) -> bool: Updates the ``status`` attribute to record that the Pipeline ran successfully. """ return self.status == PipelineStatus.SUCCEEDED + +def _format_dict(d, indent=0): + lines = [] + for key, value in d.items(): + if isinstance(value, dict): + lines.append(f"{' ' * indent}{key}:") + lines.append(_format_dict(value, indent + 4)) + else: + lines.append(f"{' ' * indent}{key}: {value}") + return "\n".join(lines) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 9af44a3..5bb4e0c 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -121,14 +121,16 @@ def __str__(self) -> str: str A string representation of the ``Pipeline`` class with its attributes. """ + stages = "\n".join(f"{str(stage)}\n" for stage in self.stages) + graph = [stage.name for stage in self.graph.stages] return ( - f"Pipeline Instance Attributes\n" + f"\nPipeline Instance Attributes\n" f"--------------------------\n" - f"Name: {self.name}\nBackend: {self.backend} \n" - f"Configuration: {self.config}\nStages: {self.stages} \n" - f"Dependencies: {self.dependencies} \nLogger: {self.logger} \n)" - f"Executor: {self.executor} \nGraph: {self.graph} \n" - f"ID: {self.id} \nManifest: {self.manifest} \nLast Run: {self.last_run}\n" + f"Name:\n {self.name}\n\nBackend:\n {self.backend} \n\n" + f"Configuration:\n{self.config}\n\nStages:\n{stages} \n" + f"Dependencies:\n {self.dependencies} \n\nLogger:\n {self.logger} \n\n" + f"Executor:\n {self.executor} \n\nGraph:\n {graph} \n\n" + f"ID:\n {self.id} \n\nManifest:\n {self.manifest} \n\nLast Run:\n {self.last_run}\n" ) def __repr__(self) -> str: diff --git a/onsrap/stage.py b/onsrap/stage.py index d581656..c6c9649 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -115,11 +115,9 @@ def __str__(self) -> str: A string representation of the ``Stage`` class with its attributes. """ return ( - f"Stage Instance Attributes\n" - f"--------------------------\n" - f"Name: {self.name}\nSource: {self.source_label} \n" - f"Dependencies: {self.dependencies}\nMetadata: {self.metadata} \n" - f"Entrypoint: {self.entrypoint} \nBackend: {self.backend} \n)" + f" Name: {self.name}\n Source: {self.source_label} \n" + f" Dependencies: {self.dependencies}\n Metadata: {self.metadata} \n" + f" Entrypoint: {self.entrypoint} \n Backend: {self.backend}" ) def __repr__(self) -> str: From 679a0e1f28949ff4d2adb9fc65f0a45d51c9ad97 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 16:08:57 +0100 Subject: [PATCH 140/332] test: tests fixed for test_pipeline.py and test_pipeline_architecture.py to account for default errors. Second set of eyes would be useful to ensure that the warnings are appropriately raised. Affects issue #29 --- tests/test_pipeline.py | 48 +++++++----- tests/test_pipeline_architecture.py | 116 +++++++++++++++++----------- 2 files changed, 100 insertions(+), 64 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 84a07f3..4d7d3f8 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,6 +4,8 @@ from pathlib import Path import pytest +from onsrap.warnings import PipelineConfigurationWarning + def test_pipeline_name(): """ Test to confirm that Pipeline instance uses either defined name from @@ -13,11 +15,15 @@ def test_pipeline_name(): PipelineConfig (shown through pipeline_no_name) """ pipeline_config = PipelineConfig(name = "test_pipeline_config") - pipeline_named = Pipeline(name = "test_pipeline_name") + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_named = Pipeline(name = "test_pipeline_name") + pipeline_config = Pipeline(name = None, config = pipeline_config) + pipeline_no_name = Pipeline() + assert pipeline_named.name == "test_pipeline_name" - pipeline_config = Pipeline(name = None, config = pipeline_config) assert pipeline_config.name == "test_pipeline_config" - pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" @@ -43,22 +49,24 @@ def example_function(): with pytest.raises(PipelineInitialisationError): Pipeline(stages = None, dependencies = dependencies_single) - - pipeline_1 = Pipeline(name = "pipeline_1", - stages = [Stage("Stage_1", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_multiple) - + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_1 = Pipeline(name = "pipeline_1", + stages = [Stage("Stage_1", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], + dependencies = dependencies_multiple) + + pipeline_2 = Pipeline(name = "pipeline_2", + stages = [Stage("Stage_1.py", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], + dependencies = dependencies_non_stage_name) + assert pipeline_1.stages[0].dependencies == ("Stage_0",) assert pipeline_1.stages[1].dependencies == ("Stage_1","Stage_0") - pipeline_2 = Pipeline(name = "pipeline_2", - stages = [Stage("Stage_1.py", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_non_stage_name) - assert pipeline_2.stages[0].dependencies == ("Stage_0",) assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) @@ -82,9 +90,11 @@ def test_add_dependencies_single_dict(tmp_path): stage_1 = Stage("Stage_1", source = path_1, dependencies = {}) stage_2 = Stage("Stage_2", source = path_2, dependencies = {}) stage_0 = Stage("Stage_0", source = path_0, dependencies = {}) - - pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], - dependencies = dependencies_multiple) + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], + dependencies = dependencies_multiple) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 131ec9b..f2d3c5d 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -10,6 +10,8 @@ from onsrap.graph import StageGraph from onsrap.pipeline import Pipeline from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning + def test_pipeline_from_files_executes_python_entrypoints(tmp_path: Path) -> None: @@ -37,17 +39,21 @@ def main(context): encoding="utf-8", ) - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {} - }, - ) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {} + }, + ) - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.succeeded is True assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] @@ -71,17 +77,23 @@ def main(context): + "\n", encoding="utf-8", ) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [writer_stage], + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, + ) - pipeline = Pipeline.from_files( - [writer_stage], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, - ) + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): - first_run = pipeline.run() - second_run = pipeline.run() + #TODO: This warning functions however from the name of the test, I would assume that the output + #directory has been set so this needs to be reviewed. + first_run = pipeline.run() + second_run = pipeline.run() first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -98,15 +110,20 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, - ) - run = pipeline.run() + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, + ) + + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.stage_results[0].outputs.strip() == "script fallback works" assert run.stage_results[0].stdout.strip() == "script fallback works" @@ -172,15 +189,16 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_config(config_file) - - #TODO: Fix above line of code. _split_config method is running twice when runs through from_config as it is called - #as part of the __init__ and part of the from_dict() that from_config() calls. Need to review how to normalise. + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.stage_outputs["0_data_validation"] == { "stage_name": "0_data_validation", @@ -202,16 +220,18 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_files( - [stage_file], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": { - "missing_stage": {"years_to_run": 2017}, + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [stage_file], + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": { + "missing_stage": {"years_to_run": 2017}, + }, }, - }, - ) + ) with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() @@ -286,7 +306,9 @@ def run(context): config_file = tmp_path / "conf.yaml" config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" assert pipeline.config.work_dir == tmp_path @@ -370,12 +392,16 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names assert sorted(pipeline.stage_configs) == stage_names - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.manifest.stages_run == stage_names assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names From a5ad358a235e183acd3b88ba0b49be2b6e51486b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 17:14:27 +0100 Subject: [PATCH 141/332] test: add pytest to check that example pipelines 1 and 2 run properly. Copilot utilised - targets issue #26 --- tests/test_pipeline_architecture.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index f2d3c5d..946bf1e 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -5,6 +5,8 @@ import pytest import yaml +import subprocess +import sys from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph @@ -414,4 +416,28 @@ def run(context): assert output["first_stage_ordinal"] == 0 assert output["known_stage_configs"] == stage_names expected_previous = None if index == 0 else index - 1 - assert output["previous_ordinal"] == expected_previous \ No newline at end of file + assert output["previous_ordinal"] == expected_previous + + +REPO_ROOT = Path(__file__).resolve().parents[1] + +MAIN_SCRIPTS = [ + REPO_ROOT / "examples" / "pipeline_1" / "main.py", + REPO_ROOT / "examples" / "pipeline_2" / "main.py", +] + +@pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) +def test_example_main_scripts_run_successfully(script_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"Script failed: {script_path}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert "completed with" in result.stdout.lower() From 47ada5192e847014c519a07f2d9d2731b303a982 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 24 Jul 2026 12:27:46 +0100 Subject: [PATCH 142/332] tests: new tests added for test_execution and test_models --- tests/test_execution.py | 43 ++++++++++++++++++++++++++++++++++------- tests/test_models.py | 7 ++++--- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 613e89a..7320ccf 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,5 +1,5 @@ from onsrap.execution import ExecutionContext, PythonStageExecutor -from onsrap.models import PipelineConfig, StageResult, StageStatus +from onsrap.models import PipelineConfig, StageConfig, StageResult, StageStatus from onsrap.logger import Logger from pathlib import Path import pytest @@ -24,8 +24,8 @@ def config() -> PipelineConfig: data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", + {"test_stage":True}, "python", - None, work_dir, project_root, None, @@ -37,7 +37,7 @@ def config() -> PipelineConfig: ) @pytest.fixture -def execution(config, logger, stageresult) -> ExecutionContext: +def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ @@ -52,8 +52,10 @@ def execution(config, logger, stageresult) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {"stage_test":stageresult}, - {} + {"test_stage":stageresult}, + {"test_stage":stage_config}, + {}, + None ) @pytest.fixture @@ -129,9 +131,9 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -def test_resolve_data_root(execution) -> None: +def test_get_data_dir(execution) -> None: """ - Tests that resolve_data_root method extracts the path from the execution context + Tests that get_data_dir method extracts the path from the execution context or, if the context is None, returns an error to indicate that additional input is required. """ @@ -261,3 +263,30 @@ def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") """CONTINUE FROM EXECUTE CLASS METHOD""" +#TODO: add tests for set_active_stage, stage_config_for, stage_config, and get_stage_config + + +@pytest.fixture +def stage_config() -> StageConfig: + """ + Return a StageConfig object for testing. + """ + return StageConfig( + name="test_stage", + _variables={"sex":"gender", + "dob":"date_of_birth"}, + datasets={}, + metadata={} + ) + +def test_set_active_stage(execution, stage_config) -> None: + """ + Tests that set_active_stage correctly sets the active_stage attribute in the + ExecutionContext instance. + """ + + execution.set_active_stage(stage_config.name) + assert execution.active_stage_name == stage_config.name + execution.set_active_stage(None) + assert execution.active_stage_name == None + diff --git a/tests/test_models.py b/tests/test_models.py index c0f44ee..7e7d0b4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -123,8 +123,6 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig) -> None: with pytest.raises(TypeError): blankpipelineconfig.from_any(11) -"""NOT SURE HOW TO TEST FROM_FILE()""" - def test_from_file(tmp_path,) -> PipelineConfig: pipeline_config = tmp_path / "configuration.py" pipeline_config.write_text( @@ -278,4 +276,7 @@ def test_result_for(pipelinerun, stageresult) -> None: def test_succeeded_pipeline(pipelinerun, status, expected) -> None: pipelinerun.status = status - assert pipelinerun.succeeded == expected \ No newline at end of file + assert pipelinerun.succeeded == expected + + +#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file From 7272250266d63dbc078d14ab59a0e59c08d14f54 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 16:08:57 +0100 Subject: [PATCH 143/332] test: tests fixed for test_pipeline.py and test_pipeline_architecture.py to account for default errors. Second set of eyes would be useful to ensure that the warnings are appropriately raised. Affects issue #29 --- tests/test_pipeline.py | 48 +++++++----- tests/test_pipeline_architecture.py | 116 +++++++++++++++++----------- 2 files changed, 100 insertions(+), 64 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 84a07f3..4d7d3f8 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,6 +4,8 @@ from pathlib import Path import pytest +from onsrap.warnings import PipelineConfigurationWarning + def test_pipeline_name(): """ Test to confirm that Pipeline instance uses either defined name from @@ -13,11 +15,15 @@ def test_pipeline_name(): PipelineConfig (shown through pipeline_no_name) """ pipeline_config = PipelineConfig(name = "test_pipeline_config") - pipeline_named = Pipeline(name = "test_pipeline_name") + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_named = Pipeline(name = "test_pipeline_name") + pipeline_config = Pipeline(name = None, config = pipeline_config) + pipeline_no_name = Pipeline() + assert pipeline_named.name == "test_pipeline_name" - pipeline_config = Pipeline(name = None, config = pipeline_config) assert pipeline_config.name == "test_pipeline_config" - pipeline_no_name = Pipeline() assert pipeline_no_name.name == "pipeline" @@ -43,22 +49,24 @@ def example_function(): with pytest.raises(PipelineInitialisationError): Pipeline(stages = None, dependencies = dependencies_single) - - pipeline_1 = Pipeline(name = "pipeline_1", - stages = [Stage("Stage_1", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_multiple) - + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_1 = Pipeline(name = "pipeline_1", + stages = [Stage("Stage_1", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], + dependencies = dependencies_multiple) + + pipeline_2 = Pipeline(name = "pipeline_2", + stages = [Stage("Stage_1.py", path_1, None,{}), + Stage("Stage_2", example_function, None,{}), + Stage("Stage_0", path_0, None,{}),], + dependencies = dependencies_non_stage_name) + assert pipeline_1.stages[0].dependencies == ("Stage_0",) assert pipeline_1.stages[1].dependencies == ("Stage_1","Stage_0") - pipeline_2 = Pipeline(name = "pipeline_2", - stages = [Stage("Stage_1.py", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_non_stage_name) - assert pipeline_2.stages[0].dependencies == ("Stage_0",) assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) @@ -82,9 +90,11 @@ def test_add_dependencies_single_dict(tmp_path): stage_1 = Stage("Stage_1", source = path_1, dependencies = {}) stage_2 = Stage("Stage_2", source = path_2, dependencies = {}) stage_0 = Stage("Stage_0", source = path_0, dependencies = {}) - - pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], - dependencies = dependencies_multiple) + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], + dependencies = dependencies_multiple) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 131ec9b..f2d3c5d 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -10,6 +10,8 @@ from onsrap.graph import StageGraph from onsrap.pipeline import Pipeline from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning + def test_pipeline_from_files_executes_python_entrypoints(tmp_path: Path) -> None: @@ -37,17 +39,21 @@ def main(context): encoding="utf-8", ) - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {} - }, - ) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {} + }, + ) - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.succeeded is True assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] @@ -71,17 +77,23 @@ def main(context): + "\n", encoding="utf-8", ) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [writer_stage], + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, + ) - pipeline = Pipeline.from_files( - [writer_stage], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, - ) + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): - first_run = pipeline.run() - second_run = pipeline.run() + #TODO: This warning functions however from the name of the test, I would assume that the output + #directory has been set so this needs to be reviewed. + first_run = pipeline.run() + second_run = pipeline.run() first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -98,15 +110,20 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, - ) - run = pipeline.run() + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": {}}, + ) + + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.stage_results[0].outputs.strip() == "script fallback works" assert run.stage_results[0].stdout.strip() == "script fallback works" @@ -172,15 +189,16 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_config(config_file) - - #TODO: Fix above line of code. _split_config method is running twice when runs through from_config as it is called - #as part of the __init__ and part of the from_dict() that from_config() calls. Need to review how to normalise. + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.stage_outputs["0_data_validation"] == { "stage_name": "0_data_validation", @@ -202,16 +220,18 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_files( - [stage_file], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": { - "missing_stage": {"years_to_run": 2017}, + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [stage_file], + config={"pipeline_config":{"work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs"}, + "stage_configuration": { + "missing_stage": {"years_to_run": 2017}, + }, }, - }, - ) + ) with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() @@ -286,7 +306,9 @@ def run(context): config_file = tmp_path / "conf.yaml" config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" assert pipeline.config.work_dir == tmp_path @@ -370,12 +392,16 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names assert sorted(pipeline.stage_configs) == stage_names - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() assert run.manifest.stages_run == stage_names assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names From b4f1ef00cb8c81a39f18dfe93d610400d1f0aa4e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 23 Jul 2026 17:14:27 +0100 Subject: [PATCH 144/332] test: add pytest to check that example pipelines 1 and 2 run properly. Copilot utilised - targets issue #26 --- tests/test_pipeline_architecture.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index f2d3c5d..946bf1e 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -5,6 +5,8 @@ import pytest import yaml +import subprocess +import sys from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph @@ -414,4 +416,28 @@ def run(context): assert output["first_stage_ordinal"] == 0 assert output["known_stage_configs"] == stage_names expected_previous = None if index == 0 else index - 1 - assert output["previous_ordinal"] == expected_previous \ No newline at end of file + assert output["previous_ordinal"] == expected_previous + + +REPO_ROOT = Path(__file__).resolve().parents[1] + +MAIN_SCRIPTS = [ + REPO_ROOT / "examples" / "pipeline_1" / "main.py", + REPO_ROOT / "examples" / "pipeline_2" / "main.py", +] + +@pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) +def test_example_main_scripts_run_successfully(script_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"Script failed: {script_path}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert "completed with" in result.stdout.lower() From 3e28b3279be32eca9109d56e75a8c6cc21360605 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 24 Jul 2026 12:27:46 +0100 Subject: [PATCH 145/332] tests: new tests added for test_execution and test_models --- tests/test_execution.py | 43 ++++++++++++++++++++++++++++++++++------- tests/test_models.py | 7 ++++--- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 613e89a..7320ccf 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,5 +1,5 @@ from onsrap.execution import ExecutionContext, PythonStageExecutor -from onsrap.models import PipelineConfig, StageResult, StageStatus +from onsrap.models import PipelineConfig, StageConfig, StageResult, StageStatus from onsrap.logger import Logger from pathlib import Path import pytest @@ -24,8 +24,8 @@ def config() -> PipelineConfig: data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", + {"test_stage":True}, "python", - None, work_dir, project_root, None, @@ -37,7 +37,7 @@ def config() -> PipelineConfig: ) @pytest.fixture -def execution(config, logger, stageresult) -> ExecutionContext: +def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ @@ -52,8 +52,10 @@ def execution(config, logger, stageresult) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {"stage_test":stageresult}, - {} + {"test_stage":stageresult}, + {"test_stage":stage_config}, + {}, + None ) @pytest.fixture @@ -129,9 +131,9 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -def test_resolve_data_root(execution) -> None: +def test_get_data_dir(execution) -> None: """ - Tests that resolve_data_root method extracts the path from the execution context + Tests that get_data_dir method extracts the path from the execution context or, if the context is None, returns an error to indicate that additional input is required. """ @@ -261,3 +263,30 @@ def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") """CONTINUE FROM EXECUTE CLASS METHOD""" +#TODO: add tests for set_active_stage, stage_config_for, stage_config, and get_stage_config + + +@pytest.fixture +def stage_config() -> StageConfig: + """ + Return a StageConfig object for testing. + """ + return StageConfig( + name="test_stage", + _variables={"sex":"gender", + "dob":"date_of_birth"}, + datasets={}, + metadata={} + ) + +def test_set_active_stage(execution, stage_config) -> None: + """ + Tests that set_active_stage correctly sets the active_stage attribute in the + ExecutionContext instance. + """ + + execution.set_active_stage(stage_config.name) + assert execution.active_stage_name == stage_config.name + execution.set_active_stage(None) + assert execution.active_stage_name == None + diff --git a/tests/test_models.py b/tests/test_models.py index c0f44ee..7e7d0b4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -123,8 +123,6 @@ def test_from_any(mapping, pipelineconfig, blankpipelineconfig) -> None: with pytest.raises(TypeError): blankpipelineconfig.from_any(11) -"""NOT SURE HOW TO TEST FROM_FILE()""" - def test_from_file(tmp_path,) -> PipelineConfig: pipeline_config = tmp_path / "configuration.py" pipeline_config.write_text( @@ -278,4 +276,7 @@ def test_result_for(pipelinerun, stageresult) -> None: def test_succeeded_pipeline(pipelinerun, status, expected) -> None: pipelinerun.status = status - assert pipelinerun.succeeded == expected \ No newline at end of file + assert pipelinerun.succeeded == expected + + +#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file From 951f0e7de90a0d7f11f849c742ec306725af5106 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 10:18:02 +0100 Subject: [PATCH 146/332] fix: Tweaked tests to resolve issues with KeyErrors making assertions fail. --- onsrap/execution.py | 11 +++++++---- tests/test_execution.py | 40 ++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index aa300bd..4d6dabe 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -169,7 +169,7 @@ def resolve_output_root(self) -> Path: raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") - def get_stage_config(self, vars_only: bool = True) -> dict | StageConfig: + def get_stage_config(self, vars_only: bool = True) -> dict[str, Any] | StageConfig: """ Returns the configuration for the stage currently being executed, with optional arguments. @@ -185,13 +185,16 @@ def get_stage_config(self, vars_only: bool = True) -> dict | StageConfig: Returns ------- - dict or StageConfig + dict[str, Any] or StageConfig The parameters contained within the configuration for the currently active stage. If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. """ + stage_config = self.stage_config + if stage_config is None: + return {} if vars_only: - return self.stage_config_for(self.active_stage_name).variables() - return self.stage_config_for(self.active_stage_name) or {} + return stage_config.variables + return stage_config def resolve_given_path(self, stage_name: str | None, path_name: str | None, diff --git a/tests/test_execution.py b/tests/test_execution.py index 7320ccf..d56bdad 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -24,7 +24,7 @@ def config() -> PipelineConfig: data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", - {"test_stage":True}, + {"stage_test":True}, "python", work_dir, project_root, @@ -52,8 +52,8 @@ def execution(config, logger, stageresult, stage_config) -> ExecutionContext: run_dir, '2024-05-06 15:45:30', work_dir, - {"test_stage":stageresult}, - {"test_stage":stage_config}, + {"stage_test":stageresult}, + {"stage_test":stage_config}, {}, None ) @@ -131,7 +131,7 @@ def test_stage_outputs(execution, stageresult) -> None: execution.record(stageresult) assert execution.stage_outputs == {"stage_test":"example output"} -def test_get_data_dir(execution) -> None: +def test_get_data_dir(execution, stageresult) -> None: """ Tests that get_data_dir method extracts the path from the execution context or, if the context is None, returns an error to indicate that additional input is @@ -263,16 +263,13 @@ def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") """CONTINUE FROM EXECUTE CLASS METHOD""" -#TODO: add tests for set_active_stage, stage_config_for, stage_config, and get_stage_config - - @pytest.fixture def stage_config() -> StageConfig: """ Return a StageConfig object for testing. """ return StageConfig( - name="test_stage", + name="stage_test", _variables={"sex":"gender", "dob":"date_of_birth"}, datasets={}, @@ -290,3 +287,30 @@ def test_set_active_stage(execution, stage_config) -> None: execution.set_active_stage(None) assert execution.active_stage_name == None +def test_stage_config_for(execution, stage_config) -> None: + """ + Tests that stage_config_for returns the StageConfig for a named stage. + """ + assert execution.stage_config_for(stage_config.name) == stage_config + assert execution.stage_config_for("missing_stage") is None + +def test_stage_config(execution, stage_config) -> None: + """ + Tests that stage_config exposes the currently active stage configuration. + """ + assert execution.stage_config is None + execution.set_active_stage(stage_config.name) + assert execution.stage_config == stage_config + +def test_get_stage_config(execution, stage_config) -> None: + """ + Tests that get_stage_config returns variables by default and the full + StageConfig object when requested. + """ + assert execution.get_stage_config() == {} + assert execution.get_stage_config(vars_only=False) == {} + + execution.set_active_stage(stage_config.name) + assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} + assert execution.get_stage_config(vars_only=False) == stage_config + From 3d7648465fc573a17847018107a2593b6d0e3f06 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:12:35 +0100 Subject: [PATCH 147/332] Added space for a local playground to dynamically test systems --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 87a11f2..bc8de98 100644 --- a/.gitignore +++ b/.gitignore @@ -909,4 +909,7 @@ docs/_linkcheck/ !examples/**/*.xls !examples/**/*.xlsx -examples/**/runs/**/ \ No newline at end of file +examples/**/runs/**/ + +# Ignore coding playground +tests/playground/ \ No newline at end of file From cd7f694fd5bab97e44e91bf6505fc3d555f9b860 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:13:22 +0100 Subject: [PATCH 148/332] Added simple context injection for example 2 (doesn't currently work due to ExecutionContext getting issue) --- examples/pipeline_2/scripts/0_clean_data.py | 27 +++++++------------- examples/pipeline_2/scripts/1_derive_vars.py | 13 +++++----- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py index 15997df..96a0af6 100644 --- a/examples/pipeline_2/scripts/0_clean_data.py +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -32,27 +32,18 @@ def standardise_columns(df): -def main(): - orders = pd.read_csv("examples/pipeline_2/data/orders.csv") - - expected_variables = ["order_id", - "customer_name", - "region", - "product", - "quantity", - "unit_price", - "order_date", - "order_method"] - - identifiable_cols = ["customer_name", - "age", - "dob", - "address"] +def main(context=None): + config = context.get_stage_config("0_clean_data") + + orders = pd.read_csv(config["input_location"]) + + expected_variables = config["expected_variables"] + identifiable_cols = config["identifiable_cols"] - check_variables(orders,expected_variables) + check_variables(orders, expected_variables) print(orders.dtypes) orders = remove_identifiable(orders, identifiable_cols) orders = standardise_columns(orders) - orders.to_csv("examples/pipeline_2/data/orders_cleaned.csv", index = False) + orders.to_csv(config["output_location"], index = False) main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py index 9673afe..d78cafd 100644 --- a/examples/pipeline_2/scripts/1_derive_vars.py +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -64,12 +64,11 @@ def profit_per_order(df): -def main(): - df = pd.read_csv("examples/pipeline_2/data/orders_cleaned.csv") - delivery_times = {"north":14, - "south":4, - "east":7, - "west":7} +def main(context=None): + config = context.get_stage_config("1_derive_vars") + + df = pd.read_csv(config["input_location"]) + delivery_times = config["delivery_times"] df = correct_date_time(df) df = estimate_delivery(df, delivery_times) @@ -79,7 +78,7 @@ def main(): df = postage_cost(df) df = production_cost(df) df = profit_per_order(df) - df.to_csv("examples/pipeline_2/data/orders_prepped.csv", index = False) + df.to_csv(config["output_location"], index = False) From 08a42de3d6a09cc1d3d3031a99833f9176c2a0f2 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:13:44 +0100 Subject: [PATCH 149/332] Added global variables section for example 2 config for later implementation of global variables for Stages --- examples/pipeline_2/conf.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index 1386391..5ae8374 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -26,6 +26,9 @@ pipeline_variables: 0_clean_data: True 1_derive_vars: True 2_reporting: True + +global_variables: + year_of_run: 2020 stage_configuration: 0_clean_data: From e38130f74131607f9494954c65f18b16166e4f34 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:15:08 +0100 Subject: [PATCH 150/332] feat: Initial implementation of dynamic Stage adjustment in Pipeline altering StageGraph and runtime behaviour with tests --- onsrap/pipeline.py | 409 +++++++++++++++++++++++++++++++++-------- tests/test_pipeline.py | 151 ++++++++++++++- 2 files changed, 478 insertions(+), 82 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 5bb4e0c..f1d28c2 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -98,9 +98,7 @@ def __init__( self.stage_configs = dict(resolved_stage_configs) self._sync_stage_configs() - self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) - - self.graph.validate() + self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None self.last_run: PipelineRun | None = None @@ -110,6 +108,7 @@ def __init__( name=self.name, backend=self.backend, stages=[stage.name for stage in self.stages], + enabled_stages=[stage.name for stage in self.graph.stages], ) def __str__(self) -> str: @@ -151,74 +150,197 @@ def __repr__(self) -> str: f"id={self.id},manifest={self.manifest},last_run={self.last_run})" ) + def add_stage( + self, + *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], + stage_configs: StageConfig | Mapping[str, Any] | str | Path | Iterable[StageConfig | Mapping[str, Any] | str | Path] | None = None, + ) -> None: + """ + Adds one or more steps to the Pipeline. + + Creates a list called ``added_stages`` that runs the _coerce_stage() method + to extract the information from the given ``stages`` parameter. It then appends + this list to the ``stages`` attribute of the ``Pipeline`` class, adds any stage + configuration that was provided alongside those stages, and updates the StageGraph + using the _rebuild_graph() method. - def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: - """ - Adds a step to the Pipeline. - - Creates a list called ``added_stages`` that runs the _coerce_stage() method - to extract the information from the given ``stages`` parameter. It then appends - this list to the ``stages`` attribute of the ``Pipeline`` class and updates the - StageGraph using the _rebuild_graph() method. A log instance is created to - reflect the changes. - - Parameters - ---------- - ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] - The new steps being added to the Pipeline. - """ - added_stages = [self._coerce_stage(stage) for stage in stages] - self.stages.extend(added_stages) - self._sync_stage_configs() - self._rebuild_graph() # TODO: This will define the StageGraph from self.stages - # We don't want this behaviour following changes to PipelineConfig.stages_to_run logic. - # See _resolve_stages_to_run() for the sort of approach we want - - # TODO: Adding Stages needs to interface with Pipeline Config's stages to run - # TODO: CARE: adding stages_to_run when stages_to_run is an empty dict changes behaviour: - # Empty dict behaviour defaults to running the entire pipeline - # Adding a stage in stages_to_run will make that one Stage run! - # TODO: If adding a stage, need to verify/check dependencies! - self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) + Parameters + ---------- + ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The new steps being added to the Pipeline. + ``stage_configs`` : StageConfig | Mapping[str, Any] | str | Path | Iterable[StageConfig | Mapping[str, Any] | str | Path] | None + Optional stage configuration payloads to add alongside the stages. + """ + if not stages: + warnings.warn( + "No stages provided to add_stage(). No changes made to the Pipeline.", + PipelineConfigurationWarning, + ) + return + + added_stages = [self._coerce_stage(stage) for stage in stages] + parsed_stage_configs: list[StageConfig] = [] + + if stage_configs is not None: + if isinstance(stage_configs, Mapping) and all(isinstance(value, Mapping) for value in stage_configs.values()): + raw_stage_configs = [ + {str(stage_name): stage_payload} + for stage_name, stage_payload in stage_configs.items() + ] + elif isinstance(stage_configs, (StageConfig, Mapping, str, Path)): + raw_stage_configs = [stage_configs] + else: + raw_stage_configs = list(stage_configs) + + if len(raw_stage_configs) != len(added_stages): + warnings.warn( + "The number of stage configurations passed to add_stage() does not match the number of stages. " + f"Received {len(raw_stage_configs)} stage configuration(s) for {len(added_stages)} stage(s).", + StageConfigurationWarning, + ) + + known_stage_names = {stage.name for stage in self.stages} + known_stage_names.update(stage.name for stage in added_stages) + + for index, raw_stage_config in enumerate(raw_stage_configs): + stage_name = added_stages[index].name if index < len(added_stages) else None + parsed_stage_config = self._coerce_stage_config(raw_stage_config, name=stage_name) + if parsed_stage_config.name not in known_stage_names: + raise StageConfigurationError( + f"Stage configuration was provided for unknown stage: {parsed_stage_config.name}." + ) + parsed_stage_configs.append(parsed_stage_config) + + self.stages.extend(added_stages) + + for conf in parsed_stage_configs: + self.add_stage_config(conf) + + self._register_added_stages_in_stage_selection(added_stages) + self._check_stage_configs(added_stages, self.stage_configs) + + self._sync_stage_configs() + self._rebuild_graph() + + self.logger.event( + "Stage added", + stages=[stage.name for stage in added_stages], + stage_configs=[stage_config.name for stage_config in parsed_stage_configs], + ) + + def add_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> None: + """ + Add or replace a ``StageConfig`` attached to the Pipeline. + + Parameters + ---------- + ``stage_config`` : StageConfig | Mapping[str, Any] | str | Path + The stage configuration information to add to the Pipeline. + ``name`` : str or None, keyword-only + Optional stage name used when the parsed configuration payload does not + identify the stage on its own. + """ + parsed_stage_config = self._coerce_stage_config(stage_config, name=name) + self.stage_configs[parsed_stage_config.name] = parsed_stage_config + self.logger.event("Stage configuration added", stage=parsed_stage_config.name) - def enable_stage(self, *stage_name: str) -> None: - # TODO: accept lists of strings - if not set(stage_name).issubset({stage.name for stage in self.stages}): - raise PipelineInitialisationError("You're trying to enable a stage that does not exist. Please add the stage to the Pipeline.") - + def enable_stage(self, *stage_name: str | list[str]) -> None: + """ + Mark one or more stages as enabled in the run selection. + + In implicit "run all" mode (``stages_to_run`` is empty), this is a no-op + because every registered stage already participates in the execution graph. + In explicit mode the requested stages are marked ``True`` in + ``stages_to_run`` and the execution graph is rebuilt to reflect the change. + + Parameters + ---------- + ``stage_name`` : str or list[str] + One or more stage names to enable. + """ + stage_names = set() for name in stage_name: + if isinstance(name, list): + stage_names.update(name) + else: + stage_names.add(name) + + if not stage_names.issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError("You're trying to enable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline.") + + if not self.config.stages_to_run: + return + + for name in stage_names: self.config.stages_to_run[name] = True - # TODO: Do something with the StageGraph! - self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) - # TODO: Look at StageGraph.validate() - # TODO: Do something with dependencies! + self._rebuild_graph() + self.logger.event("Stages enabled", stages=sorted(stage_names)) + + def disable_stage(self, *stage_name: str | list[str]) -> None: + """ + Mark one or more stages as disabled in the run selection. - def disable_stage(self, *stage_name: str) -> None: - # TODO: accept lists of strings - if not set(stage_name).issubset({stage.name for stage in self.stages}): - raise PipelineInitialisationError("You're trying to disable a stage that does not exist. Please add the stage to the Pipeline.") + When in implicit "run all" mode (``stages_to_run`` is empty), calling + ``disable_stage`` switches the pipeline into explicit stage-selection mode: + every currently registered stage is first marked enabled, then the requested + stages are set to ``False``. The execution graph is rebuilt after the change. + Parameters + ---------- + ``stage_name`` : str or list[str] + One or more stage names to disable. + """ + stage_names = set() for name in stage_name: + if isinstance(name, list): + stage_names.update(name) + else: + stage_names.add(name) + + if not stage_names.issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError("You're trying to disable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline.") + + if not self.config.stages_to_run: + self.config.stages_to_run = {stage.name: True for stage in self.stages} + + for name in stage_names: self.config.stages_to_run[name] = False - # TODO: Do something with the StageGraph! - self.graph = StageGraph.from_stages(self._resolve_stages_to_run()) - # TODO: Look at StageGraph.validate() - # TODO: Do something with dependencies! + self._rebuild_graph() + self.logger.event("Stages disabled", stages=sorted(stage_names)) def ordered_stages(self) -> list[Stage]: - """ - Runs the topological_order() method on the ``graph`` attribute to extract the - correct order for the ``stages`` to be run in. - """ - return self.graph.topological_order() + """ + Return the effective stages in dependency-respecting execution order. + + This is the primary method used by ``PipelineRunner`` to determine what to + execute. Only stages that are part of the current execution graph appear + here; stages disabled via ``PipelineConfig.stages_to_run`` are absent even + if they are registered in ``Pipeline.stages``. + """ + return self.graph.topological_order() def validate(self) -> Pipeline: """ - Confirms that the source files for the stage exist. + Confirm that the pipeline is ready to run. + + Validates source files for every stage in the current execution graph, + checks that all stage-configuration names correspond to a known stage, and + validates the execution graph for structural consistency. Disabled stages + are excluded from source-file validation because they will not be executed. """ - self.logger.event("Validating pipeline", name=self.name) + self.logger.event( + "Validating pipeline", + name=self.name, + stages=len(self.stages), + enabled_stages=len(self.graph.stages), + ) self._validate_stage_configs() - for stage in self.stages: + for stage in self.graph.stages: stage.validate() self.graph.validate() return self @@ -399,11 +521,69 @@ def _coerce_stage( raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") + def _coerce_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> StageConfig: + """ + Normalize supported stage-configuration inputs into a ``StageConfig``. + """ + if isinstance(stage_config, StageConfig): + if name is not None and stage_config.name != name: + raise StageConfigurationError( + f"Stage configuration '{stage_config.name}' does not match stage '{name}'." + ) + return stage_config + + if isinstance(stage_config, Mapping): + if name is not None and all(isinstance(value, Mapping) for value in stage_config.values()) and name not in stage_config: + available_stage_names = ", ".join(str(stage_name) for stage_name in stage_config) + raise StageConfigurationError( + f"Stage configuration '{name}' was not found. Available stage configurations are: {available_stage_names}." + ) + return self.create_stage_config(stage_config, name=name) + + if isinstance(stage_config, (str, Path)): + return self.create_stage_config(stage_config, name=name) + + raise StageConfigurationError(f"Unsupported stage configuration specification: {type(stage_config)!r}.") + def _rebuild_graph(self) -> None: """ - Updates the ``graph`` attribute with the latest stage information. + Update the execution graph while validating the full pipeline definition. + + The pipeline keeps ``self.stages`` as the complete stage registry, but + ``self.graph`` represents the effective run set after applying + ``PipelineConfig.stages_to_run`` and expanding any selected stage's + dependencies. """ - self.graph = StageGraph.from_stages(self.stages) + full_graph = StageGraph.from_stages(self.stages) + full_graph.validate() + + stages_to_run = self._resolve_stages_to_run() + if [stage.name for stage in stages_to_run] == [stage.name for stage in self.stages]: + self.graph = full_graph + return + + self.graph = StageGraph.from_stages(stages_to_run) + self.graph.validate() + + def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage]) -> None: + """ + Default newly added stages to disabled once explicit stage selection is in use. + + When ``stages_to_run`` is empty, the pipeline is in implicit "run all" + mode and new stages should immediately participate in the execution graph. + Once the configuration has switched to an explicit stage-selection mapping, + newly added stages stay out of the execution graph until they are enabled. + """ + if not self.config.stages_to_run: + return + + for stage in stages: + self.config.stages_to_run.setdefault(stage.name, False) def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: @@ -423,7 +603,7 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: git_commit=self._discover_git_commit(), stages_run=[], parameters=self._manifest_parameters(), - inputs={stage.name: list(stage.dependencies) for stage in self.stages}, + inputs={stage.name: list(stage.dependencies) for stage in self.graph.stages}, outputs={}, backend=self.backend, package_versions=self._package_versions(), @@ -666,31 +846,80 @@ def _stage_from_config_definition( def _resolve_stages_to_run(self) -> list[Stage]: """ - # TODO: Document + Resolve the effective stage subset that should populate the execution graph. + + An empty ``stages_to_run`` mapping means the pipeline runs all known stages. + Otherwise, stages explicitly marked ``True`` are selected and their transitive + dependencies are pulled in automatically. A dependency that is explicitly + disabled in ``stages_to_run`` while another enabled stage requires it raises + a ``PipelineConfigurationError``. """ - stage_lookup = {stage.name: stage - for stage in self.stages} - - if self.config.stages_to_run == {}: - warnings.warn("No stages specified to run. All stages running by default.", PipelineConfigurationWarning) - stage_names_to_run = list(stage_lookup.keys()) + stage_lookup = {stage.name: stage for stage in self.stages} + configured_stages_to_run = dict(self.config.stages_to_run or {}) - else: - stage_names_to_run = [ - stage_name - for stage_name, value in self.config.stages_to_run.items() - if value - ] - - if set(stage_lookup.keys()) == set(stage_names_to_run): + # When PipelineConfig.stages_to_run is empty, the pipeline is in implicit "run all" mode. + if not configured_stages_to_run: return self.stages - else: - # Check that all stages in stage_names_to_run exist in the Pipeline. - for stage_name in stage_names_to_run: - if stage_name not in list(stage_lookup.keys()): - raise PipelineInitialisationError(f"You're trying to run a stage that does not exist: '{stage_name}'. Please add the stage to the Pipeline.") - return [stage_lookup[name] for name in stage_names_to_run] + unknown_stage_names = sorted( + stage_name + for stage_name in configured_stages_to_run + if stage_name not in stage_lookup + ) + if unknown_stage_names: + missing = ", ".join(unknown_stage_names) + raise PipelineInitialisationError( + f"Pipeline configuration references unknown stages in stages_to_run: {missing}." + ) + + explicitly_enabled = [ + stage_name + for stage_name, value in configured_stages_to_run.items() + if value + ] + if not explicitly_enabled: + return [] + + explicitly_disabled = { + stage_name + for stage_name, value in configured_stages_to_run.items() + if not value + } + resolved_stage_names: set[str] = set() + visiting: set[str] = set() + + def add_stage_with_dependencies(stage_name: str, *, required_by: str | None = None) -> None: + """ + Add one selected stage and recursively include everything it depends on. + + ``_resolve_stages_to_run()`` uses this helper to turn the user-facing + ``stages_to_run`` selection into a runnable execution set for the + ``StageGraph``. It also guards against invalid configurations where an + enabled stage depends on a stage that has been explicitly disabled. + """ + if stage_name in resolved_stage_names: + return + if required_by is not None and stage_name in explicitly_disabled: + raise PipelineConfigurationError( + f"Stage '{required_by}' is enabled but depends on disabled stage '{stage_name}'." + ) + if stage_name in visiting: + return + if stage_name not in stage_lookup: + raise PipelineInitialisationError( + f"You're trying to run a stage that does not exist: '{stage_name}'. Please add the stage to the Pipeline." + ) + + visiting.add(stage_name) + resolved_stage_names.add(stage_name) + for dependency_name in stage_lookup[stage_name].dependencies: + add_stage_with_dependencies(dependency_name, required_by=stage_name) + visiting.remove(stage_name) + + for stage_name in explicitly_enabled: + add_stage_with_dependencies(stage_name) + + return [stage for stage in self.stages if stage.name in resolved_stage_names] @classmethod def from_files( @@ -1089,3 +1318,23 @@ def _dependencies_for_stage( return tuple(str(dependency) for dependency in dependencies[candidate]) return () + + @staticmethod + def _check_stage_configs(stages: list[Stage], stage_configs: Mapping[str, StageConfig]) -> None: + """ + Check that all stages have a corresponding stage configuration. + + Warns + ------ + ``StageConfigurationWarning`` + If any stage does not have a corresponding stage configuration. Handled in one warning instance for all stages without a configuration. + """ + stage_no_config = [] + for stage in stages: + if stage.name not in stage_configs: + stage_no_config.append(stage.name) + if stage_no_config: + warnings.warn( + f"Stage(s) {', '.join(stage_no_config)} added to Pipeline without a corresponding StageConfig. ", + StageConfigurationWarning, + ) \ No newline at end of file diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 4d7d3f8..8377229 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,6 +1,8 @@ from onsrap.pipeline import Pipeline, PipelineConfig -from onsrap.errors import PipelineInitialisationError +from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError +from onsrap.models import StageConfig from onsrap.stage import Stage +from onsrap.warnings import StageConfigurationWarning from pathlib import Path import pytest @@ -108,4 +110,149 @@ def test_add_dependencies_single_dict(tmp_path): "Stage_2":("Stage_1","Stage_0",)} - \ No newline at end of file +def test_add_stage_parses_stage_configs_keyword() -> None: + pipeline = Pipeline() + stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + + pipeline.add_stage(stage, stage_configs=[stage_config]) + + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + + +def test_add_stage_warns_when_stage_config_count_mismatches() -> None: + pipeline = Pipeline() + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} + + +def test_add_stage_config_coerces_mapping_payload_for_named_stage() -> None: + pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + + pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") + + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + + +# --------------------------------------------------------------------------- +# Stage graph and stage-selection tests +# --------------------------------------------------------------------------- + +def test_resolve_stages_to_run_includes_transitive_dependencies() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) + stage_2 = Stage("Stage_2", source=Path("Stage_2.py"), dependencies=("Stage_1",)) + + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), + ) + + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] + + +def test_resolve_stages_to_run_rejects_disabled_dependencies() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) + + with pytest.raises(PipelineConfigurationError): + Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + ) + + +def test_self_stages_is_full_registry_after_disable() -> None: + """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + pipeline = Pipeline(stages=[stage_0, stage_1]) + + pipeline.disable_stage("Stage_1") + + assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] + + +def test_disable_stage_in_implicit_mode_creates_explicit_selection() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + pipeline = Pipeline(stages=[stage_0, stage_1]) + + pipeline.disable_stage("Stage_1") + + assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} + + +def test_enable_stage_restores_stage_in_explicit_mode() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) + + pipeline.enable_stage("Stage_1") + + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + + +def test_add_stage_keeps_new_stage_out_of_explicit_selection() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) + + pipeline.add_stage( + Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), + stage_configs=[StageConfig(name="Stage_1")], + ) + + assert pipeline.config.stages_to_run["Stage_1"] is False + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + + +def test_validate_skips_source_check_for_disabled_stages(tmp_path: Path) -> None: + """Disabled stages' source files need not exist — validate() only checks the effective run set.""" + enabled_file = tmp_path / "Stage_0.py" + enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") + + stage_0 = Stage("Stage_0", source=enabled_file) + stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent + + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) + + pipeline.validate() # must not raise + + +def test_construct_manifest_inputs_contains_only_effective_stages() -> None: + """Manifest inputs should list only the stages that are part of the execution graph.""" + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) + + runtime_id = pipeline._create_runtime_id() + manifest = pipeline._construct_manifest(runtime_id=runtime_id) + + assert list(manifest.inputs.keys()) == ["Stage_0"] From ca3ec1c1c7d91a1d0f887f4035d4c585f2a3c24c Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:15:26 +0100 Subject: [PATCH 151/332] redid stage_config_for() method --- onsrap/execution.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 4d6dabe..f620e1c 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -102,12 +102,6 @@ def set_active_stage(self, stage_name: str | None) -> None: """ self.active_stage_name = stage_name - def stage_config_for(self, stage_name: str) -> StageConfig | None: - """ - Return the configuration bound to a specific stage name, if one exists. - """ - return self.stage_configs.get(stage_name) - @property def stage_config(self) -> StageConfig | None: """ @@ -123,6 +117,19 @@ def stage_config(self) -> StageConfig | None: return None return self.stage_config_for(self.active_stage_name) + def stage_config_for(self, stage_name: str) -> StageConfig | None: + """ + Return the configuration for the stage identified by ``stage_name``. + + Returns ``None`` if no configuration has been registered for that stage. + + Parameters + ---------- + ``stage_name`` : str + Name of the stage whose configuration should be returned. + """ + return self.stage_configs.get(stage_name) + @property def stage_outputs(self) -> dict[str, Any]: """ @@ -180,6 +187,8 @@ def get_stage_config(self, vars_only: bool = True) -> dict[str, Any] | StageConf Parameters ---------- + ``stage`` : str + The name of the stage to get the configuration for. ``vars_only`` : bool, default = True If True, returns only the variables dictionary from the ``StageConfig``. If False, returns the full ``StageConfig`` instance. From 09fed0fcb5df44942f3d24bd1df705abbc99fff5 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:15:41 +0100 Subject: [PATCH 152/332] added TODO for models --- onsrap/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onsrap/models.py b/onsrap/models.py index 4fddf5b..b55a228 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -434,6 +434,7 @@ class StageConfig: ``metadata`` : dict[str, Any] Additional supporting metadata for the stage configuration. """ + # TODO: output location for stages potentially problematic for output overwrites! name: str _variables: dict[str, Any] = field(default_factory=dict) datasets: dict[str, Any] = field(default_factory=dict) From bea4e45a3fdeafbdcbbc023b707c29b09fba2a8d Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:32:37 +0100 Subject: [PATCH 153/332] fix: Updated __repr__ methods that had inconsistent spacing --- onsrap/models.py | 6 +++--- onsrap/pipeline.py | 8 ++++---- onsrap/stage.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index b55a228..9a11a7b 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -180,10 +180,10 @@ def __repr__(self) -> str: return ( f"PipelineConfig(name={self.name}, stages_to_run={self.stages_to_run}, " f"backend={self.backend}, " - f"work_dir={self.work_dir},project_root={self.project_root}, " - f"output_dir={self.output_dir},log_dir={self.log_dir},data_dir={self.data_dir}, " + f"work_dir={self.work_dir}, project_root={self.project_root}, " + f"output_dir={self.output_dir}, log_dir={self.log_dir}, data_dir={self.data_dir}, " f"allow_subprocess_fallback={self.allow_subprocess_fallback}, " - f"python_executable={self.python_executable},metadata={self.metadata})" + f"python_executable={self.python_executable}, metadata={self.metadata})" ) @classmethod diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index f1d28c2..2b7fd3a 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -144,10 +144,10 @@ def __repr__(self) -> str: A string representation of the ``Pipeline`` class with its attributes. """ return ( - f"Pipeline(name={self.name},backend={self.backend}, " - f"stages={self.stages},dependencies={self.dependencies}, " - f"logger={self.logger},executor={self.executor},graph={self.graph}, " - f"id={self.id},manifest={self.manifest},last_run={self.last_run})" + f"Pipeline(name={self.name}, backend={self.backend}, " + f"stages={self.stages}, dependencies={self.dependencies}, " + f"logger={self.logger}, executor={self.executor}, graph={self.graph}, " + f"id={self.id}, manifest={self.manifest}, last_run={self.last_run})" ) def add_stage( diff --git a/onsrap/stage.py b/onsrap/stage.py index c6c9649..9cd711b 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -132,8 +132,8 @@ def __repr__(self) -> str: A string representation of the ``Stage`` class with its attributes. """ return ( - f"Stage(name={self.name},source={self.source_label}, " - f"dependencies={self.dependencies},metadata={self.metadata}, " + f"Stage(name={self.name}, source={self.source_label}, " + f"dependencies={self.dependencies}, metadata={self.metadata}, " f"entrypoint={self.entrypoint}, backend={self.backend})" ) From 341c8f31f068a4b111fab4afd41741400c2c33dd Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 14:41:44 +0100 Subject: [PATCH 154/332] fix: added 'sandbox' in tests for ignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bc8de98..961976d 100644 --- a/.gitignore +++ b/.gitignore @@ -912,4 +912,5 @@ docs/_linkcheck/ examples/**/runs/**/ # Ignore coding playground -tests/playground/ \ No newline at end of file +tests/playground/ +tests/sandbox/ \ No newline at end of file From 9c7ab9ac6f32206e96dae9a4bd8602feb05f58bf Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 15:04:51 +0100 Subject: [PATCH 155/332] feat: added enable_stages to allow users to immediate enable added stages --- onsrap/pipeline.py | 10 +++++++--- tests/test_pipeline.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 2b7fd3a..ce35229 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -154,10 +154,14 @@ def add_stage( self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], stage_configs: StageConfig | Mapping[str, Any] | str | Path | Iterable[StageConfig | Mapping[str, Any] | str | Path] | None = None, + enable_stages: bool = False, ) -> None: """ Adds one or more steps to the Pipeline. + ``enable_stages`` : bool, default False + Whether to enable the added stages immediately. Default is False and is recommended. + Creates a list called ``added_stages`` that runs the _coerce_stage() method to extract the information from the given ``stages`` parameter. It then appends this list to the ``stages`` attribute of the ``Pipeline`` class, adds any stage @@ -216,7 +220,7 @@ def add_stage( for conf in parsed_stage_configs: self.add_stage_config(conf) - self._register_added_stages_in_stage_selection(added_stages) + self._register_added_stages_in_stage_selection(added_stages, enable_stages=enable_stages) self._check_stage_configs(added_stages, self.stage_configs) self._sync_stage_configs() @@ -570,7 +574,7 @@ def _rebuild_graph(self) -> None: self.graph = StageGraph.from_stages(stages_to_run) self.graph.validate() - def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage]) -> None: + def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage], enable_stages: bool = False) -> None: """ Default newly added stages to disabled once explicit stage selection is in use. @@ -583,7 +587,7 @@ def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage]) -> return for stage in stages: - self.config.stages_to_run.setdefault(stage.name, False) + self.config.stages_to_run.setdefault(stage.name, enable_stages) def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 8377229..a3a757a 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -226,6 +226,22 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection() -> None: assert pipeline.config.stages_to_run["Stage_1"] is False assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] +def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true() -> None: + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) + + pipeline.add_stage( + Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) + + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + def test_validate_skips_source_check_for_disabled_stages(tmp_path: Path) -> None: """Disabled stages' source files need not exist — validate() only checks the effective run set.""" From 4f71e21d86e5ff3c120ca3678230d208532961ca Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Fri, 24 Jul 2026 15:05:12 +0100 Subject: [PATCH 156/332] fix: updated _rebuild_graph to remove redundant code and clarified documentation. --- onsrap/pipeline.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ce35229..7b43b4d 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -556,21 +556,17 @@ def _coerce_stage_config( def _rebuild_graph(self) -> None: """ - Update the execution graph while validating the full pipeline definition. + Update and validate the execution graph. + The execution graph is a subset of the full Stage registry + (Pipeline.stages), reflecting the stages that are actually enabled + for execution. The pipeline keeps ``self.stages`` as the complete stage registry, but ``self.graph`` represents the effective run set after applying ``PipelineConfig.stages_to_run`` and expanding any selected stage's dependencies. """ - full_graph = StageGraph.from_stages(self.stages) - full_graph.validate() - stages_to_run = self._resolve_stages_to_run() - if [stage.name for stage in stages_to_run] == [stage.name for stage in self.stages]: - self.graph = full_graph - return - self.graph = StageGraph.from_stages(stages_to_run) self.graph.validate() From 3872a618e65c976d11fe7cdc833743efdd9202b2 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 09:54:48 +0100 Subject: [PATCH 157/332] fix: warning wasn't printing full message due to structure. --- onsrap/pipeline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7b43b4d..c478cba 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -1150,8 +1150,7 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, remaining_keys = set(raw_config) - {pipeline_configuration, stage_configuration} if remaining_keys: - warnings.warn("There are remaining sections in your configuration file that have not been extracted." \ - " Please check that all your configurations are in the pipeline or stage configuration keys.", + warnings.warn("There are remaining sections in your configuration file that have not been extracted. Please check that all your configurations are in the pipeline or stage configuration keys.", PipelineConfigurationWarning) if not isinstance(pipeline_payload, Mapping): From 1583c21475ea9d22c575c6879d43ca47ff8ecae7 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 09:55:28 +0100 Subject: [PATCH 158/332] fix: added __name__==__main__ behaviour to fix main() being called at Module import erroring out --- examples/pipeline_2/scripts/0_clean_data.py | 5 ++++- examples/pipeline_2/scripts/1_derive_vars.py | 3 ++- examples/pipeline_2/scripts/2_reporting.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py index 96a0af6..885bc09 100644 --- a/examples/pipeline_2/scripts/0_clean_data.py +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -1,4 +1,5 @@ import pandas as pd +from onsrap import ExecutionContext def check_variables(df, expected_variables): @@ -34,6 +35,7 @@ def standardise_columns(df): def main(context=None): config = context.get_stage_config("0_clean_data") + print(config) orders = pd.read_csv(config["input_location"]) @@ -46,4 +48,5 @@ def main(context=None): orders = standardise_columns(orders) orders.to_csv(config["output_location"], index = False) -main() \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py index d78cafd..1b06132 100644 --- a/examples/pipeline_2/scripts/1_derive_vars.py +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -82,4 +82,5 @@ def main(context=None): -main() \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index 5f8f6de..25ac1d0 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -114,4 +114,5 @@ def main(): curate_report(report, values) write_report(report) -main() \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file From 4ff1f56a0750db96f4a871464786724670676b68 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 09:55:51 +0100 Subject: [PATCH 159/332] fix: resolved get_stage_config() behaviour and data-parsing contracts. added test. --- onsrap/execution.py | 21 +++++++++++---------- tests/test_execution.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index f620e1c..ee079d7 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -113,21 +113,22 @@ def stage_config(self) -> StageConfig | None: This property is ``None`` outside an active stage run. """ - if self.active_stage_name is None: - return None return self.stage_config_for(self.active_stage_name) - def stage_config_for(self, stage_name: str) -> StageConfig | None: + def stage_config_for(self, stage_name: str | None) -> StageConfig | None: """ - Return the configuration for the stage identified by ``stage_name``. + Return the configuration registered for ``stage_name``. - Returns ``None`` if no configuration has been registered for that stage. + Unlike ``stage_config``, this helper does not depend on the currently + active stage and can be used to inspect any known stage configuration. Parameters ---------- - ``stage_name`` : str + ``stage_name`` : str or None Name of the stage whose configuration should be returned. """ + if stage_name is None: + return None return self.stage_configs.get(stage_name) @property @@ -176,7 +177,7 @@ def resolve_output_root(self) -> Path: raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") - def get_stage_config(self, vars_only: bool = True) -> dict[str, Any] | StageConfig: + def get_stage_config(self, stage: str | None = None, vars_only: bool = True) -> dict[str, Any] | StageConfig | None: """ Returns the configuration for the stage currently being executed, with optional arguments. @@ -194,13 +195,13 @@ def get_stage_config(self, vars_only: bool = True) -> dict[str, Any] | StageConf Returns ------- - dict[str, Any] or StageConfig + dict[str, Any] or StageConfig or None The parameters contained within the configuration for the currently active stage. If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. """ - stage_config = self.stage_config + stage_config = self.stage_config_for(stage) if stage is not None else self.stage_config if stage_config is None: - return {} + return {} if vars_only else None if vars_only: return stage_config.variables return stage_config diff --git a/tests/test_execution.py b/tests/test_execution.py index d56bdad..25e1e24 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -176,6 +176,23 @@ def test_resolve_output_root(execution) -> None: with pytest.raises(PipelineConfigurationError): execution_blank_config.resolve_output_root() +def test_stage_config_accessors_return_named_and_active_configs(config, logger) -> None: + stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) + context = ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + Path("tmp/run"), + stage_configs={"stage_test": stage_config}, + active_stage_name="stage_test", + ) + + assert context.stage_config_for("stage_test") == stage_config + assert context.get_stage_config("stage_test") == {"years_to_run": 2017} + assert context.get_stage_config() == {"years_to_run": 2017} + assert context.get_stage_config(vars_only=False) == stage_config + """ Parameters for testing multiple add_folder options in test_resolve_given_path_add_folders function. From ccd908c1e82b659df28926b83418a2a4dc5872a1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 10:31:22 +0100 Subject: [PATCH 160/332] feat: add GlobalConfig and from_dict method. add GlobalConfig to StageConfig as attribute --- onsrap/models.py | 51 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 9a11a7b..54ea656 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -429,15 +429,15 @@ class StageConfig: The name of the stage that this configuration applies to. ``_variables`` : dict[str, Any] Arbitrary stage-scoped variables. - ``datasets`` : dict[str, Any] - Optional dataset-related metadata for the stage. + `_global_variables` : GlobalConfig | None = None + Variables that need to be parsed through every stage in the pipeline. ``metadata`` : dict[str, Any] Additional supporting metadata for the stage configuration. """ # TODO: output location for stages potentially problematic for output overwrites! name: str _variables: dict[str, Any] = field(default_factory=dict) - datasets: dict[str, Any] = field(default_factory=dict) + _global_variables: GlobalConfig | None = None metadata: dict[str, Any] = field(default_factory=dict) @classmethod @@ -463,11 +463,7 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> Stage """ payload = dict(data or {}) - datasets = payload.pop("datasets", {}) - if isinstance(datasets, Mapping): - datasets = dict(datasets) - else: - raise StageConfigurationError("Stage datasets must be provided as a mapping.") + globals = GlobalConfig.from_dict(payload.pop("global_variables", {})) metadata = payload.pop("metadata", {}) if isinstance(metadata, Mapping): @@ -478,7 +474,7 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> Stage return cls( name=str(name).strip(), _variables=payload, - datasets=datasets, + _global_variables=globals, metadata=metadata, ) @@ -536,12 +532,45 @@ def to_dict(self) -> dict[str, Any]: Serialize the stage configuration back to a mapping suitable for manifests. """ data = dict(self._variables) - if self.datasets: - data["datasets"] = dict(self.datasets) if self.metadata: data["metadata"] = dict(self.metadata) return data +@dataclass +class GlobalConfig: + """ + Holds configuration that should be exposed to all stages at runtime. + + Parameters + ---------- + ``_variables`` : dict[str, Any] + Variables that should be parsed to all stages throughout the pipeline. + """ + _variables: dict[str, Any] = field(default_factory=dict) + + def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: + """ + Build a ``GlobalConfig`` from a mapping loaded from code or configuration files. + + Parameters + ---------- + ``data`` : Mapping[str, Any] + Raw configuration payload for the global configuration. + + Returns + ------- + ``GlobalConfig`` + A normalized global configuration object. + """ + payload = dict(data or {}) + return cls(_variables=payload) + + @property + def variables(self) -> dict[str, Any]: + """ + Return a copy of the global variables. + """ + return dict(self._variables) @dataclass class RunManifest: From 1176083598b575d30e051df9f95b6168654903d2 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 11:03:49 +0100 Subject: [PATCH 161/332] feat: combine_vars method to combine global variables and stage variables. This function should be called by the end user when they are implementing the configuration. --- onsrap/models.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 54ea656..352dfd5 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -441,7 +441,7 @@ class StageConfig: metadata: dict[str, Any] = field(default_factory=dict) @classmethod - def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> StageConfig: + def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_vars: Mapping[str, Any] | None = None) -> StageConfig: """ Build a ``StageConfig`` from a mapping loaded from code or configuration files. @@ -455,6 +455,8 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> Stage Stage name that this configuration applies to. ``data`` : Mapping[str, Any] or None Raw configuration payload for that stage. + ``global_vars`` : Mapping[str, Any] or None + Global variables from config file that need to be parsed to every stage. Returns ------- @@ -463,7 +465,7 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> Stage """ payload = dict(data or {}) - globals = GlobalConfig.from_dict(payload.pop("global_variables", {})) + globals = GlobalConfig.from_dict(global_vars) metadata = payload.pop("metadata", {}) if isinstance(metadata, Mapping): @@ -536,6 +538,23 @@ def to_dict(self) -> dict[str, Any]: data["metadata"] = dict(self.metadata) return data + def combine_vars(self) -> dict[str, Any]: + """ + Combine the stage variables with the global variables, giving precedence to + stage variables and raises a warning in case of conflicts. + """ + combined = self._global_variables.variables if self._global_variables else {} + conflicts = self._variables.keys() & combined.keys() + if conflicts: + conflicting = ", ".join(sorted(conflicts)) + warnings.warn( + f"Stage '{self.name}' defines variable(s) that are also defined in global " + f"variables: {conflicting}. Stage variables will take precedence.", + StageConfigurationError + ) + combined.update(self._variables) + return combined + @dataclass class GlobalConfig: """ From 45b8f3a2c2391753863104b809d24d9a9b19fdf4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 11:55:28 +0100 Subject: [PATCH 162/332] tweak: adjust StageConfig so global variables aren't stored as an attribute but are instead used as a method to extract all variables required for the stage. --- onsrap/models.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 352dfd5..195747e 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -437,7 +437,7 @@ class StageConfig: # TODO: output location for stages potentially problematic for output overwrites! name: str _variables: dict[str, Any] = field(default_factory=dict) - _global_variables: GlobalConfig | None = None + _all_variables: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) @classmethod @@ -465,8 +465,6 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_v """ payload = dict(data or {}) - globals = GlobalConfig.from_dict(global_vars) - metadata = payload.pop("metadata", {}) if isinstance(metadata, Mapping): metadata = dict(metadata) @@ -476,7 +474,7 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_v return cls( name=str(name).strip(), _variables=payload, - _global_variables=globals, + _all_variables=StageConfig._combine_vars(payload, global_vars), metadata=metadata, ) @@ -538,21 +536,19 @@ def to_dict(self) -> dict[str, Any]: data["metadata"] = dict(self.metadata) return data - def combine_vars(self) -> dict[str, Any]: - """ - Combine the stage variables with the global variables, giving precedence to - stage variables and raises a warning in case of conflicts. - """ - combined = self._global_variables.variables if self._global_variables else {} - conflicts = self._variables.keys() & combined.keys() +#TODO: Add method to remove variables from global variables that are not needed based on stage. + @staticmethod + def _combine_vars(stage_vars: dict[str, Any], global_vars: dict[str, Any] | None) -> dict[str, Any]: + combined = dict(global_vars or {}) + conflicts = stage_vars.keys() & combined.keys() if conflicts: conflicting = ", ".join(sorted(conflicts)) warnings.warn( - f"Stage '{self.name}' defines variable(s) that are also defined in global " + f"Stage defines variable(s) that are also defined in global " f"variables: {conflicting}. Stage variables will take precedence.", StageConfigurationError ) - combined.update(self._variables) + combined.update(stage_vars) return combined @dataclass @@ -579,17 +575,11 @@ def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: Returns ------- ``GlobalConfig`` - A normalized global configuration object. + A global configuration object. """ payload = dict(data or {}) return cls(_variables=payload) - @property - def variables(self) -> dict[str, Any]: - """ - Return a copy of the global variables. - """ - return dict(self._variables) @dataclass class RunManifest: From 6e33a6b47c3bb6324e5d1b103781fd2775a4001f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 13:06:37 +0100 Subject: [PATCH 163/332] feat: add getter method for GlobalConfig and tweak from_dict method documentation --- onsrap/models.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 195747e..64fe4e6 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -562,8 +562,10 @@ class GlobalConfig: Variables that should be parsed to all stages throughout the pipeline. """ _variables: dict[str, Any] = field(default_factory=dict) + exclusion: dict[str, Any] = field(default_factory=dict) - def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: + @classmethod + def from_dict(cls, data: Mapping[str, Any], exclusions: dict[str, Any] | None = None) -> GlobalConfig: """ Build a ``GlobalConfig`` from a mapping loaded from code or configuration files. @@ -571,6 +573,8 @@ def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: ---------- ``data`` : Mapping[str, Any] Raw configuration payload for the global configuration. + ``exclusions`` : dict[str, Any] or None + A lookup of which global variables should be excluded from each stage. Returns ------- @@ -578,7 +582,32 @@ def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: A global configuration object. """ payload = dict(data or {}) - return cls(_variables=payload) + + return cls(_variables=payload, + exclusion=exclusions) + + def get_attributes(self, keep_exclusion: bool = True) -> dict[str, Any]: + """ + Return a copy of the global variables, optionally excluding any variables + specified in the exclusion list. + + Parameters + ---------- + ``keep_exclusion`` : bool, default = True + If True, return both _variables and exclusion. + If False, return only the variables and not the exclusion list. + + Returns + ------- + ``self._variables`` : dict[str, Any] + All global variables for the pipeline. + ``self.exclusion`` : dict[str, Any] + The exclusion list of global variables for each stage. + """ + if keep_exclusion: + return self._variables, self.exclusion + else: + return self._variables @dataclass From 86372944a4b9d48a45292ce7fd18405da1b12ee3 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 13:31:37 +0100 Subject: [PATCH 164/332] feat: move _combine_vars() method from StageConfig class to ExecutionContext class and add utilisation of exclusion attribute. --- onsrap/execution.py | 51 +++++++++++++++++++++++++++++++++++++++++++-- onsrap/models.py | 15 ------------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index ee079d7..326e272 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -7,11 +7,12 @@ from datetime import datetime from pathlib import Path from typing import Any, Protocol, TYPE_CHECKING +import warnings -from .errors import StageExecutionError, StageLoadError, PipelineConfigurationError +from .errors import StageConfigurationError, StageExecutionError, StageLoadError, PipelineConfigurationError from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable from .logger import Logger -from .models import PipelineConfig, StageConfig, StageResult, StageStatus, now +from .models import GlobalConfig, PipelineConfig, StageConfig, StageResult, StageStatus, now if TYPE_CHECKING: from .stage import Stage @@ -56,6 +57,7 @@ class ExecutionContext: working_directory: Path = field(default_factory=Path.cwd) stage_results: dict[str, StageResult] = field(default_factory=dict) stage_configs: dict[str, StageConfig] = field(default_factory=dict) + global_config: GlobalConfig | None = None variables: dict[str, Any] = field(default_factory=dict) active_stage_name: str | None = None @@ -258,6 +260,51 @@ def resolve_given_path(self, stage_name: str | None, if file_name is not None: return root / file_name return root + + def _combine_vars(self) -> dict[str, Any]: + """ + Private method that combines the global variables and the stage + variables for the current stage. + + Global variables are extracted from the ``global_config`` attribute + and any variables which are marked as to be excluded from the exclusion + attribute are removed. The global variables are then combined with the stage + specific variables and returned as a dictionary. Conflicts raise a warning + to alert the user that the stage configuration definition will be used as a + priority. + + Returns + ------- + ``combined``: dict[str, Any] + A dictionary of all variables required for the stage that are sourced + through the configuration. + + Raises + ------ + ``StageConfigurationError`` + If there are conflicting variables between the global and stage configuration, + a warning is raised to alert the user that the stage configuration will take + precedence. + """ + #Extracts variables from global config without exclusion lookup + global_vars, exclusions = self.global_config.get_attributes() if self.global_config else {} + stage_exclusions = dict(exclusions[self.active_stage_name]) + + combined = {k: v for k, v in global_vars.items() if k not in stage_exclusions} + + stage_config = self.stage_config_for(self.active_stage_name) + stage_vars = stage_config.variables + + conflicts = stage_vars.keys() & combined.keys() + if conflicts: + conflicting = ", ".join(sorted(conflicts)) + warnings.warn( + f"Stage defines variable(s) that are also defined in global " + f"variables: {conflicting}. Stage variables will take precedence.", + StageConfigurationError + ) + combined.update(stage_vars) + return combined diff --git a/onsrap/models.py b/onsrap/models.py index 64fe4e6..bddcec1 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -536,21 +536,6 @@ def to_dict(self) -> dict[str, Any]: data["metadata"] = dict(self.metadata) return data -#TODO: Add method to remove variables from global variables that are not needed based on stage. - @staticmethod - def _combine_vars(stage_vars: dict[str, Any], global_vars: dict[str, Any] | None) -> dict[str, Any]: - combined = dict(global_vars or {}) - conflicts = stage_vars.keys() & combined.keys() - if conflicts: - conflicting = ", ".join(sorted(conflicts)) - warnings.warn( - f"Stage defines variable(s) that are also defined in global " - f"variables: {conflicting}. Stage variables will take precedence.", - StageConfigurationError - ) - combined.update(stage_vars) - return combined - @dataclass class GlobalConfig: """ From 09bb2ea549a7e1e2f1b75987732db47f67fd3167 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 13:32:20 +0100 Subject: [PATCH 165/332] tweak: remove mention of global variables from StageConfig instance as this is handled in execution context --- onsrap/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index bddcec1..1c9296f 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -429,15 +429,12 @@ class StageConfig: The name of the stage that this configuration applies to. ``_variables`` : dict[str, Any] Arbitrary stage-scoped variables. - `_global_variables` : GlobalConfig | None = None - Variables that need to be parsed through every stage in the pipeline. ``metadata`` : dict[str, Any] Additional supporting metadata for the stage configuration. """ # TODO: output location for stages potentially problematic for output overwrites! name: str _variables: dict[str, Any] = field(default_factory=dict) - _all_variables: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) @classmethod @@ -474,7 +471,6 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_v return cls( name=str(name).strip(), _variables=payload, - _all_variables=StageConfig._combine_vars(payload, global_vars), metadata=metadata, ) From 61a24dc05528825ea267869e620d71086debb11e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 13:52:59 +0100 Subject: [PATCH 166/332] tweak: adjust order of attributes for ExecutionContext and add initialisation for global_configs in Pipeline instance --- onsrap/execution.py | 4 +- onsrap/pipeline.py | 116 ++++++++++++-------------------------------- 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 326e272..dc30b95 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -47,6 +47,8 @@ class ExecutionContext: Stores relevant variables regarding the stage run and their results. ``active_stage_name`` : str or None, default = None Name of the stage currently being executed. Used to expose ``stage_config``. + ``global_config`` : ``GlobalConfig`` or None, default = None + Variables which are parsed to all stages throughout the pipeline. """ pipeline_name: str run_id: str @@ -57,9 +59,9 @@ class ExecutionContext: working_directory: Path = field(default_factory=Path.cwd) stage_results: dict[str, StageResult] = field(default_factory=dict) stage_configs: dict[str, StageConfig] = field(default_factory=dict) - global_config: GlobalConfig | None = None variables: dict[str, Any] = field(default_factory=dict) active_stage_name: str | None = None + global_config: GlobalConfig | None = None def record(self, result: StageResult) -> StageResult: """ diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index c478cba..f1b1f52 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -57,7 +57,7 @@ def __init__( executor: StageExecutor | None = None, ): # if config is not None: - resolved_config, resolved_stage_configs, configured_stages = self._resolve_config(config) + resolved_config, resolved_stage_configs, configured_stages, resolved_global_config = self._resolve_config(config) self.name = name or resolved_config.name or "pipeline" self.backend = backend or resolved_config.backend or "python" @@ -95,9 +95,11 @@ def __init__( if dependencies is not None: self._assign_dependencies(dependencies, self.stages) + #TODO: This is assigned but doesn't have an attribute. Is that an issue? self.stage_configs = dict(resolved_stage_configs) + self.global_configs = dict(resolved_global_config) + self._sync_stage_configs() - self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -349,70 +351,6 @@ def validate(self) -> Pipeline: self.graph.validate() return self - def create_stage_config( - self, - s_config: Mapping[str, Any] | str | Path, - *, - name: str | None = None, - ) -> StageConfig: - """ - Create a ``StageConfig`` from direct data, a stage-name keyed mapping, or a config file. - - Parameters - ---------- - ``s_config`` : Mapping[str, Any] | str | Path - Either a single stage payload, a mapping keyed by stage name, or a config file. - Config files may contain a top-level ``stage_configuration`` section or may consist - solely of stage-name keyed configuration entries. - ``name`` : str or None, keyword-only - Stage name to extract when the input contains more than one stage configuration. - - Returns - ------- - ``StageConfig`` - The normalized stage configuration for the requested stage. - - Raises - ------ - ``StageConfigurationError`` - If the input cannot be resolved to exactly one stage configuration. - """ - # TODO: Comment the logical sections here for better readability - if isinstance(s_config, Mapping): - if "pipeline_variables" in s_config or "stage_configuration" in s_config or "stage_config" in s_config: - _, stage_config_payload = self._split_config_sections(s_config) - stage_configs = self._build_stage_configs(stage_config_payload) - elif name is not None and name in s_config and isinstance(s_config[name], Mapping): - stage_configs = self._build_stage_configs(s_config) - else: - if name is None: - if len(s_config) != 1: - raise StageConfigurationError( - "A stage configuration mapping must include exactly one stage when no name is provided." - ) - name, stage_payload = next(iter(s_config.items())) - else: - stage_payload = s_config - - if not isinstance(stage_payload, Mapping): - raise StageConfigurationError("Stage configuration values must be provided as a mapping.") - - return StageConfig.from_mapping(str(name), stage_payload) - - return self._select_stage_config(stage_configs, name=name) - - raw_payload = self._load_config_mapping(s_config) - if "pipeline_variables" in raw_payload or "stage_configuration" in raw_payload or "stage_config" in raw_payload: - _, stage_config_payload = self._split_config_sections(raw_payload) - stage_configs = self._build_stage_configs(stage_config_payload) - elif all(isinstance(value, Mapping) for value in raw_payload.values()): - stage_configs = self._build_stage_configs(raw_payload) - else: - raise StageConfigurationError( - "Config files passed to create_stage_config must define a stage-configuration section or a mapping of stage names to configuration mappings." - ) - - return self._select_stage_config(stage_configs, name=name) def run(self) -> PipelineRun: """ @@ -708,7 +646,7 @@ def _resolve_config( return config, self._build_stage_configs(stage_configuration), [] raw_config = self._load_config_mapping(config) - pipeline_payload, stage_config_payload = self._split_config_sections(raw_config) + pipeline_payload, stage_config_payload, global_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) stage_definitions = normalized_pipeline_payload.pop("stages", ()) @@ -1125,39 +1063,49 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, Contents of the pipeline configuration settings defined in the configuration file. ``stage_payload``: Mapping[str, Any] | None Contents of the stage configuration settings defined in the configuration file. + ``global_payload``: Mapping[str, Any] | None + Contents of the global configuration settings defined in the configuration file. Raises ------ ``PipelineConfigurationWarning`` - If blank values for pipeline_payload or stage_payload are detected. + If blank values for pipeline_payload or global_payload are detected. If there are remaining keys in the ``raw_config`` that have not been extracted. + ``StageConfigurationWarning`` + If blank values for stage_payload are detected. + ``PipelineConfigurationError`` - If the pipeline_payload or stage_payload are not mapping types. + If the pipeline_payload, global_payload or stage_payload are not mapping types. """ possible_stage_keys = ("stage_configuration", "stage_config") possible_pipeline_keys = ("pipeline_variables","pipeline_config") - stage_configuration = Pipeline._extract_keys(possible_stage_keys, raw_config) - pipeline_configuration = Pipeline._extract_keys(possible_pipeline_keys, raw_config) + possible_global_keys = ("global_configuration", "global_config") - pipeline_payload = raw_config.get(pipeline_configuration,{}) - if pipeline_payload is None: - warnings.warn("Blank pipeline configuration detected. Please check that this is correct.", PipelineConfigurationWarning) + stage_payload, stage_configuration = Pipeline._extract_mappings(possible_stage_keys, raw_config, StageConfigurationWarning) + pipeline_payload, pipeline_configuration = Pipeline._extract_mappings(possible_pipeline_keys, raw_config, PipelineConfigurationWarning) + global_payload, global_configuration = Pipeline._extract_mappings(possible_global_keys, raw_config, PipelineConfigurationWarning) - stage_payload = raw_config.get(stage_configuration,{}) - if stage_payload is None: - warnings.warn("Blank stage configuration detected. Please check that this is correct.", StageConfigurationWarning) - - remaining_keys = set(raw_config) - {pipeline_configuration, stage_configuration} + remaining_keys = set(raw_config) - {pipeline_configuration, stage_configuration, global_configuration} if remaining_keys: warnings.warn("There are remaining sections in your configuration file that have not been extracted. Please check that all your configurations are in the pipeline or stage configuration keys.", PipelineConfigurationWarning) - if not isinstance(pipeline_payload, Mapping): - raise PipelineConfigurationError(f"The {pipeline_configuration} section must be a mapping.") - if not isinstance(stage_payload, Mapping): - raise PipelineConfigurationError(f"The {stage_configuration} section must be a mapping.") - return pipeline_payload, stage_payload + return pipeline_payload, stage_payload, global_payload + + @staticmethod + def _extract_mappings(keys: tuple[str,...], + config: Mapping[str, Any], + warning: PipelineConfigurationWarning | StageConfigurationWarning) -> tuple[dict[str, Any], str]: + + configuration = Pipeline._extract_keys(keys, config) + payload = config.get(configuration,{}) + if payload is None: + warnings.warn(f"Blank {configuration} configuration detected. Please check that this is correct.", warning) + if not isinstance(payload, Mapping): + raise PipelineConfigurationError(f"The {configuration} section must be a mapping.") + return payload, configuration + @staticmethod def _extract_keys(possible_keys: tuple[str, ...], From 488e6c21f0354484d50c2a57e431377ad2e2bd46 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 15:57:15 +0100 Subject: [PATCH 167/332] tests: tests implemented for _combine_vars method and minor tweak to show Warning message rather than Error message --- onsrap/execution.py | 11 ++++++--- tests/test_execution.py | 55 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index dc30b95..5a8e0fc 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -9,6 +9,8 @@ from typing import Any, Protocol, TYPE_CHECKING import warnings +from onsrap.warnings import StageConfigurationWarning + from .errors import StageConfigurationError, StageExecutionError, StageLoadError, PipelineConfigurationError from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable from .logger import Logger @@ -283,14 +285,15 @@ def _combine_vars(self) -> dict[str, Any]: Raises ------ - ``StageConfigurationError`` + ``StageConfigurationWarning`` If there are conflicting variables between the global and stage configuration, a warning is raised to alert the user that the stage configuration will take precedence. """ #Extracts variables from global config without exclusion lookup - global_vars, exclusions = self.global_config.get_attributes() if self.global_config else {} - stage_exclusions = dict(exclusions[self.active_stage_name]) + global_vars, exclusions = self.global_config.get_attributes() if self.global_config else ({}, {}) + stage_exclusions_extract = exclusions.get(self.active_stage_name, []) + stage_exclusions = [exclusion for exclusion in stage_exclusions_extract] combined = {k: v for k, v in global_vars.items() if k not in stage_exclusions} @@ -303,7 +306,7 @@ def _combine_vars(self) -> dict[str, Any]: warnings.warn( f"Stage defines variable(s) that are also defined in global " f"variables: {conflicting}. Stage variables will take precedence.", - StageConfigurationError + StageConfigurationWarning ) combined.update(stage_vars) return combined diff --git a/tests/test_execution.py b/tests/test_execution.py index 25e1e24..8d67aa5 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,10 +1,11 @@ from onsrap.execution import ExecutionContext, PythonStageExecutor -from onsrap.models import PipelineConfig, StageConfig, StageResult, StageStatus +from onsrap.models import GlobalConfig, PipelineConfig, StageConfig, StageResult, StageStatus from onsrap.logger import Logger from pathlib import Path import pytest import onsrap.execution as execution_module from onsrap.errors import PipelineConfigurationError +from onsrap.warnings import StageConfigurationWarning @pytest.fixture def logger() -> Logger: @@ -279,6 +280,58 @@ def pythonstageexecutor() -> PythonStageExecutor: def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") + +def test_combine_vars(execution) -> None: + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1" + } + +def test_combine_vars_errors(execution) -> None: + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "global_var1": "value4"} + execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + + with pytest.warns(StageConfigurationWarning, + match="Stage defines variable\\(s\\) that are also defined in global " + "variables: global_var1\\. Stage variables will take precedence."): + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "global_var1": "value4" + } + +def test_combine_vars_no_exclusion(execution) -> None: + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1", + "global_var2": "value2" + } """CONTINUE FROM EXECUTE CLASS METHOD""" @pytest.fixture def stage_config() -> StageConfig: From 815e70581d11690fc9df75daa0b4573a05cc259d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 16:02:43 +0100 Subject: [PATCH 168/332] docs: documentation added for tests from previous commit. --- onsrap/models.py | 3 ++- tests/test_execution.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/onsrap/models.py b/onsrap/models.py index 1c9296f..ff0a606 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -583,7 +583,8 @@ def get_attributes(self, keep_exclusion: bool = True) -> dict[str, Any]: ``self._variables`` : dict[str, Any] All global variables for the pipeline. ``self.exclusion`` : dict[str, Any] - The exclusion list of global variables for each stage. + The exclusion list of global variables for each stage. Only returned + if ``keep_exclusion`` is True. """ if keep_exclusion: return self._variables, self.exclusion diff --git a/tests/test_execution.py b/tests/test_execution.py index 8d67aa5..a060370 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -282,6 +282,11 @@ def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: def test_combine_vars(execution) -> None: + """ + Test that checks that a dictionary is returned, combining values from a global + configuration and a stage configuration whilst removing any stage specific + exclusions. + """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} @@ -298,6 +303,10 @@ def test_combine_vars(execution) -> None: } def test_combine_vars_errors(execution) -> None: + """ + Test that confirms that a warning is raised if there is a variable defined in both + the global and the stage configurations as well as asserting the correct values. + """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} stage_vars = {"stage_var1": "value3", "global_var1": "value4"} @@ -317,6 +326,10 @@ def test_combine_vars_errors(execution) -> None: } def test_combine_vars_no_exclusion(execution) -> None: + """ + Test confirming that a dictionary is returned, combining values from a global configuration + and a stage configuration when there are no exclusions defined. + """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {} stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} From 4bf8c96f90dc94af5190ddd5e70554ab6608cfba Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 16:37:11 +0100 Subject: [PATCH 169/332] tweak: add global_config parameter to ExecutionContext creation within PipelineRunner --- onsrap/pipeline.py | 1 - onsrap/runner.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index f1b1f52..828e076 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -95,7 +95,6 @@ def __init__( if dependencies is not None: self._assign_dependencies(dependencies, self.stages) - #TODO: This is assigned but doesn't have an attribute. Is that an issue? self.stage_configs = dict(resolved_stage_configs) self.global_configs = dict(resolved_global_config) diff --git a/onsrap/runner.py b/onsrap/runner.py index 8aaac45..280a7c8 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -110,6 +110,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: started_at=started_at, working_directory=pipeline.config.work_dir, stage_configs=dict(pipeline.stage_configs), + global_config=pipeline.global_config, ) # Ensure the stages are in order and create a manifest that explains the run. From 80169d0956add72c33527e2fee148f650ca8224d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 27 Jul 2026 16:50:29 +0100 Subject: [PATCH 170/332] feat: add GlobalConfig instantiation into _resolve_config method --- onsrap/models.py | 4 +++- onsrap/pipeline.py | 27 +++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index ff0a606..c08634f 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -546,7 +546,7 @@ class GlobalConfig: exclusion: dict[str, Any] = field(default_factory=dict) @classmethod - def from_dict(cls, data: Mapping[str, Any], exclusions: dict[str, Any] | None = None) -> GlobalConfig: + def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: """ Build a ``GlobalConfig`` from a mapping loaded from code or configuration files. @@ -563,6 +563,8 @@ def from_dict(cls, data: Mapping[str, Any], exclusions: dict[str, Any] | None = A global configuration object. """ payload = dict(data or {}) + + exclusions = payload.pop("exclusions", None) return cls(_variables=payload, exclusion=exclusions) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 828e076..4916fde 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -14,7 +14,7 @@ from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger -from .models import PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now +from .models import GlobalConfig, PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now from .stage import Stage, _normalize_dependencies @@ -628,9 +628,19 @@ def _resolve_config( This method is the main normalization step for configuration injection. It accepts already-constructed config objects, raw mappings, and YAML files, and converts them into the three objects the pipeline needs before execution starts. + + Parameters + ---------- + ``config`` : PipelineConfig | Mapping[str, Any] | None + The configuration input to resolve into the three objects required for execution. + + Returns + ------- + tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig] + The resolved pipeline configuration, stage configurations, configured stages, and global configuration. """ if config is None: - return PipelineConfig.from_any(config), {}, [] + return PipelineConfig.from_any(config), {}, [], GlobalConfig() if isinstance(config, PipelineConfig): stage_configuration = config.metadata.get("stage_configuration", None) @@ -642,7 +652,14 @@ def _resolve_config( "Stage configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", StageConfigurationWarning, ) - return config, self._build_stage_configs(stage_configuration), [] + + global_configuration = config.metadata.get("global_config", None) + if global_configuration is not None: + warnings.warn( + "Global configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", + StageConfigurationWarning, + ) + return config, self._build_stage_configs(stage_configuration), [], GlobalConfig.from_dict(global_configuration) raw_config = self._load_config_mapping(config) pipeline_payload, stage_config_payload, global_config_payload = self._split_config_sections(raw_config) @@ -657,7 +674,9 @@ def _resolve_config( backend=pipeline_config.backend, work_dir=pipeline_config.work_dir, ) - return pipeline_config, stage_configs, configured_stages + global_config = GlobalConfig.from_dict(global_config_payload) + + return pipeline_config, stage_configs, configured_stages, global_config From 371b9e86c57ab4b0520b54973b57ee56eee061d3 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 14:07:29 +0100 Subject: [PATCH 171/332] fix: added GlobalConfig to init file to allow importing --- onsrap/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onsrap/__init__.py b/onsrap/__init__.py index 0be47a6..95c8203 100644 --- a/onsrap/__init__.py +++ b/onsrap/__init__.py @@ -22,6 +22,7 @@ StageConfig, StageResult, StageStatus, + GlobalConfig, ) from .pipeline import Pipeline from .runner import PipelineRunner @@ -32,6 +33,7 @@ "DependencyCycleError", "DuplicateStageError", "ExecutionContext", + "GlobalConfig", "LogConfig", "Logger", "MissingDependencyError", From d5a406fbe92297308f1ab8062e90b6826d2a2c30 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 14:08:15 +0100 Subject: [PATCH 172/332] fix: tweaks to Stage and Global Configuration handling in Pipeline --- onsrap/pipeline.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 4916fde..eaead91 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -56,7 +56,6 @@ def __init__( logger: Logger | None = None, executor: StageExecutor | None = None, ): - # if config is not None: resolved_config, resolved_stage_configs, configured_stages, resolved_global_config = self._resolve_config(config) self.name = name or resolved_config.name or "pipeline" @@ -96,7 +95,7 @@ def __init__( self._assign_dependencies(dependencies, self.stages) self.stage_configs = dict(resolved_stage_configs) - self.global_configs = dict(resolved_global_config) + self.global_config = resolved_global_config self._sync_stage_configs() self._rebuild_graph() @@ -484,10 +483,13 @@ def _coerce_stage_config( raise StageConfigurationError( f"Stage configuration '{name}' was not found. Available stage configurations are: {available_stage_names}." ) - return self.create_stage_config(stage_config, name=name) + return StageConfig.from_mapping(name=name, data=stage_config) if isinstance(stage_config, (str, Path)): - return self.create_stage_config(stage_config, name=name) + import yaml + + payload = dict(yaml.safe_load(Path(stage_config).read_text())) + return StageConfig.from_mapping(name=name, data=payload) raise StageConfigurationError(f"Unsupported stage configuration specification: {type(stage_config)!r}.") @@ -621,7 +623,7 @@ def _current_user(self) -> str | None: def _resolve_config( self, config: PipelineConfig | Mapping[str, Any] | None, - ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage]]: + ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig]: """ Resolve supported configuration inputs into pipeline config, stage config, and stages. @@ -815,6 +817,10 @@ def _resolve_stages_to_run(self) -> list[Stage]: # When PipelineConfig.stages_to_run is empty, the pipeline is in implicit "run all" mode. if not configured_stages_to_run: + warnings.warn( + "No stages specified to run. All stages running by default.", + PipelineConfigurationWarning + ) return self.stages unknown_stage_names = sorted( @@ -843,6 +849,7 @@ def _resolve_stages_to_run(self) -> list[Stage]: } resolved_stage_names: set[str] = set() visiting: set[str] = set() + def add_stage_with_dependencies(stage_name: str, *, required_by: str | None = None) -> None: """ From e7fcaae62e8f078fd58053795824e6e1b04c0a7f Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 14:08:56 +0100 Subject: [PATCH 173/332] Tweaked stageConfig.from_mapping() --- onsrap/models.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index c08634f..b6e4324 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -438,7 +438,7 @@ class StageConfig: metadata: dict[str, Any] = field(default_factory=dict) @classmethod - def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_vars: Mapping[str, Any] | None = None) -> StageConfig: + def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> StageConfig: """ Build a ``StageConfig`` from a mapping loaded from code or configuration files. @@ -452,8 +452,7 @@ def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None, global_v Stage name that this configuration applies to. ``data`` : Mapping[str, Any] or None Raw configuration payload for that stage. - ``global_vars`` : Mapping[str, Any] or None - Global variables from config file that need to be parsed to every stage. + # Removed global_vars parameter Returns ------- @@ -546,13 +545,13 @@ class GlobalConfig: exclusion: dict[str, Any] = field(default_factory=dict) @classmethod - def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: + def from_dict(cls, data: Mapping[str, Any] | None) -> GlobalConfig: """ Build a ``GlobalConfig`` from a mapping loaded from code or configuration files. Parameters ---------- - ``data`` : Mapping[str, Any] + ``data`` : Mapping[str, Any] | None Raw configuration payload for the global configuration. ``exclusions`` : dict[str, Any] or None A lookup of which global variables should be excluded from each stage. @@ -562,6 +561,9 @@ def from_dict(cls, data: Mapping[str, Any]) -> GlobalConfig: ``GlobalConfig`` A global configuration object. """ + if data is None: + return cls() + payload = dict(data or {}) exclusions = payload.pop("exclusions", None) From f940d5177842aa4f4feb0939dc43d0a565beb8dd Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Tue, 28 Jul 2026 14:09:11 +0100 Subject: [PATCH 174/332] fix: changes to execution and tests --- onsrap/execution.py | 33 ++++++++++++++++++++++++--------- onsrap/warnings.py | 6 ++++++ tests/test_execution.py | 13 +++++++++---- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/onsrap/execution.py b/onsrap/execution.py index 5a8e0fc..b065e10 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -183,7 +183,7 @@ def resolve_output_root(self) -> Path: raise PipelineConfigurationError("Please parse a run directory to " \ "the ExecutionContext.") - def get_stage_config(self, stage: str | None = None, vars_only: bool = True) -> dict[str, Any] | StageConfig | None: + def get_stage_config(self, stage: str | None = None, with_global: bool = True, vars_only: bool = True) -> dict[str, Any] | StageConfig | None: """ Returns the configuration for the stage currently being executed, with optional arguments. @@ -205,9 +205,21 @@ def get_stage_config(self, stage: str | None = None, vars_only: bool = True) -> The parameters contained within the configuration for the currently active stage. If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. """ - stage_config = self.stage_config_for(stage) if stage is not None else self.stage_config + stage_config: StageConfig | None = self.stage_config_for(stage) if stage is not None else self.stage_config + + if with_global and not vars_only: + raise PipelineConfigurationError( + "get_stage_config() cannot return a StageConfig when with_global=True. " + "Global and stage variables are combined into a dictionary; use vars_only=True " + "or set with_global=False to retrieve the raw StageConfig object." + ) + if stage_config is None: + if with_global: + return self._combine_vars() return {} if vars_only else None + if with_global: + return self._combine_vars(stage_config) if vars_only: return stage_config.variables return stage_config @@ -265,7 +277,7 @@ def resolve_given_path(self, stage_name: str | None, return root / file_name return root - def _combine_vars(self) -> dict[str, Any]: + def _combine_vars(self, stage: StageConfig | None = None) -> dict[str, Any]: """ Private method that combines the global variables and the stage variables for the current stage. @@ -290,15 +302,18 @@ def _combine_vars(self) -> dict[str, Any]: a warning is raised to alert the user that the stage configuration will take precedence. """ - #Extracts variables from global config without exclusion lookup + resolved_stage = stage or self.stage_config global_vars, exclusions = self.global_config.get_attributes() if self.global_config else ({}, {}) - stage_exclusions_extract = exclusions.get(self.active_stage_name, []) - stage_exclusions = [exclusion for exclusion in stage_exclusions_extract] + exclusions = exclusions or {} + + if resolved_stage is None: + return dict(global_vars) - combined = {k: v for k, v in global_vars.items() if k not in stage_exclusions} + stage_exclusions_extract = exclusions.get(resolved_stage.name, []) + stage_exclusions = [exclusion for exclusion in stage_exclusions_extract] - stage_config = self.stage_config_for(self.active_stage_name) - stage_vars = stage_config.variables + combined = {key: value for key, value in global_vars.items() if key not in stage_exclusions} + stage_vars = resolved_stage.variables conflicts = stage_vars.keys() & combined.keys() if conflicts: diff --git a/onsrap/warnings.py b/onsrap/warnings.py index 2d9f236..67ca367 100644 --- a/onsrap/warnings.py +++ b/onsrap/warnings.py @@ -13,4 +13,10 @@ class PipelineConfigurationWarning(OnsrapWarning): """ Raised when the pipeline configuration is not optimal. Child class with ``OnsrapWarning`` as the parent class. + """ + +class ConfigurationInjectionWarning(OnsrapWarning): + """ + Raised when the configuration injection is not optimal. + Child class with ``OnsrapWarning`` as the parent class. """ \ No newline at end of file diff --git a/tests/test_execution.py b/tests/test_execution.py index a060370..05e0f5a 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -192,7 +192,9 @@ def test_stage_config_accessors_return_named_and_active_configs(config, logger) assert context.stage_config_for("stage_test") == stage_config assert context.get_stage_config("stage_test") == {"years_to_run": 2017} assert context.get_stage_config() == {"years_to_run": 2017} - assert context.get_stage_config(vars_only=False) == stage_config + with pytest.raises(PipelineConfigurationError): + context.get_stage_config(vars_only=False) + assert context.get_stage_config(with_global=False, vars_only=False) == stage_config """ Parameters for testing multiple add_folder options in @@ -355,7 +357,6 @@ def stage_config() -> StageConfig: name="stage_test", _variables={"sex":"gender", "dob":"date_of_birth"}, - datasets={}, metadata={} ) @@ -391,9 +392,13 @@ def test_get_stage_config(execution, stage_config) -> None: StageConfig object when requested. """ assert execution.get_stage_config() == {} - assert execution.get_stage_config(vars_only=False) == {} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert execution.get_stage_config(with_global=False, vars_only=False) is None execution.set_active_stage(stage_config.name) assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} - assert execution.get_stage_config(vars_only=False) == stage_config + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert execution.get_stage_config(with_global=False, vars_only=False) == stage_config From 7cd302042d5d6e7a1371b1aa4788ea6fd0c1c643 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 08:59:49 +0100 Subject: [PATCH 175/332] fix: added "global_variables" and "global_vars" as options for keys in the configuration file --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index eaead91..e4261e0 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -1105,7 +1105,7 @@ def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, """ possible_stage_keys = ("stage_configuration", "stage_config") possible_pipeline_keys = ("pipeline_variables","pipeline_config") - possible_global_keys = ("global_configuration", "global_config") + possible_global_keys = ("global_configuration", "global_config", "global_variables", "global_vars") stage_payload, stage_configuration = Pipeline._extract_mappings(possible_stage_keys, raw_config, StageConfigurationWarning) pipeline_payload, pipeline_configuration = Pipeline._extract_mappings(possible_pipeline_keys, raw_config, PipelineConfigurationWarning) From d345aecef62cd14b415d0caf43747da036477020 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 14:51:51 +0100 Subject: [PATCH 176/332] feat: add config attribute to RunManifest, implement through _create_manifest method in Pipeline and created new method _combine_configs to collate all configurations together into a cohesive dictionary --- onsrap/models.py | 1 + onsrap/pipeline.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/onsrap/models.py b/onsrap/models.py index b6e4324..991edde 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -640,6 +640,7 @@ class RunManifest: timestamp: str = "" reason: Optional[str] = None user: Optional[str] = None + config: Optional[dict[str, Any]] = None def __str__(self) -> str: """ diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index e4261e0..f47893b 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -549,8 +549,28 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: timestamp=runtime_id.timestamp.isoformat(), reason=self.config.metadata.get("reason"), user=self._current_user(), + config = self._combine_configs(), ) + def _combine_configs(self) -> dict[str,Any]: + """ + Combines all configurations within the Pipeline into one dictionary which can + be recorded in the RunManifest for the run. + + Returns + ------- + dict[str,Any] + A dictionary containing the configuration for the Pipeline, the stages, and + the global configuration. + """ + all_stage_configuration = {name: config.to_dict() for name, config in self.stage_configs.items()} + configuration = {"pipeline_config": self.config.to_dict(), + "stage_configs": all_stage_configuration, + "global_config": self.global_config.get_attributes()} + + return configuration + + def _create_runtime_id(self) -> RuntimeID: """ Creates a RuntimeID instance for the specific run. From e7670c21da168cc20035732ce777d49642d2dc65 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 15:27:49 +0100 Subject: [PATCH 177/332] feat: add a _log_config() method and implement through PipelineRunner --- onsrap/runner.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 280a7c8..d6fe09f 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -9,7 +9,7 @@ from .warnings import StageConfigurationWarning from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, now +from .models import PipelineRun, PipelineStatus, RunManifest, now if TYPE_CHECKING: from .pipeline import Pipeline @@ -121,6 +121,8 @@ def run(self, pipeline: Pipeline) -> PipelineRun: manifest.outputs = {} pipeline.manifest = manifest + _log_config(run_dir, context, manifest) + self.logger.event( "Pipeline started", name=pipeline.name, @@ -231,3 +233,25 @@ def main(argv: list[str] | None = None) -> int: pipeline = Pipeline.from_files(args.stages, name=args.name) pipeline.run() return 0 + + +def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) -> None: + """ + Outputs the configurations used in an instance of a pipeline to a YAML file in the run directory. + + The file is kept in the block flow style typically expected of a YAML file. + + Parameters + ---------- + ``run_dir`` : str + The directory where the pipeline run is being executed. + ``context`` : ExecutionContext + The context of the current pipeline run, containing configuration and state information. + ``manifest`` : RunManifest + The manifest of the current pipeline run, containing metadata and outputs. + """ + config_file = Path(run_dir) / f"configuration_for_{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" + import yaml + with open(config_file, "w") as f: + yaml.safe_dump(manifest.config, f, default_flow_style=False) + From c8e703ef2f5dc8e7a5b1949b9bf24bfa507842a8 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 15:53:13 +0100 Subject: [PATCH 178/332] test: add formatting unit test for _log_config to check that the correct file is created and the correct information included --- onsrap/runner.py | 2 +- tests/test_runner.py | 84 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/test_runner.py diff --git a/onsrap/runner.py b/onsrap/runner.py index d6fe09f..a802816 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -250,7 +250,7 @@ def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) ``manifest`` : RunManifest The manifest of the current pipeline run, containing metadata and outputs. """ - config_file = Path(run_dir) / f"configuration_for_{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" + config_file = run_dir / f"configuration_for_{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" import yaml with open(config_file, "w") as f: yaml.safe_dump(manifest.config, f, default_flow_style=False) diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..e69eb2b --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,84 @@ +import pytest + +from pathlib import Path + +import yaml + +from onsrap.execution import ExecutionContext +from onsrap.logger import Logger +from onsrap.models import PipelineConfig, RunManifest +from onsrap.runner import _log_config + + +def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True) + + config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=config, + logger=Logger(), + run_dir=run_dir, + started_at="2026-07-29_120000", + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + manifest_config = { + "pipeline_config": { + "name": "synthetic_pipeline", + "backend": "python", + "output_dir": str(tmp_path / "outputs"), + }, + "stage_configs": { + "stage_a": { + "years_to_run": 2026, + "target_variable": "classification", + } + }, + "global_config": { + "dry_run": True, + }, + } + + manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + config=manifest_config, + ) + + _log_config(run_dir, context, manifest) + + expected_file = run_dir / ( + "configuration_for_" + f"{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" + ) + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + + assert parsed_yaml == manifest_config + assert "stage_configs:\n" in file_text + assert " stage_a:\n" in file_text + assert " years_to_run: 2026\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text \ No newline at end of file From ea26b2e1bade6ed45d80d8268148062804b5dbd9 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 17:04:57 +0100 Subject: [PATCH 179/332] tweak: adjusted _combine_configs to force the output of global_config.get_attributes() to be a dictionary rather than a tuple --- onsrap/pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index f47893b..11a9b7e 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -563,10 +563,14 @@ def _combine_configs(self) -> dict[str,Any]: A dictionary containing the configuration for the Pipeline, the stages, and the global configuration. """ + global_variables, exclusions = self.global_config.get_attributes() + global_configuration = dict(global_variables or {}) + global_configuration["exclusions"] = exclusions + all_stage_configuration = {name: config.to_dict() for name, config in self.stage_configs.items()} configuration = {"pipeline_config": self.config.to_dict(), "stage_configs": all_stage_configuration, - "global_config": self.global_config.get_attributes()} + "global_config": global_configuration} return configuration From 8bd700eb65ab585c28c31593e5fd57687ecd6a08 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 17:05:22 +0100 Subject: [PATCH 180/332] test: implement integration test for _log_configs method within PipelineRunner.run() --- onsrap/runner.py | 4 ++- tests/test_pipeline_architecture.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index a802816..08c6401 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -250,7 +250,9 @@ def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) ``manifest`` : RunManifest The manifest of the current pipeline run, containing metadata and outputs. """ - config_file = run_dir / f"configuration_for_{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" + date = context.started_at.date() + + config_file = run_dir / f"configuration_for_{context.pipeline_name}_{date}_{context.run_id[-8:]}.yaml" import yaml with open(config_file, "w") as f: yaml.safe_dump(manifest.config, f, default_flow_style=False) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 946bf1e..0338c40 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -441,3 +441,54 @@ def test_example_main_scripts_run_successfully(script_path: Path) -> None: f"stderr:\n{result.stderr}" ) assert "completed with" in result.stdout.lower() + +def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(tmp_path: Path) -> None: + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return {"status": "ok"} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + } + }, + ) + + + run = pipeline.run() + + run_dir = tmp_path / "runs" / run.manifest.run_id + config_file = run_dir / ( + f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" + ) + + assert config_file.exists() + + file_text = config_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + + assert parsed_yaml == run.manifest.config + assert "pipeline_config:\n" in file_text + assert "stage_configs:\n" in file_text + assert "global_config:\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + assert " dry_run: true" in file_text From ff6b275156e1488b8d76c4382fcf607f8ba0fbba Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 17:07:19 +0100 Subject: [PATCH 181/332] tweak: added warning messages into integration test for logging config --- tests/test_pipeline_architecture.py | 35 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 0338c40..da19806 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -455,24 +455,27 @@ def run(context): encoding="utf-8", ) - pipeline = Pipeline.from_files( - [stage_file], - name="config-export-pipeline", - config={ - "pipeline_config": { - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + } }, - "stage_configuration": {}, - "global_configuration": { - "dry_run": True, - } - }, - ) - + ) - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, + match = "Output directory is not specified. Using project root or work directory as the run output."): + run = pipeline.run() run_dir = tmp_path / "runs" / run.manifest.run_id config_file = run_dir / ( From 21b6921cddc867057e30d98275af4720b65efedb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 29 Jul 2026 17:13:21 +0100 Subject: [PATCH 182/332] docs: add documentation to pytests for configuration logging --- tests/test_pipeline_architecture.py | 8 ++++++++ tests/test_runner.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index da19806..7e1fa9b 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -443,6 +443,14 @@ def test_example_main_scripts_run_successfully(script_path: Path) -> None: assert "completed with" in result.stdout.lower() def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(tmp_path: Path) -> None: + """ + Integration test that checks that the _log_config method is correctly called within + PipelineRunner.run() and that the information is parsed in a suitable format to a YAML + file in the run directory. + + This test also captures that _combine_configs() correctly converts all configuration + information into a single dictionary that can be serialized to YAML. + """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( dedent( diff --git a/tests/test_runner.py b/tests/test_runner.py index e69eb2b..ae9e061 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,6 +11,9 @@ def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: + """ + Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + """ run_dir = tmp_path / "runs" / "synthetic_run" run_dir.mkdir(parents=True) From 3b527970f398131b4dc274c48b8d1b1dbd53775e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 30 Jul 2026 15:47:15 +0100 Subject: [PATCH 183/332] feat: added a function to differentiate two configuration files and print the differences into the terminal as well as provide them as a dictionary --- onsrap/runner.py | 63 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_runner.py | 54 +++++++++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 08c6401..fc124cc 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -257,3 +257,66 @@ def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) with open(config_file, "w") as f: yaml.safe_dump(manifest.config, f, default_flow_style=False) + +def _flatten(obj, prefix="", sep="."): + items = {} + if isinstance(obj, dict): + for k, v in obj.items(): + items.update(_flatten(v, f"{prefix}{sep}{k}" if prefix else k, sep)) + elif isinstance(obj, list): + for i, v in enumerate(obj): + items.update(_flatten(v, f"{prefix}[{i}]", sep)) + else: + items[prefix] = obj + return items + +def _diff_yaml_files(path_a, path_b): + import yaml + + with open(path_a) as f: doc_a = yaml.safe_load(f) + with open(path_b) as f: doc_b = yaml.safe_load(f) + + flat_a = _flatten(doc_a) + flat_b = _flatten(doc_b) + + keys_a, keys_b = set(flat_a), set(flat_b) + + return { + "changed": { + k: (flat_a[k], flat_b[k]) + for k in keys_a & keys_b + if flat_a[k] != flat_b[k] + }, + "added": {k: flat_b[k] for k in keys_b - keys_a}, + "removed": {k: flat_a[k] for k in keys_a - keys_b}, + } + +def _print_diff(diff: dict) -> dict: + changed = diff["changed"] + added = diff["added"] + removed = diff["removed"] + + if changed: + print(f"\nCHANGED ({len(changed)})") + for key, (val_a, val_b) in sorted(changed.items()): + print(f" {key}: {val_a!r} → {val_b!r}") + + if added: + print(f"\nADDED in second configuration ({len(added)})") + for key, val in sorted(added.items()): + print(f" {key}: {val!r}") + + if removed: + print(f"\nREMOVED in second configuration ({len(removed)})") + for key, val in sorted(removed.items()): + print(f" {key}: {val!r}") + + if not any([changed, added, removed]): + print("Files are identical.") + + return diff + +def print_config_diffs(file_1, file_2) -> dict: + + diff_dict = _diff_yaml_files(file_1, file_2) + return _print_diff(diff_dict) \ No newline at end of file diff --git a/tests/test_runner.py b/tests/test_runner.py index ae9e061..cafad2c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,3 +1,5 @@ +from textwrap import dedent + import pytest from pathlib import Path @@ -7,7 +9,7 @@ from onsrap.execution import ExecutionContext from onsrap.logger import Logger from onsrap.models import PipelineConfig, RunManifest -from onsrap.runner import _log_config +from onsrap.runner import _log_config, print_config_diffs def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: @@ -84,4 +86,52 @@ def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) - assert " years_to_run: 2026\n" in file_text assert "pipeline_config: {" not in file_text assert "stage_configs: {" not in file_text - assert "global_config: {" not in file_text \ No newline at end of file + assert "global_config: {" not in file_text + +def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + test_file_a = tmp_path / "config_a.yaml" + test_file_b = tmp_path / "config_b.yaml" + + test_file_a.write_text( + dedent(""" + pipeline_config: + name: synthetic_pipeline + output_dir: outputs #This is removed in file_b + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: classification + global_config: + dry_run: True + """).strip() + + "\n", + encoding="utf-8", + ) + + test_file_b.write_text( + dedent(""" + pipeline_config: + name: synthetic_pipeline + backend: python + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: identification + global_config: + dry_run: True + """).strip() + + "\n", + encoding="utf-8", + ) + + assert print_config_diffs(test_file_a, test_file_b) == { + "changed": {"stage_configs.stage_a.target_variable": ("classification", "identification")}, + "added": {"pipeline_config.backend": "python"}, + "removed": {"pipeline_config.output_dir": "outputs"} + } + + captured = capsys.readouterr() + assert "CHANGED (1)" in captured.out + assert "ADDED in second configuration (1)" in captured.out + assert "REMOVED in second configuration (1)" in captured.out + assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file From 4d957f3f456eea26687bd545ce90f4556fc9515d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 30 Jul 2026 16:04:13 +0100 Subject: [PATCH 184/332] docs: documentation for configuration differencing functions --- onsrap/runner.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index fc124cc..c479890 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -259,6 +259,25 @@ def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) def _flatten(obj, prefix="", sep="."): + """ + Converts nested dictionaries or lists into flat object using dot notation for keys. + Each key in the resulting dictionary represents the nested branching to get to the value + in the original dictionary. + + Parameters + ---------- + ``obj`` : dict or list + The object to flatten, which can be a dictionary or a list. + ``prefix`` : str + The prefix to use for the keys in the flattened dictionary. Defaults to an empty string. + ``sep`` : str + The separator to use between keys in the flattened dictionary. Defaults to a dot ("."). + + Returns + ------- + dict + A flattened dictionary where each key represents the path to the value in the original object. + """ items = {} if isinstance(obj, dict): for k, v in obj.items(): @@ -270,7 +289,29 @@ def _flatten(obj, prefix="", sep="."): items[prefix] = obj return items -def _diff_yaml_files(path_a, path_b): +def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: + """ + Calculates the differences between two YAML files and returns a programming oriented dictionary + describing the changes. + + This function calls the ``_flatten`` function to the loaded in dictionaries from the YAML files. + These are then differenced to account for whether a value has changed between the two files, + been added to the second file and was not present in the first, or removed from the second file + and is only present in the first. This output is structured as {changed: {}, added: {}, removed: {}}. + + Parameters + ---------- + ``path_a`` : Path + The path to the first YAML file to compare. + ``path_b`` : Path + The path to the second YAML file to compare. + + Returns + ------- + dict + A dictionary describing the differences between the two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + """ import yaml with open(path_a) as f: doc_a = yaml.safe_load(f) @@ -292,6 +333,22 @@ def _diff_yaml_files(path_a, path_b): } def _print_diff(diff: dict) -> dict: + """ + Prints the differences between two YAML files in a human-readable format and returns + the computer-readable dictionary so that it could be used for logging processes if + required. + + Parameters + ---------- + diff : dict + A dictionary describing the differences between two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + + Returns + ------- + dict + The same dictionary that was passed in as the ``diff`` parameter. + """ changed = diff["changed"] added = diff["added"] removed = diff["removed"] @@ -317,6 +374,26 @@ def _print_diff(diff: dict) -> dict: return diff def print_config_diffs(file_1, file_2) -> dict: + """ + A combining function that calculates the differences between two YAML files + and then prints the outputs to the terminal as well as returning the computer-readable + dictionary of the differences. + + This works by calling the ``_diff_yaml_files`` function to calculate the differences and + then calling the ``_print_diff`` function to print the differences. + Parameters + ---------- + ``file_1`` : Path + The path to the first YAML file to compare. + ``file_2`` : Path + The path to the second YAML file to compare. + + Returns + ------- + dict + A dictionary describing the differences between the two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + """ diff_dict = _diff_yaml_files(file_1, file_2) return _print_diff(diff_dict) \ No newline at end of file From 6dc56509aa754a900a84fffde7c1215945a65a1f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 30 Jul 2026 16:05:35 +0100 Subject: [PATCH 185/332] docs: documentation added for test created for config differencing functions. --- tests/test_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index cafad2c..49fe853 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -89,6 +89,12 @@ def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) - assert "global_config: {" not in file_text def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """ + Tests that two configuration files are correctly compared and the differences are + both returned in a computer-readable format and printed to the console. One change + for each category (changed, added, removed) is included in the test to ensure that + all cases are handled correctly. + """ test_file_a = tmp_path / "config_a.yaml" test_file_b = tmp_path / "config_b.yaml" @@ -96,7 +102,7 @@ def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) dedent(""" pipeline_config: name: synthetic_pipeline - output_dir: outputs #This is removed in file_b + output_dir: outputs stage_configs: stage_a: years_to_run: 2026 From d6701297a1782403d6b92431dd51d1b669722649 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 08:40:28 +0100 Subject: [PATCH 186/332] tweak: add warnings to _resolve_config() that alerts when an output location key is present within the stage_configuration. This is allowed to keep running however will warn that it will likely result in overwriting. --- onsrap/pipeline.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index e4261e0..0989b20 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -640,7 +640,16 @@ def _resolve_config( ------- tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig] The resolved pipeline configuration, stage configurations, configured stages, and global configuration. + + Raises + ------ + ``StageConfigurationWarning`` + If a stage configuration is found within the metadata section of a PipelineConfig instance, a warning is + raised to indicate that a composite configuration payload is preferred. + If an output location is recorded in stage configuration, a warning is raised to inform the user that + this will result in overwriting previous run outputs. """ + output_dir_keys = ("output_dir", "output_directory", "output_path", "output_location") if config is None: return PipelineConfig.from_any(config), {}, [], GlobalConfig() @@ -654,6 +663,12 @@ def _resolve_config( "Stage configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", StageConfigurationWarning, ) + if any(key in stage_configuration for key in output_dir_keys): + warnings.warn( + "Stage configuration contains output directory keys. This will result in overwriting previous run outputs. Please set your " + "output location in the stage scripts using the resolve_output_path() function to ensure unique outputs are saved for each run.", + StageConfigurationWarning, + ) global_configuration = config.metadata.get("global_config", None) if global_configuration is not None: @@ -667,6 +682,14 @@ def _resolve_config( pipeline_payload, stage_config_payload, global_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) + + if any(key in stage_config_payload for key in output_dir_keys): + warnings.warn( + "Stage configuration contains output directory keys. This will result in overwriting previous run outputs. Please set your " + "output location in the stage scripts using the resolve_output_path() function to ensure unique outputs are saved for each run.", + StageConfigurationWarning, + ) + stage_definitions = normalized_pipeline_payload.pop("stages", ()) pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) @@ -679,8 +702,7 @@ def _resolve_config( global_config = GlobalConfig.from_dict(global_config_payload) return pipeline_config, stage_configs, configured_stages, global_config - - + def _sync_stage_configs(self) -> None: """ From 78099377ce75973ff8d88de5fa17d152b05e5beb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 09:13:15 +0100 Subject: [PATCH 187/332] tweak: created a new method that assesses for output directories in stage_configs rather than being done at _resolve_config stage --- onsrap/pipeline.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0989b20..0ee0dc0 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -98,6 +98,7 @@ def __init__( self.global_config = resolved_global_config self._sync_stage_configs() + self._check_output_dir_in_stage_configs() self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -649,7 +650,7 @@ def _resolve_config( If an output location is recorded in stage configuration, a warning is raised to inform the user that this will result in overwriting previous run outputs. """ - output_dir_keys = ("output_dir", "output_directory", "output_path", "output_location") + if config is None: return PipelineConfig.from_any(config), {}, [], GlobalConfig() @@ -663,12 +664,6 @@ def _resolve_config( "Stage configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", StageConfigurationWarning, ) - if any(key in stage_configuration for key in output_dir_keys): - warnings.warn( - "Stage configuration contains output directory keys. This will result in overwriting previous run outputs. Please set your " - "output location in the stage scripts using the resolve_output_path() function to ensure unique outputs are saved for each run.", - StageConfigurationWarning, - ) global_configuration = config.metadata.get("global_config", None) if global_configuration is not None: @@ -682,14 +677,6 @@ def _resolve_config( pipeline_payload, stage_config_payload, global_config_payload = self._split_config_sections(raw_config) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) - - if any(key in stage_config_payload for key in output_dir_keys): - warnings.warn( - "Stage configuration contains output directory keys. This will result in overwriting previous run outputs. Please set your " - "output location in the stage scripts using the resolve_output_path() function to ensure unique outputs are saved for each run.", - StageConfigurationWarning, - ) - stage_definitions = normalized_pipeline_payload.pop("stages", ()) pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) @@ -707,9 +694,29 @@ def _resolve_config( def _sync_stage_configs(self) -> None: """ Ensure every known stage has a ``StageConfig`` entry, even if it is empty. + """ for stage in self.stages: self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) + + + def _check_output_dir_in_stage_configs(self) -> None: + """ + Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, + as this will result in overwriting previous run outputs. Users are advised to set their output + location in the stage scripts using the ``resolve_output_path()`` function to ensure unique + outputs are saved for each run. + """ + + output_dir_keys = ("output_dir", "output_directory", "output_path", "output_location") + for stage in self.stages: + if any(key in self.stage_configs[stage.name]._variables for key in output_dir_keys): + warnings.warn( + f"Stage configuration for {stage.name} contains output directory keys. This will result " + f"in overwriting previous run outputs. Please set your output location in the stage scripts " + f"using the resolve_output_path() function to ensure unique outputs are saved for each run.", + StageConfigurationWarning, + ) def _validate_stage_configs(self) -> None: """ From 1c44bfef919ffb8d8f58797013da5044804b0c4f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 09:15:34 +0100 Subject: [PATCH 188/332] tweak: correct name of resolve_output_root() method in warning for _check_output_dir_in_stage_configs --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0ee0dc0..4532982 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -714,7 +714,7 @@ def _check_output_dir_in_stage_configs(self) -> None: warnings.warn( f"Stage configuration for {stage.name} contains output directory keys. This will result " f"in overwriting previous run outputs. Please set your output location in the stage scripts " - f"using the resolve_output_path() function to ensure unique outputs are saved for each run.", + f"using the resolve_output_root() function to ensure unique outputs are saved for each run.", StageConfigurationWarning, ) From a4ff55f4fb841213028e6578a146a19368ff91f8 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 10:53:48 +0100 Subject: [PATCH 189/332] tweak: minor changes to output checking method --- onsrap/pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 4532982..5134581 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -225,6 +225,7 @@ def add_stage( self._check_stage_configs(added_stages, self.stage_configs) self._sync_stage_configs() + self._check_output_dir_in_stage_configs() self._rebuild_graph() self.logger.event( @@ -252,6 +253,7 @@ def add_stage_config( """ parsed_stage_config = self._coerce_stage_config(stage_config, name=name) self.stage_configs[parsed_stage_config.name] = parsed_stage_config + self._check_output_dir_in_stage_configs() self.logger.event("Stage configuration added", stage=parsed_stage_config.name) def enable_stage(self, *stage_name: str | list[str]) -> None: @@ -709,14 +711,18 @@ def _check_output_dir_in_stage_configs(self) -> None: """ output_dir_keys = ("output_dir", "output_directory", "output_path", "output_location") + #TODO: Regex? for output directory for stage in self.stages: if any(key in self.stage_configs[stage.name]._variables for key in output_dir_keys): warnings.warn( f"Stage configuration for {stage.name} contains output directory keys. This will result " f"in overwriting previous run outputs. Please set your output location in the stage scripts " - f"using the resolve_output_root() function to ensure unique outputs are saved for each run.", + f"using the resolve_output_root() method to ensure unique outputs are saved for each run.", StageConfigurationWarning, ) + self.logger.event(f"Warning: stage configuration for {stage.name} contains output directory keys. Risk of overwriting outputs.", + keys_found = [key for key in output_dir_keys if key in self.stage_configs[stage.name]._variables] + ) def _validate_stage_configs(self) -> None: """ From c5ffd4e51e9c273d4e525274d45e29d93884c7ca Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 13:28:06 +0100 Subject: [PATCH 190/332] feat: adjusted output directory search to cover regex --- onsrap/pipeline.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 5134581..973d6f1 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -8,6 +8,7 @@ from importlib import metadata as importlib_metadata from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence +import re from .errors import StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError from .warnings import StageConfigurationWarning, PipelineConfigurationWarning @@ -710,10 +711,13 @@ def _check_output_dir_in_stage_configs(self) -> None: outputs are saved for each run. """ - output_dir_keys = ("output_dir", "output_directory", "output_path", "output_location") + OUTPUT_DIR_KEY_RE = re.compile( + r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", + re.IGNORECASE + ) #TODO: Regex? for output directory for stage in self.stages: - if any(key in self.stage_configs[stage.name]._variables for key in output_dir_keys): + if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[stage.name]._variables): warnings.warn( f"Stage configuration for {stage.name} contains output directory keys. This will result " f"in overwriting previous run outputs. Please set your output location in the stage scripts " @@ -721,7 +725,7 @@ def _check_output_dir_in_stage_configs(self) -> None: StageConfigurationWarning, ) self.logger.event(f"Warning: stage configuration for {stage.name} contains output directory keys. Risk of overwriting outputs.", - keys_found = [key for key in output_dir_keys if key in self.stage_configs[stage.name]._variables] + keys_found = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] ) def _validate_stage_configs(self) -> None: From d6ffb7bb9e8d63729ba9f938348976f2cf6e35e2 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 13:50:08 +0100 Subject: [PATCH 191/332] tweak: add overwrite parameter to PipelineConfig that defaults to False --- onsrap/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onsrap/models.py b/onsrap/models.py index b6e4324..a889854 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -127,6 +127,8 @@ class PipelineConfig: pipeline. ``metadata`` : dict[str, Any] Any additional information on the pipeline. + ``overwrite`` : bool, default = False + Indicates whether the pipeline should overwrite previous outputs. """ name: Optional[str] = None stages_to_run: Optional[dict[str, bool]] = None @@ -139,6 +141,7 @@ class PipelineConfig: allow_subprocess_fallback: bool = True python_executable: Optional[str] = None metadata: dict[str, Any] = field(default_factory=dict) + overwrite: bool = False def __post_init__(self) -> None: """ From 0b124d9572f0f2d3256f8116faf4166cb185491b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 13:52:39 +0100 Subject: [PATCH 192/332] feat: add _output_dir_conflict_check() method and apply it to Pipeline.validate() --- onsrap/pipeline.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 973d6f1..e21afa0 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -100,6 +100,7 @@ def __init__( self._sync_stage_configs() self._check_output_dir_in_stage_configs() + self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -351,6 +352,9 @@ def validate(self) -> Pipeline: for stage in self.graph.stages: stage.validate() self.graph.validate() + + self._output_dir_conflict_check() + return self @@ -715,7 +719,6 @@ def _check_output_dir_in_stage_configs(self) -> None: r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", re.IGNORECASE ) - #TODO: Regex? for output directory for stage in self.stages: if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[stage.name]._variables): warnings.warn( @@ -728,6 +731,30 @@ def _check_output_dir_in_stage_configs(self) -> None: keys_found = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] ) + def _output_dir_conflict_check(self) -> None: + + OUTPUT_DIR_KEY_RE = re.compile( + r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", + re.IGNORECASE + ) + + for stage in self.stages: + available_output_dirs = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] + for directory in available_output_dirs: + output_dir = self.stage_configs[stage.name].get(directory, None) + if Path(output_dir).exists() and self.config.overwrite is False: + raise StageConfigurationError( + f"Stage configuration for {stage.name} contains an output directory path that already exists. Please set a unique " + f"output directory for this stage to prevent overwriting.") + if Path(output_dir).exists() and self.config.overwrite is True: + warnings.warn( + f"Stage configuration for {stage.name} contains an output directory path that already exists. As the overwrite " + f"parameter is True, the pipeline will proceed and will overwrite the previous run file." + ) + self.logger.event(f"Warning: Output directory {output_dir} already exists however permissions allow overwriting. The previous file " + f"will be overwritten.", overwrite = self.config.overwrite) + + def _validate_stage_configs(self) -> None: """ Confirm that every configured stage name matches a stage present in the pipeline. From 913bc1f2cb679b2ebc1fa7cc8bbb752d614ae041 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 14:33:41 +0100 Subject: [PATCH 193/332] tweak: add overwrite attribute to __str__ method and __repr method for PipelineConfig --- onsrap/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index a889854..d883fe8 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -166,7 +166,8 @@ def __str__(self) -> str: f" Work Directory: {self.work_dir}\n Project Root: {self.project_root}\n" f" Output Directory: {self.output_dir}\n Log Directory: {self.log_dir}\n" f" Data Directory: {self.data_dir}\n Allow Subprocess Fallback: {self.allow_subprocess_fallback}\n" - f" Python Executable: {self.python_executable}\n Metadata: \n{_format_dict(self.metadata, indent=8)}" + f" Python Executable: {self.python_executable}\n Overwrite: {self.overwrite}\n" + f" Metadata: \n{_format_dict(self.metadata, indent=8)}" ) def __repr__(self) -> str: @@ -186,7 +187,8 @@ def __repr__(self) -> str: f"work_dir={self.work_dir}, project_root={self.project_root}, " f"output_dir={self.output_dir}, log_dir={self.log_dir}, data_dir={self.data_dir}, " f"allow_subprocess_fallback={self.allow_subprocess_fallback}, " - f"python_executable={self.python_executable}, metadata={self.metadata})" + f"python_executable={self.python_executable}, overwrite={self.overwrite}, " + f"metadata={self.metadata})" ) @classmethod @@ -257,6 +259,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) + overwrite = payload.pop("overwrite", False) if isinstance(raw_subprocess_fallback, str): warnings.warn( "allow_subprocess_fallback should be a boolean, not a string. " @@ -281,6 +284,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: log_dir=log_dir, data_dir=data_dir, allow_subprocess_fallback=allow_subprocess_fallback, + overwrite=overwrite, python_executable=python_executable, metadata=metadata, ) From 52893b20af5664c7df2b24d4222f9c705dbf9d16 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 14:34:28 +0100 Subject: [PATCH 194/332] feat: add logging for error message with _output_dir_conflict_check() method and amend pipeline_2 config to incl. overwrite --- examples/pipeline_2/conf.yaml | 2 ++ onsrap/pipeline.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml index 5ae8374..58d076b 100644 --- a/examples/pipeline_2/conf.yaml +++ b/examples/pipeline_2/conf.yaml @@ -19,6 +19,7 @@ pipeline_variables: data_dir: examples/pipeline_2/data output_dir: examples/pipeline_2/outputs log_dir: examples/pipeline_2/logs + overwrite: True metadata: example: retail-orders-using-configuration description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." @@ -26,6 +27,7 @@ pipeline_variables: 0_clean_data: True 1_derive_vars: True 2_reporting: True + global_variables: year_of_run: 2020 diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index e21afa0..3cd4e86 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -354,7 +354,7 @@ def validate(self) -> Pipeline: self.graph.validate() self._output_dir_conflict_check() - + return self @@ -743,9 +743,12 @@ def _output_dir_conflict_check(self) -> None: for directory in available_output_dirs: output_dir = self.stage_configs[stage.name].get(directory, None) if Path(output_dir).exists() and self.config.overwrite is False: + self.logger.event(f"Error: Output directory {output_dir} already exists. Pipeline will crash to prevent overwrite.", + overwrite = self.config.overwrite) raise StageConfigurationError( f"Stage configuration for {stage.name} contains an output directory path that already exists. Please set a unique " f"output directory for this stage to prevent overwriting.") + if Path(output_dir).exists() and self.config.overwrite is True: warnings.warn( f"Stage configuration for {stage.name} contains an output directory path that already exists. As the overwrite " From 479947baa22782076093a3c28b80392ecc952ec8 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 15:21:03 +0100 Subject: [PATCH 195/332] docs: add documentation for _output_dir_conflict_check() --- onsrap/pipeline.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 3cd4e86..a8dddf8 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -732,6 +732,20 @@ def _check_output_dir_in_stage_configs(self) -> None: ) def _output_dir_conflict_check(self) -> None: + """ + Checks whether the output directory already exists and raises an error if it does, unless the overwrite parameter is set to True. + + For each stage in the Pipeline, checks whether an output directory has been defined in the stage configurations. If it has been + defined, it checks whether the Path value for the output directory already exists. If it does already exist, raise either an + error or a warning based on an overwrite configuration. Log either the error or the warning in the Logger. + + Raises + ------ + StageConfigurationError + If the overwrite parameter in the PipelineConfig is set to False and the output directory already exists. + StageConfigurationWarning + If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. + """ OUTPUT_DIR_KEY_RE = re.compile( r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", @@ -752,7 +766,7 @@ def _output_dir_conflict_check(self) -> None: if Path(output_dir).exists() and self.config.overwrite is True: warnings.warn( f"Stage configuration for {stage.name} contains an output directory path that already exists. As the overwrite " - f"parameter is True, the pipeline will proceed and will overwrite the previous run file." + f"parameter is True, the pipeline will proceed and will overwrite the previous run file.", StageConfigurationWarning ) self.logger.event(f"Warning: Output directory {output_dir} already exists however permissions allow overwriting. The previous file " f"will be overwritten.", overwrite = self.config.overwrite) From c0d75848bcb41c6025dd79c47aabad387e6c1a49 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 3 Aug 2026 15:38:30 +0100 Subject: [PATCH 196/332] feat: combine _check_output_dir_in_stage_configs into _output_dir_conflict_check. Call _output_dir_conflict_check in Pipeline.validate() and remove mention to _check_output_dir_in_stage_configs in earlier Pipeline init stage --- onsrap/pipeline.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index a8dddf8..cffd418 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -99,7 +99,6 @@ def __init__( self.global_config = resolved_global_config self._sync_stage_configs() - self._check_output_dir_in_stage_configs() self._rebuild_graph() self.id: RuntimeID | None = None @@ -227,7 +226,6 @@ def add_stage( self._check_stage_configs(added_stages, self.stage_configs) self._sync_stage_configs() - self._check_output_dir_in_stage_configs() self._rebuild_graph() self.logger.event( @@ -707,33 +705,38 @@ def _sync_stage_configs(self) -> None: self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) - def _check_output_dir_in_stage_configs(self) -> None: + def _check_output_dir_in_stage_configs(self, regex_pattern, name: str) -> None: """ Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, as this will result in overwriting previous run outputs. Users are advised to set their output location in the stage scripts using the ``resolve_output_path()`` function to ensure unique outputs are saved for each run. + + Parameters + ---------- + ``regex_pattern`` + The regular expression pattern used to identify output directory keys in the stage configurations. + + ``name`` + The stage name to check for output directory keys in the stage configurations. """ - OUTPUT_DIR_KEY_RE = re.compile( - r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", - re.IGNORECASE - ) - for stage in self.stages: - if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[stage.name]._variables): - warnings.warn( - f"Stage configuration for {stage.name} contains output directory keys. This will result " - f"in overwriting previous run outputs. Please set your output location in the stage scripts " - f"using the resolve_output_root() method to ensure unique outputs are saved for each run.", - StageConfigurationWarning, - ) - self.logger.event(f"Warning: stage configuration for {stage.name} contains output directory keys. Risk of overwriting outputs.", - keys_found = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] - ) + OUTPUT_DIR_KEY_RE = regex_pattern + if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[name]._variables): + warnings.warn( + f"Stage configuration for {name} contains output directory keys. This will result " + f"in overwriting previous run outputs. Please set your output location in the stage scripts " + f"using the resolve_output_root() method to ensure unique outputs are saved for each run.", + StageConfigurationWarning, + ) + self.logger.event(f"Warning: stage configuration for {name} contains output directory keys. Risk of overwriting outputs.", + keys_found = [key for key in self.stage_configs[name]._variables if OUTPUT_DIR_KEY_RE.match(key)] + ) def _output_dir_conflict_check(self) -> None: """ - Checks whether the output directory already exists and raises an error if it does, unless the overwrite parameter is set to True. + Checks whether the output directory has been assigned in stage configurations and already exists. It raises an error if + it does, unless the overwrite parameter is set to True. For each stage in the Pipeline, checks whether an output directory has been defined in the stage configurations. If it has been defined, it checks whether the Path value for the output directory already exists. If it does already exist, raise either an @@ -754,6 +757,8 @@ def _output_dir_conflict_check(self) -> None: for stage in self.stages: available_output_dirs = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] + self._check_output_dir_in_stage_configs(regex_pattern=OUTPUT_DIR_KEY_RE, name=stage.name) + for directory in available_output_dirs: output_dir = self.stage_configs[stage.name].get(directory, None) if Path(output_dir).exists() and self.config.overwrite is False: From 648fefd71f6684e123ac949347c7d609f2e22789 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 09:24:44 +0100 Subject: [PATCH 197/332] docs: add inline comment on what the regex covers --- onsrap/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index cffd418..5e676ba 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -750,6 +750,7 @@ def _output_dir_conflict_check(self) -> None: If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. """ + #captures long and short versions of output, directory, path, location, and file that's not case sensitive. OUTPUT_DIR_KEY_RE = re.compile( r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", re.IGNORECASE From e4f7c11198259486ab26cbf9b0e0bf3317834d5f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 09:27:37 +0100 Subject: [PATCH 198/332] tweak: add output_dir_regex as a module level global variable --- onsrap/pipeline.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 5e676ba..67ca9cd 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -21,6 +21,11 @@ ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") AVAILABLE_EXECUTORS = ("python",) +#captures long and short versions of output, directory, path, location, and file that's not case sensitive. +OUTPUT_DIR_KEY_RE = re.compile( + r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", + re.IGNORECASE + ) class Pipeline: """ @@ -705,7 +710,7 @@ def _sync_stage_configs(self) -> None: self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) - def _check_output_dir_in_stage_configs(self, regex_pattern, name: str) -> None: + def _check_output_dir_in_stage_configs(self, name: str) -> None: """ Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, as this will result in overwriting previous run outputs. Users are advised to set their output @@ -721,7 +726,6 @@ def _check_output_dir_in_stage_configs(self, regex_pattern, name: str) -> None: The stage name to check for output directory keys in the stage configurations. """ - OUTPUT_DIR_KEY_RE = regex_pattern if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[name]._variables): warnings.warn( f"Stage configuration for {name} contains output directory keys. This will result " @@ -750,15 +754,10 @@ def _output_dir_conflict_check(self) -> None: If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. """ - #captures long and short versions of output, directory, path, location, and file that's not case sensitive. - OUTPUT_DIR_KEY_RE = re.compile( - r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", - re.IGNORECASE - ) for stage in self.stages: available_output_dirs = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] - self._check_output_dir_in_stage_configs(regex_pattern=OUTPUT_DIR_KEY_RE, name=stage.name) + self._check_output_dir_in_stage_configs(name=stage.name) for directory in available_output_dirs: output_dir = self.stage_configs[stage.name].get(directory, None) From 339e2de5c9a8b9f0e1fdfeb2c8bf5d2997ce3854 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:42:25 +0100 Subject: [PATCH 199/332] Apply suggestions from code review Applied suggestions from CoPilot and Alex to config logging. These are minor changes reflecting corrections to testing assertions, typing, and docstrings. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> --- onsrap/pipeline.py | 2 +- onsrap/runner.py | 23 ++++++++++++----------- tests/test_runner.py | 3 +-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 11a9b7e..7de65d8 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -554,7 +554,7 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: def _combine_configs(self) -> dict[str,Any]: """ - Combines all configurations within the Pipeline into one dictionary which can + Combines all configurations (PipelineConfig, GlobalConfig, StageConfig) within the Pipeline into one dictionary which can be recorded in the RunManifest for the run. Returns diff --git a/onsrap/runner.py b/onsrap/runner.py index c479890..5641150 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -235,16 +235,16 @@ def main(argv: list[str] | None = None) -> int: return 0 -def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) -> None: +def _log_config(run_dir: Path, context: ExecutionContext, manifest: RunManifest) -> None: """ - Outputs the configurations used in an instance of a pipeline to a YAML file in the run directory. + Outputs the configurations used in an instance of a pipeline to a YAML file in the run directory. - The file is kept in the block flow style typically expected of a YAML file. + The file is kept in the block flow style typically expected of a YAML file. Parameters ---------- - ``run_dir`` : str - The directory where the pipeline run is being executed. + ``run_dir`` : Path + The directory where the pipeline run is being executed. ``context`` : ExecutionContext The context of the current pipeline run, containing configuration and state information. ``manifest`` : RunManifest @@ -254,11 +254,11 @@ def _log_config(run_dir: str, context: ExecutionContext, manifest: RunManifest) config_file = run_dir / f"configuration_for_{context.pipeline_name}_{date}_{context.run_id[-8:]}.yaml" import yaml - with open(config_file, "w") as f: - yaml.safe_dump(manifest.config, f, default_flow_style=False) + with open(config_file, "w", encoding="utf-8") as f: + yaml.safe_dump(manifest.config or {}, f, default_flow_style=False) -def _flatten(obj, prefix="", sep="."): +def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: """ Converts nested dictionaries or lists into flat object using dot notation for keys. Each key in the resulting dictionary represents the nested branching to get to the value @@ -314,12 +314,13 @@ def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: """ import yaml - with open(path_a) as f: doc_a = yaml.safe_load(f) - with open(path_b) as f: doc_b = yaml.safe_load(f) + with open(path_a, encoding="utf-8") as f: + doc_a = yaml.safe_load(f) or {} + with open(path_b, encoding="utf-8") as f: + doc_b = yaml.safe_load(f) or {} flat_a = _flatten(doc_a) flat_b = _flatten(doc_b) - keys_a, keys_b = set(flat_a), set(flat_b) return { diff --git a/tests/test_runner.py b/tests/test_runner.py index 49fe853..225413f 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -39,7 +39,6 @@ def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) - config=config, logger=Logger(), run_dir=run_dir, - started_at="2026-07-29_120000", working_directory=tmp_path, stage_configs={}, global_config=None, @@ -72,7 +71,7 @@ def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) - expected_file = run_dir / ( "configuration_for_" - f"{context.pipeline_name}_{context.started_at}_{context.run_id}.yaml" + f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" ) assert expected_file.exists() From 6cc4068eceec75149f4ab955f8a6e8ddd196c205 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:29:32 +0100 Subject: [PATCH 200/332] Apply suggestions from code review Committing suggestions from CoPilot review of changes. Good to merge Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- onsrap/models.py | 2 +- onsrap/pipeline.py | 44 ++++++++++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index d883fe8..c7619b4 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -259,7 +259,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) - overwrite = payload.pop("overwrite", False) + overwrite = PipelineConfig._to_bool(payload.pop("overwrite", False)) if isinstance(raw_subprocess_fallback, str): warnings.warn( "allow_subprocess_fallback should be a boolean, not a string. " diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 67ca9cd..afeb926 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -256,9 +256,9 @@ def add_stage_config( Optional stage name used when the parsed configuration payload does not identify the stage on its own. """ - parsed_stage_config = self._coerce_stage_config(stage_config, name=name) +parsed_stage_config = self._coerce_stage_config(stage_config, name=name) self.stage_configs[parsed_stage_config.name] = parsed_stage_config - self._check_output_dir_in_stage_configs() + self._check_output_dir_in_stage_configs(name=parsed_stage_config.name) self.logger.event("Stage configuration added", stage=parsed_stage_config.name) def enable_stage(self, *stage_name: str | list[str]) -> None: @@ -657,8 +657,7 @@ def _resolve_config( ``StageConfigurationWarning`` If a stage configuration is found within the metadata section of a PipelineConfig instance, a warning is raised to indicate that a composite configuration payload is preferred. - If an output location is recorded in stage configuration, a warning is raised to inform the user that - this will result in overwriting previous run outputs. + Output-location warnings are emitted during ``Pipeline.validate()`` when stage configurations are inspected. """ if config is None: @@ -760,21 +759,38 @@ def _output_dir_conflict_check(self) -> None: self._check_output_dir_in_stage_configs(name=stage.name) for directory in available_output_dirs: - output_dir = self.stage_configs[stage.name].get(directory, None) - if Path(output_dir).exists() and self.config.overwrite is False: - self.logger.event(f"Error: Output directory {output_dir} already exists. Pipeline will crash to prevent overwrite.", - overwrite = self.config.overwrite) + output_dir = self.stage_configs[stage.name].get(directory) + if not output_dir: + continue + + output_path = Path(output_dir) + if not output_path.is_absolute(): + output_path = self.config.work_dir / output_path + + exists = output_path.exists() + overwrite = bool(self.config.overwrite) + + if exists and not overwrite: + self.logger.event( + f"Error: Output directory {output_path} already exists. Pipeline will crash to prevent overwrite.", + overwrite=overwrite, + ) raise StageConfigurationError( f"Stage configuration for {stage.name} contains an output directory path that already exists. Please set a unique " - f"output directory for this stage to prevent overwriting.") - - if Path(output_dir).exists() and self.config.overwrite is True: + f"output directory for this stage to prevent overwriting." + ) + + if exists and overwrite: warnings.warn( f"Stage configuration for {stage.name} contains an output directory path that already exists. As the overwrite " - f"parameter is True, the pipeline will proceed and will overwrite the previous run file.", StageConfigurationWarning + f"parameter is True, the pipeline will proceed and will overwrite the previous run file.", + StageConfigurationWarning, + ) + self.logger.event( + f"Warning: Output directory {output_path} already exists however permissions allow overwriting. The previous file " + f"will be overwritten.", + overwrite=overwrite, ) - self.logger.event(f"Warning: Output directory {output_dir} already exists however permissions allow overwriting. The previous file " - f"will be overwritten.", overwrite = self.config.overwrite) def _validate_stage_configs(self) -> None: From d7ef508f78f5d248b81cdc8c06196ad499ca3b5d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 14:33:34 +0100 Subject: [PATCH 201/332] tweak: correct indenting error in add_stage_config --- onsrap/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ccfa77a..7b17244 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -256,7 +256,7 @@ def add_stage_config( Optional stage name used when the parsed configuration payload does not identify the stage on its own. """ -parsed_stage_config = self._coerce_stage_config(stage_config, name=name) + parsed_stage_config = self._coerce_stage_config(stage_config, name=name) self.stage_configs[parsed_stage_config.name] = parsed_stage_config self._check_output_dir_in_stage_configs(name=parsed_stage_config.name) self.logger.event("Stage configuration added", stage=parsed_stage_config.name) From d1e892857e090e9653b80425332c455147394d5c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 16:21:23 +0100 Subject: [PATCH 202/332] feat: add method to generate an ExecutionContext type for a pipeline based on the backend and validates stage backends to ensure compatability through a second method --- onsrap/pipeline.py | 67 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7b17244..a890c2a 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -60,7 +60,7 @@ def __init__( stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, logger: Logger | None = None, - executor: StageExecutor | None = None, + executor: StageExecutor | PythonStageExecutor | None = None, ): resolved_config, resolved_stage_configs, configured_stages, resolved_global_config = self._resolve_config(config) @@ -74,13 +74,11 @@ def __init__( self.config.name = self.name self.logger = logger or Logger(log_dir=self.config.log_dir) + if executor is not None: self.executor = executor else: - if self.backend == "python": - self.executor = PythonStageExecutor() - else: - raise PipelineInitialisationError(f"Requested backend does not have a compatible executor. Available executors are: {', '.join(AVAILABLE_EXECUTORS)}.") + self._generate_context() if stages is not None and configured_stages: raise PipelineInitialisationError( @@ -1012,6 +1010,65 @@ def add_stage_with_dependencies(stage_name: str, *, required_by: str | None = No return [stage for stage in self.stages if stage.name in resolved_stage_names] + def _generate_context(self) -> None: + """ + Generates the execution context for the Pipeline based on values parsed. + + Validates stage backends and then uses the pipeline backend to generate + the expected StageExecutor class. If this class does not exist within + the orchestration tool, an error is raised. If the class does exist, + it is assigned to the executor attribute of the Pipeline instance. + + Raises + ------ + ``PipelineInitialisationError`` + If the backend for the Pipeline does not have a compatible executor class + """ + # Check all stages have the same backend as Pipeline + self._validate_stage_backends() + + execution_class_name = f"{self.backend.capitalize()}StageExecutor" + + if (executor_class := globals().get(execution_class_name)) is not None: + self.executor = executor_class() + else: + raise PipelineInitialisationError( + f"Requested backend {self.backend} does not have a compatible executor. " + f"Available executors are: {', '.join(AVAILABLE_EXECUTORS)}." + ) + + def _validate_stage_backends(self) -> str: + """ + Checks the backends that have been assigned to each stage. + + Raise an error if the backends for a stage do not match the Pipeline + backend or if there are multiple backends across the stages. This + ensures that the ExecutionContext will run correctly on all stages. + + Raises + ------ + ``PipelineInitialisationError`` + If there are multiple backends across the stages or if the stage + backend does not match the Pipeline backend. + """ + backends = [] + for stage in self.stages: + backends.append(stage.backend) + + backends = [backend.lower() for backend in backends] + + if len(set(backends)) > 1: + raise PipelineInitialisationError( + f"Not all stages have the same backend. Found backends: {', '.join(set(backends))}. This means " + "that the execution context will not work on all stages." + ) + + if set(backends) != {self.backend}: + raise PipelineInitialisationError( + f"Stages have backends '{', '.join(set(backends))}' which do not match pipeline backend '{self.backend}'." + ) + + @classmethod def from_files( cls, From 415b5c8c9516afcbe48c53d9f21fcded1ba7ecf7 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 16:55:42 +0100 Subject: [PATCH 203/332] tweak: reorder where _generate_context is called to allow for stage parsing --- onsrap/pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index a890c2a..0a51cf1 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -75,11 +75,6 @@ def __init__( self.logger = logger or Logger(log_dir=self.config.log_dir) - if executor is not None: - self.executor = executor - else: - self._generate_context() - if stages is not None and configured_stages: raise PipelineInitialisationError( "Stages parsed through both Pipeline construction and configuration file. Either provide stages through the constructor or the configuration file, not both." @@ -90,6 +85,11 @@ def __init__( else [self._coerce_stage(stage) for stage in stages] ) + if executor is not None: + self.executor = executor + else: + self._generate_context() + self.dependencies = dependencies if dependencies is not None and stages is None: raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " From bb715fed1c93cf2f4e765ffd79394cd997c50711 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 16:55:56 +0100 Subject: [PATCH 204/332] test: add test for generate_context() --- tests/test_pipeline.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a3a757a..73e1951 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,4 +1,5 @@ from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.execution import PythonStageExecutor from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError from onsrap.models import StageConfig from onsrap.stage import Stage @@ -272,3 +273,20 @@ def test_construct_manifest_inputs_contains_only_effective_stages() -> None: manifest = pipeline._construct_manifest(runtime_id=runtime_id) assert list(manifest.inputs.keys()) == ["Stage_0"] + +def test_generate_context_correctly_assigns_executor() -> None: + """ + Test that the correct executor class is assigned to the Pipeline instance + based on the backend specified. If the backend does not have a compatible + executor, an error is raised. + """ + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + assert isinstance(pipeline.executor, PythonStageExecutor) + + with pytest.raises(PipelineInitialisationError): + Pipeline(backend="nonexistent_backend", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) \ No newline at end of file From 0854c7ddf846e7f63641373a01f6821bbaa9de57 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 4 Aug 2026 16:59:16 +0100 Subject: [PATCH 205/332] test: implement test for _validate_stage_backends() --- tests/test_pipeline.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 73e1951..7305209 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -289,4 +289,22 @@ def test_generate_context_correctly_assigns_executor() -> None: with pytest.raises(PipelineInitialisationError): Pipeline(backend="nonexistent_backend", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) \ No newline at end of file + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + +def test_validate_stage_backends_errors() -> None: + """ + Test that the _validate_stage_backends method correctly raises an error if + the backends for a stage do not match the Pipeline backend or if there are + multiple backends across the stages. + + Successful runs not tested here as they are covered in test_generate_context_ + correctly_assigns_executor(). + """ + + with pytest.raises(PipelineInitialisationError): + Pipeline(backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=(), backend="nonexistent_backend")]) + with pytest.raises(PipelineInitialisationError): + Pipeline(backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=(), backend="nonexistent_backend"), + Stage("Stage_1", source=Path("Stage_1.py"), dependencies=(), backend="python")]) \ No newline at end of file From 85da7d74368e9452167908937ef5a74badabe578 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 09:40:05 +0100 Subject: [PATCH 206/332] test: correct errors in test_pipeline and test_pipeline_architecture caused by adding global_config requirement in configuration files. --- tests/test_pipeline.py | 52 ++++++++++++++++++----------- tests/test_pipeline_architecture.py | 21 ++++++++---- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a3a757a..bf3ad14 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -111,34 +111,40 @@ def test_add_dependencies_single_dict(tmp_path): def test_add_stage_parses_stage_configs_keyword() -> None: - pipeline = Pipeline() - stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline() + stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) - pipeline.add_stage(stage, stage_configs=[stage_config]) + pipeline.add_stage(stage, stage_configs=[stage_config]) - assert pipeline.stages[-1].name == "Stage_1" - assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 def test_add_stage_warns_when_stage_config_count_mismatches() -> None: - pipeline = Pipeline() - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline() + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) - assert any( - "does not match the number of stages" in str(recorded_warning.message) - for recorded_warning in recorded_warnings - ) - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - assert pipeline.stage_configs["Stage_1"].to_dict() == {} + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} def test_add_stage_config_coerces_mapping_payload_for_named_stage() -> None: - pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") @@ -154,6 +160,7 @@ def test_resolve_stages_to_run_includes_transitive_dependencies() -> None: stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) stage_2 = Stage("Stage_2", source=Path("Stage_2.py"), dependencies=("Stage_1",)) + pipeline = Pipeline( stages=[stage_0, stage_1, stage_2], config=PipelineConfig(stages_to_run={"Stage_2": True}), @@ -178,7 +185,10 @@ def test_self_stages_is_full_registry_after_disable() -> None: """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline(stages=[stage_0, stage_1]) + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -190,7 +200,9 @@ def test_self_stages_is_full_registry_after_disable() -> None: def test_disable_stage_in_implicit_mode_creates_explicit_selection() -> None: stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 7e1fa9b..02e9892 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -49,7 +49,8 @@ def main(context): config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {} + "stage_configuration": {}, + "global_config":{} }, ) @@ -86,7 +87,8 @@ def main(context): config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, + "stage_configuration": {}, + "global_config":{}}, ) with pytest.warns(StageConfigurationWarning, @@ -120,7 +122,8 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, + "stage_configuration": {}, + "global_config":{}}, ) with pytest.warns(StageConfigurationWarning, @@ -185,6 +188,8 @@ def run(context): 0_data_validation: years_to_run: 2017 target_variable: "classification" + global_configuration: + dry_run: true """ ).strip() + "\n", @@ -232,6 +237,7 @@ def run(context): "stage_configuration": { "missing_stage": {"years_to_run": 2017}, }, + "global_config":{} }, ) @@ -303,6 +309,7 @@ def run(context): }, }, }, + "global_config":{} } config_file = tmp_path / "conf.yaml" @@ -320,10 +327,9 @@ def run(context): assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() assert pipeline.stages[0].metadata["owner"] == "analytics" assert pipeline.stages[1].dependencies == ("0_extract",) - assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017} - assert pipeline.stage_configs["0_extract"].datasets == {"orders": {"path": "data/orders.csv"}} + assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017, "datasets": {"orders": {"path": "data/orders.csv"}}} assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.create_stage_config(config_file, name="1_transform").require("target_variable") == "classification" + assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" def test_pipeline_from_config_scales_stage_configuration_to_many_stages(tmp_path: Path) -> None: @@ -388,6 +394,9 @@ def run(context): "stages": stage_definitions, }, "stage_configuration": stage_configuration, + "global_config": { + "dry_run": True, + }, }, sort_keys=False, ), From 709bb26021f290f787d5673f1d97e86cca6d3baf Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 10:30:14 +0100 Subject: [PATCH 207/332] tweak: CoPilot refactor (human review) tests into class structure --- tests/test_pipeline.py | 496 +++++++++++++++++++++-------------------- 1 file changed, 251 insertions(+), 245 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index bf3ad14..a2656bb 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -8,279 +8,285 @@ from onsrap.warnings import PipelineConfigurationWarning -def test_pipeline_name(): - """ - Test to confirm that Pipeline instance uses either defined name from - instance creation (shown in pipeline_named), utilises name from PipelineConfig - if no name was given (shown in pipeline_config), or defaults to "pipeline" if - no name is provided through Pipeline instance creation or through the - PipelineConfig (shown through pipeline_no_name) - """ - pipeline_config = PipelineConfig(name = "test_pipeline_config") - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_named = Pipeline(name = "test_pipeline_name") - pipeline_config = Pipeline(name = None, config = pipeline_config) - pipeline_no_name = Pipeline() - - assert pipeline_named.name == "test_pipeline_name" - assert pipeline_config.name == "test_pipeline_config" - assert pipeline_no_name.name == "pipeline" - - -def test_assign_dependencies(tmp_path): - """ - Test to ensure that different formats of dependencies can be parsed to the - Pipeline creation and appropriately assigned to each stage within the - Pipeline. Will also check for error raise if the dependencies are defined - but there are no defined stages. - """ - def example_function(): - pass - - path_1 = tmp_path/"Stage_1.py" - path_0 = tmp_path/"Stage_0.py" - - dependencies_single = {"Stage_2":("Stage_1",)} - dependencies_multiple = {"Stage_1":["Stage_0"], - "Stage_2":("Stage_1", "Stage_0")} - dependencies_non_stage_name = {"Stage_1.py":("Stage_0",), - "example_function":("Stage_1.py",)} - - with pytest.raises(PipelineInitialisationError): - Pipeline(stages = None, - dependencies = dependencies_single) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_1 = Pipeline(name = "pipeline_1", - stages = [Stage("Stage_1", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_multiple) - - pipeline_2 = Pipeline(name = "pipeline_2", - stages = [Stage("Stage_1.py", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_non_stage_name) - - assert pipeline_1.stages[0].dependencies == ("Stage_0",) - assert pipeline_1.stages[1].dependencies == ("Stage_1","Stage_0") - - assert pipeline_2.stages[0].dependencies == ("Stage_0",) - assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) - - -def test_add_dependencies_single_dict(tmp_path): - """ - Tests that a dictionary correctly assigns dependencies to - individual stages and the Pipeline instance. - """ - - path_1 = tmp_path/"Stage_1.py" - path_2 = tmp_path/"Stage_2.py" - path_0 = tmp_path/"Stage_0.py" - - dependencies_multiple = {"Stage_0":(), - "Stage_1":(), - "Stage_2":("Stage_1",)} - dep_dict = {"Stage_1":("Stage_0",), - "Stage_2":("Stage_0","Stage_1")} - dep_tuple = ("Stage_0.25",) - stage_1 = Stage("Stage_1", source = path_1, dependencies = {}) - stage_2 = Stage("Stage_2", source = path_2, dependencies = {}) - stage_0 = Stage("Stage_0", source = path_0, dependencies = {}) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], - dependencies = dependencies_multiple) - - with pytest.raises(PipelineInitialisationError): - pipeline_dict.add_dependencies(dep_tuple) - - pipeline_dict.add_dependencies(dep_dict) - assert stage_1.dependencies == ("Stage_0",) - assert stage_2.dependencies == ("Stage_1","Stage_0",) - assert stage_0.dependencies == () - assert pipeline_dict.dependencies == {"Stage_0":(), - "Stage_1":("Stage_0",), - "Stage_2":("Stage_1","Stage_0",)} - - -def test_add_stage_parses_stage_configs_keyword() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline() - stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) - - pipeline.add_stage(stage, stage_configs=[stage_config]) - - assert pipeline.stages[-1].name == "Stage_1" - assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 - - -def test_add_stage_warns_when_stage_config_count_mismatches() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline() - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - - with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) - - assert any( - "does not match the number of stages" in str(recorded_warning.message) - for recorded_warning in recorded_warnings - ) - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - assert pipeline.stage_configs["Stage_1"].to_dict() == {} - - -def test_add_stage_config_coerces_mapping_payload_for_named_stage() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - - pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") - - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - +NO_STAGES_WARNING = "No stages specified to run. All stages running by default." + + +@pytest.fixture +def stage_factory(): + def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: + resolved_source = source if source is not None else Path(f"{name}.py") + return Stage(name, source=resolved_source, dependencies=dependencies) + + return _build_stage + + +class TestPipelineNamingAndInit: + def test_pipeline_name(self): + """ + Test to confirm that Pipeline instance uses either defined name from + instance creation (shown in pipeline_named), utilises name from PipelineConfig + if no name was given (shown in pipeline_config), or defaults to "pipeline" if + no name is provided through Pipeline instance creation or through the + PipelineConfig (shown through pipeline_no_name) + """ + pipeline_config = PipelineConfig(name="test_pipeline_config") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_named = Pipeline(name="test_pipeline_name") + pipeline_config = Pipeline(name=None, config=pipeline_config) + pipeline_no_name = Pipeline() + + assert pipeline_named.name == "test_pipeline_name" + assert pipeline_config.name == "test_pipeline_config" + assert pipeline_no_name.name == "pipeline" + + def test_assign_dependencies(self, tmp_path): + """ + Test to ensure that different formats of dependencies can be parsed to the + Pipeline creation and appropriately assigned to each stage within the + Pipeline. Will also check for error raise if the dependencies are defined + but there are no defined stages. + """ + + def example_function(): + pass + + path_1 = tmp_path / "Stage_1.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_single = {"Stage_2": ("Stage_1",)} + dependencies_multiple = { + "Stage_1": ["Stage_0"], + "Stage_2": ("Stage_1", "Stage_0"), + } + dependencies_non_stage_name = { + "Stage_1.py": ("Stage_0",), + "example_function": ("Stage_1.py",), + } + + with pytest.raises(PipelineInitialisationError): + Pipeline(stages=None, dependencies=dependencies_single) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_1 = Pipeline( + name="pipeline_1", + stages=[ + Stage("Stage_1", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_multiple, + ) + + pipeline_2 = Pipeline( + name="pipeline_2", + stages=[ + Stage("Stage_1.py", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_non_stage_name, + ) + + assert pipeline_1.stages[0].dependencies == ("Stage_0",) + assert pipeline_1.stages[1].dependencies == ("Stage_1", "Stage_0") + + assert pipeline_2.stages[0].dependencies == ("Stage_0",) + assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + + def test_add_dependencies_single_dict(self, tmp_path): + """ + Tests that a dictionary correctly assigns dependencies to + individual stages and the Pipeline instance. + """ + + path_1 = tmp_path / "Stage_1.py" + path_2 = tmp_path / "Stage_2.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_multiple = {"Stage_0": (), "Stage_1": (), "Stage_2": ("Stage_1",)} + dep_dict = {"Stage_1": ("Stage_0",), "Stage_2": ("Stage_0", "Stage_1")} + dep_tuple = ("Stage_0.25",) + stage_1 = Stage("Stage_1", source=path_1, dependencies={}) + stage_2 = Stage("Stage_2", source=path_2, dependencies={}) + stage_0 = Stage("Stage_0", source=path_0, dependencies={}) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_dict = Pipeline(stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple) + + with pytest.raises(PipelineInitialisationError): + pipeline_dict.add_dependencies(dep_tuple) + + pipeline_dict.add_dependencies(dep_dict) + assert stage_1.dependencies == ("Stage_0",) + assert stage_2.dependencies == ("Stage_1", "Stage_0",) + assert stage_0.dependencies == () + assert pipeline_dict.dependencies == { + "Stage_0": (), + "Stage_1": ("Stage_0",), + "Stage_2": ("Stage_1", "Stage_0",), + } + + +class TestPipelineStageConfigHandling: + def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline() + stage = stage_factory("Stage_1") + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + + pipeline.add_stage(stage, stage_configs=[stage_config]) + + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + + def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline() + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} + + def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_factory("Stage_0")]) + + pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") -# --------------------------------------------------------------------------- -# Stage graph and stage-selection tests -# --------------------------------------------------------------------------- - -def test_resolve_stages_to_run_includes_transitive_dependencies() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) - stage_2 = Stage("Stage_2", source=Path("Stage_2.py"), dependencies=("Stage_1",)) - - - pipeline = Pipeline( - stages=[stage_0, stage_1, stage_2], - config=PipelineConfig(stages_to_run={"Stage_2": True}), - ) - - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 -def test_resolve_stages_to_run_rejects_disabled_dependencies() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) +class TestPipelineStageSelectionAndGraph: + def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) - with pytest.raises(PipelineConfigurationError): - Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), ) + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] -def test_self_stages_is_full_registry_after_disable() -> None: - """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[stage_0, stage_1]) + def test_resolve_stages_to_run_rejects_disabled_dependencies(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) - pipeline.disable_stage("Stage_1") + with pytest.raises(PipelineConfigurationError): + Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + ) - assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] + def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: + """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_0, stage_1]) -def test_disable_stage_in_implicit_mode_creates_explicit_selection() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[stage_0, stage_1]) + pipeline.disable_stage("Stage_1") - pipeline.disable_stage("Stage_1") + assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] - assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} + def test_disable_stage_in_implicit_mode_creates_explicit_selection(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_0, stage_1]) + pipeline.disable_stage("Stage_1") -def test_enable_stage_restores_stage_in_explicit_mode() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) - - pipeline.enable_stage("Stage_1") - - assert pipeline.config.stages_to_run["Stage_1"] is True - assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} + def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) -def test_add_stage_keeps_new_stage_out_of_explicit_selection() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + pipeline.enable_stage("Stage_1") - pipeline.add_stage( - Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), - stage_configs=[StageConfig(name="Stage_1")], - ) + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} - assert pipeline.config.stages_to_run["Stage_1"] is False - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + def test_add_stage_keeps_new_stage_out_of_explicit_selection(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) -def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + ) - pipeline.add_stage( - Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), - stage_configs=[StageConfig(name="Stage_1")], - enable_stages=True, - ) + assert pipeline.config.stages_to_run["Stage_1"] is False + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + + def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true( + self, + stage_factory, + ) -> None: + stage_0 = stage_factory("Stage_0") + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - assert pipeline.config.stages_to_run["Stage_1"] is True - assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} -def test_validate_skips_source_check_for_disabled_stages(tmp_path: Path) -> None: - """Disabled stages' source files need not exist — validate() only checks the effective run set.""" - enabled_file = tmp_path / "Stage_0.py" - enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") - stage_0 = Stage("Stage_0", source=enabled_file) - stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent +class TestPipelineValidationAndManifest: + def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) -> None: + """Disabled stages' source files need not exist - validate() only checks the effective run set.""" + enabled_file = tmp_path / "Stage_0.py" + enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + stage_0 = Stage("Stage_0", source=enabled_file) + stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent - pipeline.validate() # must not raise + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) + pipeline.validate() # must not raise -def test_construct_manifest_inputs_contains_only_effective_stages() -> None: - """Manifest inputs should list only the stages that are part of the execution graph.""" - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + def test_construct_manifest_inputs_contains_only_effective_stages(self, stage_factory) -> None: + """Manifest inputs should list only the stages that are part of the execution graph.""" + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) - runtime_id = pipeline._create_runtime_id() - manifest = pipeline._construct_manifest(runtime_id=runtime_id) + runtime_id = pipeline._create_runtime_id() + manifest = pipeline._construct_manifest(runtime_id=runtime_id) - assert list(manifest.inputs.keys()) == ["Stage_0"] + assert list(manifest.inputs.keys()) == ["Stage_0"] From b6c8c852370d2b07f69cfd1645a8cf90c90afbaf Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 10:40:37 +0100 Subject: [PATCH 208/332] tweak: ruff formatting and linting applied --- tests/test_pipeline.py | 103 ++++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a2656bb..65cee3f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,12 +1,12 @@ -from onsrap.pipeline import Pipeline, PipelineConfig -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError -from onsrap.models import StageConfig -from onsrap.stage import Stage -from onsrap.warnings import StageConfigurationWarning from pathlib import Path + import pytest -from onsrap.warnings import PipelineConfigurationWarning +from onsrap.errors import PipelineConfigurationError, PipelineInitialisationError +from onsrap.models import StageConfig +from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning NO_STAGES_WARNING = "No stages specified to run. All stages running by default." @@ -112,19 +112,27 @@ def test_add_dependencies_single_dict(self, tmp_path): stage_0 = Stage("Stage_0", source=path_0, dependencies={}) with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline_dict = Pipeline(stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple) + pipeline_dict = Pipeline( + stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple + ) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) pipeline_dict.add_dependencies(dep_dict) assert stage_1.dependencies == ("Stage_0",) - assert stage_2.dependencies == ("Stage_1", "Stage_0",) + assert stage_2.dependencies == ( + "Stage_1", + "Stage_0", + ) assert stage_0.dependencies == () assert pipeline_dict.dependencies == { "Stage_0": (), "Stage_1": ("Stage_0",), - "Stage_2": ("Stage_1", "Stage_0",), + "Stage_2": ( + "Stage_1", + "Stage_0", + ), } @@ -133,21 +141,27 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage = stage_factory("Stage_1") - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + stage_config = StageConfig( + name="Stage_1", _variables={"years_to_run": 2017} + ) pipeline.add_stage(stage, stage_configs=[stage_config]) assert pipeline.stages[-1].name == "Stage_1" assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 - def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) -> None: + def test_add_stage_warns_when_stage_config_count_mismatches( + self, stage_factory + ) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + pipeline.add_stage( + stage_0, stage_1, stage_configs=[{"years_to_run": 2017}] + ) assert any( "does not match the number of stages" in str(recorded_warning.message) @@ -156,7 +170,9 @@ def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 assert pipeline.stage_configs["Stage_1"].to_dict() == {} - def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_factory) -> None: + def test_add_stage_config_coerces_mapping_payload_for_named_stage( + self, stage_factory + ) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -166,7 +182,9 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_fa class TestPipelineStageSelectionAndGraph: - def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_factory) -> None: + def test_resolve_stages_to_run_includes_transitive_dependencies( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) @@ -176,21 +194,36 @@ def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_fact config=PipelineConfig(stages_to_run={"Stage_2": True}), ) - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] - - def test_resolve_stages_to_run_rejects_disabled_dependencies(self, stage_factory) -> None: + assert [stage.name for stage in pipeline.graph.stages] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + assert [stage.name for stage in pipeline.ordered_stages()] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + + def test_resolve_stages_to_run_rejects_disabled_dependencies( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) with pytest.raises(PipelineConfigurationError): Pipeline( stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + config=PipelineConfig( + stages_to_run={"Stage_0": False, "Stage_1": True} + ), ) def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: - """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" + """ + Pipeline.stages always holds all stages; only graph.stages is the effective run + set. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -203,7 +236,9 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] - def test_disable_stage_in_implicit_mode_creates_explicit_selection(self, stage_factory) -> None: + def test_disable_stage_in_implicit_mode_creates_explicit_selection( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): @@ -226,7 +261,9 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No assert pipeline.config.stages_to_run["Stage_1"] is True assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} - def test_add_stage_keeps_new_stage_out_of_explicit_selection(self, stage_factory) -> None: + def test_add_stage_keeps_new_stage_out_of_explicit_selection( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], @@ -262,13 +299,20 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr class TestPipelineValidationAndManifest: - def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) -> None: - """Disabled stages' source files need not exist - validate() only checks the effective run set.""" + def test_validate_skips_source_check_for_disabled_stages( + self, tmp_path: Path + ) -> None: + """ + Disabled stages' source files need not exist - validate() only checks the + effective run set. + """ enabled_file = tmp_path / "Stage_0.py" enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") stage_0 = Stage("Stage_0", source=enabled_file) - stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent + stage_1 = Stage( + "Stage_1", source=tmp_path / "missing.py" + ) # file intentionally absent pipeline = Pipeline( stages=[stage_0, stage_1], @@ -277,8 +321,13 @@ def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) - pipeline.validate() # must not raise - def test_construct_manifest_inputs_contains_only_effective_stages(self, stage_factory) -> None: - """Manifest inputs should list only the stages that are part of the execution graph.""" + def test_construct_manifest_inputs_contains_only_effective_stages( + self, stage_factory + ) -> None: + """ + Manifest inputs should list only the stages that are part of the execution + graph. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) pipeline = Pipeline( From fc9c79206960301d1349111684544d7d10399699 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 11:27:58 +0100 Subject: [PATCH 209/332] doc: add doc strings for each test in test_pipeline --- tests/test_pipeline.py | 52 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 65cee3f..80dbf1f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -14,6 +14,11 @@ @pytest.fixture def stage_factory(): def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: + """ + Function that builds a Stage object with a given name, dependencies, and a + source file path that's built out of the name if it is not provided. This + standardises the creation of Stage objects for testing. + """ resolved_source = source if source is not None else Path(f"{name}.py") return Stage(name, source=resolved_source, dependencies=dependencies) @@ -138,6 +143,12 @@ def test_add_dependencies_single_dict(self, tmp_path): class TestPipelineStageConfigHandling: def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: + """ + Tests that when a stage is added after a Pipeline has been initialised, + the stage and the stage_configurations are correctly added to the + Pipeline instance and the stage_configurations are correctly associated + with the stage. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage = stage_factory("Stage_1") @@ -153,6 +164,11 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: def test_add_stage_warns_when_stage_config_count_mismatches( self, stage_factory ) -> None: + """ + Tests that when a stage is added but there is not the correct number of + stage_configs provided, a warning is raised and the stage_configuration + for that stage is added as a blank StageConfig object. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage_0 = stage_factory("Stage_0") @@ -173,6 +189,11 @@ def test_add_stage_warns_when_stage_config_count_mismatches( def test_add_stage_config_coerces_mapping_payload_for_named_stage( self, stage_factory ) -> None: + """ + Tests that when a stage_configuration is added to a Pipeline instance, + the configuration is correctly associated with the named stage and that + the configuration is coerced into a StageConfig object if it is provided. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -185,6 +206,11 @@ class TestPipelineStageSelectionAndGraph: def test_resolve_stages_to_run_includes_transitive_dependencies( self, stage_factory ) -> None: + """ + Tests that when resolving stages_to_run, the Pipeline instance correctly + includes all dependent stages required in the StageGraph even if these + are not explicitly called out in the configuration. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) @@ -208,6 +234,10 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( def test_resolve_stages_to_run_rejects_disabled_dependencies( self, stage_factory ) -> None: + """ + Checks that when resolving stages_to_run, the Pipeline init raises an + error if a stage is enabled but one of its dependencies is disabled. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -239,6 +269,10 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None def test_disable_stage_in_implicit_mode_creates_explicit_selection( self, stage_factory ) -> None: + """ + Tests that when a stage is manually disabled in a Pipeline instance, it + is initialised in the stages_to_run configuration. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): @@ -249,6 +283,11 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> None: + """ + Tests that when a stage is manually enabled in a Pipeline instance, it + is correctly reflected in the stages_to_run configuration and the stage + is included in the execution graph. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") pipeline = Pipeline( @@ -264,6 +303,10 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No def test_add_stage_keeps_new_stage_out_of_explicit_selection( self, stage_factory ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance, it is + kept out of the explicit selection. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], @@ -272,8 +315,9 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( pipeline.add_stage( stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")], - ) + stage_configs=[StageConfig(name="Stage_1")] + ) + assert pipeline.config.stages_to_run["Stage_1"] is False assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] @@ -282,6 +326,10 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr self, stage_factory, ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance with + enable_stages=True, it is included in the explicit selection. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], From ce8f252ec773b26db3a848777719245874b0353d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 11:53:49 +0100 Subject: [PATCH 210/332] tweak: refactor test_stage.py to class structure --- tests/test_stage.py | 342 ++++++++++++++++++++++---------------------- 1 file changed, 172 insertions(+), 170 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 759bfc1..15f3f3d 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -3,22 +3,23 @@ from pathlib import Path from textwrap import dedent -def test_normalize_dependencies_none() -> None: - """ - Tests that None values return empty tuple. - """ - assert _normalize_dependencies(None) == () - -def test_normalize_dependencies_str() -> None: - """ - Tests single string and list of string values including - where whitespace appears before and after main text body - """ - assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) - assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) - assert _normalize_dependencies( - ["Stage_1.py"," Stage_2.py", "Stage_3.py "] - ) == ("Stage_1.py","Stage_2.py", "Stage_3.py") +class TestNormalizeDependencies: + def test_normalize_dependencies_none(self) -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + + def test_normalize_dependencies_str(self) -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py", " Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") @pytest.fixture def example_function(): @@ -34,163 +35,164 @@ def stage_test() -> Stage: """ return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) -def test_stage_creation_callable(stage_test) -> None: - """ - Tests that attributes have been appropriately assigned to Stage class. - """ - assert stage_test.name == "callable_stage" - assert stage_test.source == example_function - assert stage_test.dependencies == ("stage_1",) - assert stage_test.metadata == {"info":"example"} - assert stage_test.entrypoint == None - assert stage_test.backend == "python" - -def test_stage_name_error(example_function) -> None: - """ - Tests that a StageConfigurationError is raised if the name is left blank - in a Stage class instance. - """ - with pytest.raises(StageConfigurationError): - stage = Stage("",example_function,["stage_1"],{"info":"example"}) - -def test_stage_source_type() -> None: - """ - Tests that a non-valid source type returns a StageConfigurationError. - """ - with pytest.raises(StageConfigurationError): - stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) - -def test_stage_backend(example_function) -> None: - """ - Tests that backend can be any string, None, and corrects for whitespace. - """ - stage_diff = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "java") - stage = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "") - stage_white_space = Stage("callable_stage", example_function,["stage_1"], - {"info":"example"}, backend = "python ") - assert stage_diff.backend == "java" - assert stage.backend == "python" - assert stage_white_space.backend == "python" - -def test_stage_from_files_error(tmp_path: Path) -> None: - """ - Tests that if the file doesn't exist, a StageConfigurationError is raised. - """ - source_file = tmp_path / "not_an_actual_file.py" - with pytest.raises(StageConfigurationError): - stage = Stage.from_file(source_file) - -def test_stage_from_callable_name() -> None: - """ - Tests that a stage name is extracted from a callable object stage. - """ - def example_function(): - pass - test = Stage.from_callable(example_function) - assert test.name == "example_function" - -def test_from_dict_norm() -> None: - """ - Tests that a stage instance is created from a dictionary item. - """ - def example_function(): - pass - data = {"name":"test_Stage", - "callable" : example_function} - stage = Stage.from_dict(data) - assert stage.source == example_function - -def test_with_dependencies_list(stage_test) -> None: - """ - Tests adding different types of dependencies when the original dependency is - a list. - """ - new_deps = ["stage2","stage3"] - new_deps_blank = [] - stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') - assert stage_test_blank.dependencies == ("stage_1", ) - stage_test = stage_test.with_dependencies("stage2","stage3") - assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') - - -def test_validate(stage_test, tmp_path) -> None: - """ - Tests whether an error is raised if the source file isn't suitable. - """ - stage_test.source = None - with pytest.raises(StageConfigurationError): - stage_test.validate() - not_file_path = tmp_path - stage_test.source = not_file_path - with pytest.raises(StageConfigurationError): - stage_test.validate() - stage_test.source = "" - with pytest.raises(StageConfigurationError): - stage_test.validate() - -def test_source_path(stage_test, tmp_path) -> None: - """ - Tests whether source_path detects a path vs other valid and invalid source types. - """ - stage_test.source = tmp_path/"fake_file.py" - assert stage_test.source_path == tmp_path/"fake_file.py" - stage_test.source = 11 - assert stage_test.source_path == None - stage_test.source = "not a file path" - assert stage_test.source_path == None - - def example_function(): - pass - stage_test.source = example_function - assert stage_test.source_path == None - -def test_source_label(stage_test, tmp_path) -> None: - """ - Tests that source_label is created if the source is a Path or a callable and is None if it is - another type. - """ - stage_test.source = tmp_path/"fake_file.py" - temp_path_str = str(tmp_path/"fake_file.py") - assert stage_test.source_label == temp_path_str - - def example_function(): - pass - stage_test.source = example_function - assert stage_test.source_label == "tests.test_stage.example_function" - - stage_test.source = 11 - assert stage_test.source_label == None +class TestStage: + def test_stage_creation_callable(self, stage_test) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info":"example"} + assert stage_test.entrypoint == None + assert stage_test.backend == "python" + + def test_stage_name_error(self, example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("",example_function,["stage_1"],{"info":"example"}) + + def test_stage_source_type(self) -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + + def test_stage_backend(self, example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + """ + stage_diff = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "java") + stage = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "") + stage_white_space = Stage("callable_stage", example_function,["stage_1"], + {"info":"example"}, backend = "python ") + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + + def test_stage_from_files_error(self, tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + stage = Stage.from_file(source_file) + + def test_stage_from_callable_name(self) -> None: + """ + Tests that a stage name is extracted from a callable object stage. + """ + def example_function(): + pass + test = Stage.from_callable(example_function) + assert test.name == "example_function" + + def test_from_dict_norm(self) -> None: + """ + Tests that a stage instance is created from a dictionary item. + """ + def example_function(): + pass + data = {"name":"test_Stage", + "callable" : example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + + def test_with_dependencies_list(self, stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. + """ + new_deps = ["stage2","stage3"] + new_deps_blank = [] + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_blank.dependencies == ("stage_1", ) + stage_test = stage_test.with_dependencies("stage2","stage3") + assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + + def test_validate(self, stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + + def test_source_path(self, stage_test, tmp_path) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source types. + """ + stage_test.source = tmp_path/"fake_file.py" + assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = 11 + assert stage_test.source_path == None + stage_test.source = "not a file path" + assert stage_test.source_path == None + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_path == None + + def test_source_label(self, stage_test, tmp_path) -> None: + """ + Tests that source_label is created if the source is a Path or a callable and is None if it is + another type. + """ + stage_test.source = tmp_path/"fake_file.py" + temp_path_str = str(tmp_path/"fake_file.py") + assert stage_test.source_label == temp_path_str + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label == None """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ -def test_stage_instance_from_file(tmp_path) -> None: - """ - Tests that a Stage instance is created from a filepath. - """ - test_stage = tmp_path / "test_stage.py" - test_stage.write_text( - dedent( - """ - def main(): - variable = "Hello world" - return variable - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - assert Stage.from_file(test_stage, - entrypoint = "main") == Stage("test_stage", - test_stage.resolve(), - (), - {}, - "main", - "python") +class TestStageFactories: + def test_stage_instance_from_file(self, tmp_path) -> None: + """ + Tests that a Stage instance is created from a filepath. + """ + test_stage = tmp_path / "test_stage.py" + test_stage.write_text( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + assert Stage.from_file(test_stage, + entrypoint = "main") == Stage("test_stage", + test_stage.resolve(), + (), + {}, + "main", + "python") From 00b7f091c75621f21dbed2fddb60ed2334a69486 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 12:08:33 +0100 Subject: [PATCH 211/332] tweak: ruff format and check test_stage.py --- tests/test_stage.py | 106 +++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 15f3f3d..b29a182 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -1,8 +1,11 @@ -import pytest -from onsrap.stage import _normalize_dependencies, Stage, StageConfigurationError from pathlib import Path from textwrap import dedent +import pytest + +from onsrap.stage import Stage, StageConfigurationError, _normalize_dependencies + + class TestNormalizeDependencies: def test_normalize_dependencies_none(self) -> None: """ @@ -21,6 +24,7 @@ def test_normalize_dependencies_str(self) -> None: ["Stage_1.py", " Stage_2.py", "Stage_3.py "] ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") + @pytest.fixture def example_function(): """ @@ -28,12 +32,14 @@ def example_function(): """ pass + @pytest.fixture def stage_test() -> Stage: """ - Stage object for testing Stage class methods and construction. + Stage object for testing Stage class methods and construction. """ - return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) + return Stage("callable_stage", example_function, ["stage_1"], {"info": "example"}) + class TestStage: def test_stage_creation_callable(self, stage_test) -> None: @@ -43,8 +49,8 @@ def test_stage_creation_callable(self, stage_test) -> None: assert stage_test.name == "callable_stage" assert stage_test.source == example_function assert stage_test.dependencies == ("stage_1",) - assert stage_test.metadata == {"info":"example"} - assert stage_test.entrypoint == None + assert stage_test.metadata == {"info": "example"} + assert stage_test.entrypoint is None assert stage_test.backend == "python" def test_stage_name_error(self, example_function) -> None: @@ -53,25 +59,40 @@ def test_stage_name_error(self, example_function) -> None: in a Stage class instance. """ with pytest.raises(StageConfigurationError): - stage = Stage("",example_function,["stage_1"],{"info":"example"}) + Stage("", example_function, ["stage_1"], {"info": "example"}) def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. """ with pytest.raises(StageConfigurationError): - stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + Stage("callable_stage", 11, ["stage_1"], {"info": "example"}) def test_stage_backend(self, example_function) -> None: """ Tests that backend can be any string, None, and corrects for whitespace. """ - stage_diff = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "java") - stage = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "") - stage_white_space = Stage("callable_stage", example_function,["stage_1"], - {"info":"example"}, backend = "python ") + stage_diff = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="java", + ) + stage = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="", + ) + stage_white_space = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="python ", + ) assert stage_diff.backend == "java" assert stage.backend == "python" assert stage_white_space.backend == "python" @@ -82,14 +103,16 @@ def test_stage_from_files_error(self, tmp_path: Path) -> None: """ source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): - stage = Stage.from_file(source_file) + Stage.from_file(source_file) def test_stage_from_callable_name(self) -> None: """ Tests that a stage name is extracted from a callable object stage. """ + def example_function(): pass + test = Stage.from_callable(example_function) assert test.name == "example_function" @@ -97,10 +120,11 @@ def test_from_dict_norm(self) -> None: """ Tests that a stage instance is created from a dictionary item. """ + def example_function(): pass - data = {"name":"test_Stage", - "callable" : example_function} + + data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) assert stage.source == example_function @@ -109,14 +133,14 @@ def test_with_dependencies_list(self, stage_test) -> None: Tests adding different types of dependencies when the original dependency is a list. """ - new_deps = ["stage2","stage3"] + new_deps = ["stage2", "stage3"] new_deps_blank = [] stage_test_list = stage_test.with_dependencies(new_deps) stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') - assert stage_test_blank.dependencies == ("stage_1", ) - stage_test = stage_test.with_dependencies("stage2","stage3") - assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") + assert stage_test_blank.dependencies == ("stage_1",) + stage_test = stage_test.with_dependencies("stage2", "stage3") + assert stage_test.dependencies == ("stage_1", "stage2", "stage3") def test_validate(self, stage_test, tmp_path) -> None: """ @@ -135,41 +159,46 @@ def test_validate(self, stage_test, tmp_path) -> None: def test_source_path(self, stage_test, tmp_path) -> None: """ - Tests whether source_path detects a path vs other valid and invalid source types. + Tests whether source_path detects a path vs other valid and invalid source + types. """ - stage_test.source = tmp_path/"fake_file.py" - assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = tmp_path / "fake_file.py" + assert stage_test.source_path == tmp_path / "fake_file.py" stage_test.source = 11 - assert stage_test.source_path == None + assert stage_test.source_path is None stage_test.source = "not a file path" - assert stage_test.source_path == None + assert stage_test.source_path is None def example_function(): pass + stage_test.source = example_function - assert stage_test.source_path == None + assert stage_test.source_path is None def test_source_label(self, stage_test, tmp_path) -> None: """ - Tests that source_label is created if the source is a Path or a callable and is None if it is - another type. + Tests that source_label is created if the source is a Path or a callable + and is None if it is another type. """ - stage_test.source = tmp_path/"fake_file.py" - temp_path_str = str(tmp_path/"fake_file.py") + stage_test.source = tmp_path / "fake_file.py" + temp_path_str = str(tmp_path / "fake_file.py") assert stage_test.source_label == temp_path_str def example_function(): pass + stage_test.source = example_function assert stage_test.source_label == "tests.test_stage.example_function" stage_test.source = 11 - assert stage_test.source_label == None + assert stage_test.source_label is None + """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ + class TestStageFactories: def test_stage_instance_from_file(self, tmp_path) -> None: """ @@ -188,11 +217,6 @@ def main(): encoding="utf-8", ) - assert Stage.from_file(test_stage, - entrypoint = "main") == Stage("test_stage", - test_stage.resolve(), - (), - {}, - "main", - "python") - + assert Stage.from_file(test_stage, entrypoint="main") == Stage( + "test_stage", test_stage.resolve(), (), {}, "main", "python" + ) From bf1e80de48b24a3977f7a5eedf880c9878fdd5f2 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 13:17:13 +0100 Subject: [PATCH 212/332] tweak: CoPilot refactor (human review) of test_execution to add class structure. Ruff formatting and checking completed --- tests/test_execution.py | 673 +++++++++++++++++++++------------------- 1 file changed, 351 insertions(+), 322 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 05e0f5a..496ee37 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,12 +1,20 @@ -from onsrap.execution import ExecutionContext, PythonStageExecutor -from onsrap.models import GlobalConfig, PipelineConfig, StageConfig, StageResult, StageStatus -from onsrap.logger import Logger from pathlib import Path + import pytest -import onsrap.execution as execution_module + from onsrap.errors import PipelineConfigurationError +from onsrap.execution import ExecutionContext, PythonStageExecutor +from onsrap.logger import Logger +from onsrap.models import ( + GlobalConfig, + PipelineConfig, + StageConfig, + StageResult, + StageStatus, +) from onsrap.warnings import StageConfigurationWarning + @pytest.fixture def logger() -> Logger: """ @@ -14,51 +22,66 @@ def logger() -> Logger: """ return Logger() + @pytest.fixture def config() -> PipelineConfig: """ Return a PipelineConfig object for testing. """ - work_dir = Path('tmp/work_dir') - project_root = Path('tmp/project') - log_dir = Path('tmp/log') + work_dir = Path("tmp/work_dir") + project_root = Path("tmp/project") + log_dir = Path("tmp/log") data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", - {"stage_test":True}, + {"stage_test": True}, "python", - work_dir, + work_dir, project_root, None, - log_dir, - data_dir, - True, + log_dir, + data_dir, + True, None, - {} + {}, ) + +@pytest.fixture +def stage_config() -> StageConfig: + """ + Return a StageConfig object for testing. + """ + return StageConfig( + name="stage_test", + _variables={"sex": "gender", "dob": "date_of_birth"}, + metadata={}, + ) + + @pytest.fixture def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ run_dir = Path("tmp/run") - work_dir = Path('tmp/work_dir') + work_dir = Path("tmp/work_dir") return ExecutionContext( "test_pipeline", "run_id_1234", - config, + config, logger, run_dir, - '2024-05-06 15:45:30', + "2024-05-06 15:45:30", work_dir, - {"stage_test":stageresult}, - {"stage_test":stage_config}, + {"stage_test": stageresult}, + {"stage_test": stage_config}, {}, - None + None, ) + @pytest.fixture def stageresult() -> StageResult: """ @@ -67,338 +90,344 @@ def stageresult() -> StageResult: return StageResult( "stage_test", StageStatus.PENDING, - '2024-05-06 15:45:30', - '2024-05-07 15:45:30', + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", metadata={}, - outputs = "example output" + outputs="example output", ) -def test_executioncontext_creation(execution, logger, config, stageresult) -> None: - """ - Test that the ExecutionContext creates the right attributes. - """ - assert execution.pipeline_name == "test_pipeline" - assert execution.run_id == "run_id_1234" - assert execution.config == config - assert execution.logger == logger - assert execution.run_dir == Path("tmp/run") - assert execution.started_at == '2024-05-06 15:45:30' - assert execution.working_directory == Path('tmp/work_dir') - assert execution.stage_results == {"stage_test":stageresult} - assert execution.variables == {} - -def test_record(stageresult, execution) -> None: - """ - Tests that StageResult attributes are attached to stage_results and variables - attributes in the ExecutionContext instance. - """ - execution.record(stageresult) - assert execution.stage_results == {'stage_test':StageResult(name='stage_test', - status='pending', - started_at='2024-05-06 15:45:30', - finished_at='2024-05-07 15:45:30', - outputs="example output", - stdout='', - stderr='', - return_code=None, - metadata={}, - error=None, - source=None)} - assert execution.variables == {'stage_test':"example output"} - -def test_result_for(execution, stageresult) -> None: - """ - Tests that result_for correctly extracts the results of a requested stage. - """ - execution.record(stageresult) - assert execution.result_for("stage_test") == StageResult(name='stage_test', - status='pending', - started_at='2024-05-06 15:45:30', - finished_at='2024-05-07 15:45:30', - outputs="example output", - stdout='', - stderr='', - return_code=None, - metadata={}, - error=None, - source=None) - -def test_stage_outputs(execution, stageresult) -> None: +@pytest.fixture +def expected_recorded_stage_result() -> StageResult: """ - Tests that stage_outputs shows the outputs attribute of the StageResult - instance for a requested stage is extracted. + Expected StageResult after recording for assertions. """ - execution.record(stageresult) - assert execution.stage_outputs == {"stage_test":"example output"} + return StageResult( + name="stage_test", + status="pending", + started_at="2024-05-06 15:45:30", + finished_at="2024-05-07 15:45:30", + outputs="example output", + stdout="", + stderr="", + return_code=None, + metadata={}, + error=None, + source=None, + ) -def test_get_data_dir(execution, stageresult) -> None: - """ - Tests that get_data_dir method extracts the path from the execution context - or, if the context is None, returns an error to indicate that additional input is - required. - """ - assert execution.get_data_dir() == Path("tmp/config_data") - - run_dir = Path("tmp/run") - work_dir = Path('tmp/work_dir') - execution_blank_config = ExecutionContext("test_pipeline", - "run_id_1234", - None, - Logger(), - run_dir, - '2024-05-06 15:45:30', - work_dir, - {"stage_test":stageresult}, - {} ) - - with pytest.raises(PipelineConfigurationError): - execution_blank_config.get_data_dir() -def test_resolve_output_root(execution) -> None: +class TestExecutionContext: + def test_executioncontext_creation( + self, execution, logger, config, stageresult + ) -> None: + """ + Test that the ExecutionContext creates the right attributes. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == Path("tmp/run") + assert execution.started_at == "2024-05-06 15:45:30" + assert execution.working_directory == Path("tmp/work_dir") + assert execution.stage_results == {"stage_test": stageresult} + assert execution.variables == {} + + def test_record( + self, stageresult, execution, expected_recorded_stage_result + ) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + """ + execution.record(stageresult) + assert execution.stage_results == {"stage_test": expected_recorded_stage_result} + assert execution.variables == {"stage_test": "example output"} + + def test_result_for( + self, execution, stageresult, expected_recorded_stage_result + ) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + """ + execution.record(stageresult) + assert execution.result_for("stage_test") == expected_recorded_stage_result + + def test_stage_outputs(self, execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + """ + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test": "example output"} + + @pytest.fixture + def blank_context_with_config_none(self, stageresult) -> ExecutionContext: + run_dir = Path("tmp/run") + work_dir = Path("tmp/work_dir") + return ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + run_dir, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: + """ + Tests that get_data_dir method extracts the path from the execution context + or, if the context is None, returns an error to indicate that additional input + is required. + """ + assert execution.get_data_dir() == Path("tmp/config_data") + + with pytest.raises(PipelineConfigurationError): + blank_context_with_config_none.get_data_dir() + + def test_resolve_output_root(self, execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, raises an error to indicate additional input + is required.. + """ + work_dir = Path("tmp/work_dir") + assert execution.resolve_output_root() == Path("tmp/run") + + execution_blank_config = ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + None, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + with pytest.raises(PipelineConfigurationError): + execution_blank_config.resolve_output_root() + + def test_stage_config_accessors_return_named_and_active_configs( + self, config, logger + ) -> None: + stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) + context = ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + Path("tmp/run"), + stage_configs={"stage_test": stage_config}, + active_stage_name="stage_test", + ) + + assert context.stage_config_for("stage_test") == stage_config + assert context.get_stage_config("stage_test") == {"years_to_run": 2017} + assert context.get_stage_config() == {"years_to_run": 2017} + with pytest.raises(PipelineConfigurationError): + context.get_stage_config(vars_only=False) + assert ( + context.get_stage_config(with_global=False, vars_only=False) == stage_config + ) + + def test_set_active_stage(self, execution, stage_config) -> None: + """ + Tests that set_active_stage correctly sets the active_stage attribute in the + ExecutionContext instance. + """ + + execution.set_active_stage(stage_config.name) + assert execution.active_stage_name == stage_config.name + execution.set_active_stage(None) + assert execution.active_stage_name is None + + def test_stage_config_for(self, execution, stage_config) -> None: + """ + Tests that stage_config_for returns the StageConfig for a named stage. + """ + assert execution.stage_config_for(stage_config.name) == stage_config + assert execution.stage_config_for("missing_stage") is None + + def test_stage_config(self, execution, stage_config) -> None: + """ + Tests that stage_config exposes the currently active stage configuration. + """ + assert execution.stage_config is None + execution.set_active_stage(stage_config.name) + assert execution.stage_config == stage_config + + def test_get_stage_config(self, execution, stage_config) -> None: + """ + Tests that get_stage_config returns variables by default and the full + StageConfig object when requested. + """ + assert execution.get_stage_config() == {} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert execution.get_stage_config(with_global=False, vars_only=False) is None + + execution.set_active_stage(stage_config.name) + assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert ( + execution.get_stage_config(with_global=False, vars_only=False) + == stage_config + ) + + +class TestResolveGivenPath: """ - Tests that resolve_output_root method extracts the path from the given run - directory or, if None are given, raises an error to indicate additional input - is required.. + Parameters for testing multiple add_folder options in + test_resolve_given_path_add_folders function. """ - work_dir = Path('tmp/work_dir') - assert execution.resolve_output_root() == Path("tmp/run") - execution_blank_config = ExecutionContext("test_pipeline", - "run_id_1234", - None, - Logger(), - None, - '2024-05-06 15:45:30', - work_dir, - {"stage_test":stageresult}, - {} ) - - with pytest.raises(PipelineConfigurationError): - execution_blank_config.resolve_output_root() - -def test_stage_config_accessors_return_named_and_active_configs(config, logger) -> None: - stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) - context = ExecutionContext( - "test_pipeline", - "run_id_1234", - config, - logger, - Path("tmp/run"), - stage_configs={"stage_test": stage_config}, - active_stage_name="stage_test", - ) - - assert context.stage_config_for("stage_test") == stage_config - assert context.get_stage_config("stage_test") == {"years_to_run": 2017} - assert context.get_stage_config() == {"years_to_run": 2017} - with pytest.raises(PipelineConfigurationError): - context.get_stage_config(vars_only=False) - assert context.get_stage_config(with_global=False, vars_only=False) == stage_config - -""" -Parameters for testing multiple add_folder options in -test_resolve_given_path_add_folders function. -""" -@pytest.mark.parametrize( + @pytest.mark.parametrize( "add_folder,file_name,expected", [ ( - ["interim","testing_files"], + ["interim", "testing_files"], "clean.py", - Path("tmp/data/interim/testing_files/clean.py") + Path("tmp/data/interim/testing_files/clean.py"), ), + ("interim", "clean.py", Path("tmp/data/interim/clean.py")), + (None, "clean.py", Path("tmp/data/clean.py")), ( - "interim", - "clean.py", - Path("tmp/data/interim/clean.py") - ), - ( - None, - "clean.py", - Path("tmp/data/clean.py") - ), - ( - ["interim","testing_files"], + ["interim", "testing_files"], None, - Path("tmp/data/interim/testing_files") + Path("tmp/data/interim/testing_files"), ), - ( - "interim", - None, - Path("tmp/data/interim") - ), - ( - None, - None, - Path("tmp/data") - ) + ("interim", None, Path("tmp/data/interim")), + (None, None, Path("tmp/data")), ], -) + ) + def test_resolve_given_path_add_folders( + self, execution, add_folder, file_name, expected + ) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid + string or None type. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert ( + execution.resolve_given_path(None, path_name, file_name, root, add_folder) + == expected + ) + def test_resolve_given_path_norm(self, execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + """ + execution.record( + StageResult( + "stage_test2", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs={"data_path": "clean.py"}, + ) + ) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") -def test_resolve_given_path_add_folders(execution, add_folder, file_name, expected) -> None: - """ - Tests the add_folder functionality for lists, single strings, or None type in - the resolve_given_path class method as well as when the file_name is a valid string - or None type. - """ - path_name = "data_path" - root = Path("tmp/data") + assert execution.resolve_given_path( + stage_name, path_name, None, root, None + ) == Path("clean.py") - assert execution.resolve_given_path(None, - path_name, - file_name, - root, - add_folder) == expected -def test_resolve_given_path_norm(execution) -> None: - """ - Tests that resolve_given_path returns a file path that has been output in a - StageResult instance. - """ - execution.record(StageResult("stage_test2", - StageStatus.PENDING, - '2024-05-06 15:45:30', - '2024-05-07 15:45:30', - metadata={}, - outputs = {"data_path":"clean.py"} )) - stage_name = "stage_test2" - path_name = "data_path" - root = Path("tmp/data") - - assert execution.resolve_given_path(stage_name, - path_name, - None, - root, - None) == Path("clean.py") - """TEST NOT RUN FOR StageExecutor AS COVERED UNDER PythonStageExecutor""" + @pytest.fixture def pythonstageexecutor() -> PythonStageExecutor: - return PythonStageExecutor(("main.py","run.py")) - -def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: - assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") - - -def test_combine_vars(execution) -> None: - """ - Test that checks that a dictionary is returned, combining values from a global - configuration and a stage configuration whilst removing any stage specific - exclusions. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {"stage_1": ["global_var2"]} - stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - combined_vars = execution._combine_vars() - assert combined_vars == { - "stage_var1": "value3", - "stage_var2": "value4", - "global_var1": "value1" - } - -def test_combine_vars_errors(execution) -> None: - """ - Test that confirms that a warning is raised if there is a variable defined in both - the global and the stage configurations as well as asserting the correct values. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {"stage_1": ["global_var2"]} - stage_vars = {"stage_var1": "value3", "global_var1": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - - with pytest.warns(StageConfigurationWarning, - match="Stage defines variable\\(s\\) that are also defined in global " - "variables: global_var1\\. Stage variables will take precedence."): - combined_vars = execution._combine_vars() - assert combined_vars == { + return PythonStageExecutor(("main.py", "run.py")) + + +class TestPythonStageExecutor: + def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: + assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") + + +class TestCombineVars: + def test_combine_vars(self, execution) -> None: + """ + Test that checks that a dictionary is returned, combining values from a global + configuration and a stage configuration whilst removing any stage specific + exclusions. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { "stage_var1": "value3", - "global_var1": "value4" + "stage_var2": "value4", + "global_var1": "value1", } -def test_combine_vars_no_exclusion(execution) -> None: - """ - Test confirming that a dictionary is returned, combining values from a global configuration - and a stage configuration when there are no exclusions defined. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {} - stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - combined_vars = execution._combine_vars() - assert combined_vars == { - "stage_var1": "value3", - "stage_var2": "value4", - "global_var1": "value1", - "global_var2": "value2" - } -"""CONTINUE FROM EXECUTE CLASS METHOD""" -@pytest.fixture -def stage_config() -> StageConfig: - """ - Return a StageConfig object for testing. - """ - return StageConfig( - name="stage_test", - _variables={"sex":"gender", - "dob":"date_of_birth"}, - metadata={} + def test_combine_vars_errors(self, execution) -> None: + """ + Test that confirms that a warning is raised if there is a variable defined in + both the global and the stage configurations as well as asserting the correct + values. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "global_var1": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions ) - -def test_set_active_stage(execution, stage_config) -> None: - """ - Tests that set_active_stage correctly sets the active_stage attribute in the - ExecutionContext instance. - """ - - execution.set_active_stage(stage_config.name) - assert execution.active_stage_name == stage_config.name - execution.set_active_stage(None) - assert execution.active_stage_name == None - -def test_stage_config_for(execution, stage_config) -> None: - """ - Tests that stage_config_for returns the StageConfig for a named stage. - """ - assert execution.stage_config_for(stage_config.name) == stage_config - assert execution.stage_config_for("missing_stage") is None - -def test_stage_config(execution, stage_config) -> None: - """ - Tests that stage_config exposes the currently active stage configuration. - """ - assert execution.stage_config is None - execution.set_active_stage(stage_config.name) - assert execution.stage_config == stage_config - -def test_get_stage_config(execution, stage_config) -> None: - """ - Tests that get_stage_config returns variables by default and the full - StageConfig object when requested. - """ - assert execution.get_stage_config() == {} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) - assert execution.get_stage_config(with_global=False, vars_only=False) is None - - execution.set_active_stage(stage_config.name) - assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) - assert execution.get_stage_config(with_global=False, vars_only=False) == stage_config - + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + + with pytest.warns( + StageConfigurationWarning, + match="Stage defines variable\\(s\\) that are also defined in global " + "variables: global_var1\\. Stage variables will take precedence.", + ): + combined_vars = execution._combine_vars() + assert combined_vars == {"stage_var1": "value3", "global_var1": "value4"} + + def test_combine_vars_no_exclusion(self, execution) -> None: + """ + Test confirming that a dictionary is returned, combining values from a global + configuration and a stage configuration when there are no exclusions defined. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1", + "global_var2": "value2", + } From 029fb6e17b9008eec015127f3c90b2d071ee9592 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 13:22:44 +0100 Subject: [PATCH 213/332] docs: added docstrings to tests in test_execution.py --- tests/test_execution.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_execution.py b/tests/test_execution.py index 496ee37..618e112 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -216,6 +216,10 @@ def test_resolve_output_root(self, execution) -> None: def test_stage_config_accessors_return_named_and_active_configs( self, config, logger ) -> None: + """ + Tests that getter methods to return the stage_config for a named stage + returns correct attributes based on given parameters. + """ stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) context = ExecutionContext( "test_pipeline", @@ -357,6 +361,10 @@ def pythonstageexecutor() -> PythonStageExecutor: class TestPythonStageExecutor: def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: + """ + Checks that entrypoints are set correctly in the PythonStageExecutor + instance. + """ assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") From 2d008d1a7b394d6f0977ce6b5157a6c8c49b2fcf Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:03:21 +0100 Subject: [PATCH 214/332] tweal: test_models.py refactor tests into classes with CoPilot, utilises Ruff formatting and checking --- tests/test_models.py | 515 ++++++++++++++++++++++--------------------- 1 file changed, 269 insertions(+), 246 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 7e7d0b4..21f45f4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,282 +1,305 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig -import pytest import datetime from pathlib import Path from textwrap import dedent -from tests.test_execution import stageresult -def test_stagestatus() -> None: - """ - Test that stagestatus outputs the correct values. - """ - assert StageStatus.PENDING == "pending" - assert StageStatus.RUNNING == "running" - assert StageStatus.SUCCEEDED == "succeeded" - assert StageStatus.FAILED == "failed" - assert StageStatus.SKIPPED == "skipped" +import pytest + +from onsrap.models import ( + PipelineConfig, + PipelineRun, + PipelineStatus, + RunManifest, + RuntimeID, + StageStatus, +) + +STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) +FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) + + +class TestStatuses: + def test_stagestatus(self) -> None: + """ + Test that stagestatus outputs the correct values. + """ + assert StageStatus.PENDING == "pending" + assert StageStatus.RUNNING == "running" + assert StageStatus.SUCCEEDED == "succeeded" + assert StageStatus.FAILED == "failed" + assert StageStatus.SKIPPED == "skipped" + + def test_pipeline_status(self) -> None: + """ + Test that pipeline status outputs the correct values. + """ + assert PipelineStatus.PENDING == "pending" + assert PipelineStatus.RUNNING == "running" + assert PipelineStatus.SUCCEEDED == "succeeded" + assert PipelineStatus.FAILED == "failed" -def test_pipeline_status() -> None: - """ - Test that pipeline status outputs the correct values. - """ - assert PipelineStatus.PENDING == "pending" - assert PipelineStatus.RUNNING == "running" - assert PipelineStatus.SUCCEEDED == "succeeded" - assert PipelineStatus.FAILED == "failed" @pytest.fixture def runtimeID() -> RuntimeID: """ - Example RuntimeID instance for testing of other methods. + Example RuntimeID instance for testing of other methods. """ - return RuntimeID(id = "abc123", - timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), - hash = "fnruw9574893ghkwq234h5kg", - short_hash = "4h5kg") + return RuntimeID( + id="abc123", + timestamp=datetime.datetime(2026, 7, 7, 13, 5, 46), + hash="fnruw9574893ghkwq234h5kg", + short_hash="4h5kg", + ) -def test_runtimeID_creation(runtimeID) -> None: - """ - Test that a RuntimeID is correctly created. - """ - assert runtimeID.id == "abc123" - assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) - assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" - assert runtimeID.short_hash == "4h5kg" -def test_getter_functions_runtimeID(runtimeID) -> None: - """ - Tests all the getter functions for the RuntimeID instance. - """ - assert runtimeID.get_id() == "abc123" - assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) - assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" - assert runtimeID.get_short_hash() == "4h5kg" +class TestRuntimeID: + def test_runtimeID_creation(self, runtimeID) -> None: + """ + Test that a RuntimeID is correctly created. + """ + assert runtimeID.id == "abc123" + assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" + assert runtimeID.short_hash == "4h5kg" + + def test_getter_functions_runtimeID(self, runtimeID) -> None: + """ + Tests all the getter functions for the RuntimeID instance. + """ + assert runtimeID.get_id() == "abc123" + assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" + assert runtimeID.get_short_hash() == "4h5kg" + @pytest.fixture def blankpipelineconfig() -> PipelineConfig: """ - Blank PipelineConfig instance for class method testing. + Blank PipelineConfig instance for class method testing. """ return PipelineConfig() + @pytest.fixture -def pipelineconfig() -> PipelineConfig: +def expected_pipeline_config() -> PipelineConfig: """ - Example PipelineConfig completed class instance for method testing. + Example PipelineConfig completed class instance for method testing. """ - return PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) + return PipelineConfig( + name="test_rap", + backend="python", + work_dir=Path("tmp/work"), + project_root=Path("project"), + log_dir=Path("tmp/logs"), + data_dir=Path("tmp/data"), + allow_subprocess_fallback=True, + python_executable=None, + metadata={"variables": ["name", "age"], "num_stages": 6}, + ) + + +@pytest.fixture +def pipelineconfig(expected_pipeline_config) -> PipelineConfig: + return expected_pipeline_config + @pytest.fixture def mapping() -> dict: """ - Example mapping dictionary for use in testing from_mapping() method. - """ - return {"name":"test_rap", - "backend":"python", - "work_dir":Path("tmp/work"), - "project_root":Path("project"), - "log_dir":Path("tmp/logs"), - "data_dir":Path("tmp/data"), - "allow_subprocess_fallback":True, - "python_executable":None, - "metadata":{"variables":["name","age"], - "num_stages":6}} - - -def test_from_any(mapping, pipelineconfig, blankpipelineconfig) -> None: - """ - Test derivation for a PipelineConfig instance using the from_any() method. This test - checks all methods EXCEPT from_file as this will be covered in another test due to - creation of a mock file being required. - """ - assert blankpipelineconfig.from_any(None) == PipelineConfig() - assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - assert blankpipelineconfig.from_any(mapping) == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - - with pytest.raises(TypeError): - blankpipelineconfig.from_any(11) - -def test_from_file(tmp_path,) -> PipelineConfig: - pipeline_config = tmp_path / "configuration.py" - pipeline_config.write_text( - dedent( - """ - {"name":"test_rap", - "backend":"python", - "work_dir":"tmp/work", - "project_root":"project", - "log_dir":"tmp/logs", - "data_dir":"tmp/data", - "allow_subprocess_fallback":True, - "python_executable": , - "metadata":{"variables":["name","age"], - "num_stages":6} - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) - no_map_pipeline_config = tmp_path / "not_valid.py" - no_map_pipeline_config.write_text( - dedent( - """ - variable = "Hello world" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - configuration = PipelineConfig.from_file(pipeline_config) - assert configuration == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - - fake_file = "path_not_real" - with pytest.raises(FileNotFoundError): - PipelineConfig.from_file(fake_file) - with pytest.raises(TypeError): - PipelineConfig.from_file(no_map_pipeline_config) - -def test_to_dict(pipelineconfig) -> None: - """ - Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values - as a dictionary. + Example mapping dictionary for use in testing from_mapping() method. """ + return { + "name": "test_rap", + "backend": "python", + "work_dir": Path("tmp/work"), + "project_root": Path("project"), + "log_dir": Path("tmp/logs"), + "data_dir": Path("tmp/data"), + "allow_subprocess_fallback": True, + "python_executable": None, + "metadata": {"variables": ["name", "age"], "num_stages": 6}, + } + + +class TestPipelineConfig: + def test_from_any( + self, mapping, pipelineconfig, blankpipelineconfig, expected_pipeline_config + ) -> None: + """ + Test derivation for a PipelineConfig instance using the from_any() method. This + test checks all methods EXCEPT from_file as this will be covered in another + test due to creation of a mock file being required. + """ + assert blankpipelineconfig.from_any(None) == PipelineConfig() + assert blankpipelineconfig.from_any(pipelineconfig) == expected_pipeline_config + assert blankpipelineconfig.from_any(mapping) == expected_pipeline_config + + with pytest.raises(TypeError): + blankpipelineconfig.from_any(11) + + def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config + + fake_file = "path_not_real" + with pytest.raises(FileNotFoundError): + PipelineConfig.from_file(fake_file) + with pytest.raises(TypeError): + PipelineConfig.from_file(no_map_pipeline_config) + + def test_to_dict(self, pipelineconfig) -> None: + """ + Test of to_dict() class method for PipelineConfig that it outputs the + PipelineConfig values as a dictionary. + """ + + assert pipelineconfig.to_dict() == { + "name": "test_rap", + "backend": "python", + "work_dir": "tmp\\work", + "project_root": "project", + "output_dir": None, + "log_dir": "tmp\\logs", + "data_dir": "tmp\\data", + "allow_subprocess_fallback": True, + "python_executable": None, + "variables": ["name", "age"], + "num_stages": 6, + } + - assert pipelineconfig.to_dict() == {"name":"test_rap", - "backend":"python", - "work_dir":"tmp\\work", - "project_root":"project", - "output_dir":None, - "log_dir":"tmp\\logs", - "data_dir":"tmp\\data", - "allow_subprocess_fallback":True, - "python_executable":None, - "variables":["name","age"], - "num_stages":6} - - @pytest.fixture def runmanifest() -> RunManifest: """ - Example RunManifest class instance for testing of class method. - """ - return RunManifest("pipeline", - "1", - None, - ["stage1","stage2"], - {"uniqueID":"example"}, - {"input_path":"input/data/example.csv"}, - {"output_path":"output/data/example.csv"}, - "python", - ["1.3.2"], - "", - None, - None) - -def test_stage_result(stageresult) -> None: - """ - Uses a StageResult instance created in test_execution to ensure that - the class instance is created suitably with required defaults. - """ - assert stageresult.name == "stage_test" - assert stageresult.status == "pending" - assert stageresult.started_at == '2024-05-06 15:45:30' - assert stageresult.finished_at == '2024-05-07 15:45:30' - assert stageresult.outputs == "example output" - assert stageresult.stdout == "" - assert stageresult.stderr == "" - assert stageresult.return_code == None - assert stageresult.metadata == {} - assert stageresult.error == None - assert stageresult.source == None - -@pytest.mark.parametrize("status_stage,expected_stage", - [(StageStatus.PENDING, False), - (StageStatus.RUNNING, False), - (StageStatus.FAILED, False), - (StageStatus.SUCCEEDED, True), - (StageStatus.SKIPPED, False)]) - -def test_succeeded(stageresult, status_stage, expected_stage) -> None: - """ - Tests succeeded() method for StageResult which outputs True or False depending on - the status of the StageResult. + Example RunManifest class instance for testing of class method. """ - stageresult.status = status_stage - assert stageresult.succeeded == expected_stage + return RunManifest( + "pipeline", + "1", + None, + ["stage1", "stage2"], + {"uniqueID": "example"}, + {"input_path": "input/data/example.csv"}, + {"output_path": "output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None, + ) + + +class TestStageResult: + def test_stage_result(self, stageresult) -> None: + """ + Uses a StageResult instance created in test_execution to ensure that + the class instance is created suitably with required defaults. + """ + assert stageresult.name == "stage_test" + assert stageresult.status == "pending" + assert stageresult.started_at == "2024-05-06 15:45:30" + assert stageresult.finished_at == "2024-05-07 15:45:30" + assert stageresult.outputs == "example output" + assert stageresult.stdout == "" + assert stageresult.stderr == "" + assert stageresult.return_code is None + assert stageresult.metadata == {} + assert stageresult.error is None + assert stageresult.source is None + + @pytest.mark.parametrize( + "status_stage,expected_stage", + [ + (StageStatus.PENDING, False), + (StageStatus.RUNNING, False), + (StageStatus.FAILED, False), + (StageStatus.SUCCEEDED, True), + (StageStatus.SKIPPED, False), + ], + ) + def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: + """ + Tests succeeded() method for StageResult which outputs True or False depending + on the status of the StageResult. + """ + stageresult.status = status_stage + assert stageresult.succeeded == expected_stage + + def test_duration_seconds(self, stageresult) -> None: + stageresult.started_at = STARTED_AT + stageresult.finished_at = FINISHED_AT + seconds_value = (FINISHED_AT - STARTED_AT).total_seconds() + assert stageresult.duration_seconds == seconds_value -def test_duration_seconds(stageresult) -> None: - stageresult.started_at = datetime.datetime(2024,5,6,15,45,30) - stageresult.finished_at = datetime.datetime(2024,5,7,15,45,30) - seconds_value = (datetime.datetime(2024,5,7,15,45,30) - datetime.datetime(2024,5,6,15,45,30)).total_seconds() - assert stageresult.duration_seconds == seconds_value @pytest.fixture def pipelinerun(stageresult, runmanifest) -> PipelineRun: - return PipelineRun(runmanifest, - PipelineStatus.SUCCEEDED, - datetime.datetime(2024,5,6,15,45,30), - datetime.datetime(2024,5,7,15,45,30), - [stageresult], - {"stage_test":"example output"}) - -def test_pipelinerun_configuration(pipelinerun, runmanifest, stageresult) -> None: - assert pipelinerun.manifest == runmanifest - assert pipelinerun.status == PipelineStatus.SUCCEEDED - assert pipelinerun.started_at == datetime.datetime(2024,5,6,15,45,30) - assert pipelinerun.completed_at == datetime.datetime(2024,5,7,15,45,30) - assert pipelinerun.stage_results == [stageresult] - assert pipelinerun.stage_outputs == {"stage_test":"example output"} - -def test_result_for(pipelinerun, stageresult) -> None: - assert pipelinerun.result_for("stage_test") == stageresult - assert pipelinerun.result_for("not_a_stage") == None - -@pytest.mark.parametrize("status,expected", - [(PipelineStatus.PENDING, False), - (PipelineStatus.RUNNING, False), - (PipelineStatus.FAILED, False), - (PipelineStatus.SUCCEEDED, True)]) - -def test_succeeded_pipeline(pipelinerun, status, expected) -> None: - pipelinerun.status = status - assert pipelinerun.succeeded == expected - - -#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file + return PipelineRun( + runmanifest, + PipelineStatus.SUCCEEDED, + STARTED_AT, + FINISHED_AT, + [stageresult], + {"stage_test": "example output"}, + ) + + +class TestPipelineRun: + def test_pipelinerun_configuration( + self, pipelinerun, runmanifest, stageresult + ) -> None: + assert pipelinerun.manifest == runmanifest + assert pipelinerun.status == PipelineStatus.SUCCEEDED + assert pipelinerun.started_at == STARTED_AT + assert pipelinerun.completed_at == FINISHED_AT + assert pipelinerun.stage_results == [stageresult] + assert pipelinerun.stage_outputs == {"stage_test": "example output"} + + def test_result_for(self, pipelinerun, stageresult) -> None: + assert pipelinerun.result_for("stage_test") == stageresult + assert pipelinerun.result_for("not_a_stage") is None + + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.PENDING, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.FAILED, False), + (PipelineStatus.SUCCEEDED, True), + ], + ) + def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: + pipelinerun.status = status + assert pipelinerun.succeeded == expected + + +# TODO: Test _extract_stages_run and all methods in StageConfig class From 8ad6173fb75da40b1990833c1988bd0ed0532d4e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:27:24 +0100 Subject: [PATCH 215/332] tweak: split test_from_file into two methods to act as singular unit tests rather than doing multiple tasks in one function --- tests/test_models.py | 72 ++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 21f45f4..b831773 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -13,6 +13,8 @@ StageStatus, ) +from tests.test_execution import stageresult + STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) @@ -136,27 +138,12 @@ def test_from_any( with pytest.raises(TypeError): blankpipelineconfig.from_any(11) - def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: - pipeline_config = tmp_path / "configuration.py" - pipeline_config.write_text( - dedent( - """ - {"name":"test_rap", - "backend":"python", - "work_dir":"tmp/work", - "project_root":"project", - "log_dir":"tmp/logs", - "data_dir":"tmp/data", - "allow_subprocess_fallback":True, - "python_executable": , - "metadata":{"variables":["name","age"], - "num_stages":6} - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) + def test_from_file_errors(self, tmp_path) -> PipelineConfig: + """ + Checks that a PipelineConfig instance raises the correct exceptions when a + file is not found or the file does not contain a dictionary mapping. + """ + no_map_pipeline_config = tmp_path / "not_valid.py" no_map_pipeline_config.write_text( dedent( @@ -167,8 +154,6 @@ def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: + "\n", encoding="utf-8", ) - configuration = PipelineConfig.from_file(pipeline_config) - assert configuration == expected_pipeline_config fake_file = "path_not_real" with pytest.raises(FileNotFoundError): @@ -176,6 +161,47 @@ def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: with pytest.raises(TypeError): PipelineConfig.from_file(no_map_pipeline_config) + def test_from_file_success( + self, tmp_path, expected_pipeline_config + ) -> PipelineConfig: + """ + Checks that a PipelineConfig instance is created successfully from a mock + file. + """ + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config + + def test_to_dict(self, pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the From 6dc30eaf37425259784fec1a83afa1d8371b966a Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:31:26 +0100 Subject: [PATCH 216/332] docs: complete docstrings for test_models.py --- tests/test_models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index b831773..7486f47 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -281,6 +281,10 @@ def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: assert stageresult.succeeded == expected_stage def test_duration_seconds(self, stageresult) -> None: + """ + Tests that duration_seconds() method calculates the correct duration in seconds + between the started_at and finished_at attributes of the StageResult instance. + """ stageresult.started_at = STARTED_AT stageresult.finished_at = FINISHED_AT seconds_value = (FINISHED_AT - STARTED_AT).total_seconds() @@ -303,6 +307,10 @@ class TestPipelineRun: def test_pipelinerun_configuration( self, pipelinerun, runmanifest, stageresult ) -> None: + """ + Checks that the PipelineRun instance is created successfully with the correct + attributes and values. + """ assert pipelinerun.manifest == runmanifest assert pipelinerun.status == PipelineStatus.SUCCEEDED assert pipelinerun.started_at == STARTED_AT @@ -324,6 +332,10 @@ def test_result_for(self, pipelinerun, stageresult) -> None: ], ) def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: + """ + Checks that the succeeded() method of the PipelineRun instance returns the + correct boolean value based on its status. + """ pipelinerun.status = status assert pipelinerun.succeeded == expected From e52183903d7642d697708da741297415337c85ca Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 15:07:52 +0100 Subject: [PATCH 217/332] tweak: refactor with CoPilot to structure tests as classes --- tests/test_pipeline_architecture.py | 878 ++++++++++++++-------------- 1 file changed, 438 insertions(+), 440 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 02e9892..15add2d 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -1,12 +1,12 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from textwrap import dedent import pytest import yaml -import subprocess -import sys from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph @@ -14,418 +14,415 @@ from onsrap.stage import Stage from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning +NO_STAGES_SPECIFIED_WARNING = "No stages specified to run. All stages running by default." +OUTPUT_DIRECTORY_WARNING = ( + "Output directory is not specified. Using project root or work directory as the run output." +) -def test_pipeline_from_files_executes_python_entrypoints(tmp_path: Path) -> None: - first_stage = tmp_path / "first_stage.py" - first_stage.write_text( - dedent( - """ - def run(context): - return "alpha" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - second_stage = tmp_path / "second_stage.py" - second_stage.write_text( - dedent( - """ - def main(context): - return context.result_for("first_stage").outputs + "-beta" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{} - }, - ) +def _base_pipeline_config(tmp_path: Path) -> dict: + return { + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_config": {}, + } - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.succeeded is True - assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] - assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} - - -def test_pipeline_uses_run_specific_output_directory(tmp_path: Path) -> None: - writer_stage = tmp_path / "writer_stage.py" - writer_stage.write_text( - dedent( - """ - from pathlib import Path - - def main(context): - output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(context.run_id, encoding="utf-8") - return {"output_path": str(output_path), "run_id": context.run_id} - """ - ).strip() - + "\n", - encoding="utf-8", - ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [writer_stage], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{}}, - ) - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - - #TODO: This warning functions however from the name of the test, I would assume that the output - #directory has been set so this needs to be reviewed. - first_run = pipeline.run() - second_run = pipeline.run() - - first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) - second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) - - assert first_run.manifest.run_id != second_run.manifest.run_id - assert first_output != second_output - assert first_output.exists() - assert second_output.exists() - assert first_output.parents[2].name == first_run.manifest.run_id - assert second_output.parents[2].name == second_run.manifest.run_id - - -def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Path) -> None: - script_stage = tmp_path / "script_stage.py" - script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{}}, +class TestPipelineFromFiles: + def test_pipeline_from_files_executes_python_entrypoints( + self, + tmp_path: Path + ) -> None: + """ + + """ + first_stage = tmp_path / "first_stage.py" + first_stage.write_text( + dedent( + """ + def run(context): + return "alpha" + """ + ).strip() + + "\n", + encoding="utf-8", ) - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.stage_results[0].outputs.strip() == "script fallback works" - assert run.stage_results[0].stdout.strip() == "script fallback works" - - -def test_stage_graph_detects_cycles() -> None: - first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) - second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) - - graph = StageGraph.from_stages([first_stage, second_stage]) + second_stage = tmp_path / "second_stage.py" + second_stage.write_text( + dedent( + """ + def main(context): + return context.result_for("first_stage").outputs + "-beta" + """ + ).strip() + + "\n", + encoding="utf-8", + ) - try: - graph.topological_order() - except Exception as exc: # noqa: BLE001 - assert exc.__class__.__name__ == "DependencyCycleError" - else: - raise AssertionError("Expected a dependency cycle error") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config=_base_pipeline_config(tmp_path), + ) + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() -def test_pipeline_from_config_builds_stages_and_injects_stage_config(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() - - stage_file = scripts_dir / "0_data_validation.py" - stage_file.write_text( - dedent( - """ - def run(context): - return { - "stage_name": context.stage_config.name, - "years_to_run": context.stage_config.get("years_to_run"), - "target_variable": context.stage_config.require("target_variable"), - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - config_file = tmp_path / "conf.yaml" - config_file.write_text( - dedent( - f""" - pipeline_variables: - name: "configured-pipeline" - backend: python - working_dir: "{tmp_path.as_posix()}" - project_root: "{tmp_path.as_posix()}" - log_dir: "{(tmp_path / 'logs').as_posix()}" - stages: - - 0_data_validation: - location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" - run: true - dependencies: [] - - stage_configuration: - 0_data_validation: - years_to_run: 2017 - target_variable: "classification" - global_configuration: - dry_run: true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) - - assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] - assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 - - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.stage_outputs["0_data_validation"] == { - "stage_name": "0_data_validation", - "years_to_run": 2017, - "target_variable": "classification", - } + assert run.succeeded is True + assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] + assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} + def test_pipeline_uses_run_specific_output_directory(self, tmp_path: Path) -> None: + writer_stage = tmp_path / "writer_stage.py" + writer_stage.write_text( + dedent( + """ + from pathlib import Path -def test_pipeline_rejects_unknown_stage_configuration(tmp_path: Path) -> None: - stage_file = tmp_path / "single_stage.py" - stage_file.write_text( - dedent( - """ - def run(context): - return "ok" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [stage_file], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": { - "missing_stage": {"years_to_run": 2017}, - }, - "global_config":{} - }, + def main(context): + output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(context.run_id, encoding="utf-8") + return {"output_path": str(output_path), "run_id": context.run_id} + """ + ).strip() + + "\n", + encoding="utf-8", ) - - with pytest.raises(StageConfigurationError, match="unknown stages"): - pipeline.validate() - - -def test_pipeline_from_config_parses_stage_configuration_payloads(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() - - for stage_name in ("0_extract", "1_transform"): - (scripts_dir / f"{stage_name}.py").write_text( + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [writer_stage], + config=_base_pipeline_config(tmp_path), + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + # TODO: This warning functions however from the name of the test, I would assume that the output + # directory has been set so this needs to be reviewed. + first_run = pipeline.run() + second_run = pipeline.run() + + first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) + second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) + + assert first_run.manifest.run_id != second_run.manifest.run_id + assert first_output != second_output + assert first_output.exists() + assert second_output.exists() + assert first_output.parents[2].name == first_run.manifest.run_id + assert second_output.parents[2].name == second_run.manifest.run_id + + def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + script_stage = tmp_path / "script_stage.py" + script_stage.write_text("print('script fallback works')\n", encoding="utf-8") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config=_base_pipeline_config(tmp_path), + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() + + assert run.stage_results[0].outputs.strip() == "script fallback works" + assert run.stage_results[0].stdout.strip() == "script fallback works" + + +class TestStageGraph: + def test_stage_graph_detects_cycles(self) -> None: + first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) + second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) + + graph = StageGraph.from_stages([first_stage, second_stage]) + + try: + graph.topological_order() + except Exception as exc: # noqa: BLE001 + assert exc.__class__.__name__ == "DependencyCycleError" + else: + raise AssertionError("Expected a dependency cycle error") + + +class TestPipelineFromConfig: + def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_file = scripts_dir / "0_data_validation.py" + stage_file.write_text( dedent( """ def run(context): - return context.stage_config.to_dict() + return { + "stage_name": context.stage_config.name, + "years_to_run": context.stage_config.get("years_to_run"), + "target_variable": context.stage_config.require("target_variable"), + } """ ).strip() + "\n", encoding="utf-8", ) - config_payload = { - "pipeline_variables": { - "name": "parse-test", - "backend": "python", - "working_dir": tmp_path.as_posix(), - "project_root": tmp_path.as_posix(), - "data_dir": (tmp_path / "data").as_posix(), - "log_dir": (tmp_path / "logs").as_posix(), - "metadata": { - "description": "configuration parsing test", - }, - "stages": [ - { - "0_extract": { - "location": "", - "run": True, - "dependencies": [], - "owner": "analytics", - } - }, - { - "1_transform": { - "location": "", - "run": True, - "dependencies": ["0_extract"], - } - }, - ], - }, - "stage_configuration": { - "0_extract": { - "years_to_run": 2017, - "datasets": { - "orders": { - "path": "data/orders.csv", - } - }, - "metadata": { - "purpose": "extract", - }, - }, - "1_transform": { - "target_variable": "classification", - "metadata": { - "purpose": "transform", - }, - }, - }, - "global_config":{} - } - - config_file = tmp_path / "conf.yaml" - config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + config_file = tmp_path / "conf.yaml" + config_file.write_text( + dedent( + f""" + pipeline_variables: + name: "configured-pipeline" + backend: python + working_dir: "{tmp_path.as_posix()}" + project_root: "{tmp_path.as_posix()}" + log_dir: "{(tmp_path / 'logs').as_posix()}" + stages: + - 0_data_validation: + location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" + run: true + dependencies: [] + + stage_configuration: + 0_data_validation: + years_to_run: 2017 + target_variable: "classification" + global_configuration: + dry_run: true + """ + ).strip() + + "\n", + encoding="utf-8", + ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) - assert pipeline.name == "parse-test" - assert pipeline.config.work_dir == tmp_path - assert pipeline.config.project_root == tmp_path - assert pipeline.config.log_dir == tmp_path / "logs" - assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] - assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() - assert pipeline.stages[0].metadata["owner"] == "analytics" - assert pipeline.stages[1].dependencies == ("0_extract",) - assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017, "datasets": {"orders": {"path": "data/orders.csv"}}} - assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" + assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] + assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() -def test_pipeline_from_config_scales_stage_configuration_to_many_stages(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() + assert run.stage_outputs["0_data_validation"] == { + "stage_name": "0_data_validation", + "years_to_run": 2017, + "target_variable": "classification", + } - stage_count = 6 - stage_names = [f"{index}_stage" for index in range(stage_count)] - for index, stage_name in enumerate(stage_names): - stage_file = scripts_dir / f"{stage_name}.py" - previous_stage_name = stage_names[index - 1] if index > 0 else None + def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: + stage_file = tmp_path / "single_stage.py" stage_file.write_text( dedent( - f""" + """ def run(context): - previous_ordinal = None - if {index} > 0: - previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] - return {{ - "stage_name": context.stage_config.name, - "ordinal": context.stage_config.require("ordinal"), - "label": context.stage_config.require("label"), - "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), - "known_stage_configs": sorted(context.stage_configs), - "previous_ordinal": previous_ordinal, - }} + return "ok" """ ).strip() + "\n", encoding="utf-8", ) - stage_definitions = [] - stage_configuration = {} - for index, stage_name in enumerate(stage_names): - dependencies = [stage_names[index - 1]] if index > 0 else [] - stage_definitions.append( - { - stage_name: { - "location": "", - "run": True, - "dependencies": dependencies, - } - } - ) - stage_configuration[stage_name] = { - "ordinal": index, - "label": f"label-{index}", + config = _base_pipeline_config(tmp_path) + config["stage_configuration"] = { + "missing_stage": {"years_to_run": 2017}, } - - config_file = tmp_path / "conf.yaml" - config_file.write_text( - yaml.safe_dump( - { - "pipeline_variables": { - "name": "many-stage-pipeline", - "backend": "python", - "working_dir": tmp_path.as_posix(), - "project_root": tmp_path.as_posix(), - "log_dir": (tmp_path / "logs").as_posix(), - "stages": stage_definitions, + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [stage_file], + config=config, + ) + + with pytest.raises(StageConfigurationError, match="unknown stages"): + pipeline.validate() + + + def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + for stage_name in ("0_extract", "1_transform"): + (scripts_dir / f"{stage_name}.py").write_text( + dedent( + """ + def run(context): + return context.stage_config.to_dict() + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_payload = { + "pipeline_variables": { + "name": "parse-test", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "data_dir": (tmp_path / "data").as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "metadata": { + "description": "configuration parsing test", }, - "stage_configuration": stage_configuration, - "global_config": { - "dry_run": True, + "stages": [ + { + "0_extract": { + "location": "", + "run": True, + "dependencies": [], + "owner": "analytics", + } + }, + { + "1_transform": { + "location": "", + "run": True, + "dependencies": ["0_extract"], + } + } + ], + }, + "stage_configuration": { + "0_extract": { + "years_to_run": 2017, + "datasets": { + "orders": { + "path": "data/orders.csv", + } + }, + "metadata": { + "purpose": "extract", + }, + }, + "1_transform": { + "target_variable": "classification", + "metadata": { + "purpose": "transform", + }, }, }, - sort_keys=False, - ), - encoding="utf-8", - ) + "global_config": {}, + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) + + assert pipeline.name == "parse-test" + assert pipeline.config.work_dir == tmp_path + assert pipeline.config.project_root == tmp_path + assert pipeline.config.log_dir == tmp_path / "logs" + assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] + assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + assert pipeline.stages[0].metadata["owner"] == "analytics" + assert pipeline.stages[1].dependencies == ("0_extract",) + assert pipeline.stage_configs["0_extract"].variables == { + "years_to_run": 2017, + "datasets": {"orders": {"path": "data/orders.csv"}}, + } + assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} + assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" + + + def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_count = 6 + stage_names = [f"{index}_stage" for index in range(stage_count)] + + for index, stage_name in enumerate(stage_names): + stage_file = scripts_dir / f"{stage_name}.py" + previous_stage_name = stage_names[index - 1] if index > 0 else None + stage_file.write_text( + dedent( + f""" + def run(context): + previous_ordinal = None + if {index} > 0: + previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] + return {{ + "stage_name": context.stage_config.name, + "ordinal": context.stage_config.require("ordinal"), + "label": context.stage_config.require("label"), + "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), + "known_stage_configs": sorted(context.stage_configs), + "previous_ordinal": previous_ordinal, + }} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + stage_definitions = [] + stage_configuration = {} + for index, stage_name in enumerate(stage_names): + dependencies = [stage_names[index - 1]] if index > 0 else [] + stage_definitions.append( + { + stage_name: { + "location": "", + "run": True, + "dependencies": dependencies, + } + } + ) + stage_configuration[stage_name] = { + "ordinal": index, + "label": f"label-{index}", + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + yaml.safe_dump( + { + "pipeline_variables": { + "name": "many-stage-pipeline", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "stages": stage_definitions, + }, + "stage_configuration": stage_configuration, + "global_config": { + "dry_run": True, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) - assert [stage.name for stage in pipeline.stages] == stage_names - assert sorted(pipeline.stage_configs) == stage_names + assert [stage.name for stage in pipeline.stages] == stage_names + assert sorted(pipeline.stage_configs) == stage_names - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() - assert run.manifest.stages_run == stage_names - assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names + assert run.manifest.stages_run == stage_names + assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names - for index, stage_name in enumerate(stage_names): - output = run.stage_outputs[stage_name] - assert output["stage_name"] == stage_name - assert output["ordinal"] == index - assert output["label"] == f"label-{index}" - assert output["first_stage_ordinal"] == 0 - assert output["known_stage_configs"] == stage_names - expected_previous = None if index == 0 else index - 1 - assert output["previous_ordinal"] == expected_previous + for index, stage_name in enumerate(stage_names): + output = run.stage_outputs[stage_name] + assert output["stage_name"] == stage_name + assert output["ordinal"] == index + assert output["label"] == f"label-{index}" + assert output["first_stage_ordinal"] == 0 + assert output["known_stage_configs"] == stage_names + expected_previous = None if index == 0 else index - 1 + assert output["previous_ordinal"] == expected_previous REPO_ROOT = Path(__file__).resolve().parents[1] @@ -435,80 +432,81 @@ def run(context): REPO_ROOT / "examples" / "pipeline_2" / "main.py", ] -@pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) -def test_example_main_scripts_run_successfully(script_path: Path) -> None: - result = subprocess.run( - [sys.executable, str(script_path)], - cwd=REPO_ROOT, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, ( - f"Script failed: {script_path}\n" - f"stdout:\n{result.stdout}\n" - f"stderr:\n{result.stderr}" - ) - assert "completed with" in result.stdout.lower() - -def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(tmp_path: Path) -> None: - """ - Integration test that checks that the _log_config method is correctly called within - PipelineRunner.run() and that the information is parsed in a suitable format to a YAML - file in the run directory. - - This test also captures that _combine_configs() correctly converts all configuration - information into a single dictionary that can be serialized to YAML. - """ - stage_file = tmp_path / "single_stage.py" - stage_file.write_text( - dedent( - """ - def run(context): - return {"status": "ok"} - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [stage_file], - name="config-export-pipeline", - config={ - "pipeline_config": { - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", - }, - "stage_configuration": {}, - "global_configuration": { - "dry_run": True, - } - }, +class TestExamples: + @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) + def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"Script failed: {script_path}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" ) + assert "completed with" in result.stdout.lower() + - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() +class TestPipelineRunConfigurationLogging: + def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(self, tmp_path: Path) -> None: + """ + Integration test that checks that the _log_config method is correctly called within + PipelineRunner.run() and that the information is parsed in a suitable format to a YAML + file in the run directory. - run_dir = tmp_path / "runs" / run.manifest.run_id - config_file = run_dir / ( - f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" - ) + This test also captures that _combine_configs() correctly converts all configuration + information into a single dictionary that can be serialized to YAML. + """ + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return {"status": "ok"} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + }, + }, + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() + + run_dir = tmp_path / "runs" / run.manifest.run_id + config_file = run_dir / ( + f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" + ) - assert config_file.exists() + assert config_file.exists() - file_text = config_file.read_text(encoding="utf-8") - parsed_yaml = yaml.safe_load(file_text) + file_text = config_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) - assert parsed_yaml == run.manifest.config - assert "pipeline_config:\n" in file_text - assert "stage_configs:\n" in file_text - assert "global_config:\n" in file_text - assert "pipeline_config: {" not in file_text - assert "stage_configs: {" not in file_text - assert "global_config: {" not in file_text - assert " dry_run: true" in file_text + assert parsed_yaml == run.manifest.config + assert "pipeline_config:\n" in file_text + assert "stage_configs:\n" in file_text + assert "global_config:\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + assert " dry_run: true" in file_text From 00459e25f4d51a47ee07856f197610e15e60c397 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:20:58 +0100 Subject: [PATCH 218/332] docs: add docstrings for all tests in test_pipeline_architecture.py --- tests/test_pipeline_architecture.py | 55 ++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 15add2d..ffd9eb8 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -38,7 +38,8 @@ def test_pipeline_from_files_executes_python_entrypoints( tmp_path: Path ) -> None: """ - + Checks that the pipeline entrypoints are run successfully by reviewing + the outputs of the stages. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -78,7 +79,24 @@ def main(context): assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} - def test_pipeline_uses_run_specific_output_directory(self, tmp_path: Path) -> None: + def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> None: + """ + Checks that the pipelines produce outputs in unique locations based on runs. + As each run produces a unique run_id, the outputs should be written to unique + directories. This test checks that when the same pipeline is run twice, the + outputs are saved into two locations. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage specification in the Pipeline + configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the parent directory for the run + output. This does not affect the test capability. + """ writer_stage = tmp_path / "writer_stage.py" writer_stage.write_text( dedent( @@ -102,11 +120,10 @@ def main(context): ) with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - # TODO: This warning functions however from the name of the test, I would assume that the output - # directory has been set so this needs to be reviewed. first_run = pipeline.run() second_run = pipeline.run() + first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -118,6 +135,10 @@ def main(context): assert second_output.parents[2].name == second_run.manifest.run_id def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + """ + Checks that a pipeline will run with a non-module based Python script by running the + entire script. + """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") @@ -137,6 +158,10 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_pa class TestStageGraph: def test_stage_graph_detects_cycles(self) -> None: + """ + Checks that the stage graph appropriately detects required orders + based on dependencies in the stages. + """ first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) @@ -152,6 +177,10 @@ def test_stage_graph_detects_cycles(self) -> None: class TestPipelineFromConfig: def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + """ + Checks that from_config() method appropriately builds the configurations + for the pipeline and uses the configurations to run the Pipeline. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -191,6 +220,7 @@ def run(context): 0_data_validation: years_to_run: 2017 target_variable: "classification" + global_configuration: dry_run: true """ @@ -216,6 +246,10 @@ def run(context): def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: + """ + Checks that the pipeline raises an error when a stage configuration is provided + for a stage that is not within the pipeline. + """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( dedent( @@ -243,6 +277,12 @@ def run(context): def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + """ + Checks that the correct information from a configuration file is parsed into + the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig + instance. Also covers that the stage configuration is correctly injected into + the stage. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -332,6 +372,10 @@ def run(context): def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + """ + Checks that multiple stage configurations can be parsed from a configuration + file and input in the correct order into the pipeline. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -435,6 +479,9 @@ def run(context): class TestExamples: @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: + """ + Checks that a main script in a pipeline is successfully run. + """ result = subprocess.run( [sys.executable, str(script_path)], cwd=REPO_ROOT, From 328469044712595c9531bfb888c012d522d55ee7 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:33:16 +0100 Subject: [PATCH 219/332] tweak: ruff format and check test_pipeline_architecture.py --- tests/test_pipeline_architecture.py | 163 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 54 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index ffd9eb8..adaf028 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -14,10 +14,11 @@ from onsrap.stage import Stage from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning -NO_STAGES_SPECIFIED_WARNING = "No stages specified to run. All stages running by default." -OUTPUT_DIRECTORY_WARNING = ( - "Output directory is not specified. Using project root or work directory as the run output." +NO_STAGES_SPECIFIED_WARNING = ( + "No stages specified to run. All stages running by default." ) +OUTPUT_DIRECTORY_WARNING = "Output directory is not specified. Using project root or " \ + "work directory as the run output." def _base_pipeline_config(tmp_path: Path) -> dict: @@ -34,12 +35,11 @@ def _base_pipeline_config(tmp_path: Path) -> dict: class TestPipelineFromFiles: def test_pipeline_from_files_executes_python_entrypoints( - self, - tmp_path: Path - ) -> None: + self, tmp_path: Path + ) -> None: """ Checks that the pipeline entrypoints are run successfully by reviewing - the outputs of the stages. + the outputs of the stages. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -65,7 +65,9 @@ def main(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [first_stage, second_stage], dependencies={"second_stage": ("first_stage",)}, @@ -76,24 +78,30 @@ def main(context): run = pipeline.run() assert run.succeeded is True - assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] - assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} + assert [result.name for result in run.stage_results] == [ + "first_stage", + "second_stage", + ] + assert run.stage_outputs == { + "first_stage": "alpha", + "second_stage": "alpha-beta", + } def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> None: """ Checks that the pipelines produce outputs in unique locations based on runs. As each run produces a unique run_id, the outputs should be written to unique directories. This test checks that when the same pipeline is run twice, the - outputs are saved into two locations. + outputs are saved into two locations. Raises ------ 'PipelineConfigurationWarning' Expected and asserted as there is no stage specification in the Pipeline - configuration. This does not affect the test capability. + configuration. This does not affect the test capability. 'StageConfigurationWarning' - Expected and asserted as there is no output directory specified - in the configuration. This means that the Pipeline defaults to using + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using the project root or working directory as the parent directory for the run output. This does not affect the test capability. """ @@ -104,7 +112,9 @@ def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> Non from pathlib import Path def main(context): - output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" + output_path = Path( + context.run_dir + ) / "data" / "interim" / "artifact.txt" output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(context.run_id, encoding="utf-8") return {"output_path": str(output_path), "run_id": context.run_id} @@ -113,7 +123,9 @@ def main(context): + "\n", encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [writer_stage], config=_base_pipeline_config(tmp_path), @@ -123,7 +135,6 @@ def main(context): first_run = pipeline.run() second_run = pipeline.run() - first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -134,15 +145,19 @@ def main(context): assert first_output.parents[2].name == first_run.manifest.run_id assert second_output.parents[2].name == second_run.manifest.run_id - def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( + self, tmp_path: Path + ) -> None: """ - Checks that a pipeline will run with a non-module based Python script by running the - entire script. + Checks that a pipeline will run with a non-module based Python script by + running the entire script. """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [script_stage], name="script-pipeline", @@ -160,10 +175,18 @@ class TestStageGraph: def test_stage_graph_detects_cycles(self) -> None: """ Checks that the stage graph appropriately detects required orders - based on dependencies in the stages. + based on dependencies in the stages. """ - first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) - second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) + first_stage = Stage( + name="first_stage", + source=lambda context: None, + dependencies=("second_stage",), + ) + second_stage = Stage( + name="second_stage", + source=lambda context: None, + dependencies=("first_stage",), + ) graph = StageGraph.from_stages([first_stage, second_stage]) @@ -176,7 +199,9 @@ def test_stage_graph_detects_cycles(self) -> None: class TestPipelineFromConfig: - def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + def test_pipeline_from_config_builds_stages_and_injects_stage_config( + self, tmp_path: Path + ) -> None: """ Checks that from_config() method appropriately builds the configurations for the pipeline and uses the configurations to run the Pipeline. @@ -192,7 +217,9 @@ def run(context): return { "stage_name": context.stage_config.name, "years_to_run": context.stage_config.get("years_to_run"), - "target_variable": context.stage_config.require("target_variable"), + "target_variable": context.stage_config.require( + "target_variable" + ), } """ ).strip() @@ -209,10 +236,12 @@ def run(context): backend: python working_dir: "{tmp_path.as_posix()}" project_root: "{tmp_path.as_posix()}" - log_dir: "{(tmp_path / 'logs').as_posix()}" + log_dir: "{(tmp_path / "logs").as_posix()}" stages: - 0_data_validation: - location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" + location: "{ + (tmp_path / "scripts" / "0_data_validation.py").as_posix() + }" run: true dependencies: [] @@ -229,7 +258,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] @@ -244,11 +275,10 @@ def run(context): "target_variable": "classification", } - def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: """ Checks that the pipeline raises an error when a stage configuration is provided - for a stage that is not within the pipeline. + for a stage that is not within the pipeline. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -266,7 +296,9 @@ def run(context): config["stage_configuration"] = { "missing_stage": {"years_to_run": 2017}, } - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [stage_file], config=config, @@ -275,11 +307,12 @@ def run(context): with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() - - def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + def test_pipeline_from_config_parses_stage_configuration_payloads( + self, tmp_path: Path + ) -> None: """ - Checks that the correct information from a configuration file is parsed into - the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig + Checks that the correct information from a configuration file is parsed into + the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig instance. Also covers that the stage configuration is correctly injected into the stage. """ @@ -324,7 +357,7 @@ def run(context): "run": True, "dependencies": ["0_extract"], } - } + }, ], }, "stage_configuration": { @@ -350,9 +383,13 @@ def run(context): } config_file = tmp_path / "conf.yaml" - config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + config_file.write_text( + yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8" + ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" @@ -360,7 +397,9 @@ def run(context): assert pipeline.config.project_root == tmp_path assert pipeline.config.log_dir == tmp_path / "logs" assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] - assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + assert ( + pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + ) assert pipeline.stages[0].metadata["owner"] == "analytics" assert pipeline.stages[1].dependencies == ("0_extract",) assert pipeline.stage_configs["0_extract"].variables == { @@ -368,12 +407,16 @@ def run(context): "datasets": {"orders": {"path": "data/orders.csv"}}, } assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" - + assert ( + pipeline.stage_configs["1_transform"].require("target_variable") + == "classification" + ) - def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + def test_pipeline_from_config_scales_stage_configuration_to_many_stages( + self, tmp_path: Path + ) -> None: """ - Checks that multiple stage configurations can be parsed from a configuration + Checks that multiple stage configurations can be parsed from a configuration file and input in the correct order into the pipeline. """ scripts_dir = tmp_path / "scripts" @@ -391,12 +434,16 @@ def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tm def run(context): previous_ordinal = None if {index} > 0: - previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] + previous_ordinal = context.result_for( + "{previous_stage_name}" + ).outputs["ordinal"] return {{ "stage_name": context.stage_config.name, "ordinal": context.stage_config.require("ordinal"), "label": context.stage_config.require("label"), - "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), + "first_stage_ordinal": context.stage_config_for( + "{stage_names[0]}" + ).require("ordinal"), "known_stage_configs": sorted(context.stage_configs), "previous_ordinal": previous_ordinal, }} @@ -446,7 +493,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names @@ -476,6 +525,7 @@ def run(context): REPO_ROOT / "examples" / "pipeline_2" / "main.py", ] + class TestExamples: @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: @@ -498,14 +548,17 @@ def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: class TestPipelineRunConfigurationLogging: - def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(self, tmp_path: Path) -> None: + def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( + self, tmp_path: Path + ) -> None: """ - Integration test that checks that the _log_config method is correctly called within - PipelineRunner.run() and that the information is parsed in a suitable format to a YAML - file in the run directory. + Integration test that checks that the _log_config method is correctly called + within PipelineRunner.run() and that the information is parsed in a suitable + format to a YAML file in the run directory. - This test also captures that _combine_configs() correctly converts all configuration - information into a single dictionary that can be serialized to YAML. + This test also captures that _combine_configs() correctly converts all + configuration information into a single dictionary that can be serialized + to YAML. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -519,7 +572,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [stage_file], name="config-export-pipeline", From c7311c847905cce5e1f289d3391a7cb9a0b36e27 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:41:40 +0100 Subject: [PATCH 220/332] tweak: refactor test_runner.py with CoPilot (human review) to be structured as classes --- tests/test_runner.py | 272 ++++++++++++++++++++++--------------------- 1 file changed, 142 insertions(+), 130 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 225413f..6333af2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,9 +1,7 @@ +from pathlib import Path from textwrap import dedent import pytest - -from pathlib import Path - import yaml from onsrap.execution import ExecutionContext @@ -12,131 +10,145 @@ from onsrap.runner import _log_config, print_config_diffs -def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: - """ - Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. - """ - run_dir = tmp_path / "runs" / "synthetic_run" - run_dir.mkdir(parents=True) - - config = PipelineConfig( - name="synthetic_pipeline", - stages_to_run={"stage_a": True}, - backend="python", - work_dir=tmp_path / "work", - project_root=tmp_path, - output_dir=tmp_path / "outputs", - log_dir=tmp_path / "logs", - data_dir=tmp_path / "data", - allow_subprocess_fallback=True, - python_executable=None, - metadata={"reason": "unit test"}, - ) - - context = ExecutionContext( - pipeline_name="synthetic_pipeline", - run_id="run_1234", - config=config, - logger=Logger(), - run_dir=run_dir, - working_directory=tmp_path, - stage_configs={}, - global_config=None, - ) - - manifest_config = { - "pipeline_config": { - "name": "synthetic_pipeline", - "backend": "python", - "output_dir": str(tmp_path / "outputs"), - }, - "stage_configs": { - "stage_a": { - "years_to_run": 2026, - "target_variable": "classification", - } - }, - "global_config": { - "dry_run": True, - }, - } - - manifest = RunManifest( - rap_name="synthetic_pipeline", - run_id="run_1234", - config=manifest_config, - ) - - _log_config(run_dir, context, manifest) - - expected_file = run_dir / ( - "configuration_for_" - f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" - ) - - assert expected_file.exists() - - file_text = expected_file.read_text(encoding="utf-8") - parsed_yaml = yaml.safe_load(file_text) - - assert parsed_yaml == manifest_config - assert "stage_configs:\n" in file_text - assert " stage_a:\n" in file_text - assert " years_to_run: 2026\n" in file_text - assert "pipeline_config: {" not in file_text - assert "stage_configs: {" not in file_text - assert "global_config: {" not in file_text - -def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """ - Tests that two configuration files are correctly compared and the differences are - both returned in a computer-readable format and printed to the console. One change - for each category (changed, added, removed) is included in the test to ensure that - all cases are handled correctly. - """ - test_file_a = tmp_path / "config_a.yaml" - test_file_b = tmp_path / "config_b.yaml" - - test_file_a.write_text( - dedent(""" - pipeline_config: - name: synthetic_pipeline - output_dir: outputs - stage_configs: - stage_a: - years_to_run: 2026 - target_variable: classification - global_config: - dry_run: True - """).strip() - + "\n", - encoding="utf-8", +class TestLogConfig: + def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + """ + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True) + + config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + manifest_config = { + "pipeline_config": { + "name": "synthetic_pipeline", + "backend": "python", + "output_dir": str(tmp_path / "outputs"), + }, + "stage_configs": { + "stage_a": { + "years_to_run": 2026, + "target_variable": "classification", + } + }, + "global_config": { + "dry_run": True, + }, + } + + manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + config=manifest_config, + ) + + _log_config(run_dir, context, manifest) + + expected_file = run_dir / ( + "configuration_for_" + f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" + ) + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + + assert parsed_yaml == manifest_config + assert "stage_configs:\n" in file_text + assert " stage_a:\n" in file_text + assert " years_to_run: 2026\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + + +class TestPrintConfigDiffs: + @staticmethod + def _write_yaml(path: Path, content: str) -> None: + path.write_text(dedent(content).strip() + "\n", encoding="utf-8") + + def test_returns_and_prints_differences( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + """ + Tests that two configuration files are correctly compared and the differences are + both returned in a computer-readable format and printed to the console. One change + for each category (changed, added, removed) is included in the test to ensure that + all cases are handled correctly. + """ + test_file_a = tmp_path / "config_a.yaml" + test_file_b = tmp_path / "config_b.yaml" + + self._write_yaml( + test_file_a, + """ + pipeline_config: + name: synthetic_pipeline + output_dir: outputs + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: classification + global_config: + dry_run: True + """, + ) + + self._write_yaml( + test_file_b, + """ + pipeline_config: + name: synthetic_pipeline + backend: python + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: identification + global_config: + dry_run: True + """, + ) + + assert print_config_diffs(test_file_a, test_file_b) == { + "changed": { + "stage_configs.stage_a.target_variable": ( + "classification", + "identification", ) - - test_file_b.write_text( - dedent(""" - pipeline_config: - name: synthetic_pipeline - backend: python - stage_configs: - stage_a: - years_to_run: 2026 - target_variable: identification - global_config: - dry_run: True - """).strip() - + "\n", - encoding="utf-8", - ) - - assert print_config_diffs(test_file_a, test_file_b) == { - "changed": {"stage_configs.stage_a.target_variable": ("classification", "identification")}, - "added": {"pipeline_config.backend": "python"}, - "removed": {"pipeline_config.output_dir": "outputs"} - } - - captured = capsys.readouterr() - assert "CHANGED (1)" in captured.out - assert "ADDED in second configuration (1)" in captured.out - assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + }, + "added": {"pipeline_config.backend": "python"}, + "removed": {"pipeline_config.output_dir": "outputs"}, + } + + captured = capsys.readouterr() + assert "CHANGED (1)" in captured.out + assert "ADDED in second configuration (1)" in captured.out + assert "REMOVED in second configuration (1)" in captured.out + assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file From 44ebe476d327a6b34107f2c661fa851ee045d508 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:45:04 +0100 Subject: [PATCH 221/332] docs: add doc strings to test_runner.py methods --- tests/test_runner.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 6333af2..e48f74c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,9 +11,13 @@ class TestLogConfig: - def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: + def test_writes_manifest_config_as_block_style_yaml( + self, + tmp_path: Path + ) -> None: """ - Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + Tests that the ``_log_config`` function correctly writes the manifest + configuration to a YAML file in block style format. """ run_dir = tmp_path / "runs" / "synthetic_run" run_dir.mkdir(parents=True) @@ -70,7 +74,8 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non expected_file = run_dir / ( "configuration_for_" - f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" + f"{context.pipeline_name}_{context.started_at.date()}_" + f"{context.run_id[-8:]}.yaml" ) assert expected_file.exists() @@ -90,6 +95,12 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non class TestPrintConfigDiffs: @staticmethod def _write_yaml(path: Path, content: str) -> None: + """ + Helper function that writes a YAML file to the specified path with + the provided content. The content is dedented and stripped of + leading/trailing whitespace before being written to the file. A newline is + added at the end of the file. + """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") def test_returns_and_prints_differences( @@ -98,10 +109,10 @@ def test_returns_and_prints_differences( capsys: pytest.CaptureFixture[str], ) -> None: """ - Tests that two configuration files are correctly compared and the differences are - both returned in a computer-readable format and printed to the console. One change - for each category (changed, added, removed) is included in the test to ensure that - all cases are handled correctly. + Tests that two configuration files are correctly compared and the differences + are both returned in a computer-readable format and printed to the console. + One change for each category (changed, added, removed) is included in the test + to ensure that all cases are handled correctly. """ test_file_a = tmp_path / "config_a.yaml" test_file_b = tmp_path / "config_b.yaml" From 2635d517440ce052a5342a74e95241d8287278e6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:46:03 +0100 Subject: [PATCH 222/332] tweak: Ruff formatting and checking on test_runner.py --- tests/test_runner.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index e48f74c..9265465 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,12 +11,9 @@ class TestLogConfig: - def test_writes_manifest_config_as_block_style_yaml( - self, - tmp_path: Path - ) -> None: + def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: """ - Tests that the ``_log_config`` function correctly writes the manifest + Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. """ run_dir = tmp_path / "runs" / "synthetic_run" @@ -96,9 +93,9 @@ class TestPrintConfigDiffs: @staticmethod def _write_yaml(path: Path, content: str) -> None: """ - Helper function that writes a YAML file to the specified path with - the provided content. The content is dedented and stripped of - leading/trailing whitespace before being written to the file. A newline is + Helper function that writes a YAML file to the specified path with + the provided content. The content is dedented and stripped of + leading/trailing whitespace before being written to the file. A newline is added at the end of the file. """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") @@ -162,4 +159,4 @@ def test_returns_and_prints_differences( assert "CHANGED (1)" in captured.out assert "ADDED in second configuration (1)" in captured.out assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + assert "stage_configs.stage_a.target_variable" in captured.out From 9a8e79caf66dcd2d0b40c46f684844370e402197 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 17:40:03 +0100 Subject: [PATCH 223/332] docs: add parameters and raises sections to docstrings of all test files --- tests/test_execution.py | 178 +++++++++++++++++++++++++++- tests/test_models.py | 109 +++++++++++++++++ tests/test_pipeline.py | 143 ++++++++++++++++++++++ tests/test_pipeline_architecture.py | 111 ++++++++++++++++- tests/test_runner.py | 19 +++ tests/test_stage.py | 116 +++++++++++++++--- 6 files changed, 652 insertions(+), 24 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 618e112..19e191f 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -63,6 +63,17 @@ def stage_config() -> StageConfig: def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ run_dir = Path("tmp/run") work_dir = Path("tmp/work_dir") @@ -123,6 +134,17 @@ def test_executioncontext_creation( ) -> None: """ Test that the ExecutionContext creates the right attributes. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. """ assert execution.pipeline_name == "test_pipeline" assert execution.run_id == "run_id_1234" @@ -140,6 +162,15 @@ def test_record( """ Tests that StageResult attributes are attached to stage_results and variables attributes in the ExecutionContext instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. """ execution.record(stageresult) assert execution.stage_results == {"stage_test": expected_recorded_stage_result} @@ -150,6 +181,15 @@ def test_result_for( ) -> None: """ Tests that result_for correctly extracts the results of a requested stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. """ execution.record(stageresult) assert execution.result_for("stage_test") == expected_recorded_stage_result @@ -158,12 +198,28 @@ def test_stage_outputs(self, execution, stageresult) -> None: """ Tests that stage_outputs shows the outputs attribute of the StageResult instance for a requested stage is extracted. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. """ execution.record(stageresult) assert execution.stage_outputs == {"stage_test": "example output"} @pytest.fixture def blank_context_with_config_none(self, stageresult) -> ExecutionContext: + """ + Fixture that returns a test ExecutionContext instance with a None config for + testing error handling. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + """ run_dir = Path("tmp/run") work_dir = Path("tmp/work_dir") return ExecutionContext( @@ -183,6 +239,19 @@ def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: Tests that get_data_dir method extracts the path from the execution context or, if the context is None, returns an error to indicate that additional input is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``blank_context_with_config_none`` : ExecutionContext + An ``ExecutionContext`` object with a None config for testing error + handling. + + Raises + ------ + ``PipelineConfigurationError`` + If the config attribute of the ExecutionContext instance is None. """ assert execution.get_data_dir() == Path("tmp/config_data") @@ -193,7 +262,17 @@ def test_resolve_output_root(self, execution) -> None: """ Tests that resolve_output_root method extracts the path from the given run directory or, if None are given, raises an error to indicate additional input - is required.. + is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + If the run_dir attribute of the ExecutionContext instance is None. """ work_dir = Path("tmp/work_dir") assert execution.resolve_output_root() == Path("tmp/run") @@ -219,6 +298,20 @@ def test_stage_config_accessors_return_named_and_active_configs( """ Tests that getter methods to return the stage_config for a named stage returns correct attributes based on given parameters. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. """ stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) context = ExecutionContext( @@ -244,8 +337,14 @@ def test_set_active_stage(self, execution, stage_config) -> None: """ Tests that set_active_stage correctly sets the active_stage attribute in the ExecutionContext instance. - """ + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ execution.set_active_stage(stage_config.name) assert execution.active_stage_name == stage_config.name execution.set_active_stage(None) @@ -254,6 +353,13 @@ def test_set_active_stage(self, execution, stage_config) -> None: def test_stage_config_for(self, execution, stage_config) -> None: """ Tests that stage_config_for returns the StageConfig for a named stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ assert execution.stage_config_for(stage_config.name) == stage_config assert execution.stage_config_for("missing_stage") is None @@ -261,6 +367,13 @@ def test_stage_config_for(self, execution, stage_config) -> None: def test_stage_config(self, execution, stage_config) -> None: """ Tests that stage_config exposes the currently active stage configuration. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ assert execution.stage_config is None execution.set_active_stage(stage_config.name) @@ -270,6 +383,21 @@ def test_get_stage_config(self, execution, stage_config) -> None: """ Tests that get_stage_config returns variables by default and the full StageConfig object when requested. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. + """ assert execution.get_stage_config() == {} with pytest.raises(PipelineConfigurationError): @@ -278,8 +406,7 @@ def test_get_stage_config(self, execution, stage_config) -> None: execution.set_active_stage(stage_config.name) assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) + assert ( execution.get_stage_config(with_global=False, vars_only=False) == stage_config @@ -318,6 +445,18 @@ def test_resolve_given_path_add_folders( Tests the add_folder functionality for lists, single strings, or None type in the resolve_given_path class method as well as when the file_name is a valid string or None type. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``add_folder`` : Union[str, List[str], None] + A string, list of strings, or None type to specify additional folders to + add to the path. + ``file_name`` : Union[str, None] + A string or None type to specify the file name to append to the path. + ``expected`` : Path + The expected Path object that should be returned by the method. """ path_name = "data_path" root = Path("tmp/data") @@ -331,6 +470,11 @@ def test_resolve_given_path_norm(self, execution) -> None: """ Tests that resolve_given_path returns a file path that has been output in a StageResult instance. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ execution.record( StageResult( @@ -364,6 +508,11 @@ def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: """ Checks that entrypoints are set correctly in the PythonStageExecutor instance. + + Parameters + ---------- + ``pythonstageexecutor`` : PythonStageExecutor + A ``PythonStageExecutor`` object for testing. """ assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") @@ -374,6 +523,11 @@ def test_combine_vars(self, execution) -> None: Test that checks that a dictionary is returned, combining values from a global configuration and a stage configuration whilst removing any stage specific exclusions. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} @@ -397,6 +551,17 @@ def test_combine_vars_errors(self, execution) -> None: Test that confirms that a warning is raised if there is a variable defined in both the global and the stage configurations as well as asserting the correct values. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``StageConfigurationWarning`` + If a variable is defined in both the global and stage configurations, a + warning is raised to indicate that the stage variable will take precedence. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} @@ -421,6 +586,11 @@ def test_combine_vars_no_exclusion(self, execution) -> None: """ Test confirming that a dictionary is returned, combining values from a global configuration and a stage configuration when there are no exclusions defined. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {} diff --git a/tests/test_models.py b/tests/test_models.py index 7486f47..f2aa837 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -57,6 +57,11 @@ class TestRuntimeID: def test_runtimeID_creation(self, runtimeID) -> None: """ Test that a RuntimeID is correctly created. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. """ assert runtimeID.id == "abc123" assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) @@ -66,6 +71,11 @@ def test_runtimeID_creation(self, runtimeID) -> None: def test_getter_functions_runtimeID(self, runtimeID) -> None: """ Tests all the getter functions for the RuntimeID instance. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. """ assert runtimeID.get_id() == "abc123" assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) @@ -101,6 +111,12 @@ def expected_pipeline_config() -> PipelineConfig: @pytest.fixture def pipelineconfig(expected_pipeline_config) -> PipelineConfig: + """ + Returns a PipelineConfig instance for testing that is derived + fromthe expected_pipeline_config fixture. Used as a separate + fixture to ensure that the behaviour of the from_any() method is + tested correctly in the TestPipelineConfig class. + """ return expected_pipeline_config @@ -130,6 +146,16 @@ def test_from_any( Test derivation for a PipelineConfig instance using the from_any() method. This test checks all methods EXCEPT from_file as this will be covered in another test due to creation of a mock file being required. + + Parameters + ---------- + ``mapping`` : dict + A dictionary mapping of values for a PipelineConfig instance. + + Raises + ------ + TypeError + If the input to from_any() is not of a supported type. """ assert blankpipelineconfig.from_any(None) == PipelineConfig() assert blankpipelineconfig.from_any(pipelineconfig) == expected_pipeline_config @@ -142,6 +168,20 @@ def test_from_file_errors(self, tmp_path) -> PipelineConfig: """ Checks that a PipelineConfig instance raises the correct exceptions when a file is not found or the file does not contain a dictionary mapping. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``FileNotFoundError`` + If the file path provided does not exist. + + ``TypeError`` + If the file does not contain a dictionary mapping. """ no_map_pipeline_config = tmp_path / "not_valid.py" @@ -167,6 +207,15 @@ def test_from_file_success( """ Checks that a PipelineConfig instance is created successfully from a mock file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``expected_pipeline_config`` : PipelineConfig + A PipelineConfig instance that is expected to be created from the mock + file. """ pipeline_config = tmp_path / "configuration.py" pipeline_config.write_text( @@ -206,6 +255,11 @@ def test_to_dict(self, pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values as a dictionary. + + Parameters + ---------- + ``pipelineconfig`` : PipelineConfig + A PipelineConfig instance for testing. """ assert pipelineconfig.to_dict() == { @@ -249,6 +303,11 @@ def test_stage_result(self, stageresult) -> None: """ Uses a StageResult instance created in test_execution to ensure that the class instance is created suitably with required defaults. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. """ assert stageresult.name == "stage_test" assert stageresult.status == "pending" @@ -276,6 +335,16 @@ def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: """ Tests succeeded() method for StageResult which outputs True or False depending on the status of the StageResult. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``status_stage`` : StageStatus + A StageStatus value to set the status of the StageResult instance. + ``expected_stage`` : bool + The expected boolean output from the succeeded() method based on the + status of the StageResult instance. """ stageresult.status = status_stage assert stageresult.succeeded == expected_stage @@ -284,6 +353,11 @@ def test_duration_seconds(self, stageresult) -> None: """ Tests that duration_seconds() method calculates the correct duration in seconds between the started_at and finished_at attributes of the StageResult instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. """ stageresult.started_at = STARTED_AT stageresult.finished_at = FINISHED_AT @@ -293,6 +367,17 @@ def test_duration_seconds(self, stageresult) -> None: @pytest.fixture def pipelinerun(stageresult, runmanifest) -> PipelineRun: + """ + Creates a PipelineRun instance for testing that is used in the TestPipelineRun + class. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + """ return PipelineRun( runmanifest, PipelineStatus.SUCCEEDED, @@ -310,6 +395,15 @@ def test_pipelinerun_configuration( """ Checks that the PipelineRun instance is created successfully with the correct attributes and values. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + ``stageresult`` : StageResult + A StageResult instance for testing. """ assert pipelinerun.manifest == runmanifest assert pipelinerun.status == PipelineStatus.SUCCEEDED @@ -319,6 +413,11 @@ def test_pipelinerun_configuration( assert pipelinerun.stage_outputs == {"stage_test": "example output"} def test_result_for(self, pipelinerun, stageresult) -> None: + """ + Checks that the result_for() method of the PipelineRun instance returns the + correct StageResult instance when provided with a valid stage name, and returns + None when the stage name is not found. + """ assert pipelinerun.result_for("stage_test") == stageresult assert pipelinerun.result_for("not_a_stage") is None @@ -335,6 +434,16 @@ def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: """ Checks that the succeeded() method of the PipelineRun instance returns the correct boolean value based on its status. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``status`` : PipelineStatus + A PipelineStatus value to set the status of the PipelineRun instance. + ``expected`` : bool + The expected boolean output from the succeeded() method based on the + status of the PipelineRun instance. """ pipelinerun.status = status assert pipelinerun.succeeded == expected diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 80dbf1f..02d4bf8 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -18,6 +18,16 @@ def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stag Function that builds a Stage object with a given name, dependencies, and a source file path that's built out of the name if it is not provided. This standardises the creation of Stage objects for testing. + + Parameters + ---------- + ``name`` : str + The name of the stage to be created. + ``dependencies`` : tuple + A tuple of stage names that the created stage depends on. + ``source`` : Path | None + A Path object representing the source file for the stage. If None, a default + source file path is created based on the stage name. """ resolved_source = source if source is not None else Path(f"{name}.py") return Stage(name, source=resolved_source, dependencies=dependencies) @@ -33,6 +43,13 @@ def test_pipeline_name(self): if no name was given (shown in pipeline_config), or defaults to "pipeline" if no name is provided through Pipeline instance creation or through the PipelineConfig (shown through pipeline_no_name) + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ pipeline_config = PipelineConfig(name="test_pipeline_config") @@ -51,6 +68,20 @@ def test_assign_dependencies(self, tmp_path): Pipeline creation and appropriately assigned to each stage within the Pipeline. Will also check for error raise if the dependencies are defined but there are no defined stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as there are no stages defined in the Pipeline + but there are dependencies. """ def example_function(): @@ -103,6 +134,20 @@ def test_add_dependencies_single_dict(self, tmp_path): """ Tests that a dictionary correctly assigns dependencies to individual stages and the Pipeline instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as dependencies are specified for stages that do not + exist in the Pipeline instance. """ path_1 = tmp_path / "Stage_1.py" @@ -148,6 +193,17 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: the stage and the stage_configurations are correctly added to the Pipeline instance and the stage_configurations are correctly associated with the stage. + + Parameter + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() @@ -168,6 +224,17 @@ def test_add_stage_warns_when_stage_config_count_mismatches( Tests that when a stage is added but there is not the correct number of stage_configs provided, a warning is raised and the stage_configuration for that stage is added as a blank StageConfig object. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() @@ -193,6 +260,17 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage( Tests that when a stage_configuration is added to a Pipeline instance, the configuration is correctly associated with the named stage and that the configuration is coerced into a StageConfig object if it is provided. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -210,6 +288,11 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( Tests that when resolving stages_to_run, the Pipeline instance correctly includes all dependent stages required in the StageGraph even if these are not explicitly called out in the configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -237,6 +320,16 @@ def test_resolve_stages_to_run_rejects_disabled_dependencies( """ Checks that when resolving stages_to_run, the Pipeline init raises an error if a stage is enabled but one of its dependencies is disabled. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Raised when a stage is enabled but one of its dependencies is disabled. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -253,6 +346,17 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None """ Pipeline.stages always holds all stages; only graph.stages is the effective run set. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -272,6 +376,17 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( """ Tests that when a stage is manually disabled in a Pipeline instance, it is initialised in the stages_to_run configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -287,6 +402,12 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No Tests that when a stage is manually enabled in a Pipeline instance, it is correctly reflected in the stages_to_run configuration and the stage is included in the execution graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -306,6 +427,12 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( """ Tests that when a new stage is added to a Pipeline instance, it is kept out of the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( @@ -329,6 +456,12 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr """ Tests that when a new stage is added to a Pipeline instance with enable_stages=True, it is included in the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( @@ -353,6 +486,11 @@ def test_validate_skips_source_check_for_disabled_stages( """ Disabled stages' source files need not exist - validate() only checks the effective run set. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. """ enabled_file = tmp_path / "Stage_0.py" enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") @@ -375,6 +513,11 @@ def test_construct_manifest_inputs_contains_only_effective_stages( """ Manifest inputs should list only the stages that are part of the execution graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index adaf028..4515107 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -40,6 +40,21 @@ def test_pipeline_from_files_executes_python_entrypoints( """ Checks that the pipeline entrypoints are run successfully by reviewing the outputs of the stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the run output. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -94,11 +109,16 @@ def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> Non directories. This test checks that when the same pipeline is run twice, the outputs are saved into two locations. + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage specification in the Pipeline - configuration. This does not affect the test capability. + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified in the configuration. This means that the Pipeline defaults to using @@ -151,6 +171,20 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( """ Checks that a pipeline will run with a non-module based Python script by running the entire script. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") @@ -205,6 +239,20 @@ def test_pipeline_from_config_builds_stages_and_injects_stage_config( """ Checks that from_config() method appropriately builds the configurations for the pipeline and uses the configurations to run the Pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -279,6 +327,20 @@ def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> N """ Checks that the pipeline raises an error when a stage configuration is provided for a stage that is not within the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -315,6 +377,18 @@ def test_pipeline_from_config_parses_stage_configuration_payloads( the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig instance. Also covers that the stage configuration is correctly injected into the stage. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -418,6 +492,20 @@ def test_pipeline_from_config_scales_stage_configuration_to_many_stages( """ Checks that multiple stage configurations can be parsed from a configuration file and input in the correct order into the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -531,6 +619,11 @@ class TestExamples: def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: """ Checks that a main script in a pipeline is successfully run. + + Parameters + ---------- + ``script_path`` : Path + The path to the main.py script of a pipeline example. """ result = subprocess.run( [sys.executable, str(script_path)], @@ -559,6 +652,20 @@ def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( This test also captures that _combine_configs() correctly converts all configuration information into a single dictionary that can be serialized to YAML. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( diff --git a/tests/test_runner.py b/tests/test_runner.py index 9265465..faab644 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -15,6 +15,11 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non """ Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + + Parameters + ---------- + tmp_path : Path + A temporary directory provided by pytest for creating test files. """ run_dir = tmp_path / "runs" / "synthetic_run" run_dir.mkdir(parents=True) @@ -97,6 +102,13 @@ def _write_yaml(path: Path, content: str) -> None: the provided content. The content is dedented and stripped of leading/trailing whitespace before being written to the file. A newline is added at the end of the file. + + Parameters + ---------- + ``path`` : Path + The path where the YAML file will be written. + ``content`` : str + The YAML content to write to the file. """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") @@ -110,6 +122,13 @@ def test_returns_and_prints_differences( are both returned in a computer-readable format and printed to the console. One change for each category (changed, added, removed) is included in the test to ensure that all cases are handled correctly. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. + ``capsys`` : pytest.CaptureFixture[str] + A pytest fixture that captures output to stdout and stderr during the test. """ test_file_a = tmp_path / "config_a.yaml" test_file_b = tmp_path / "config_b.yaml" diff --git a/tests/test_stage.py b/tests/test_stage.py index b29a182..9f61b33 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -45,6 +45,11 @@ class TestStage: def test_stage_creation_callable(self, stage_test) -> None: """ Tests that attributes have been appropriately assigned to Stage class. + + Parameter + --------- + stage_test : Stage + A ``Stage`` object created with a callable source for testing. """ assert stage_test.name == "callable_stage" assert stage_test.source == example_function @@ -57,6 +62,16 @@ def test_stage_name_error(self, example_function) -> None: """ Tests that a StageConfigurationError is raised if the name is left blank in a Stage class instance. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the name is left blank in a ``Stage`` class instance. """ with pytest.raises(StageConfigurationError): Stage("", example_function, ["stage_1"], {"info": "example"}) @@ -64,6 +79,12 @@ def test_stage_name_error(self, example_function) -> None: def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. """ with pytest.raises(StageConfigurationError): Stage("callable_stage", 11, ["stage_1"], {"info": "example"}) @@ -71,6 +92,11 @@ def test_stage_source_type(self) -> None: def test_stage_backend(self, example_function) -> None: """ Tests that backend can be any string, None, and corrects for whitespace. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_diff = Stage( "callable_stage", @@ -100,29 +126,44 @@ def test_stage_backend(self, example_function) -> None: def test_stage_from_files_error(self, tmp_path: Path) -> None: """ Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance """ source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): Stage.from_file(source_file) - def test_stage_from_callable_name(self) -> None: + def test_stage_from_callable_name(self, example_function) -> None: """ Tests that a stage name is extracted from a callable object stage. - """ - - def example_function(): - pass + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ test = Stage.from_callable(example_function) assert test.name == "example_function" - def test_from_dict_norm(self) -> None: + def test_from_dict_norm(self, example_function) -> None: """ Tests that a stage instance is created from a dictionary item. - """ - def example_function(): - pass + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) @@ -132,6 +173,11 @@ def test_with_dependencies_list(self, stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is a list. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. """ new_deps = ["stage2", "stage3"] new_deps_blank = [] @@ -145,6 +191,21 @@ def test_with_dependencies_list(self, stage_test) -> None: def test_validate(self, stage_test, tmp_path) -> None: """ Tests whether an error is raised if the source file isn't suitable. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. In this instance, it raises if the source is None, an empty + string, or a Path object that is not a file. """ stage_test.source = None with pytest.raises(StageConfigurationError): @@ -157,10 +218,20 @@ def test_validate(self, stage_test, tmp_path) -> None: with pytest.raises(StageConfigurationError): stage_test.validate() - def test_source_path(self, stage_test, tmp_path) -> None: + def test_source_path(self, stage_test, tmp_path, example_function) -> None: """ Tests whether source_path detects a path vs other valid and invalid source types. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_test.source = tmp_path / "fake_file.py" assert stage_test.source_path == tmp_path / "fake_file.py" @@ -168,25 +239,28 @@ def test_source_path(self, stage_test, tmp_path) -> None: assert stage_test.source_path is None stage_test.source = "not a file path" assert stage_test.source_path is None - - def example_function(): - pass - stage_test.source = example_function assert stage_test.source_path is None - def test_source_label(self, stage_test, tmp_path) -> None: + def test_source_label(self, stage_test, tmp_path, example_function) -> None: """ Tests that source_label is created if the source is a Path or a callable and is None if it is another type. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_test.source = tmp_path / "fake_file.py" temp_path_str = str(tmp_path / "fake_file.py") assert stage_test.source_label == temp_path_str - def example_function(): - pass - stage_test.source = example_function assert stage_test.source_label == "tests.test_stage.example_function" @@ -203,6 +277,12 @@ class TestStageFactories: def test_stage_instance_from_file(self, tmp_path) -> None: """ Tests that a Stage instance is created from a filepath. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. """ test_stage = tmp_path / "test_stage.py" test_stage.write_text( From da2a8cea618597b6fc4a49f04a391a604f70bb61 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 09:18:17 +0100 Subject: [PATCH 224/332] tweak: switch around tests in test_stage.py to more logically fit the class structure --- tests/test_stage.py | 92 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 9f61b33..c68ebc9 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -123,52 +123,6 @@ def test_stage_backend(self, example_function) -> None: assert stage.backend == "python" assert stage_white_space.backend == "python" - def test_stage_from_files_error(self, tmp_path: Path) -> None: - """ - Tests that if the file doesn't exist, a StageConfigurationError is raised. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and - manipulation. - - Raises - ------ - ``StageConfigurationError`` - If the source file doesn't exist when attempting to create a - ``Stage`` instance - """ - source_file = tmp_path / "not_an_actual_file.py" - with pytest.raises(StageConfigurationError): - Stage.from_file(source_file) - - def test_stage_from_callable_name(self, example_function) -> None: - """ - Tests that a stage name is extracted from a callable object stage. - - Parameters - ---------- - ``example_function`` : callable - A callable function to pass as a source for a ``Stage`` class instance. - """ - test = Stage.from_callable(example_function) - assert test.name == "example_function" - - def test_from_dict_norm(self, example_function) -> None: - """ - Tests that a stage instance is created from a dictionary item. - - Parameters - ---------- - ``example_function`` : callable - A callable function to pass as a source for a ``Stage`` class instance. - """ - - data = {"name": "test_Stage", "callable": example_function} - stage = Stage.from_dict(data) - assert stage.source == example_function - def test_with_dependencies_list(self, stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is @@ -300,3 +254,49 @@ def main(): assert Stage.from_file(test_stage, entrypoint="main") == Stage( "test_stage", test_stage.resolve(), (), {}, "main", "python" ) + + def test_stage_from_files_error(self, tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + Stage.from_file(source_file) + + def test_stage_from_callable_name(self, example_function) -> None: + """ + Tests that a stage name is extracted from a callable object stage. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function) + assert test.name == "example_function" + + def test_from_dict_norm(self, example_function) -> None: + """ + Tests that a stage instance is created from a dictionary item. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + + data = {"name": "test_Stage", "callable": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function From 1975371d460c8a2383bc069944fdc3ef12b6aa58 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 14:54:35 +0100 Subject: [PATCH 225/332] tests: added additional tests to test_stage.py on recommendation from CoPilot. Some tests not added and these are detailed in issue #40 --- tests/test_stage.py | 618 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 572 insertions(+), 46 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index c68ebc9..9be5702 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -3,6 +3,7 @@ import pytest +from onsrap.errors import StageDependencyError from onsrap.stage import Stage, StageConfigurationError, _normalize_dependencies @@ -24,17 +25,65 @@ def test_normalize_dependencies_str(self) -> None: ["Stage_1.py", " Stage_2.py", "Stage_3.py "] ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") + def test_normalize_dependencies_dedupe(self) -> None: + """ + Tests that duplicate values are removed from the normalized dependencies + whilst preserving first seen order. + """ + assert _normalize_dependencies( + ["Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_1.py", "Stage_2.py") + + assert _normalize_dependencies( + ["Stage_2.py","Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_2.py","Stage_1.py") + + def test_normalize_dependencies_whitespace_handling(self) -> None: + """ + Tests that whitespace only or blank dependency values are removed from + the normalised dependencies. + """ + assert _normalize_dependencies( + [" ", ""] + ) == () + + def test_normalize_dependencies_type_check(self) -> None: + """ + Tests that normalize_dependencies works with other iterables such as tuples + and sets, and raises a TypeError for non-iterable types. + """ + assert _normalize_dependencies(("Stage_1.py", "Stage_2.py")) == ( + "Stage_1.py", + "Stage_2.py", + ) + + result = _normalize_dependencies({"Stage_1.py", "Stage_2.py"}) + assert set(result) == {"Stage_1.py", "Stage_2.py"} + + with pytest.raises(TypeError): + _normalize_dependencies(11) + + def test_normalize_dependencies_mixed_types(self) -> None: + """ + Tests that normalize_dependencies stringifies non-string types in a + dependency iterable. + """ + assert _normalize_dependencies(["Stage_1.py", 11, "Stage_2.py"]) == ( + "Stage_1.py", "11", "Stage_2.py" + ) + + @pytest.fixture def example_function(): """ Test function to pass as a callable stage for stage testing. """ - pass + return example_function @pytest.fixture -def stage_test() -> Stage: +def stage_test(example_function) -> Stage: """ Stage object for testing Stage class methods and construction. """ @@ -61,7 +110,7 @@ def test_stage_creation_callable(self, stage_test) -> None: def test_stage_name_error(self, example_function) -> None: """ Tests that a StageConfigurationError is raised if the name is left blank - in a Stage class instance. + in a Stage class instance. This also includes whitespace only names. Parameters ---------- @@ -71,11 +120,15 @@ def test_stage_name_error(self, example_function) -> None: Raises ------ ``StageConfigurationError`` - If the name is left blank in a ``Stage`` class instance. + If the name is left blank or entirely whitespace in a ``Stage`` class + instance. """ with pytest.raises(StageConfigurationError): Stage("", example_function, ["stage_1"], {"info": "example"}) + with pytest.raises(StageConfigurationError): + Stage(" ", example_function, ["stage_1"], {"info": "example"}) + def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. @@ -123,54 +176,132 @@ def test_stage_backend(self, example_function) -> None: assert stage.backend == "python" assert stage_white_space.backend == "python" - def test_with_dependencies_list(self, stage_test) -> None: + def test_stage_backend_irregular_values(self, example_function) -> None: """ - Tests adding different types of dependencies when the original dependency is - a list. + Tests that backend defaults with a None or whitespace only string to "python" + and converts any non-string type (other than None) to a string. Parameters ---------- - ``stage_test`` : Stage - A ``Stage`` object created with a callable source for testing. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ - new_deps = ["stage2", "stage3"] - new_deps_blank = [] - stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") - assert stage_test_blank.dependencies == ("stage_1",) - stage_test = stage_test.with_dependencies("stage2", "stage3") - assert stage_test.dependencies == ("stage_1", "stage2", "stage3") + stage_none = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=None, + ) - def test_validate(self, stage_test, tmp_path) -> None: + stage_blank = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=" ", + ) + stage_non_string = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=11, + ) + + assert stage_none.backend == "python" + assert stage_blank.backend == "python" + assert stage_non_string.backend == "11" + + + def test_stage_constructor_expands_string_source_with_home( + self, + monkeypatch, + tmp_path): """ - Tests whether an error is raised if the source file isn't suitable. + Check that a string source is converted to a Path and expanded with + expanduser(). This uses fake environmental variables to make sure that the + tests are not dependent on the actual user's home directory. Parameters ---------- - ``stage_test`` : Stage - A ``Stage`` object created with a callable source for testing. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. ``tmp_path`` : Path A temporary path provided by pytest for testing file creation and manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() - Raises - ------ - ``StageConfigurationError`` - If the source is not a valid callable or file path in a ``Stage`` class - instance. In this instance, it raises if the source is None, an empty - string, or a Path object that is not a file. + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="string_source_stage", + source="~/scripts/my_stage.py", + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + + def test_stage_constructor_expands_path_source_with_home( + self, + monkeypatch, + tmp_path): """ - stage_test.source = None - with pytest.raises(StageConfigurationError): - stage_test.validate() - not_file_path = tmp_path - stage_test.source = not_file_path - with pytest.raises(StageConfigurationError): - stage_test.validate() - stage_test.source = "" - with pytest.raises(StageConfigurationError): - stage_test.validate() + Check that a Path source is expanded with expanduser(). This uses fake + environmental variables to make sure that the tests are not dependent on the + actual user's home directory. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="path_source_stage", + source=Path("~/scripts/my_stage.py"), + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + def test_normalise_dependencies_within_stage_init(self, example_function) -> None: + """ + Thin smoke test to check that _normalize_dependencies is called within the Stage + post_init method. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage = Stage( + name="test_stage", + source=example_function, + dependencies=["dep1", "dep2", "dep1", " dep3 ", "", " "], + metadata={}, + ) + assert stage.dependencies == ("dep1", "dep2", "dep3") def test_source_path(self, stage_test, tmp_path, example_function) -> None: """ @@ -199,7 +330,8 @@ def test_source_path(self, stage_test, tmp_path, example_function) -> None: def test_source_label(self, stage_test, tmp_path, example_function) -> None: """ Tests that source_label is created if the source is a Path or a callable - and is None if it is another type. + and is None if it is another type. This also checks that if the callable + has no name attribute, the stage name is used as the source label. Parameters ---------- @@ -221,16 +353,236 @@ def test_source_label(self, stage_test, tmp_path, example_function) -> None: stage_test.source = 11 assert stage_test.source_label is None + class NoName: + def __call__(self): pass + + stage_test.source = NoName() + assert stage_test.source_label == f"tests.test_stage.{stage_test.name}" + + def test_metadata_copy_safely(self) -> None: + """ + Checks that if the original metadata dictionary is modified after the Stage + instance is created, the Stage instance's metadata remains unchanged. + """ + #TODO: do we want this to be how it works? Or would the user assume that if + #they modify the original dict, it modifies the Stage instance. + original_metadata = {"info": "example"} + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=[], + metadata=original_metadata, + ) + original_metadata["info"] = "modified" + assert stage.metadata["info"] == "example" + + def test_repr_function(self) -> None: + """ + Tests that the __repr__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_repr = ( + "Stage(name=callable_stage, " + "source=tests.test_stage., " + "dependencies=('stage_1',), " + "metadata={'info': 'example'}, " + "entrypoint=main, " + "backend=python)" + ) + assert repr(stage) == expected_repr + + def test_str_function(self) -> None: + """ + Tests that the __str__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_str = ( + " Name: callable_stage\n" + " Source: tests.test_stage. \n" + " Dependencies: ('stage_1',)\n" + " Metadata: {'info': 'example'} \n" + " Entrypoint: main \n" + " Backend: python" + ) + assert str(stage) == expected_str + +class TestValidateStage: + def test_validate(self, stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. In this instance, it raises if the source is None, an empty + string, or a Path object that is not a file. + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + + def test_validate_successes(self, stage_test, example_function, temp_script) -> None: + """ + Tests that validate successfully approves of callables and file paths as + sources for a stage instance. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + stage_test.source = example_function + assert stage_test.validate() == None + + stage_test.source = temp_script(filename="valid_script.py") + assert stage_test.validate() == None + +class TestWithDependencies: + def test_with_dependencies_list(self, stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. Also checks that the original stage_test instance is not modified when + with_dependencies is called. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + new_deps_blank = [] + original_deps = stage_test.dependencies + + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") + assert stage_test_blank.dependencies == ("stage_1",) + stage_test_2 = stage_test.with_dependencies("stage2", "stage3") + assert stage_test_2.dependencies == ("stage_1", "stage2", "stage3") + + stage_test.with_dependencies("stage2", "stage3") + assert stage_test.dependencies == original_deps + + def test_with_dependencies_errors(self, stage_test) -> None: + """ + Tests that a StageDependencyError is raised if a nested list is + provided in dependencies. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + with pytest.raises(StageDependencyError): + stage_test.with_dependencies(["stage2", ["nested_stage"]]) + + def test_with_dependencies_list_positional_args(self, stage_test) -> None: + """ + Tests that with_dependencies can accept a list and positional arguments in the + same call and combine them into a single normalized dependencies tuple. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + stage_test_combined = stage_test.with_dependencies(new_deps, "stage4") + assert stage_test_combined.dependencies == ( + "stage_1", + "stage2", + "stage3", + "stage4" + ) + + def test_with_dependencies_duplicates(self, stage_test) -> None: + """ + Tests that when the same dependency is added through with_dependencies, + it is not duplicated in the dependencies tuple of the new Stage instance. + + Caution that this only deduplicates due to Stage post_init calling + _normalize_dependencies however if that moves, + test_normalise_dependencies_within_stage_init will capture the issue. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + print(f"before: {stage_test.dependencies}") + new_stage = stage_test.with_dependencies(["stage_1"],) + print(f"after: {new_stage.dependencies}") + assert new_stage.dependencies == ("stage_1",) + """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ +@pytest.fixture +def temp_script(tmp_path): + """ + Fixture factory that creates temporary Python scripts. + + Usage: + script = temp_script("def main(): pass") + script = temp_script("def process(): return 42", "processor.py") + """ + def _create_script(content="def main(): pass\n", filename="temp_script.py"): + script = tmp_path / filename + script.write_text(content, encoding="utf-8") + return script + return _create_script class TestStageFactories: - def test_stage_instance_from_file(self, tmp_path) -> None: + """ + Parent class for tests which create Stage class instances from different methods. + """ + +class TestStageFromFile(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a file path. + """ + def test_stage_instance_from_file(self, temp_script) -> None: """ - Tests that a Stage instance is created from a filepath. + Tests that a Stage instance is created from a filepath where name is either + default value from file stem or a user defined name. Parameters ---------- @@ -238,8 +590,7 @@ def test_stage_instance_from_file(self, tmp_path) -> None: A temporary path provided by pytest for testing file creation and manipulation. """ - test_stage = tmp_path / "test_stage.py" - test_stage.write_text( + test_stage = temp_script( dedent( """ def main(): @@ -248,13 +599,17 @@ def main(): """ ).strip() + "\n", - encoding="utf-8", + "test_stage.py", ) assert Stage.from_file(test_stage, entrypoint="main") == Stage( "test_stage", test_stage.resolve(), (), {}, "main", "python" ) + assert Stage.from_file(test_stage, name="Stage_1", entrypoint="main") == Stage( + "Stage_1", test_stage.resolve(), (), {}, "main", "python" + ) + def test_stage_from_files_error(self, tmp_path: Path) -> None: """ Tests that if the file doesn't exist, a StageConfigurationError is raised. @@ -274,7 +629,64 @@ def test_stage_from_files_error(self, tmp_path: Path) -> None: source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): Stage.from_file(source_file) - + + def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch): + """ + Tests that a relative path passed to from_file is resolved to an + absolute path on the Stage source attribute. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + script = temp_script() + + monkeypatch.chdir(script.parent) + + stage = Stage.from_file(script.name) + + assert stage.source.is_absolute() + assert stage.source == script.resolve() + + def test_from_file_expands_source_path(self, tmp_path, monkeypatch): + """ + Tests that a path with a tilde (~) is expanded to the user's home directory + when passed to from_file. Uses monkeypatch to set a fake home directory for + testing purposes. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + script = fake_home / "scripts" / "my_stage.py" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("def main(): pass\n", encoding="utf-8") + + stage = Stage.from_file("~/scripts/my_stage.py") + + expected = fake_home / "scripts" / "my_stage.py" + assert stage.source == expected + + +class TestStageFromCallable(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a callable object. + """ def test_stage_from_callable_name(self, example_function) -> None: """ Tests that a stage name is extracted from a callable object stage. @@ -287,9 +699,40 @@ def test_stage_from_callable_name(self, example_function) -> None: test = Stage.from_callable(example_function) assert test.name == "example_function" - def test_from_dict_norm(self, example_function) -> None: + def test_from_callable_fallback_name(self): + """ + Tests that a fallback name is assigned to a stage instance if the callable + object does not have a name attribute. + """ + class NoName: + def __call__(self): pass + + stage = Stage.from_callable(NoName()) + assert stage.name == "stage" + + def test_from_callable_explicit_name(self, example_function) -> None: """ - Tests that a stage instance is created from a dictionary item. + Tests that an explicit name is assigned to a stage instance if provided. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function, name="explicit_name") + assert test.name == "explicit_name" + +class TestStageFromDict(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a dictionary. + """ + + def test_from_dict_callable_sources(self, example_function) -> None: + """ + Tests that callable sources are correctly used to create a stage instance + from a dictionary regardless of whether the key is source or callable. + Also validates that the name is correctly assigned from the dictionary or + derived from the callable. Parameters ---------- @@ -300,3 +743,86 @@ def test_from_dict_norm(self, example_function) -> None: data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) assert stage.source == example_function + assert stage.name == "test_Stage" + + data = {"source": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + assert stage.name == "example_function" + + def test_from_dict_aliases(self, temp_script) -> None: + """ + Tests that a stage instance is created from a dictionary item with aliases + source and path as source options. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + script = temp_script( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + "test_stage.py", + ) + + data = { + "name": "test_Stage", + "source": script, + "entrypoint": "main", + } + + data_2 = { + "name": "test_Stage2", + "path": script, + "entrypoint": "main", + } + stage = Stage.from_dict(data) + stage_2 = Stage.from_dict(data_2) + assert stage.source == script.resolve() + assert stage.name == "test_Stage" + assert stage_2.source == script.resolve() + assert stage_2.name == "test_Stage2" + + def test_from_dict_errors(self) -> None: + """ + Tests that a StageConfigurationError is raised if the dictionary does not + contain a valid source or callable key. + + Raises + ------ + ``StageConfigurationError`` + If the dictionary does not contain a valid source or callable key. + """ + data = {"name": "test_Stage"} + with pytest.raises(StageConfigurationError): + Stage.from_dict(data) + + def test_all_keys_from_dict_in_stage(self, example_function) -> None: + """ + Checks that from_dict does not change the originally parsed dictionary so + that if the dictionary is needed later, it is not permanently changed when + creating a stage instance from it. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + data = { + "name": "test_Stage", + "source": example_function, + "dependencies": ["dep1", "dep2"], + "metadata": {"info": "example"}, + "entrypoint": "main", + "backend": "python", + } + original = dict(data) + Stage.from_dict(data) + assert original == data From 40c232fc857ae88fcef70f7708f28587f85690cc Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 15:53:46 +0100 Subject: [PATCH 226/332] feat: adds to_dict and from_dict methods to PipelineRun, StageResult, and RunManifest in prep for saving out to a YAML. Also added documentation to _format_dict() --- onsrap/models.py | 197 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 8 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8fe1c21..7640d66 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -690,6 +690,67 @@ def __repr__(self) -> str: f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" ) + def runmanifest_to_dict(self) -> dict[str, Any]: + """ + Converts the RunManifest instance into a dictionary representation. + This is needed to allow a RunManifest instance to be serialized into a + JSON format for later methods on RunManifest instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the RunManifest instance. + """ + return { + "rap_name": self.rap_name, + "run_id": self.run_id, + "git_commit": self.git_commit, + "stages_run": self.stages_run, + "parameters": self.parameters, + "inputs": self.inputs, + "outputs": self.outputs, + "backend": self.backend, + "package_versions": self.package_versions, + "timestamp": self.timestamp, + "reason": self.reason, + "user": self.user, + "config": self.config + } + + @classmethod + def runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: + """ + Converts a dictionary representation of a RunManifest instance back into a + RunManifest instance. Allows for RunManifest instances to be created from + a JSON representation of a RunManifest instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a RunManifest instance. + + Returns + ------- + ``RunManifest`` class instance + A RunManifest instance created from the dictionary representation. + """ + return cls( + rap_name=data.get("rap_name", ""), + run_id=data.get("run_id", ""), + git_commit=data.get("git_commit"), + stages_run=data.get("stages_run", []), + parameters=data.get("parameters", {}), + inputs=data.get("inputs", {}), + outputs=data.get("outputs", {}), + backend=data.get("backend", "python"), + package_versions=data.get("package_versions", []), + timestamp=data.get("timestamp", ""), + reason=data.get("reason"), + user=data.get("user"), + config=data.get("config") + ) + class RAPDataset: def __init__(self): @@ -746,6 +807,62 @@ class StageResult: error: Optional[str] = None source: Optional[str] = None + def _stage_result_to_dict(self, name = True) -> dict[str, Any]: + """ + Converts the StageResult instance into a dictionary representation. + This is needed to allow a StageResult instance to be serialized into a + JSON format for later methods on StageResult instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the StageResult instance. + """ + return { + **({"name": self.name} if name else {}), + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "finished_at": self.finished_at.isoformat(), + "outputs": self.outputs, + "stdout": self.stdout, + "stderr": self.stderr, + "return_code": self.return_code, + "metadata": self.metadata, + "error": self.error, + "source": self.source, + } + + def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: + """ + Converts a dictionary representation of a StageResult instance back into a + StageResult instance. Allows for StageResult instances to be created from + a JSON representation of a StageResult instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a StageResult instance. + + Returns + ------- + ``StageResult`` class instance + A StageResult instance created from the dictionary representation. + """ + return cls( + name=data["name"], + status=StageStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + finished_at=datetime.fromisoformat(data["finished_at"]), + outputs=data.get("outputs"), + stdout=data.get("stdout", ""), + stderr=data.get("stderr", ""), + return_code=data.get("return_code"), + metadata=data.get("metadata", {}), + error=data.get("error"), + source=data.get("source"), + ) + @property def succeeded(self) -> bool: """ @@ -805,6 +922,55 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: return result return None + def pipeline_run_to_dict(self) -> dict[str, Any]: + """ + Converts the PipelineRun instance into a dictionary representation. + This is needed to allow a PipelineRun instance to be serialized into a + JSON format for later methods on PipelineRun instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the PipelineRun instance. + """ + return { + "manifest": self.manifest.runmanifest_to_dict(), + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "completed_at": self.completed_at.isoformat(), + "stage_results": {result.name:result._stage_result_to_dict(name = False) + for result in self.stage_results}, + "stage_outputs": self.stage_outputs, + } + + @classmethod + def pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: + """ + Converts a dictionary representation of a PipelineRun instance back into a + PipelineRun instance. Allows for PipelineRun instances to be created from + a JSON representation of a PipelineRun instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` class instance + A PipelineRun instance created from the dictionary representation. + """ + return cls( + manifest=RunManifest.runmanifest_from_dict(data["manifest"]), + status=PipelineStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + completed_at=datetime.fromisoformat(data["completed_at"]), + stage_results=[StageResult._stage_result_from_dict(result) + for result in data.get("stage_results", {}).values()], + stage_outputs=data.get("stage_outputs", {}), + ) + @property def succeeded(self) -> bool: """ @@ -815,11 +981,26 @@ def succeeded(self) -> bool: return self.status == PipelineStatus.SUCCEEDED def _format_dict(d, indent=0): - lines = [] - for key, value in d.items(): - if isinstance(value, dict): - lines.append(f"{' ' * indent}{key}:") - lines.append(_format_dict(value, indent + 4)) - else: - lines.append(f"{' ' * indent}{key}: {value}") - return "\n".join(lines) + """ + Helper function to format dictionaries for __str__ methods. + + Parameters + ---------- + ``d`` : dict + The dictionary to format. + ``indent`` : int, default = 0 + The number of spaces to indent the dictionary representation. + + Returns + ------- + str + A formatted string representation of the dictionary. + """ + lines = [] + for key, value in d.items(): + if isinstance(value, dict): + lines.append(f"{' ' * indent}{key}:") + lines.append(_format_dict(value, indent + 4)) + else: + lines.append(f"{' ' * indent}{key}: {value}") + return "\n".join(lines) From 746384dd913aadf3efc9a1ff9dbf0a68d33e4fa4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 16:26:21 +0100 Subject: [PATCH 227/332] tests: adds tests for to_dict and from_dict methods for PipelineRun, StageResult, RunManifest --- onsrap/models.py | 11 +-- tests/test_models.py | 188 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 192 insertions(+), 7 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 7640d66..72b7bed 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -807,7 +807,7 @@ class StageResult: error: Optional[str] = None source: Optional[str] = None - def _stage_result_to_dict(self, name = True) -> dict[str, Any]: + def _stage_result_to_dict(self) -> dict[str, Any]: """ Converts the StageResult instance into a dictionary representation. This is needed to allow a StageResult instance to be serialized into a @@ -820,7 +820,7 @@ def _stage_result_to_dict(self, name = True) -> dict[str, Any]: A dictionary representation of the StageResult instance. """ return { - **({"name": self.name} if name else {}), + "name": self.name, "status": self.status.value, "started_at": self.started_at.isoformat(), "finished_at": self.finished_at.isoformat(), @@ -833,6 +833,7 @@ def _stage_result_to_dict(self, name = True) -> dict[str, Any]: "source": self.source, } + @classmethod def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: """ Converts a dictionary representation of a StageResult instance back into a @@ -922,7 +923,7 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: return result return None - def pipeline_run_to_dict(self) -> dict[str, Any]: + def _pipeline_run_to_dict(self) -> dict[str, Any]: """ Converts the PipelineRun instance into a dictionary representation. This is needed to allow a PipelineRun instance to be serialized into a @@ -939,13 +940,13 @@ def pipeline_run_to_dict(self) -> dict[str, Any]: "status": self.status.value, "started_at": self.started_at.isoformat(), "completed_at": self.completed_at.isoformat(), - "stage_results": {result.name:result._stage_result_to_dict(name = False) + "stage_results": {result.name:result._stage_result_to_dict() for result in self.stage_results}, "stage_outputs": self.stage_outputs, } @classmethod - def pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: + def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: """ Converts a dictionary representation of a PipelineRun instance back into a PipelineRun instance. Allows for PipelineRun instances to be created from diff --git a/tests/test_models.py b/tests/test_models.py index 7e7d0b4..0f5f0ff 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,4 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig +from onsrap.models import StageResult, StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig import pytest import datetime from pathlib import Path @@ -279,4 +279,188 @@ def test_succeeded_pipeline(pipelinerun, status, expected) -> None: assert pipelinerun.succeeded == expected -#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file +#TODO: Test _extract_stages_run and all methods in StageConfig class + +class TestToFromDictMethods: + """ + Class to store testing methods for to_dict and from_dict, specifically + for PipelineRun, RunManifest, and StageResult classes. + """ + @pytest.fixture + def runmanifest(self) -> RunManifest: + return RunManifest("pipeline", + "1", + None, + ["stage1","stage2"], + {"uniqueID":"example"}, + {"input_path":"input/data/example.csv"}, + {"output_path":"output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None) + @pytest.fixture + def stageresult(self) -> StageResult: + return StageResult("stage_test", + StageStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + "example output", + "", + "", + None, + {}, + None, + None) + + @pytest.fixture + def pipelinerun(self, runmanifest, stageresult) -> PipelineRun: + return PipelineRun(runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + [stageresult], + {"stage_test":"example output"}) + + def test_runmanifest_to_dict(self, runmanifest) -> None: + """ + Test that the to_dict method for RunManifest outputs the correct dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + expected_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config":None + } + assert runmanifest.runmanifest_to_dict() == expected_dict + + def test_runmanifest_from_dict(self, runmanifest) -> None: + """ + Test that the from_dict method for RunManifest correctly creates a RunManifest instance from a dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + runmanifest_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config":None + } + new_runmanifest = RunManifest.runmanifest_from_dict(runmanifest_dict) + assert new_runmanifest == runmanifest + + def test_stageresult_to_dict(self, stageresult) -> None: + """ + Test that the to_dict method for StageResult outputs the correct dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + expected_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None + } + assert stageresult._stage_result_to_dict() == expected_dict + + def test_stageresult_from_dict(self, stageresult) -> None: + """ + Test that the from_dict method for StageResult correctly creates a StageResult instance from a dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + stageresult_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None + } + new_stageresult = StageResult._stage_result_from_dict(stageresult_dict) + assert new_stageresult == stageresult + + def test_pipelinerun_to_dict(self, pipelinerun) -> None: + """ + Test that the to_dict method for PipelineRun outputs the correct dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + expected_dict = { + "manifest": pipelinerun.manifest.runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, + "stage_outputs": {"stage_test": "example output"} + } + assert pipelinerun._pipeline_run_to_dict() == expected_dict + + def test_pipelinerun_from_dict(self, pipelinerun) -> None: + """ + Test that the from_dict method for PipelineRun correctly creates a PipelineRun instance from a dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + pipelinerun_dict = { + "manifest": pipelinerun.manifest.runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, + "stage_outputs": {"stage_test": "example output"} + } + new_pipelinerun = PipelineRun._pipeline_run_from_dict(pipelinerun_dict) + assert new_pipelinerun == pipelinerun + From eeafbc6a9fbb706c4f7842e92765d2977b925f98 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:00:33 +0100 Subject: [PATCH 228/332] test: add test to check that log_pipeline_attributes correctly writes out to a YAML and the contents of that YAML can be correctly parsed back to a PipelineRun instance. --- tests/test_runner.py | 89 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 225413f..db0e25c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -8,8 +8,8 @@ from onsrap.execution import ExecutionContext from onsrap.logger import Logger -from onsrap.models import PipelineConfig, RunManifest -from onsrap.runner import _log_config, print_config_diffs +from onsrap.models import PipelineConfig, PipelineStatus, RunManifest, PipelineRun, StageResult, StageStatus, now +from onsrap.runner import _log_config, print_config_diffs, _log_pipeline_attributes def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: @@ -139,4 +139,87 @@ def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) assert "CHANGED (1)" in captured.out assert "ADDED in second configuration (1)" in captured.out assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + assert "stage_configs.stage_a.target_variable" in captured.out + +class TestRunInfoWriteOut: + def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_pipeline_attributes`` function correctly writes the pipeline attributes to a YAML file. + """ + + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True, exist_ok=True) + + pipeline_config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=pipeline_config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + stage_results = [ + StageResult( + name="stage_a", + status=StageStatus.SUCCEEDED, + started_at=now(), + finished_at= now(), + error=None, + source=None, + ) + ] + + run_manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + ) + + pipeline_run = PipelineRun( + manifest=run_manifest, + status=PipelineStatus.SUCCEEDED, + started_at=context.started_at, + completed_at=now(), + stage_results=stage_results, + stage_outputs={}, + ) + + _log_pipeline_attributes( + pipeline_run=pipeline_run, + run_dir=run_dir, + context=context + ) + + expected_file = run_dir / ( + "pipeline_attributes_for_" + f"{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + + expected_contents = pipeline_run._pipeline_run_to_dict() + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + assert parsed_yaml == expected_contents + + assert PipelineRun._pipeline_run_from_dict(parsed_yaml) == pipeline_run + + + \ No newline at end of file From 3101545f922ed462e137d7c85e749c1a655ee28c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:01:00 +0100 Subject: [PATCH 229/332] tweak: minor change to _log_pipeline_attributes() to remove StageResult parameter as this is covered in PipelineRun already --- onsrap/runner.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 5641150..8658a38 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -9,7 +9,7 @@ from .warnings import StageConfigurationWarning from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, RunManifest, now +from .models import PipelineRun, PipelineStatus, RunManifest, StageResult, now if TYPE_CHECKING: from .pipeline import Pipeline @@ -165,6 +165,13 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ) pipeline.manifest = manifest pipeline.last_run = run + + #Creates attributes file in the run_directory to log information for later + #analysis of pipeline runs + _log_pipeline_attributes(pipeline_run = run, + run_dir = run_dir, + context = context) + self.logger.event( "Pipeline failed", name=pipeline.name, @@ -186,6 +193,12 @@ def run(self, pipeline: Pipeline) -> PipelineRun: pipeline.manifest = manifest pipeline.last_run = run + #Creates attributes file in the run_directory to log information for later + #analysis of pipeline runs + _log_pipeline_attributes(pipeline_run = run, + run_dir = run_dir, + context = context) + self.logger.event( "Pipeline completed", name=pipeline.name, @@ -234,6 +247,36 @@ def main(argv: list[str] | None = None) -> int: pipeline.run() return 0 +def _log_pipeline_attributes(pipeline_run: PipelineRun, + run_dir: Path, + context: ExecutionContext) -> None: + """ + Creates a YAML file within the run directory that contains information + regarding PipelineRun and StageResult instances for the run. This is + later used to extract information about previous runs which are not + currently stored in memory. + + Parameters + ---------- + ``pipeline_run`` : PipelineRun + The PipelineRun instance for the current run of the pipeline. + ``stage_results`` : list[StageResult] + A list of StageResult instances for the current run of the pipeline. + ``run_dir`` : Path + The directory where the pipeline run is being currently being executed. + ``context`` : ExecutionContext + The context of the current pipeline run, containing configuration and + state information. + """ + attributes_file = run_dir / f"pipeline_attributes_for_{context.pipeline_name}_{context.run_id[-8:]}.yaml" + import yaml + with open(attributes_file, "w", encoding="utf-8") as f: + yaml.safe_dump( + pipeline_run._pipeline_run_to_dict(), + f, + default_flow_style=False + ) + def _log_config(run_dir: Path, context: ExecutionContext, manifest: RunManifest) -> None: """ From 0a3de03605d3191cad854cfe3fe3972636c2b540 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:11:14 +0100 Subject: [PATCH 230/332] feat: method for loading a PipelineRun instance from a previous run file --- onsrap/loader.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/onsrap/loader.py b/onsrap/loader.py index 80c01ab..25f1504 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -8,6 +8,8 @@ from types import ModuleType from typing import Any +from onsrap.models import PipelineRun + from .errors import StageConfigurationError, StageLoadError PREFERRED_ENTRYPOINTS = ("run", "main", "execute") @@ -158,3 +160,36 @@ def load_python_module(path: Path) -> ModuleType: ) from exc return module + +def load_pipeline_run_for_historical_run(file_path: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + This function is used to load the state of a pipeline run that has been + saved to a YAML file. It reads the file, parses the YAML content, and + reconstructs the PipelineRun object. + + Parameters + ---------- + ``file_path`` : Path + The path to the YAML file containing the saved pipeline run. + + Returns + ------- + ``PipelineRun`` + The reconstructed PipelineRun object. + + Raises + ------ + ``FileNotFoundError`` + If the specified file does not exist. + """ + import yaml + + if not file_path.exists(): + raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file From adad82433693408aee2202046e3dc8447029381e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 08:28:47 +0100 Subject: [PATCH 231/332] tweak: move load_pipeline_run_for_historical_run() method into PipelineRun class as a class method rather than in loader.py --- onsrap/loader.py | 32 -------------------------------- onsrap/models.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/onsrap/loader.py b/onsrap/loader.py index 25f1504..69098e7 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -161,35 +161,3 @@ def load_python_module(path: Path) -> ModuleType: return module -def load_pipeline_run_for_historical_run(file_path: Path) -> PipelineRun: - """ - Load a previously executed pipeline run from a YAML file. - - This function is used to load the state of a pipeline run that has been - saved to a YAML file. It reads the file, parses the YAML content, and - reconstructs the PipelineRun object. - - Parameters - ---------- - ``file_path`` : Path - The path to the YAML file containing the saved pipeline run. - - Returns - ------- - ``PipelineRun`` - The reconstructed PipelineRun object. - - Raises - ------ - ``FileNotFoundError`` - If the specified file does not exist. - """ - import yaml - - if not file_path.exists(): - raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") - - with open(file_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - - return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file diff --git a/onsrap/models.py b/onsrap/models.py index 72b7bed..9bdf534 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -972,6 +972,40 @@ def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: stage_outputs=data.get("stage_outputs", {}), ) + @classmethod + def load_pipeline_run_for_historical_run(cls, file_path: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + This function is used to load the state of a pipeline run that has been + saved to a YAML file. It reads the file, parses the YAML content, and + reconstructs the PipelineRun object. + + Parameters + ---------- + ``file_path`` : Path + The path to the YAML file containing the saved pipeline run. + + Returns + ------- + ``PipelineRun`` + The reconstructed PipelineRun object. + + Raises + ------ + ``FileNotFoundError`` + If the specified file does not exist. + """ + import yaml + + if not file_path.exists(): + raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return cls._pipeline_run_from_dict(data) + @property def succeeded(self) -> bool: """ From b36638787f7fcb773b67a83cb175ba561f66b480 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 08:47:12 +0100 Subject: [PATCH 232/332] tweak: add leading _ to runmanifest_to_dict and runmanifest_from_dict methods --- onsrap/models.py | 4 ++-- tests/test_models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 9bdf534..67f2f32 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -690,7 +690,7 @@ def __repr__(self) -> str: f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" ) - def runmanifest_to_dict(self) -> dict[str, Any]: + def _runmanifest_to_dict(self) -> dict[str, Any]: """ Converts the RunManifest instance into a dictionary representation. This is needed to allow a RunManifest instance to be serialized into a @@ -719,7 +719,7 @@ def runmanifest_to_dict(self) -> dict[str, Any]: } @classmethod - def runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: + def _runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: """ Converts a dictionary representation of a RunManifest instance back into a RunManifest instance. Allows for RunManifest instances to be created from diff --git a/tests/test_models.py b/tests/test_models.py index 0f5f0ff..b397497 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -347,7 +347,7 @@ def test_runmanifest_to_dict(self, runmanifest) -> None: "user": None, "config":None } - assert runmanifest.runmanifest_to_dict() == expected_dict + assert runmanifest._runmanifest_to_dict() == expected_dict def test_runmanifest_from_dict(self, runmanifest) -> None: """ @@ -373,7 +373,7 @@ def test_runmanifest_from_dict(self, runmanifest) -> None: "user": None, "config":None } - new_runmanifest = RunManifest.runmanifest_from_dict(runmanifest_dict) + new_runmanifest = RunManifest._runmanifest_from_dict(runmanifest_dict) assert new_runmanifest == runmanifest def test_stageresult_to_dict(self, stageresult) -> None: From c6efc58010130754fe8a51d35fd72654e715a3c0 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:15:37 +0100 Subject: [PATCH 233/332] tweak: move run_output derivation into Pipeline init phase --- onsrap/pipeline.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7b17244..b45600e 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -110,6 +110,8 @@ def __init__( self.manifest: RunManifest | None = None self.last_run: PipelineRun | None = None + self.run_output = self._set_run_output() + self.logger.event( "Pipeline initialized", name=self.name, @@ -421,7 +423,31 @@ def add_dependencies(self, self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + def _set_run_output(self) -> Path: + """ + Private method that sets the run output directory for the Pipeline. + + Returns + ------- + ``Path`` + The path to the run output directory for the Pipeline. + Raises + ------ + ``StageConfigurationWarning`` + If the output_dir is not specified in the PipelineConfig, a warning is + raised to show that the project root or work directory will be used as + the directory for the run outputs. + """ + if self.config.output_dir is not None: + run_output = Path(self.config.output_dir) + else: + warnings.warn( + "Output directory is not specified. Using project root or work directory as the run output.", + StageConfigurationWarning + ) # TODO: fill with warnings from Pipeline branch + run_output = Path(self.config.project_root or self.config.work_dir) + return run_output / "runs" def _assign_dependencies(self, dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, From 458d286b03ed0588de9603c777d72ca796d47135 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:15:57 +0100 Subject: [PATCH 234/332] tweak: add an error message to cover where historical pipeline runs load incorrectly --- onsrap/errors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/onsrap/errors.py b/onsrap/errors.py index bd29f9f..2377f87 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -83,4 +83,10 @@ class PipelineConfigurationError(OnsrapError): """ Raised when there has been an issue with the PipelineConfig instance. + """ + +class HistoricalPipelineLoadError(OnsrapError): + """ + Raised when there is an issue loading a previous PipelineRun + instance. """ \ No newline at end of file From a0f10be9ceeb77a0b57e9284ccbd2dbbfa53e790 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:33:31 +0100 Subject: [PATCH 235/332] feat: add a function that reviews the log file and extracts all run_ids and timestamps for every run where a log entry contains "Pipeline started" --- onsrap/logger.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index ff065f1..4fe13da 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any +from .errors import HistoricalPipelineLoadError + @dataclass class LogConfig: @@ -142,4 +144,61 @@ def warning(self, message: str, **kwargs: Any) -> None: if kwargs: self._logger.warning("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) else: - self._logger.warning(message) \ No newline at end of file + self._logger.warning(message) + + def extract_historical_run_ids(self, run_root: Path) -> list[str]: + """ + + """ + + #ensure that logger is writing to a file and extract filepath + if not self._logger.root.hasHandlers(): + raise HistoricalPipelineLoadError("The logger does not write to a" \ + "filepath. Please ensure that your logger writes to a file path so that" \ + "we can extract the run_id for historical runs.") + + logfile_path = self._logger.root.handlers[0].baseFilename + + if not Path(logfile_path).exists(): + raise HistoricalPipelineLoadError("The log file does not exist at this" \ + " location.") + + print(logfile_path) + + matches: list[dict[str, Any]] = [] + + for raw_line in reversed(Path(logfile_path).read_text(encoding="utf-8").splitlines()): + if "Pipeline started" not in raw_line or " | " not in raw_line: + continue + + # left: timestamp + message, right: JSON context + left, right = raw_line.split(" | ", 1) + + try: + payload = json.loads(right) + except json.JSONDecodeError: + continue + + run_id = payload.get("run_id") + if not run_id: + continue + + # timestamp is the first two space-separated tokens: YYYY-MM-DD HH:MM:SS,mmm + parts = left.split(" ", 2) + if len(parts) < 2: + continue + timestamp = f"{parts[0]} {parts[1]}" + + run_dir = run_root / run_id + if run_dir.exists(): + matches.append({ + "run_id": run_id, + "timestamp": timestamp, + "run_dir": run_dir, + }) + + return matches + + + + From 266744e0d3b11055c4d21d963d51996cdb5286f1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:35:14 +0100 Subject: [PATCH 236/332] feat: add method that allows historical runs to be loaded as PipelineRun instances --- onsrap/loader.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/onsrap/loader.py b/onsrap/loader.py index 69098e7..1b9d348 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -8,9 +8,8 @@ from types import ModuleType from typing import Any -from onsrap.models import PipelineRun - from .errors import StageConfigurationError, StageLoadError +from .models import PipelineRun PREFERRED_ENTRYPOINTS = ("run", "main", "execute") @@ -161,3 +160,23 @@ def load_python_module(path: Path) -> ModuleType: return module +def load_historical_run(run_dir: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + Returns + ------- + ``PipelineRun`` + An instance of ``PipelineRun`` representing the historical run. + """ + import glob + files = glob.glob(str(run_dir / "pipeline_attributes_for_*.yaml")) + if not files: + raise StageLoadError("Historical run file does not exist in: {0}".format(run_dir)) + file_path = Path(files[0]) + + import yaml + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file From 02ff9df1d8c6270da629370363341a37c364b9b7 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:59:08 +0100 Subject: [PATCH 237/332] feat: added in methods to load the latest run from a cold run log into the last_run attribute of a Pipeline --- onsrap/logger.py | 25 +++++++++++++++++-------- onsrap/pipeline.py | 30 +++++++++++++++++++++++++++++- onsrap/runner.py | 10 +--------- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 4fe13da..48dee4c 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -146,24 +146,31 @@ def warning(self, message: str, **kwargs: Any) -> None: else: self._logger.warning(message) - def extract_historical_run_ids(self, run_root: Path) -> list[str]: + def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: """ - + Extracts historical run IDs from the log files. + + Returns + ------- + list[dict[str, Any]] + A list of dictionaries containing run_id, timestamp, and run_dir for each historical run. """ #ensure that logger is writing to a file and extract filepath - if not self._logger.root.hasHandlers(): + if not self._logger.hasHandlers(): raise HistoricalPipelineLoadError("The logger does not write to a" \ "filepath. Please ensure that your logger writes to a file path so that" \ "we can extract the run_id for historical runs.") - logfile_path = self._logger.root.handlers[0].baseFilename + logfile_handler = next((h for h in self._logger.handlers if isinstance(h, logging.FileHandler)), None) + if logfile_handler is None: + raise HistoricalPipelineLoadError("The logger does not have a FileHandler. " \ + "Please ensure that your logger writes to a file path so that we can extract the run_id " + "for historical runs.") + logfile_path = logfile_handler.baseFilename if not Path(logfile_path).exists(): - raise HistoricalPipelineLoadError("The log file does not exist at this" \ - " location.") - - print(logfile_path) + raise HistoricalPipelineLoadError("The log file does not exist at this location.") matches: list[dict[str, Any]] = [] @@ -190,6 +197,8 @@ def extract_historical_run_ids(self, run_root: Path) -> list[str]: timestamp = f"{parts[0]} {parts[1]}" run_dir = run_root / run_id + + #only returns run_ids for runs where a run_directory is still present. if run_dir.exists(): matches.append({ "run_id": run_id, diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index b45600e..a2ae446 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -17,6 +17,7 @@ from .logger import Logger from .models import GlobalConfig, PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now from .stage import Stage, _normalize_dependencies +from .loader import load_historical_run ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") @@ -108,10 +109,12 @@ def __init__( self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None - self.last_run: PipelineRun | None = None self.run_output = self._set_run_output() + self.last_run: PipelineRun | None = None + self.last_run = self._load_latest_run() + self.logger.event( "Pipeline initialized", name=self.name, @@ -120,6 +123,31 @@ def __init__( enabled_stages=[stage.name for stage in self.graph.stages], ) + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + if previous_run_logs == []: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + + def __str__(self) -> str: """ String method that returns a human-readable representation of the ``Pipeline`` class. diff --git a/onsrap/runner.py b/onsrap/runner.py index 8658a38..8e769cd 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -87,15 +87,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: runtime_id = pipeline._create_runtime_id() pipeline.id = runtime_id - if pipeline.config.output_dir is not None: - run_output = Path(pipeline.config.output_dir) - else: - warnings.warn( - "Output directory is not specified. Using project root or work directory as the run output.", - StageConfigurationWarning - ) # TODO: fill with warnings from Pipeline branch - run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) - run_dir = run_output / "runs" / runtime_id.get_id() + run_dir = pipeline.run_output / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) # Initialise the ExecutionContext which will be passed to each stage as it runs. This From 82eea4122a5a8fdcf83cbc2b9535e0fc0b8498e5 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 08:47:54 +0100 Subject: [PATCH 238/332] tweak: reorder _load_latest_run method in pipeline.py to sit with rest of private methods --- onsrap/pipeline.py | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index a2ae446..e548162 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -123,30 +123,6 @@ def __init__( enabled_stages=[stage.name for stage in self.graph.stages], ) - def _load_latest_run(self) -> PipelineRun | None: - """ - Load the most recent run of the Pipeline as a PipelineRun instance. - - Returns - ------- - ``PipelineRun`` or None - An instance of ``PipelineRun`` representing the most recent run of the - Pipeline, or None if no previous runs are found. - """ - previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) - if previous_run_logs == []: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) - return None - latest_run_log = previous_run_logs[0] - latest_run_id = latest_run_log["run_id"] - if latest_run_id is None: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) - return None - - return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) - def __str__(self) -> str: """ @@ -449,7 +425,31 @@ def add_dependencies(self, self.graph = StageGraph.from_stages(self.stages) self.graph.validate() - self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + if previous_run_logs == []: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) def _set_run_output(self) -> Path: """ From 3a6345d5b06eacbcd5af652f96b821c2b1e0c0d0 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 12:13:02 +0100 Subject: [PATCH 239/332] tweak: correct error message in _load_latest_run --- onsrap/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index e548162..e4b8054 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -440,13 +440,13 @@ def _load_latest_run(self) -> PipelineRun | None: previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) if previous_run_logs == []: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) + " will be None.", PipelineConfigurationWarning) return None latest_run_log = previous_run_logs[0] latest_run_id = latest_run_log["run_id"] if latest_run_id is None: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) + " will be None.", PipelineConfigurationWarning) return None return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) From 776e52d1102d3aed6e43fadf32e50c6848633068 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 12:13:20 +0100 Subject: [PATCH 240/332] tests: add testing for load_latest_run method with fully mocked data --- tests/test_pipeline.py | 275 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 273 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a3a757a..55d6135 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,6 +1,9 @@ +from unittest import mock +import warnings + from onsrap.pipeline import Pipeline, PipelineConfig -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError -from onsrap.models import StageConfig +from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError +from onsrap.models import PipelineRun, PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning from pathlib import Path @@ -272,3 +275,271 @@ def test_construct_manifest_inputs_contains_only_effective_stages() -> None: manifest = pipeline._construct_manifest(runtime_id=runtime_id) assert list(manifest.inputs.keys()) == ["Stage_0"] + + +class TestLoadLatestRunIntegration: + @pytest.fixture + def pipeline_log_line(self): + def _make(run_id: str, timestamp: str) -> str: + """ + Returns a false log file line to simulate a historical run in the log file. + The line is formatted to match the expected log output. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + ``timestamp`` : str + The timestamp of when the historical run was initiated. + """ + return f"{timestamp} Pipeline started | " \ + f"{{\"run_id\": \"{run_id}\", \"run_dir\": \"/path/to/run\"}}" + return _make + + @pytest.fixture + def minimal_pipeline_yaml(self): + def _make(run_id: str) -> str: + """ + Returns a minimal YAML configuration for a historical run. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + """ + return f""" + manifest: + run_id: {run_id} + status: succeeded + started_at: '2026-08-06T17:03:30.000077' + completed_at: '2026-08-06T17:03:30.031654' + stage_results: [] + stage_outputs: {{}} + """ + return _make + +class TestLoadLatestRun(TestLoadLatestRunIntegration): + @pytest.fixture + def pipeline_no_history(self, tmp_path: Path) -> Pipeline: + """ + Sets up a blank pipeline instance for testing that accounts for warnings + in init phase rather than dealing with these in the tests. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline(config=PipelineConfig( + output_dir = tmp_path/"outputs", + )) + pipeline.run_output = tmp_path/"runs" + return pipeline + + def test_blank_historical_run_ids(self, + pipeline_no_history: Pipeline, + monkeypatch) -> None: + """ + Tests that if extract_historical_run_ids returns a blank list, the + _load_latest_run method will return None and raise a warning. Assert + that it will also store None in the last_run attribute of the Pipeline + instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: []) + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) == [] + with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ + "found for this Pipeline. Last_run attribute will be None."): + assert pipeline_no_history._load_latest_run() == None + assert pipeline_no_history.last_run == None + + def test_blank_run_ids(self, + pipeline_no_history: Pipeline, + monkeypatch) -> None: + """ + Tests that if the found log record does not have a run_id, _load_latest_run + will return None and raise a warning. Assert that it will also store None + in the last_run attribute of the Pipeline instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [{ + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/")}] + ) + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) == [{ + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/") + }] + with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ + "found for this Pipeline. Last_run attribute will be None."): + assert pipeline_no_history._load_latest_run() == None + assert pipeline_no_history.last_run == None + + def test_load_latest_run_success(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + + """ + Tests that load_latest_run works successfully with fully mocked data. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [{ + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run") + }] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + result = pipeline_no_history._load_latest_run() + + assert result is expected_run + + expected_path = pipeline_no_history.run_output / run_id + mock_load_historical_run.assert_called_once_with(run_dir = expected_path) + + def test_which_run_is_selected_load_latest_run(self, + monkeypatch, + pipeline_no_history: Pipeline + ) -> None: + + """ + Checks that the first item is selected from the list of historical runs + returned by extract_historical_run_ids. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + expected_run = mock.MagicMock(spec=PipelineRun) + run_id_1 = "2026-08-10_100000_abc12345" + run_id_2 = "2026-08-10_100000_def67890" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [ + { + "run_id": run_id_1, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_A" + }, + { + "run_id": run_id_2, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_B" + } + ] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + pipeline_no_history._load_latest_run() + + expected_path = pipeline_no_history.run_output / run_id_1 + mock_load_historical_run.assert_called_once_with(run_dir = expected_path) + + #does not refer to run_dir in the extract_historical_run_ids list but the + #parameter required in load_historical_run. + assert mock_load_historical_run.call_args.kwargs["run_dir"].name == run_id_1 + + def test_no_errors_raised_success_load_latest_run(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + + """ + Tests that no errors are raised when load_latest_run is successful. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [{ + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run") + }] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + with warnings.catch_warnings(record=True) as w: + pipeline_no_history._load_latest_run() + + assert not any(issubclass(warning.category, PipelineConfigurationWarning) + for warning in w) + + + + From 490aac0c9bdef93666d2dd115ea191cf33ba1707 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 13:13:55 +0100 Subject: [PATCH 241/332] docs: edit documentation to clarify what the parameters refer to --- onsrap/logger.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 48dee4c..7fad6fa 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -150,6 +150,11 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: """ Extracts historical run IDs from the log files. + Parameters + ---------- + ``run_root`` : Path + The root directory where the historical runs are stored. + Returns ------- list[dict[str, Any]] From 84e8c6bf45c1365c1aa590c5903df042d8666b59 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 13:53:07 +0100 Subject: [PATCH 242/332] tweak: added init values and changed code and documentation style to be more consistent. Added stricter typing enforcement --- onsrap/__init__.py | 2 +- onsrap/execution.py | 267 ++++++++++++++++++++++++++------------------ onsrap/stage.py | 16 ++- onsrap/warnings.py | 6 +- 4 files changed, 175 insertions(+), 116 deletions(-) diff --git a/onsrap/__init__.py b/onsrap/__init__.py index 95c8203..b23aab9 100644 --- a/onsrap/__init__.py +++ b/onsrap/__init__.py @@ -13,6 +13,7 @@ from .logger import LogConfig, Logger from .models import ( Catalog, + GlobalConfig, PipelineConfig, PipelineRun, PipelineStatus, @@ -22,7 +23,6 @@ StageConfig, StageResult, StageStatus, - GlobalConfig, ) from .pipeline import Pipeline from .runner import PipelineRunner diff --git a/onsrap/execution.py b/onsrap/execution.py index b065e10..504e241 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -3,18 +3,33 @@ import inspect import subprocess import sys +import warnings from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import Any, Protocol, TYPE_CHECKING -import warnings +from typing import TYPE_CHECKING, Any, Protocol from onsrap.warnings import StageConfigurationWarning -from .errors import StageConfigurationError, StageExecutionError, StageLoadError, PipelineConfigurationError -from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable +from .errors import ( + PipelineConfigurationError, + StageExecutionError, + StageLoadError, +) +from .loader import ( + PREFERRED_ENTRYPOINTS, + discover_python_entrypoint, + load_python_callable, +) from .logger import Logger -from .models import GlobalConfig, PipelineConfig, StageConfig, StageResult, StageStatus, now +from .models import ( + GlobalConfig, + PipelineConfig, + StageConfig, + StageResult, + StageStatus, + now, +) if TYPE_CHECKING: from .stage import Stage @@ -52,6 +67,7 @@ class ExecutionContext: ``global_config`` : ``GlobalConfig`` or None, default = None Variables which are parsed to all stages throughout the pipeline. """ + pipeline_name: str run_id: str config: PipelineConfig @@ -72,7 +88,7 @@ def record(self, result: StageResult) -> StageResult: Saves all information on the results of the Stage to the ``stage_results`` attribute and exclusively metadata outputs regarding the run to the ``variables`` attribute. - Parameters + Parameters ---------- ``result`` : ``StageResult`` An instance of a ``StageResult`` class which is created from the Executor classes (StageExecutor, PythonStageExecutor). @@ -81,7 +97,7 @@ def record(self, result: StageResult) -> StageResult: ------ ``result`` An unchanged ``StageResult`` instance. - """ + """ self.stage_results[result.name] = result self.variables[result.name] = result.outputs return result @@ -95,9 +111,9 @@ def result_for(self, stage_name: str) -> StageResult | None: ``stage_name`` : str The name of the ``Stage`` that you are calling the results for. - Returns + Returns ------- - ``stage_results`` + ``stage_results`` Attribute for the specific `Stage` named. """ return self.stage_results.get(stage_name) @@ -114,7 +130,7 @@ def stage_config(self) -> StageConfig | None: Return the configuration for the stage currently being executed. The preferred access method for this is ``get_stage_config()`` which allows - for optional arguments to return the full ``StageConfig`` instance or just + for optional arguments to return the full ``StageConfig`` instance or just the variables dictionary. This property is ``None`` outside an active stage run. @@ -140,50 +156,54 @@ def stage_config_for(self, stage_name: str | None) -> StageConfig | None: @property def stage_outputs(self) -> dict[str, Any]: """ - Creates a ``stage_outputs`` attribute for the ``ExecutionContext`` class. + Creates a ``stage_outputs`` attribute for the ``ExecutionContext`` class. Extracts the ```outputs`` attribute from the ``stage_results`` class for each ``Stage`` name. - Returns - ------- + Returns + ------- ``stage_outputs`` - Dictionary containing the name of the stage and the associated outputs of + Dictionary containing the name of the stage and the associated outputs of the run. """ return {name: result.outputs for name, result in self.stage_results.items()} - + def get_data_dir(self) -> Path: """ - Establishes the filepath that the data is held in. - + Establishes the filepath that the data is held in. + Returns ------- Path - The file path for the location of the data being used in the pipeline. + The file path for the location of the data being used in the pipeline. """ if self.config is not None: return Path(self.config.data_dir) - - raise PipelineConfigurationError("Please parse a PipelineConfig instance to " \ - "the ExecutionContext.") - + + raise PipelineConfigurationError( + "Please parse a PipelineConfig instance to the ExecutionContext." + ) + def resolve_output_root(self) -> Path: """ - Establishes the filepath that the outputs are going to be saved to. + Establishes the filepath that the outputs are going to be saved to. Returns ------- Path - The file path for the outputs of the run to be saved to. + The file path for the outputs of the run to be saved to. """ if self.run_dir is not None: return Path(self.run_dir) - - raise PipelineConfigurationError("Please parse a run directory to " \ - "the ExecutionContext.") - def get_stage_config(self, stage: str | None = None, with_global: bool = True, vars_only: bool = True) -> dict[str, Any] | StageConfig | None: + raise PipelineConfigurationError( + "Please parse a run directory to the ExecutionContext." + ) + + def get_stage_config( + self, stage: str | None = None, with_global: bool = True, vars_only: bool = True + ) -> dict[str, Any] | StageConfig | None: """ Returns the configuration for the stage currently being executed, with optional arguments. @@ -198,14 +218,16 @@ def get_stage_config(self, stage: str | None = None, with_global: bool = True, v The name of the stage to get the configuration for. ``vars_only`` : bool, default = True If True, returns only the variables dictionary from the ``StageConfig``. If False, returns the full ``StageConfig`` instance. - + Returns ------- dict[str, Any] or StageConfig or None - The parameters contained within the configuration for the currently active stage. + The parameters contained within the configuration for the currently active stage. If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. """ - stage_config: StageConfig | None = self.stage_config_for(stage) if stage is not None else self.stage_config + stage_config: StageConfig | None = ( + self.stage_config_for(stage) if stage is not None else self.stage_config + ) if with_global and not vars_only: raise PipelineConfigurationError( @@ -223,77 +245,79 @@ def get_stage_config(self, stage: str | None = None, with_global: bool = True, v if vars_only: return stage_config.variables return stage_config - - def resolve_given_path(self, stage_name: str | None, - path_name: str | None, - file_name: str | None, - root: Path, - add_folder: list[str] | str | None = None - ) -> Path: + + def resolve_given_path( + self, + stage_name: str | None, + path_name: str | None, + file_name: str | None, + root: Path, + add_folder: list[str] | str | None = None, + ) -> Path: """ Returns a file path for a requested item. - - This investigates the result of a previous stage to extract a selected path. - If the path is not available, it creates a path using a root previously derived - in main.py, the chosen directory within the root (optional), and the file path. + + This investigates the result of a previous stage to extract a selected path. + If the path is not available, it creates a path using a root previously derived + in main.py, the chosen directory within the root (optional), and the file path. Parameters ---------- ``stage_name`` : str - The name of the stage where the path was outputted. + The name of the stage where the path was outputted. ``path_name`` : str - The name for the path within the stage results. This will be the key from the - key/value pair within the output of the previous stage. + The name for the path within the stage results. This will be the key from the + key/value pair within the output of the previous stage. ``file_name`` : str - The name of the file that you are trying to access the Path for. + The name of the file that you are trying to access the Path for. ``root`` : Path - The file path for the root of the directory. This should be denoted through - other methods. + The file path for the root of the directory. This should be denoted through + other methods. ``add_folder`` : list[str] | str | None, default = None Additional folder name/s to add into the returned file path. Returns ------- Path - The file path where data has previously been saved to to allow for extraction of - that data throughout the pipeline. + The file path where data has previously been saved to to allow for extraction of + that data throughout the pipeline. """ - result = self.result_for(stage_name) + result = self.result_for(stage_name) if stage_name is not None else None if result is not None and path_name is not None: selected_path = result.outputs.get(path_name) - if selected_path: + if selected_path: return Path(selected_path) if isinstance(add_folder, list): - if file_name is not None: + if file_name is not None: new_path = root.joinpath(*add_folder, file_name) return new_path new_path = root.joinpath(*add_folder) return new_path if isinstance(add_folder, str): if file_name is not None: - return root/ add_folder/ file_name + return root / add_folder / file_name return root / add_folder - if file_name is not None: + if file_name is not None: return root / file_name return root def _combine_vars(self, stage: StageConfig | None = None) -> dict[str, Any]: """ - Private method that combines the global variables and the stage - variables for the current stage. + Private method that combines the global variables and the stage + variables for the current stage. - Global variables are extracted from the ``global_config`` attribute + Global variables are extracted from the ``global_config`` attribute and any variables which are marked as to be excluded from the exclusion attribute are removed. The global variables are then combined with the stage specific variables and returned as a dictionary. Conflicts raise a warning - to alert the user that the stage configuration definition will be used as a - priority. + to alert the user that the stage configuration definition will be used as a + priority. Returns ------- ``combined``: dict[str, Any] - A dictionary of all variables required for the stage that are sourced - through the configuration. + A dictionary of all variables required for the stage that are sourced + through the configuration. Raises ------ @@ -303,7 +327,12 @@ def _combine_vars(self, stage: StageConfig | None = None) -> dict[str, Any]: precedence. """ resolved_stage = stage or self.stage_config - global_vars, exclusions = self.global_config.get_attributes() if self.global_config else ({}, {}) + global_vars: dict[str, Any] + exclusions: dict[str, Any] + if self.global_config is not None: + global_vars, exclusions = self.global_config.get_attributes() + else: + global_vars, exclusions = {}, {} exclusions = exclusions or {} if resolved_stage is None: @@ -312,27 +341,31 @@ def _combine_vars(self, stage: StageConfig | None = None) -> dict[str, Any]: stage_exclusions_extract = exclusions.get(resolved_stage.name, []) stage_exclusions = [exclusion for exclusion in stage_exclusions_extract] - combined = {key: value for key, value in global_vars.items() if key not in stage_exclusions} + combined = { + key: value + for key, value in global_vars.items() + if key not in stage_exclusions + } stage_vars = resolved_stage.variables - + conflicts = stage_vars.keys() & combined.keys() if conflicts: conflicting = ", ".join(sorted(conflicts)) warnings.warn( f"Stage defines variable(s) that are also defined in global " f"variables: {conflicting}. Stage variables will take precedence.", - StageConfigurationWarning + StageConfigurationWarning, ) combined.update(stage_vars) return combined - class StageExecutor(Protocol): """ - Child class of ``Protocol`` + Child class of ``Protocol`` Implementation required """ + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Method to run ``Stage`` however implementation required @@ -345,22 +378,25 @@ class PythonStageExecutor: Class to run Python `Stage`. Contains methods that allow automatic running of individual `Stage` processes for - a pipeline. + a pipeline. """ + def __init__(self, preferred_entrypoints: tuple[str, ...] = PREFERRED_ENTRYPOINTS): self.preferred_entrypoints = preferred_entrypoints - def __str__(self) -> str: + def __str__(self) -> str: return f"PythonStageExecutor: \n Preferred Entrypoints: {self.preferred_entrypoints})" - def __repr__(self) -> str: - return f"PythonStageExecutor(preferred_entrypoints={self.preferred_entrypoints})" + def __repr__(self) -> str: + return ( + f"PythonStageExecutor(preferred_entrypoints={self.preferred_entrypoints})" + ) def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Main function to select how ``Stage`` is run. - Identifies the type of ``source`` within the ``Stage`` and runs the relevant + Identifies the type of ``source`` within the ``Stage`` and runs the relevant function for that type. Parameters @@ -378,10 +414,12 @@ def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: Raise ----- ``StageExecutionError`` - If the ``source`` is not a Path or a callable object. + If the ``source`` is not a Path or a callable object. """ if callable(stage.source): - return self._execute_callable(stage, context, stage.source, stage.source_label) + return self._execute_callable( + stage, context, stage.source, stage.source_label + ) if isinstance(stage.source, Path): return self._execute_file(stage, context) @@ -402,11 +440,11 @@ def _execute_callable( """ Attempt to run a callable object. - Calls the logger.event() method to record an event and attempts to - run the callable parsed. If the callable cannot be run, an error is flagged - and the ``StageResult`` instance created shows a failure. If it can be run, - the callable is run and the ``StageResult`` instance shows a success. - Metadata is kept for the attempt including ``duration``, ``name``, ``outputs``, + Calls the logger.event() method to record an event and attempts to + run the callable parsed. If the callable cannot be run, an error is flagged + and the ``StageResult`` instance created shows a failure. If it can be run, + the callable is run and the ``StageResult`` instance shows a success. + Metadata is kept for the attempt including ``duration``, ``name``, ``outputs``, ``source``, ``mode`` attempted, and ``errors``. Parameters @@ -480,15 +518,15 @@ def _execute_file(self, stage: Stage, context: ExecutionContext) -> StageResult: Attempt to run a file. Attempt to run a callable object. - Calls the logger.event() method to record an event and attempts to run - the callable parsed. If the callable cannot be run, an error is flagged and - the ``StageResult`` instance created shows a failure. If it can be run, the - callable is run and the ``StageResult`` instance shows a success. Metadata + Calls the logger.event() method to record an event and attempts to run + the callable parsed. If the callable cannot be run, an error is flagged and + the ``StageResult`` instance created shows a failure. If it can be run, the + callable is run and the ``StageResult`` instance shows a success. Metadata is kept for the attempt including ``duration``, ``name``, ``outputs``, ``source``, ``mode`` attempted, and ``errors``. - If there is no entrypoint or the entrypoint is not a callable object, an error will be - raised. ``_execute_subprocess()`` method called if no entrypoint is found. A - ``StageResult`` instance will be created to log the results of the ``Stage``run regardless + If there is no entrypoint or the entrypoint is not a callable object, an error will be + raised. ``_execute_subprocess()`` method called if no entrypoint is found. A + ``StageResult`` instance will be created to log the results of the ``Stage``run regardless of success or failure. Parameters @@ -561,8 +599,8 @@ def _execute_file(self, stage: Stage, context: ExecutionContext) -> StageResult: def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageResult: """ Run the entire Python file for the ``Stage`` from the top. - - Not desired method. Uses black-box design and obfuscates Pipeline running. Please refer + + Not desired method. Uses black-box design and obfuscates Pipeline running. Please refer to Wiki documentation on how to implement callable solutions instead. If the ``Stage`` source is a file but does not have a callable entrypoint, this method @@ -575,7 +613,7 @@ def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageR A ``Stage`` class instance for the stage being run. ``context`` : ``ExecutionContext`` class The metadata required to run the ``Stage``. - + Return ------ ``result`` @@ -612,7 +650,9 @@ def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageR finished_at = now() result = StageResult( name=stage.name, - status=StageStatus.SUCCEEDED if completed.returncode == 0 else StageStatus.FAILED, + status=StageStatus.SUCCEEDED + if completed.returncode == 0 + else StageStatus.FAILED, started_at=started_at, finished_at=finished_at, outputs=completed.stdout, @@ -622,7 +662,8 @@ def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageR metadata=dict(stage.metadata), error=None if completed.returncode == 0 - else completed.stderr.strip() or "Subprocess returned a non-zero exit code.", + else completed.stderr.strip() + or "Subprocess returned a non-zero exit code.", source=str(path), ) @@ -647,20 +688,20 @@ def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageR def _invoke_callable(callable_object: Any, stage: Stage, context: ExecutionContext) -> Any: """ - Assigns appropriate parameters for a callable and runs it. + Assigns appropriate parameters for a callable and runs it. - Searches for parameter terms that likely refer to context or stage. If none of these are found, - assigns ``context`` as the first parameter and ``stage`` as the second. + Searches for parameter terms that likely refer to context or stage. If none of these are found, + assigns ``context`` as the first parameter and ``stage`` as the second. Parameters ---------- - ``callable_object`` : Any - The callable item that is going to be run. - ``stage`` : ``Stage`` class - The ``Stage`` class instance to be a parameter for the ``callable_object``. + ``callable_object`` : Any + The callable item that is going to be run. + ``stage`` : ``Stage`` class + The ``Stage`` class instance to be a parameter for the ``callable_object``. ``context`` : ``ExecutionContext`` class - The ``ExecutionContext`` class instance to be a parameter for the ``callable_object``. - + The ``ExecutionContext`` class instance to be a parameter for the ``callable_object``. + Returns ------- ``callable_object`` @@ -689,7 +730,9 @@ def _invoke_callable(callable_object: Any, stage: Stage, context: ExecutionConte if parameter.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) ] - has_varargs = any(parameter.kind == inspect.Parameter.VAR_POSITIONAL for parameter in parameters) + has_varargs = any( + parameter.kind == inspect.Parameter.VAR_POSITIONAL for parameter in parameters + ) if not positional_parameters and not has_varargs: return callable_object() @@ -701,8 +744,14 @@ def _invoke_callable(callable_object: Any, stage: Stage, context: ExecutionConte return callable_object(context) if len(positional_parameters) >= 2 or has_varargs: - first_name = positional_parameters[0].name.lower() if positional_parameters else "" - second_name = positional_parameters[1].name.lower() if len(positional_parameters) > 1 else "" + first_name = ( + positional_parameters[0].name.lower() if positional_parameters else "" + ) + second_name = ( + positional_parameters[1].name.lower() + if len(positional_parameters) > 1 + else "" + ) if first_name in ("stage", "task") and second_name in ("context", "ctx"): return callable_object(stage, context) if first_name in ("context", "ctx") and second_name in ("stage", "task"): @@ -724,25 +773,25 @@ def _build_success_result( Create a ``StageResult`` instance showing a successful stage run. If the output of a ``Stage`` run is a ``StageResult`` class, set missing attributes to relevant - information from the ``Stage``. + information from the ``Stage``. Parameters ---------- ``stage`` : ``Stage`` class The ``Stage`` class instance being run. ``started_at`` : datetime - The time and date that the run started. + The time and date that the run started. ``finished_at`` : datetime The time and date that the run ended. ``output`` : Any The output produced from the stage run. ``source`` : str or None The file/callable being run in the stage. - + Return ------ ``StageResult`` instance - Containing metadata for the stage run and showing that the run was a success. + Containing metadata for the stage run and showing that the run was a success. """ if isinstance(output, StageResult): if output.name != stage.name: @@ -761,4 +810,4 @@ def _build_success_result( outputs=output, metadata=dict(stage.metadata), source=source, - ) \ No newline at end of file + ) diff --git a/onsrap/stage.py b/onsrap/stage.py index 9cd711b..f524129 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -1,4 +1,3 @@ - from __future__ import annotations from dataclasses import dataclass, field, replace @@ -13,7 +12,7 @@ from .models import StageResult -def _normalize_dependencies(dependencies: list[str] | str | None) -> tuple[str, ...]: +def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: """ Standardise the names of any stages dependant on other stages/processes. @@ -36,6 +35,7 @@ def _normalize_dependencies(dependencies: list[str] | str | None) -> tuple[str, if isinstance(dependencies, list) and dependencies == []: return () if isinstance(dependencies, str): + candidate_items: Iterable[str] candidate_items = [dependencies] else: @@ -227,7 +227,7 @@ def from_callable( ``Stage class`` instance with collected Stage ``name``, normalised ``dependencies`` and ``metadata``, and defined the source as the callable_object. """ - stage_name = name or getattr(callable_object, "__name__", "stage") + stage_name = str(name or getattr(callable_object, "__name__", "stage")) return cls( name=stage_name, source=callable_object, @@ -267,7 +267,13 @@ def from_dict(cls, data: Mapping[str, Any]) -> Stage: metadata = payload.pop("metadata", {}) entrypoint = payload.pop("entrypoint", None) backend = payload.pop("backend", "python") - name = payload.pop("name", None) + raw_name = payload.pop("name", None) + name = str(raw_name).strip() if raw_name is not None else None + + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} if callable(source): return cls.from_callable( @@ -313,7 +319,7 @@ def with_dependencies(self, *dependencies: str) -> Stage: ``Stage`` ``Stage`` class instance with normalised ``dependencies`` attribute. """ - unpacked_deps: list = [] + unpacked_deps: list[str] = [] for dependency in dependencies: if isinstance(dependency, list): unpacked_deps = unpacked_deps + dependency diff --git a/onsrap/warnings.py b/onsrap/warnings.py index 67ca367..0e67739 100644 --- a/onsrap/warnings.py +++ b/onsrap/warnings.py @@ -1,22 +1,26 @@ from __future__ import annotations + class OnsrapWarning(Warning): """Base warning for onsrap.""" + class StageConfigurationWarning(OnsrapWarning): """ Raised when the stage configuration is not optimal. Child class with ``OnsrapWarning`` as the parent class. """ + class PipelineConfigurationWarning(OnsrapWarning): """ Raised when the pipeline configuration is not optimal. Child class with ``OnsrapWarning`` as the parent class. """ + class ConfigurationInjectionWarning(OnsrapWarning): """ Raised when the configuration injection is not optimal. Child class with ``OnsrapWarning`` as the parent class. - """ \ No newline at end of file + """ From be48e0d0fdeebd14c9f98a840dad427c44b193c7 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 13:53:57 +0100 Subject: [PATCH 243/332] feat: commit-hooks: added mypy checks for static type checking --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03ad356..f185fa3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,13 @@ repos: args: [ --fix ] # Run ruff formatter. - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy - Type check onsrap package + entry: python -m mypy onsrap + language: system + pass_filenames: false - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 hooks: @@ -36,5 +43,5 @@ repos: hooks: - id: bandit name: bandit - Checks for vulnerabilities - args: ["-c", "pyproject.toml"] + args: ["-ll", "-c", "pyproject.toml"] additional_dependencies: ["bandit[toml]"] From 9ac640b0bf39e6d1fbd9b8a618e09b19e38b39f5 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 13:54:23 +0100 Subject: [PATCH 244/332] fix: overhauled & updated Package metadata files --- pyproject.toml | 10 +++++++++- setup.cfg | 13 ++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 21b441b..83f0f80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ select = [ # isort "I", ] -ignore = ["D203", "E203"] +ignore = ["B028", "D203", "E203", "E501"] [tool.ruff] line-length = 88 @@ -50,3 +50,11 @@ exclude = [] [tool.ruff.format] line-ending = "auto" + +[tool.mypy] +python_version = "3.10" +files = ["onsrap"] +show_error_codes = true +warn_redundant_casts = true +warn_unused_configs = true +warn_unused_ignores = true diff --git a/setup.cfg b/setup.cfg index 65415b7..aa8718a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,14 +22,21 @@ install_requires = include = onsrap* +[options.package_data] +onsrap = + py.typed + [options.extras_require] dev = + bandit[toml] coverage - detect-secrets == 1.0.3 + detect-secrets==1.0.3 + mypy myst-parser pre-commit pytest - detect-secrets python-dotenv + ruff Sphinx - toml \ No newline at end of file + toml + types-PyYAML \ No newline at end of file From a99a5ff97838b903be6034cc2f5df25cab8d8769 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 13:54:53 +0100 Subject: [PATCH 245/332] tweak: updated python package yaml --- .github/workflows/python-package.yml | 53 +++++++++++++++++++--------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f321dd8..2ca9e55 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,40 +1,59 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python package +name: Python CI on: push: branches: [ "main", "development" ] pull_request: branches: [ "main", "development" ] + workflow_dispatch: jobs: - build: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install development dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Lint with Ruff + run: | + python -m ruff check onsrap + python -m ruff format --check onsrap + - name: Run security checks + run: | + python -m bandit -ll -c pyproject.toml -r onsrap + - name: Type check with mypy + run: | + python -m mypy onsrap + - name: Build documentation + run: | + python -m sphinx -b html docs docs/_build/html + + tests: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + cache: pip + - name: Install development dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + python -m pip install -e ".[dev]" - name: Test with pytest run: | - pytest + python -m pytest From 2c4d43e23dcf7b8c03352816c38a3e0b89fa0ff3 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 14:01:52 +0100 Subject: [PATCH 246/332] tweak: stricter typing and code/documentation standardised style --- onsrap/models.py | 288 ++++++++++++++++++++++++++--------------------- onsrap/runner.py | 120 +++++++++++--------- 2 files changed, 226 insertions(+), 182 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8fe1c21..6610126 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,20 +1,20 @@ from __future__ import annotations -from textwrap import indent import warnings from dataclasses import dataclass, field from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Iterable, Mapping, Optional +from typing import Any, Iterable, Literal, Mapping, Optional, overload -from .errors import StageConfigurationError, PipelineConfigurationError +from .errors import PipelineConfigurationError, StageConfigurationError class StageStatus(str, Enum): """ Class to hold information on how the Stage has run. """ + PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -26,6 +26,7 @@ class PipelineStatus(str, Enum): """ Class to hold information on how the Pipeline has run. """ + PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -49,7 +50,7 @@ def utcnow() -> datetime: @dataclass class RuntimeID: """ - Holds information regarding individual runs. + Holds information regarding individual runs. Parameters ---------- @@ -61,9 +62,10 @@ class RuntimeID: A hashed identifier created with the combined ID and timestamp to create a unique identifier for the run. ``short_hash`` : str - A shortened version of the ``hash`` attribute to be used - in file names for the runs. + A shortened version of the ``hash`` attribute to be used + in file names for the runs. """ + id: str timestamp: datetime hash: str @@ -97,25 +99,25 @@ def get_short_hash(self) -> str: @dataclass class PipelineConfig: """ - Holds information required to run the whole pipeline. + Holds information required to run the whole pipeline. Parameters ---------- ``name`` : str, optional The name of the pipeline. - ``stages_to_run`` : dict[str, bool], optional - A dictionary of all stage names alongside a boolean value that indicates + ``stages_to_run`` : dict[str, bool], optional + A dictionary of all stage names alongside a boolean value that indicates whether the stage should be run or not. ``backend`` : str, default = "python" - The system that the pipeline is run on. - ``work_dir`` : Path - The directory to run the Pipeline in. + The system that the pipeline is run on. + ``work_dir`` : Path + The directory to run the Pipeline in. ``project_root`` : Path - The top level directory for the whole project. + The top level directory for the whole project. ``log_dir`` : Path - The directory to store the logs in. + The directory to store the logs in. ``data_dir`` : Path - The directory where the data is stored. + The directory where the data is stored. ``output_dir`` : Path, optional The directory where pipeline outputs should be written. Not used internally by the runner; exposed for stage code to read via ``context.config.output_dir``. @@ -123,13 +125,14 @@ class PipelineConfig: Indicates whether the subprocess system (running the whole file rather than an entrypoint function) should be allowed. ``python_executable`` : str, optional - The name of the executable function for the entrypoint of the - pipeline. + The name of the executable function for the entrypoint of the + pipeline. ``metadata`` : dict[str, Any] - Any additional information on the pipeline. + Any additional information on the pipeline. ``overwrite`` : bool, default = False - Indicates whether the pipeline should overwrite previous outputs. + Indicates whether the pipeline should overwrite previous outputs. """ + name: Optional[str] = None stages_to_run: Optional[dict[str, bool]] = None backend: str = "python" @@ -146,7 +149,7 @@ class PipelineConfig: def __post_init__(self) -> None: """ Post-initialization method to ensure that the ``work_dir`` and ``project_root`` - attributes are set correctly. + attributes are set correctly. """ if self.stages_to_run is None: self.stages_to_run = {} @@ -172,11 +175,11 @@ def __str__(self) -> str: def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the ``PipelineConfig`` class. - This method is structured to be more concise than the ``__str__`` method and is + Representation method that returns a human readable representation of the ``PipelineConfig`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``PipelineConfig`` class with its attributes. @@ -197,19 +200,19 @@ def from_any( value: PipelineConfig | Mapping[str, Any] | str | Path | None, ) -> PipelineConfig: """ - Converts one of several datatypes into a PipelineConfig class instance. + Converts one of several datatypes into a PipelineConfig class instance. Parameters ---------- ``value`` : PipelineConfig, Mapping[str, Any], str, Path, or None - The object holding metadata on how the Pipeline should run to be converted - into a PipelineConfig class instance. - + The object holding metadata on how the Pipeline should run to be converted + into a PipelineConfig class instance. + Raises ------ ``TypeError`` If the datatype for the object holding information on how the pipeline is run - is not a datatype that can be converted to a PipelineConfig. + is not a datatype that can be converted to a PipelineConfig. """ if value is None: return cls() @@ -228,14 +231,14 @@ def from_any( @classmethod def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: """ - Extracts information from a mapping datatype and returns a PipelineConfig - instance. + Extracts information from a mapping datatype and returns a PipelineConfig + instance. Parameters ---------- ``data`` : Mapping[str, Any] The information to be converted into a ``PipelineConfig`` instance. - + Returns ------- ``PipelineConfig`` class instance @@ -249,13 +252,15 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: metadata = {"metadata": metadata} name = payload.pop("name", None) - + backend = payload.pop("backend", "python") stages_to_run = PipelineConfig._extract_stages_run(payload) work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) output_dir_value = payload.pop("output_dir", None) - project_root = Path(project_root_value) if project_root_value is not None else work_dir + project_root = ( + Path(project_root_value) if project_root_value is not None else work_dir + ) log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) @@ -267,7 +272,12 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: UserWarning, stacklevel=2, ) - allow_subprocess_fallback = raw_subprocess_fallback.strip().lower() not in ("false", "0", "no", "off") + allow_subprocess_fallback = raw_subprocess_fallback.strip().lower() not in ( + "false", + "0", + "no", + "off", + ) else: allow_subprocess_fallback = bool(raw_subprocess_fallback) python_executable = payload.pop("python_executable", None) @@ -276,7 +286,7 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: return cls( name=name, - stages_to_run = stages_to_run, + stages_to_run=stages_to_run, backend=backend, work_dir=work_dir, project_root=project_root, @@ -288,36 +298,38 @@ def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: python_executable=python_executable, metadata=metadata, ) - + @classmethod def from_file(cls, path: Path) -> PipelineConfig: """ - Extracts a mapping item from a file containing information about how the - pipeline should run. + Extracts a mapping item from a file containing information about how the + pipeline should run. - Then calls the from_mapping() method to extract the information. + Then calls the from_mapping() method to extract the information. Parameters ---------- ``path`` : Path - The file path containing information to be converted into a PipelineConfig - instance. - + The file path containing information to be converted into a PipelineConfig + instance. + Returns ------- ``PipelineConfig`` class instance. - Raises + Raises ------ ``FileNotFoundError`` If the file path does not exist. - ``TypeError`` - If the file containing information about how the Pipeline runs does not + ``TypeError`` + If the file containing information about how the Pipeline runs does not contain a mapping type. """ config_path = Path(path).expanduser() if not config_path.exists(): - raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) + raise FileNotFoundError( + "Config file does not exist: {0}".format(config_path) + ) import yaml @@ -326,20 +338,24 @@ def from_file(cls, path: Path) -> PipelineConfig: return cls() if not isinstance(raw_config, Mapping): - raise TypeError("Pipeline config file must contain a mapping at the top level.") + raise TypeError( + "Pipeline config file must contain a mapping at the top level." + ) return cls.from_mapping(raw_config) def to_dict(self) -> dict[str, Any]: """ Returns a prescriptive expression of the attributes within the PipelineConfig instance - that allows for easier processing by the user. + that allows for easier processing by the user. """ data = { "name": self.name, "backend": self.backend, "work_dir": str(self.work_dir), - "project_root": str(self.project_root) if self.project_root is not None else None, + "project_root": str(self.project_root) + if self.project_root is not None + else None, "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), @@ -349,60 +365,59 @@ def to_dict(self) -> dict[str, Any]: } data.update(self.metadata) return data - + @staticmethod - def _extract_stages_run(payload: Mapping[str, Any] - ) -> dict[str, bool] | None: + def _extract_stages_run(payload: dict[str, Any]) -> dict[str, bool] | None: """ - Method to extract stages_to_run configuration and convert all values to boolean values. + Method to extract stages_to_run configuration and convert all values to boolean values. Parameters ---------- ``payload`` : Mapping[str, Any] The dictionary where the stages_to_run configuration is being extracted from. - Returns + Returns ------- ``boolean_dict`` - A dictionary of stage_name:bool to indicate whether a stage is being run. - - None + A dictionary of stage_name:bool to indicate whether a stage is being run. + + None If stages_to_run does not exist within the configuration. """ stages_to_run = payload.pop("stages_to_run", None) - if stages_to_run is None: + if stages_to_run is None: return None - + boolean_dict = { - stage_name: PipelineConfig._to_bool(value) + stage_name: PipelineConfig._to_bool(value) for stage_name, value in stages_to_run.items() - } - + } + return boolean_dict @staticmethod - def _to_bool(value): + def _to_bool(value: bool | int | str) -> bool: """ - Method to convert values to boolean True/False values. + Method to convert values to boolean True/False values. Integers convert to boolean where 0 = False and 1 = True. A certain subset of strings are - accepted for conversion. Any other strings will. raise an error. + accepted for conversion. Any other strings will. raise an error. Parameters ---------- ``value`` : bool | int | str - Returns + Returns ------- ``value`` - The value input but converted to a boolean value. + The value input but converted to a boolean value. - Raises + Raises ------ - ``ValueError`` - When the value has not been able to be converted to a boolean. + ``ValueError`` + When the value has not been able to be converted to a boolean. """ - + if isinstance(value, bool): return value @@ -417,14 +432,12 @@ def _to_bool(value): if value in {"false", "no", "n", "0"}: return False - + raise ValueError(f"Cannot convert {value!r} to bool") raise ValueError(f"Cannot convert {value!r} to bool") - - @dataclass class StageConfig: """ @@ -439,6 +452,7 @@ class StageConfig: ``metadata`` : dict[str, Any] Additional supporting metadata for the stage configuration. """ + # TODO: output location for stages potentially problematic for output overwrites! name: str _variables: dict[str, Any] = field(default_factory=dict) @@ -538,6 +552,7 @@ def to_dict(self) -> dict[str, Any]: data["metadata"] = dict(self.metadata) return data + @dataclass class GlobalConfig: """ @@ -548,6 +563,7 @@ class GlobalConfig: ``_variables`` : dict[str, Any] Variables that should be parsed to all stages throughout the pipeline. """ + _variables: dict[str, Any] = field(default_factory=dict) exclusion: dict[str, Any] = field(default_factory=dict) @@ -561,7 +577,7 @@ def from_dict(cls, data: Mapping[str, Any] | None) -> GlobalConfig: ``data`` : Mapping[str, Any] | None Raw configuration payload for the global configuration. ``exclusions`` : dict[str, Any] or None - A lookup of which global variables should be excluded from each stage. + A lookup of which global variables should be excluded from each stage. Returns ------- @@ -570,13 +586,26 @@ def from_dict(cls, data: Mapping[str, Any] | None) -> GlobalConfig: """ if data is None: return cls() - + payload = dict(data or {}) - exclusions = payload.pop("exclusions", None) - - return cls(_variables=payload, - exclusion=exclusions) + exclusions = payload.pop("exclusions", {}) + if exclusions is None: + exclusions = {} + elif not isinstance(exclusions, Mapping): + raise PipelineConfigurationError("Global exclusions must be a mapping.") + else: + exclusions = dict(exclusions) + + return cls(_variables=payload, exclusion=exclusions) + + @overload + def get_attributes( + self, keep_exclusion: Literal[True] = True + ) -> tuple[dict[str, Any], dict[str, Any]]: ... + + @overload + def get_attributes(self, keep_exclusion: Literal[False]) -> dict[str, Any]: ... def get_attributes(self, keep_exclusion: bool = True) -> dict[str, Any]: """ @@ -594,30 +623,29 @@ def get_attributes(self, keep_exclusion: bool = True) -> dict[str, Any]: ``self._variables`` : dict[str, Any] All global variables for the pipeline. ``self.exclusion`` : dict[str, Any] - The exclusion list of global variables for each stage. Only returned + The exclusion list of global variables for each stage. Only returned if ``keep_exclusion`` is True. """ - if keep_exclusion: - return self._variables, self.exclusion - else: - return self._variables + if keep_exclusion: + return dict(self._variables), dict(self.exclusion or {}) + return dict(self._variables) @dataclass class RunManifest: """ - Holds metadata information about the run. + Holds metadata information about the run. Parameters ---------- ``rap_name`` : str, default = "" The name of the Pipeline. ``run_id`` : str, default = "" - The unique ID of the run. + The unique ID of the run. ``git_commit`` : str, default = None The git commit number for the run, indicating the exact state of the code. ``stages_run`` : list[str] - List of the names of stages that were included in this run. + List of the names of stages that were included in this run. ``parameters`` : dict[str, Any] ``inputs`` : dict[str, Any] @@ -625,16 +653,17 @@ class RunManifest: ``outputs`` : dict[str, Any] ``backend`` : str, default = "python" - The system that the Pipeline will run in. + The system that the Pipeline will run in. ``package_versions``: list[str] or str - The package versions that are used in this run. + The package versions that are used in this run. ``timestamp`` : str, default = "" The time that this run started. ``reason`` : str, optional, default = None - The reason that this run took place. + The reason that this run took place. ``user`` : str, optional, default = None - The person running this specific run. + The person running this specific run. """ + rap_name: str = "" run_id: str = "" git_commit: Optional[str] = None @@ -665,18 +694,18 @@ def __str__(self) -> str: f"Git Commit: {self.git_commit}\nStages Run: {self.stages_run} \n" f"Parameters: \n{_format_dict(self.parameters, indent=4)} \n" f"Inputs: \n{_format_dict(self.inputs, indent=4)} \n" - f"Outputs: \n{_format_dict(self.outputs, indent = 4)} \nBackend: {self.backend} \n" + f"Outputs: \n{_format_dict(self.outputs, indent=4)} \nBackend: {self.backend} \n" f"Package Versions: {self.package_versions} \nTimestamp: {self.timestamp}\n" f"Reason: {self.reason} \nUser: {self.user}\n" ) def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the - ``RunManifest`` class. This method is structured to be more concise than + Representation method that returns a human readable representation of the + ``RunManifest`` class. This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``RunManifest`` class with its attributes. @@ -711,29 +740,30 @@ class StageResult: Parameters ---------- ``name`` : str - The name of the Stage run. + The name of the Stage run. ``status`` : StageStatus - The status of the run at completion. + The status of the run at completion. ``started_at`` : datetime - The date and time that the Stage started. + The date and time that the Stage started. ``finished_at`` : datetime - The date and time that the Stage finished. + The date and time that the Stage finished. ``outputs`` : Any, default = None Captures outputs of the stage being run. ``stdout`` : str, default = "" Captures outputs of the stage being run. ``stderr`` : str, default = "" - Captures any errors produced during the run. + Captures any errors produced during the run. ``return_code`` : int, optional, default = None Indicates whether the stage has run successfully or if there - was an error. + was an error. ``metadata``: dict[str, Any] - Holds information about the Stage such as file directories. + Holds information about the Stage such as file directories. ``error`` : str, optional, default = None - Any errors produced during the run. + Any errors produced during the run. ``source`` : str, optional, default = None - The name/location of the code for that Stage run. + The name/location of the code for that Stage run. """ + name: str status: StageStatus started_at: datetime @@ -749,9 +779,9 @@ class StageResult: @property def succeeded(self) -> bool: """ - Creates a new attribute in the ``StageResult`` class called ``succeeded`` that - contains a boolean value indicating if the run was a success or not. - Updates the ``status`` attribute to record that the Stage ran successfully. + Creates a new attribute in the ``StageResult`` class called ``succeeded`` that + contains a boolean value indicating if the run was a success or not. + Updates the ``status`` attribute to record that the Stage ran successfully. """ return self.status == StageStatus.SUCCEEDED @@ -759,7 +789,7 @@ def succeeded(self) -> bool: def duration_seconds(self) -> float: """ Creates a new attribute in the ``StageResult`` class called ``duration_seconds`` - that holds the exact duration of the stage in seconds. + that holds the exact duration of the stage in seconds. """ return max((self.finished_at - self.started_at).total_seconds(), 0.0) @@ -767,7 +797,7 @@ def duration_seconds(self) -> float: @dataclass class PipelineRun: """ - Holds information about how the whole Pipeline ran. + Holds information about how the whole Pipeline ran. Parameters ---------- @@ -776,14 +806,15 @@ class PipelineRun: ``status`` : PipelineStatus class instance Whether the Pipeline ran successfully or if there were errors. ``started_at`` : datetime - The date and time the Pipeline started. + The date and time the Pipeline started. ``completed_at`` : datetime - The date and time the Pipeline ended. + The date and time the Pipeline ended. ``stage_results`` : list[StageResult] - Holds the results for every stage run as part of the Pipeline. + Holds the results for every stage run as part of the Pipeline. ``stage_outputs`` : dict[str, Any] - Holds the outputs from all stages run as part of the Pipeline. + Holds the outputs from all stages run as part of the Pipeline. """ + manifest: RunManifest status: PipelineStatus started_at: datetime @@ -793,12 +824,12 @@ class PipelineRun: def result_for(self, stage_name: str) -> Optional[StageResult]: """ - Extracts the results for a specific stage. - + Extracts the results for a specific stage. + Parameters ---------- ``stage_name`` : str - The name of the Stage that you are requesting the results for. + The name of the Stage that you are requesting the results for. """ for result in self.stage_results: if result.name == stage_name: @@ -808,18 +839,19 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: @property def succeeded(self) -> bool: """ - Creates a new attribute in the ``PipelineRun`` class called ``succeeded`` that - contains a boolean value indicating if the Pipeline was a success or not. - Updates the ``status`` attribute to record that the Pipeline ran successfully. + Creates a new attribute in the ``PipelineRun`` class called ``succeeded`` that + contains a boolean value indicating if the Pipeline was a success or not. + Updates the ``status`` attribute to record that the Pipeline ran successfully. """ return self.status == PipelineStatus.SUCCEEDED + def _format_dict(d, indent=0): - lines = [] - for key, value in d.items(): - if isinstance(value, dict): - lines.append(f"{' ' * indent}{key}:") - lines.append(_format_dict(value, indent + 4)) - else: - lines.append(f"{' ' * indent}{key}: {value}") - return "\n".join(lines) + lines = [] + for key, value in d.items(): + if isinstance(value, dict): + lines.append(f"{' ' * indent}{key}:") + lines.append(_format_dict(value, indent + 4)) + else: + lines.append(f"{' ' * indent}{key}: {value}") + return "\n".join(lines) diff --git a/onsrap/runner.py b/onsrap/runner.py index 5641150..00381c1 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -6,10 +6,10 @@ from typing import TYPE_CHECKING from .errors import StageExecutionError -from .warnings import StageConfigurationWarning from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, RunManifest, now +from .models import PipelineRun, PipelineStatus, RunManifest, StageResult, now +from .warnings import StageConfigurationWarning if TYPE_CHECKING: from .pipeline import Pipeline @@ -17,13 +17,14 @@ class PipelineRunner: """ - Represents the information required to run the Pipeline. + Represents the information required to run the Pipeline. Parameters ---------- ``logger`` : Logger class type - Information used to log progress throughout the Pipeline. + Information used to log progress throughout the Pipeline. """ + def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() @@ -44,42 +45,40 @@ def __str__(self) -> str: def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the ``PipelineRunner`` class. - This method is structured to be more concise than the ``__str__`` method and is + Representation method that returns a human readable representation of the ``PipelineRunner`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``PipelineRunner`` class with its attributes. """ - return ( - f"PipelineRunner(logger={self.logger})" - ) + return f"PipelineRunner(logger={self.logger})" def run(self, pipeline: Pipeline) -> PipelineRun: """ - Method that runs a ``Pipeline`` instance. + Method that runs a ``Pipeline`` instance. - This method validates the source information, establishes the directories and + This method validates the source information, establishes the directories and the context to run the pipeline within, sets out the manifest for the run, attempts to run the stages in the order outlined by the ``StageGraph`` instance and logs all progress alongside relevant statuses. Before each stage executes, the runner binds the current stage name onto the ``ExecutionContext`` so ``context.stage_config`` resolves to the correct stage-specific configuration. - It returns a PipelineRun instance containing metadata and logging information for the - specific run of the whole Pipeline. + It returns a PipelineRun instance containing metadata and logging information for the + specific run of the whole Pipeline. Parameters ---------- ``pipeline`` : Pipeline - A Pipeline instance that this method will run. + A Pipeline instance that this method will run. Raises ------ ``StageExecutionError`` - If the stage is unable to be run. Logs will be created to show a failed stage. + If the stage is unable to be run. Logs will be created to show a failed stage. """ # Initial Pipeline steps - validate, create run ID and any relevant directories. pipeline.validate() @@ -92,7 +91,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: else: warnings.warn( "Output directory is not specified. Using project root or work directory as the run output.", - StageConfigurationWarning + StageConfigurationWarning, ) # TODO: fill with warnings from Pipeline branch run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) run_dir = run_output / "runs" / runtime_id.get_id() @@ -131,10 +130,12 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ) # Execution of the stages in the dependency-driven order. - stage_results = [] + stage_results: list[StageResult] = [] try: for stage in ordered_stages: - self.logger.event("Executing stage", name=stage.name, source=stage.source_label) + self.logger.event( + "Executing stage", name=stage.name, source=stage.source_label + ) context.set_active_stage(stage.name) try: result = stage.run(context, pipeline.executor) @@ -198,34 +199,38 @@ def run(self, pipeline: Pipeline) -> PipelineRun: def build_parser() -> argparse.ArgumentParser: """ - Determines what arguments are needed when running a Pipeline from the command line. + Determines what arguments are needed when running a Pipeline from the command line. - Enables stages to be input, followed by a name if provided. + Enables stages to be input, followed by a name if provided. """ - parser = argparse.ArgumentParser(description="Run an onsrap pipeline from Python files.") - parser.add_argument("stages", nargs="+", help="One or more Python stage files to run.") + parser = argparse.ArgumentParser( + description="Run an onsrap pipeline from Python files." + ) + parser.add_argument( + "stages", nargs="+", help="One or more Python stage files to run." + ) parser.add_argument("--name", default=None, help="Optional pipeline name.") return parser def main(argv: list[str] | None = None) -> int: """ - Entrypoint to the pipeline. + Entrypoint to the pipeline. This function can be called from the command line. It builds a parser which enables the arguments to be held before using those arguments to build a Pipeline instance. - The pipeline.run() method is then run which runs the entire pipeline. If this runs - successfully, a 0 is returned which is the success code. + The pipeline.run() method is then run which runs the entire pipeline. If this runs + successfully, a 0 is returned which is the success code. Parameters ---------- ``argv`` : list[str] or None - Command line arguments to parse. - + Command line arguments to parse. + Returns ------- - int - Success code for completion of the run. + int + Success code for completion of the run. """ from .pipeline import Pipeline @@ -235,7 +240,9 @@ def main(argv: list[str] | None = None) -> int: return 0 -def _log_config(run_dir: Path, context: ExecutionContext, manifest: RunManifest) -> None: +def _log_config( + run_dir: Path, context: ExecutionContext, manifest: RunManifest +) -> None: """ Outputs the configurations used in an instance of a pipeline to a YAML file in the run directory. @@ -252,17 +259,21 @@ def _log_config(run_dir: Path, context: ExecutionContext, manifest: RunManifest) """ date = context.started_at.date() - config_file = run_dir / f"configuration_for_{context.pipeline_name}_{date}_{context.run_id[-8:]}.yaml" + config_file = ( + run_dir + / f"configuration_for_{context.pipeline_name}_{date}_{context.run_id[-8:]}.yaml" + ) import yaml + with open(config_file, "w", encoding="utf-8") as f: yaml.safe_dump(manifest.config or {}, f, default_flow_style=False) def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: """ - Converts nested dictionaries or lists into flat object using dot notation for keys. + Converts nested dictionaries or lists into flat object using dot notation for keys. Each key in the resulting dictionary represents the nested branching to get to the value - in the original dictionary. + in the original dictionary. Parameters ---------- @@ -273,7 +284,7 @@ def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: ``sep`` : str The separator to use between keys in the flattened dictionary. Defaults to a dot ("."). - Returns + Returns ------- dict A flattened dictionary where each key represents the path to the value in the original object. @@ -289,13 +300,14 @@ def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: items[prefix] = obj return items + def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: """ - Calculates the differences between two YAML files and returns a programming oriented dictionary - describing the changes. - + Calculates the differences between two YAML files and returns a programming oriented dictionary + describing the changes. + This function calls the ``_flatten`` function to the loaded in dictionaries from the YAML files. - These are then differenced to account for whether a value has changed between the two files, + These are then differenced to account for whether a value has changed between the two files, been added to the second file and was not present in the first, or removed from the second file and is only present in the first. This output is structured as {changed: {}, added: {}, removed: {}}. @@ -309,7 +321,7 @@ def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: Returns ------- dict - A dictionary describing the differences between the two YAML files, structured as + A dictionary describing the differences between the two YAML files, structured as {changed: {}, added: {}, removed: {}}. """ import yaml @@ -325,24 +337,23 @@ def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: return { "changed": { - k: (flat_a[k], flat_b[k]) - for k in keys_a & keys_b - if flat_a[k] != flat_b[k] + k: (flat_a[k], flat_b[k]) for k in keys_a & keys_b if flat_a[k] != flat_b[k] }, - "added": {k: flat_b[k] for k in keys_b - keys_a}, + "added": {k: flat_b[k] for k in keys_b - keys_a}, "removed": {k: flat_a[k] for k in keys_a - keys_b}, } + def _print_diff(diff: dict) -> dict: """ - Prints the differences between two YAML files in a human-readable format and returns - the computer-readable dictionary so that it could be used for logging processes if + Prints the differences between two YAML files in a human-readable format and returns + the computer-readable dictionary so that it could be used for logging processes if required. Parameters ---------- diff : dict - A dictionary describing the differences between two YAML files, structured as + A dictionary describing the differences between two YAML files, structured as {changed: {}, added: {}, removed: {}}. Returns @@ -351,7 +362,7 @@ def _print_diff(diff: dict) -> dict: The same dictionary that was passed in as the ``diff`` parameter. """ changed = diff["changed"] - added = diff["added"] + added = diff["added"] removed = diff["removed"] if changed: @@ -374,13 +385,14 @@ def _print_diff(diff: dict) -> dict: return diff + def print_config_diffs(file_1, file_2) -> dict: """ A combining function that calculates the differences between two YAML files and then prints the outputs to the terminal as well as returning the computer-readable - dictionary of the differences. - - This works by calling the ``_diff_yaml_files`` function to calculate the differences and + dictionary of the differences. + + This works by calling the ``_diff_yaml_files`` function to calculate the differences and then calling the ``_print_diff`` function to print the differences. Parameters @@ -390,11 +402,11 @@ def print_config_diffs(file_1, file_2) -> dict: ``file_2`` : Path The path to the second YAML file to compare. - Returns + Returns ------- dict - A dictionary describing the differences between the two YAML files, structured as + A dictionary describing the differences between the two YAML files, structured as {changed: {}, added: {}, removed: {}}. """ diff_dict = _diff_yaml_files(file_1, file_2) - return _print_diff(diff_dict) \ No newline at end of file + return _print_diff(diff_dict) From c116d68b3d31746d4be50ab39185f161bbf6ee77 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 14:02:50 +0100 Subject: [PATCH 247/332] tweak: tweaked approach to loggers to improve memory efficiency --- onsrap/logger.py | 80 +++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index ff065f1..013420a 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -15,12 +15,13 @@ class LogConfig: Parameters ---------- ``log_dir`` : str, default = "logs/" - The directory where all logs are stored for the Pipeline. + The directory where all logs are stored for the Pipeline. ``log_level`` : str, default = "INFO" - Denotes how severe the log message is. + Denotes how severe the log message is. ``logger_name`` : str, default = "onsrap" The name of the logging system. """ + log_dir: str = "logs/" log_level: str = "INFO" logger_name: str = "onsrap" @@ -28,21 +29,24 @@ class LogConfig: class Logger: """ - Creates a logging system. + Creates a logging system. This system creates a logging directory and enables writing the log messages to both console and the logging files. It allows configurable logging levels - to adjust for severity and avoids duplicating logging messages or handlers. - If the logger is unable to write to a file, the logging continues using only + to adjust for severity and avoids duplicating logging messages or handlers. + If the logger is unable to write to a file, the logging continues using only the console handler. Parameters ---------- ``log_dir`` : str or Path, default = "logs/" - The directory where you'd like your logs stored. - ``log_level`` : str, default = "INFO" - The severity of the log. + The directory where you'd like your logs stored. + ``log_level`` : str, default = "INFO" + The severity of the log. """ + + _configured_loggers: set[str] = set() + def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.config = LogConfig(log_dir=str(log_dir), log_level=log_level) self.log_dir = Path(self.config.log_dir) @@ -50,8 +54,10 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): logger_name = f"{self.config.logger_name}:{self.log_dir.resolve()}" self._logger = logging.getLogger(logger_name) - if not getattr(self._logger, "_onsrap_configured", False): - self._logger.setLevel(getattr(logging, self.config.log_level.upper(), logging.INFO)) + if logger_name not in self._configured_loggers: + self._logger.setLevel( + getattr(logging, self.config.log_level.upper(), logging.INFO) + ) self._logger.propagate = False stream_handler = logging.StreamHandler() @@ -59,21 +65,23 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self._logger.addHandler(stream_handler) try: - file_handler = logging.FileHandler(self.log_dir / "onsrap.log", encoding="utf-8") + file_handler = logging.FileHandler( + self.log_dir / "onsrap.log", encoding="utf-8" + ) file_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self._logger.addHandler(file_handler) except OSError: pass - setattr(self._logger, "_onsrap_configured", True) + self._configured_loggers.add(logger_name) def __call__(self, *args: Any, **kwargs: Any) -> None: """ Converts Logger instances to be callable, enabling easier implementation - of logging. + of logging. - Positional arguemnts are converted to strings and joined with spaces. - Keyword arguments are serialised as JSON and appended as structured + Positional arguemnts are converted to strings and joined with spaces. + Keyword arguments are serialised as JSON and appended as structured context. """ message = " ".join(str(arg) for arg in args) @@ -99,47 +107,49 @@ def __str__(self) -> str: def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the - ``Logger`` class. This method is structured to be more concise than + Representation method that returns a human readable representation of the + ``Logger`` class. This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``Logger`` class with its attributes. """ - return ( - f"Logger(log_dir={self.log_dir.resolve()}, log_level={self.config.log_level})" - ) - + return f"Logger(log_dir={self.log_dir.resolve()}, log_level={self.config.log_level})" + def event(self, message: str, **kwargs: Any) -> None: """ - Logs a named event with optional structured context. + Logs a named event with optional structured context. Parameters ---------- - ``message`` : str - The main description of the event to be logged. - ``**kwargs`` : Any - Additional information to be recorded in the log record. + ``message`` : str + The main description of the event to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. """ if kwargs: - self._logger.info("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) + self._logger.info( + "%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True) + ) else: self._logger.info(message) def warning(self, message: str, **kwargs: Any) -> None: """ - Logs a warning message with optional structured context. + Logs a warning message with optional structured context. Parameters ---------- - ``message`` : str - The main description of the warning to be logged. - ``**kwargs`` : Any - Additional information to be recorded in the log record. + ``message`` : str + The main description of the warning to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. """ if kwargs: - self._logger.warning("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) + self._logger.warning( + "%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True) + ) else: - self._logger.warning(message) \ No newline at end of file + self._logger.warning(message) From 22cd71c6996d9534a8d023495081f77669ff5bf7 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Mon, 10 Aug 2026 14:03:33 +0100 Subject: [PATCH 248/332] tweak: minor changes in code style and specificities. --- onsrap/errors.py | 16 ++++++++++++---- onsrap/graph.py | 19 +++++++++++-------- onsrap/loader.py | 36 +++++++++++++++++++----------------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index bd29f9f..68650d7 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -1,5 +1,10 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .models import StageResult + class OnsrapError(Exception): """Base exception for onsrap.""" @@ -52,7 +57,7 @@ def __init__( stage_name: str | None = None, source: str | None = None, original_exception: Exception | None = None, - result: object | None = None, + result: StageResult | None = None, ): super().__init__(message) self.stage_name = stage_name @@ -67,20 +72,23 @@ class StageLoadError(StageExecutionError): Child class with ``StageExecutionError`` as the parent class. """ + class StageDependencyError(OnsrapError): """ Raised when incorrect inputs are provided to the dependency attribute of a Stage. """ + class PipelineInitialisationError(OnsrapError): """ - Raised when there is an error in definition of the Pipeline + Raised when there is an error in definition of the Pipeline instance """ + class PipelineConfigurationError(OnsrapError): """ Raised when there has been an issue with the PipelineConfig - instance. - """ \ No newline at end of file + instance. + """ diff --git a/onsrap/graph.py b/onsrap/graph.py index 6da5931..efacc13 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -12,12 +12,13 @@ class StageGraph: """ Represents an order to run stages. - Holds an order that stages need to run in based on dependencies and logic. + Holds an order that stages need to run in based on dependencies and logic. Parameters ---------- ``stages`` : list of ``Stage`` class items """ + stages: list[Stage] = field(default_factory=list) @classmethod @@ -42,7 +43,7 @@ def validate(self) -> None: Raises ------ ``DuplicateStageError`` - If the stage name appears multiple times in the stage list. + If the stage name appears multiple times in the stage list. ``MissingDependencyError`` If there are unknown dependencies. """ @@ -94,10 +95,10 @@ def topological_order(self) -> list[Stage]: stages that depend on it. That may free up more stages, which are then added to the ready list. - Returns + Returns ------- - A list of stages ordered in the way that they need to be run through the - pipeline. + A list of stages ordered in the way that they need to be run through the + pipeline. Raises ------ @@ -106,8 +107,10 @@ def topological_order(self) -> list[Stage]: cycle or a dependency that could not be resolved. """ stage_by_name = {stage.name: stage for stage in self.stages} - incoming = {stage.name: set(stage.dependencies) for stage in self.stages} - dependents = {stage.name: set() for stage in self.stages} + incoming: dict[str, set[str]] = { + stage.name: set(stage.dependencies) for stage in self.stages + } + dependents: dict[str, set[str]] = {stage.name: set() for stage in self.stages} for stage in self.stages: for dependency in stage.dependencies: @@ -119,7 +122,7 @@ def topological_order(self) -> list[Stage]: original_order = [stage.name for stage in self.stages] ready = [name for name in original_order if not incoming[name]] - ordered = [] + ordered: list[str] = [] while ready: current = ready.pop(0) diff --git a/onsrap/loader.py b/onsrap/loader.py index 80c01ab..444d84b 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -26,12 +26,12 @@ def discover_python_entrypoint(path: Path) -> str | None: Parameters ---------- - ``path`` : Path - File path for the stage being run. + ``path`` : Path + File path for the stage being run. Returns ------- - String item containing the name of the ``PREFERRED_ENTRYPOINTS`` item relevant + String item containing the name of the ``PREFERRED_ENTRYPOINTS`` item relevant for the stages. ``None`` when the file exists but does not define a preferred callable, which signals to the executor that it should treat the file as a @@ -42,10 +42,12 @@ def discover_python_entrypoint(path: Path) -> str | None: ``StageConfigurationError`` If the file path requested for the ``Stage`` does not exist. """ - + file_path = Path(path) if not file_path.exists(): - raise StageConfigurationError("Stage source file does not exist: {0}".format(file_path)) + raise StageConfigurationError( + "Stage source file does not exist: {0}".format(file_path) + ) try: tree = ast.parse(file_path.read_text(encoding="utf-8"), filename=str(file_path)) @@ -79,19 +81,19 @@ def load_python_callable(path: Path, entrypoint: str) -> Any: Parameters ---------- ``path`` : Path - The file path for the stage being run. + The file path for the stage being run. ``entrypoint`` : str The name of the entrypoint function defined in the stage script. Raises ------ - ``StageConfigurationError`` - If the chosen entrypoint does not exist or is not callable, because that means + ``StageConfigurationError`` + If the chosen entrypoint does not exist or is not callable, because that means the stage definition and the executable surface no longer agree. Returns ------- - ``target`` + ``target`` The ``entrypoint`` attribute of the module called to run the stage. """ module = load_python_module(path) @@ -116,21 +118,21 @@ def load_python_module(path: Path) -> ModuleType: The generated name is derived from the file path so repeated loads of the same stage remain stable during a run, while still avoiding collisions with - other Python modules. - + other Python modules. + Parameters ---------- - ``path`` : Path - The path for the stage. + ``path`` : Path + The path for the stage. Returns ------- ``module`` - The set of code being run for the stage. - + The set of code being run for the stage. + Raises ------ - ``StageLoadError`` + ``StageLoadError`` If the file is unable to be imported so callers can report a stage-specific problem rather than a raw import exception. """ @@ -140,7 +142,7 @@ def load_python_module(path: Path) -> ModuleType: module_name = "onsrap_stage_{0}_{1}".format( file_path.stem, - hashlib.sha1(str(file_path.resolve()).encode("utf-8")).hexdigest()[:12], + hashlib.sha256(str(file_path.resolve()).encode("utf-8")).hexdigest()[:12], ) spec = importlib.util.spec_from_file_location(module_name, str(file_path)) if spec is None or spec.loader is None: From 22cc45150c77f3edfd80ed832103bf4c5fe684fc Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:17:55 +0100 Subject: [PATCH 249/332] tweak: adds potential issue for review in PR --- onsrap/logger.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 7fad6fa..658f8d1 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -179,6 +179,9 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: matches: list[dict[str, Any]] = [] + #TODO: This method works if the logs are recorded in chronological order. Would there + #ever be a case where a record would appear below another and not be chronological? + #If so, we may need to sort based on the timestamp rather than the ordering. for raw_line in reversed(Path(logfile_path).read_text(encoding="utf-8").splitlines()): if "Pipeline started" not in raw_line or " | " not in raw_line: continue From b11e2617964ac2d59da2a2351421730b51981f86 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:32:05 +0100 Subject: [PATCH 250/332] tweak: extract_historical_run_ids to exclude if the timestamp is incorrect --- onsrap/logger.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 658f8d1..fb384b7 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json import logging from dataclasses import dataclass @@ -189,6 +190,17 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: # left: timestamp + message, right: JSON context left, right = raw_line.split(" | ", 1) + #catches where Pipeline started is not recorded in the correct place. + if not left.endswith(" Pipeline started"): + continue + + #checks that datetime is valid + timestamp = left[:-len(" Pipeline started")].strip() + try: + datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S,%f") + except ValueError: + continue + try: payload = json.loads(right) except json.JSONDecodeError: From 52193d5104e0ca14cf3607039bbcabab5fd4c305 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:47:48 +0100 Subject: [PATCH 251/332] tests: implement tests for extract_historical_run_ids --- tests/test_pipeline.py | 289 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 287 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 55d6135..bcb1a07 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,11 +1,14 @@ +import logging from unittest import mock import warnings from onsrap.pipeline import Pipeline, PipelineConfig -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError +from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError from onsrap.models import PipelineRun, PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning +from onsrap.logger import Logger + from pathlib import Path import pytest @@ -540,6 +543,288 @@ def test_no_errors_raised_success_load_latest_run(self, assert not any(issubclass(warning.category, PipelineConfigurationWarning) for warning in w) +class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): + def test_logger_no_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. - + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers.clear() # Remove all handlers to simulate no file logging + logger._logger.propagate = False # Prevent checking root logger handlers + with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_no_file_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no file handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers = [logging.StreamHandler()] + with pytest.raises(HistoricalPipelineLoadError, match="does not have a FileHandler"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_does_not_exist(self, + tmp_path:Path) -> None: + """ + Checks that the method raises an error if the log doesn't exist at the + location specified. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + file_handler.close() + log_path.unlink(missing_ok=True) # Remove the log file to simulate non-existence + + with pytest.raises(HistoricalPipelineLoadError, + match="does not exist at this location"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_return_blank_list_no_matches_in_log(self, + tmp_path) -> None: + """ + Tests that a log file that does not have a record covering "Pipeline started" + will return a blank list from extract_historical_run_ids. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Some unrelated log entry\n" \ + "2026-08-10 10:00:01,000 Another unrelated log entry\n") + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] + + def test_skips_poor_json_in_log(self, + tmp_path: Path) -> None: + """ + Tests that if a JSON record in the log file is not valid, it will e skipped + and the next valid entry will be extracted. Assert that the returned list + contains only the valid entry. Confirms that only the incorrect record is + skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" \ + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_abc1234", + "timestamp": "2026-08-10 10:00:02,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_abc1234" + }, + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + @pytest.mark.parametrize("string, expected", + [('{"some_key":"some_value"}', []), + ('{"run_id":""}',[])]) + + def test_run_id_absent_falsy(self, + tmp_path: Path, + string: str, + expected: list) -> None: + """ + Tests that if the JSON record in the log file does not have a run_id, it will be skipped + and the returned list will be empty. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``string`` : str + A dictionary representing a valid JSON record in the log file that excludes + run_id. + ``expected`` : list + The expected output from extract_historical_run_ids when the log file + contains a record without a run_id. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") + + #creates directory for runs to avoid removal given the directory doesn't exist + (tmp_path/"runs").mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == expected + + def test_records_only_if_directory_exists(self, + tmp_path) -> None: + """ + Checks that a record is only output if the run directory exists. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + def test_reverse_chronological_order(self, + tmp_path) -> None: + """ + Checks that the run_ids are output in reverse chronological order + based on their positioning in the log file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result[0]["run_id"] == "2026-06-23_101719_abc1234" + assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" + + def test_skip_poor_timestamps(self, + tmp_path) -> None: + """ + Checks that entries with poor timestamps are skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "BADTIMESTAMP Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] From 9c40364bb7b3dbbb55816617220fe95ad6edddd3 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:55:46 +0100 Subject: [PATCH 252/332] Apply suggestions from code review Apply copilot suggestions Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- onsrap/pipeline.py | 7 ++++--- tests/test_pipeline.py | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0a51cf1..1e32b40 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -1027,17 +1027,18 @@ def _generate_context(self) -> None: # Check all stages have the same backend as Pipeline self._validate_stage_backends() - execution_class_name = f"{self.backend.capitalize()}StageExecutor" + backend_key = str(self.backend).strip().lower() + execution_class_name = f"{backend_key.capitalize()}StageExecutor" if (executor_class := globals().get(execution_class_name)) is not None: self.executor = executor_class() else: raise PipelineInitialisationError( - f"Requested backend {self.backend} does not have a compatible executor. " + f"Requested backend {backend_key} does not have a compatible executor. " f"Available executors are: {', '.join(AVAILABLE_EXECUTORS)}." ) - def _validate_stage_backends(self) -> str: + def _validate_stage_backends(self) -> None: """ Checks the backends that have been assigned to each stage. diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7305209..f66f8d5 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -302,9 +302,33 @@ def test_validate_stage_backends_errors() -> None: """ with pytest.raises(PipelineInitialisationError): - Pipeline(backend="python", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=(), backend="nonexistent_backend")]) - with pytest.raises(PipelineInitialisationError): - Pipeline(backend="python", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=(), backend="nonexistent_backend"), - Stage("Stage_1", source=Path("Stage_1.py"), dependencies=(), backend="python")]) \ No newline at end of file + Pipeline( + backend="python", + stages=[ + Stage( + "Stage_0", + source=Path("Stage_0.py"), + dependencies=(), + backend="nonexistent_backend", + ) + ], + ) + + with pytest.raises(PipelineInitialisationError): + Pipeline( + backend="python", + stages=[ + Stage( + "Stage_0", + source=Path("Stage_0.py"), + dependencies=(), + backend="nonexistent_backend", + ), + Stage( + "Stage_1", + source=Path("Stage_1.py"), + dependencies=(), + backend="python", + ), + ], + ) \ No newline at end of file From 108a7c89d27a1078c38211c59145a257db60717a Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 15:53:46 +0100 Subject: [PATCH 253/332] feat: adds to_dict and from_dict methods to PipelineRun, StageResult, and RunManifest in prep for saving out to a YAML. Also added documentation to _format_dict() --- onsrap/models.py | 197 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 8 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8fe1c21..7640d66 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -690,6 +690,67 @@ def __repr__(self) -> str: f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" ) + def runmanifest_to_dict(self) -> dict[str, Any]: + """ + Converts the RunManifest instance into a dictionary representation. + This is needed to allow a RunManifest instance to be serialized into a + JSON format for later methods on RunManifest instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the RunManifest instance. + """ + return { + "rap_name": self.rap_name, + "run_id": self.run_id, + "git_commit": self.git_commit, + "stages_run": self.stages_run, + "parameters": self.parameters, + "inputs": self.inputs, + "outputs": self.outputs, + "backend": self.backend, + "package_versions": self.package_versions, + "timestamp": self.timestamp, + "reason": self.reason, + "user": self.user, + "config": self.config + } + + @classmethod + def runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: + """ + Converts a dictionary representation of a RunManifest instance back into a + RunManifest instance. Allows for RunManifest instances to be created from + a JSON representation of a RunManifest instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a RunManifest instance. + + Returns + ------- + ``RunManifest`` class instance + A RunManifest instance created from the dictionary representation. + """ + return cls( + rap_name=data.get("rap_name", ""), + run_id=data.get("run_id", ""), + git_commit=data.get("git_commit"), + stages_run=data.get("stages_run", []), + parameters=data.get("parameters", {}), + inputs=data.get("inputs", {}), + outputs=data.get("outputs", {}), + backend=data.get("backend", "python"), + package_versions=data.get("package_versions", []), + timestamp=data.get("timestamp", ""), + reason=data.get("reason"), + user=data.get("user"), + config=data.get("config") + ) + class RAPDataset: def __init__(self): @@ -746,6 +807,62 @@ class StageResult: error: Optional[str] = None source: Optional[str] = None + def _stage_result_to_dict(self, name = True) -> dict[str, Any]: + """ + Converts the StageResult instance into a dictionary representation. + This is needed to allow a StageResult instance to be serialized into a + JSON format for later methods on StageResult instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the StageResult instance. + """ + return { + **({"name": self.name} if name else {}), + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "finished_at": self.finished_at.isoformat(), + "outputs": self.outputs, + "stdout": self.stdout, + "stderr": self.stderr, + "return_code": self.return_code, + "metadata": self.metadata, + "error": self.error, + "source": self.source, + } + + def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: + """ + Converts a dictionary representation of a StageResult instance back into a + StageResult instance. Allows for StageResult instances to be created from + a JSON representation of a StageResult instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a StageResult instance. + + Returns + ------- + ``StageResult`` class instance + A StageResult instance created from the dictionary representation. + """ + return cls( + name=data["name"], + status=StageStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + finished_at=datetime.fromisoformat(data["finished_at"]), + outputs=data.get("outputs"), + stdout=data.get("stdout", ""), + stderr=data.get("stderr", ""), + return_code=data.get("return_code"), + metadata=data.get("metadata", {}), + error=data.get("error"), + source=data.get("source"), + ) + @property def succeeded(self) -> bool: """ @@ -805,6 +922,55 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: return result return None + def pipeline_run_to_dict(self) -> dict[str, Any]: + """ + Converts the PipelineRun instance into a dictionary representation. + This is needed to allow a PipelineRun instance to be serialized into a + JSON format for later methods on PipelineRun instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the PipelineRun instance. + """ + return { + "manifest": self.manifest.runmanifest_to_dict(), + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "completed_at": self.completed_at.isoformat(), + "stage_results": {result.name:result._stage_result_to_dict(name = False) + for result in self.stage_results}, + "stage_outputs": self.stage_outputs, + } + + @classmethod + def pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: + """ + Converts a dictionary representation of a PipelineRun instance back into a + PipelineRun instance. Allows for PipelineRun instances to be created from + a JSON representation of a PipelineRun instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` class instance + A PipelineRun instance created from the dictionary representation. + """ + return cls( + manifest=RunManifest.runmanifest_from_dict(data["manifest"]), + status=PipelineStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + completed_at=datetime.fromisoformat(data["completed_at"]), + stage_results=[StageResult._stage_result_from_dict(result) + for result in data.get("stage_results", {}).values()], + stage_outputs=data.get("stage_outputs", {}), + ) + @property def succeeded(self) -> bool: """ @@ -815,11 +981,26 @@ def succeeded(self) -> bool: return self.status == PipelineStatus.SUCCEEDED def _format_dict(d, indent=0): - lines = [] - for key, value in d.items(): - if isinstance(value, dict): - lines.append(f"{' ' * indent}{key}:") - lines.append(_format_dict(value, indent + 4)) - else: - lines.append(f"{' ' * indent}{key}: {value}") - return "\n".join(lines) + """ + Helper function to format dictionaries for __str__ methods. + + Parameters + ---------- + ``d`` : dict + The dictionary to format. + ``indent`` : int, default = 0 + The number of spaces to indent the dictionary representation. + + Returns + ------- + str + A formatted string representation of the dictionary. + """ + lines = [] + for key, value in d.items(): + if isinstance(value, dict): + lines.append(f"{' ' * indent}{key}:") + lines.append(_format_dict(value, indent + 4)) + else: + lines.append(f"{' ' * indent}{key}: {value}") + return "\n".join(lines) From b90d66fa7d58040bb0ab0a5d8dc3f49b84c5aeba Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 16:26:21 +0100 Subject: [PATCH 254/332] tests: adds tests for to_dict and from_dict methods for PipelineRun, StageResult, RunManifest --- onsrap/models.py | 11 +-- tests/test_models.py | 188 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 192 insertions(+), 7 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 7640d66..72b7bed 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -807,7 +807,7 @@ class StageResult: error: Optional[str] = None source: Optional[str] = None - def _stage_result_to_dict(self, name = True) -> dict[str, Any]: + def _stage_result_to_dict(self) -> dict[str, Any]: """ Converts the StageResult instance into a dictionary representation. This is needed to allow a StageResult instance to be serialized into a @@ -820,7 +820,7 @@ def _stage_result_to_dict(self, name = True) -> dict[str, Any]: A dictionary representation of the StageResult instance. """ return { - **({"name": self.name} if name else {}), + "name": self.name, "status": self.status.value, "started_at": self.started_at.isoformat(), "finished_at": self.finished_at.isoformat(), @@ -833,6 +833,7 @@ def _stage_result_to_dict(self, name = True) -> dict[str, Any]: "source": self.source, } + @classmethod def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: """ Converts a dictionary representation of a StageResult instance back into a @@ -922,7 +923,7 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: return result return None - def pipeline_run_to_dict(self) -> dict[str, Any]: + def _pipeline_run_to_dict(self) -> dict[str, Any]: """ Converts the PipelineRun instance into a dictionary representation. This is needed to allow a PipelineRun instance to be serialized into a @@ -939,13 +940,13 @@ def pipeline_run_to_dict(self) -> dict[str, Any]: "status": self.status.value, "started_at": self.started_at.isoformat(), "completed_at": self.completed_at.isoformat(), - "stage_results": {result.name:result._stage_result_to_dict(name = False) + "stage_results": {result.name:result._stage_result_to_dict() for result in self.stage_results}, "stage_outputs": self.stage_outputs, } @classmethod - def pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: + def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: """ Converts a dictionary representation of a PipelineRun instance back into a PipelineRun instance. Allows for PipelineRun instances to be created from diff --git a/tests/test_models.py b/tests/test_models.py index 7e7d0b4..0f5f0ff 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,4 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig +from onsrap.models import StageResult, StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig import pytest import datetime from pathlib import Path @@ -279,4 +279,188 @@ def test_succeeded_pipeline(pipelinerun, status, expected) -> None: assert pipelinerun.succeeded == expected -#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file +#TODO: Test _extract_stages_run and all methods in StageConfig class + +class TestToFromDictMethods: + """ + Class to store testing methods for to_dict and from_dict, specifically + for PipelineRun, RunManifest, and StageResult classes. + """ + @pytest.fixture + def runmanifest(self) -> RunManifest: + return RunManifest("pipeline", + "1", + None, + ["stage1","stage2"], + {"uniqueID":"example"}, + {"input_path":"input/data/example.csv"}, + {"output_path":"output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None) + @pytest.fixture + def stageresult(self) -> StageResult: + return StageResult("stage_test", + StageStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + "example output", + "", + "", + None, + {}, + None, + None) + + @pytest.fixture + def pipelinerun(self, runmanifest, stageresult) -> PipelineRun: + return PipelineRun(runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024,5,6,15,45,30), + datetime.datetime(2024,5,7,15,45,30), + [stageresult], + {"stage_test":"example output"}) + + def test_runmanifest_to_dict(self, runmanifest) -> None: + """ + Test that the to_dict method for RunManifest outputs the correct dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + expected_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config":None + } + assert runmanifest.runmanifest_to_dict() == expected_dict + + def test_runmanifest_from_dict(self, runmanifest) -> None: + """ + Test that the from_dict method for RunManifest correctly creates a RunManifest instance from a dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + runmanifest_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config":None + } + new_runmanifest = RunManifest.runmanifest_from_dict(runmanifest_dict) + assert new_runmanifest == runmanifest + + def test_stageresult_to_dict(self, stageresult) -> None: + """ + Test that the to_dict method for StageResult outputs the correct dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + expected_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None + } + assert stageresult._stage_result_to_dict() == expected_dict + + def test_stageresult_from_dict(self, stageresult) -> None: + """ + Test that the from_dict method for StageResult correctly creates a StageResult instance from a dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + stageresult_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None + } + new_stageresult = StageResult._stage_result_from_dict(stageresult_dict) + assert new_stageresult == stageresult + + def test_pipelinerun_to_dict(self, pipelinerun) -> None: + """ + Test that the to_dict method for PipelineRun outputs the correct dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + expected_dict = { + "manifest": pipelinerun.manifest.runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, + "stage_outputs": {"stage_test": "example output"} + } + assert pipelinerun._pipeline_run_to_dict() == expected_dict + + def test_pipelinerun_from_dict(self, pipelinerun) -> None: + """ + Test that the from_dict method for PipelineRun correctly creates a PipelineRun instance from a dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + pipelinerun_dict = { + "manifest": pipelinerun.manifest.runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, + "stage_outputs": {"stage_test": "example output"} + } + new_pipelinerun = PipelineRun._pipeline_run_from_dict(pipelinerun_dict) + assert new_pipelinerun == pipelinerun + From f058b8fab635c4f7b36408e99e9003da324a48a3 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:00:33 +0100 Subject: [PATCH 255/332] test: add test to check that log_pipeline_attributes correctly writes out to a YAML and the contents of that YAML can be correctly parsed back to a PipelineRun instance. --- tests/test_runner.py | 89 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 225413f..db0e25c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -8,8 +8,8 @@ from onsrap.execution import ExecutionContext from onsrap.logger import Logger -from onsrap.models import PipelineConfig, RunManifest -from onsrap.runner import _log_config, print_config_diffs +from onsrap.models import PipelineConfig, PipelineStatus, RunManifest, PipelineRun, StageResult, StageStatus, now +from onsrap.runner import _log_config, print_config_diffs, _log_pipeline_attributes def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: @@ -139,4 +139,87 @@ def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) assert "CHANGED (1)" in captured.out assert "ADDED in second configuration (1)" in captured.out assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + assert "stage_configs.stage_a.target_variable" in captured.out + +class TestRunInfoWriteOut: + def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_pipeline_attributes`` function correctly writes the pipeline attributes to a YAML file. + """ + + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True, exist_ok=True) + + pipeline_config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=pipeline_config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + stage_results = [ + StageResult( + name="stage_a", + status=StageStatus.SUCCEEDED, + started_at=now(), + finished_at= now(), + error=None, + source=None, + ) + ] + + run_manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + ) + + pipeline_run = PipelineRun( + manifest=run_manifest, + status=PipelineStatus.SUCCEEDED, + started_at=context.started_at, + completed_at=now(), + stage_results=stage_results, + stage_outputs={}, + ) + + _log_pipeline_attributes( + pipeline_run=pipeline_run, + run_dir=run_dir, + context=context + ) + + expected_file = run_dir / ( + "pipeline_attributes_for_" + f"{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + + expected_contents = pipeline_run._pipeline_run_to_dict() + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + assert parsed_yaml == expected_contents + + assert PipelineRun._pipeline_run_from_dict(parsed_yaml) == pipeline_run + + + \ No newline at end of file From dc41be7619309bbed9c1d35fca7c56cce2cc0d40 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:01:00 +0100 Subject: [PATCH 256/332] tweak: minor change to _log_pipeline_attributes() to remove StageResult parameter as this is covered in PipelineRun already --- onsrap/runner.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/onsrap/runner.py b/onsrap/runner.py index 5641150..8658a38 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -9,7 +9,7 @@ from .warnings import StageConfigurationWarning from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, RunManifest, now +from .models import PipelineRun, PipelineStatus, RunManifest, StageResult, now if TYPE_CHECKING: from .pipeline import Pipeline @@ -165,6 +165,13 @@ def run(self, pipeline: Pipeline) -> PipelineRun: ) pipeline.manifest = manifest pipeline.last_run = run + + #Creates attributes file in the run_directory to log information for later + #analysis of pipeline runs + _log_pipeline_attributes(pipeline_run = run, + run_dir = run_dir, + context = context) + self.logger.event( "Pipeline failed", name=pipeline.name, @@ -186,6 +193,12 @@ def run(self, pipeline: Pipeline) -> PipelineRun: pipeline.manifest = manifest pipeline.last_run = run + #Creates attributes file in the run_directory to log information for later + #analysis of pipeline runs + _log_pipeline_attributes(pipeline_run = run, + run_dir = run_dir, + context = context) + self.logger.event( "Pipeline completed", name=pipeline.name, @@ -234,6 +247,36 @@ def main(argv: list[str] | None = None) -> int: pipeline.run() return 0 +def _log_pipeline_attributes(pipeline_run: PipelineRun, + run_dir: Path, + context: ExecutionContext) -> None: + """ + Creates a YAML file within the run directory that contains information + regarding PipelineRun and StageResult instances for the run. This is + later used to extract information about previous runs which are not + currently stored in memory. + + Parameters + ---------- + ``pipeline_run`` : PipelineRun + The PipelineRun instance for the current run of the pipeline. + ``stage_results`` : list[StageResult] + A list of StageResult instances for the current run of the pipeline. + ``run_dir`` : Path + The directory where the pipeline run is being currently being executed. + ``context`` : ExecutionContext + The context of the current pipeline run, containing configuration and + state information. + """ + attributes_file = run_dir / f"pipeline_attributes_for_{context.pipeline_name}_{context.run_id[-8:]}.yaml" + import yaml + with open(attributes_file, "w", encoding="utf-8") as f: + yaml.safe_dump( + pipeline_run._pipeline_run_to_dict(), + f, + default_flow_style=False + ) + def _log_config(run_dir: Path, context: ExecutionContext, manifest: RunManifest) -> None: """ From 4d152555b4b8fc01bee6999f1fb31d1c1b1410f8 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 17:11:14 +0100 Subject: [PATCH 257/332] feat: method for loading a PipelineRun instance from a previous run file --- onsrap/loader.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/onsrap/loader.py b/onsrap/loader.py index 80c01ab..25f1504 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -8,6 +8,8 @@ from types import ModuleType from typing import Any +from onsrap.models import PipelineRun + from .errors import StageConfigurationError, StageLoadError PREFERRED_ENTRYPOINTS = ("run", "main", "execute") @@ -158,3 +160,36 @@ def load_python_module(path: Path) -> ModuleType: ) from exc return module + +def load_pipeline_run_for_historical_run(file_path: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + This function is used to load the state of a pipeline run that has been + saved to a YAML file. It reads the file, parses the YAML content, and + reconstructs the PipelineRun object. + + Parameters + ---------- + ``file_path`` : Path + The path to the YAML file containing the saved pipeline run. + + Returns + ------- + ``PipelineRun`` + The reconstructed PipelineRun object. + + Raises + ------ + ``FileNotFoundError`` + If the specified file does not exist. + """ + import yaml + + if not file_path.exists(): + raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file From b881574127c18d7f0011683b85d831dcd3525cce Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 08:28:47 +0100 Subject: [PATCH 258/332] tweak: move load_pipeline_run_for_historical_run() method into PipelineRun class as a class method rather than in loader.py --- onsrap/loader.py | 32 -------------------------------- onsrap/models.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/onsrap/loader.py b/onsrap/loader.py index 25f1504..69098e7 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -161,35 +161,3 @@ def load_python_module(path: Path) -> ModuleType: return module -def load_pipeline_run_for_historical_run(file_path: Path) -> PipelineRun: - """ - Load a previously executed pipeline run from a YAML file. - - This function is used to load the state of a pipeline run that has been - saved to a YAML file. It reads the file, parses the YAML content, and - reconstructs the PipelineRun object. - - Parameters - ---------- - ``file_path`` : Path - The path to the YAML file containing the saved pipeline run. - - Returns - ------- - ``PipelineRun`` - The reconstructed PipelineRun object. - - Raises - ------ - ``FileNotFoundError`` - If the specified file does not exist. - """ - import yaml - - if not file_path.exists(): - raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") - - with open(file_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - - return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file diff --git a/onsrap/models.py b/onsrap/models.py index 72b7bed..9bdf534 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -972,6 +972,40 @@ def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: stage_outputs=data.get("stage_outputs", {}), ) + @classmethod + def load_pipeline_run_for_historical_run(cls, file_path: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + This function is used to load the state of a pipeline run that has been + saved to a YAML file. It reads the file, parses the YAML content, and + reconstructs the PipelineRun object. + + Parameters + ---------- + ``file_path`` : Path + The path to the YAML file containing the saved pipeline run. + + Returns + ------- + ``PipelineRun`` + The reconstructed PipelineRun object. + + Raises + ------ + ``FileNotFoundError`` + If the specified file does not exist. + """ + import yaml + + if not file_path.exists(): + raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return cls._pipeline_run_from_dict(data) + @property def succeeded(self) -> bool: """ From 7949826588ebd9487a75856e6e1774bfdf45a5ef Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 08:47:12 +0100 Subject: [PATCH 259/332] tweak: add leading _ to runmanifest_to_dict and runmanifest_from_dict methods --- onsrap/models.py | 4 ++-- tests/test_models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 9bdf534..67f2f32 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -690,7 +690,7 @@ def __repr__(self) -> str: f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" ) - def runmanifest_to_dict(self) -> dict[str, Any]: + def _runmanifest_to_dict(self) -> dict[str, Any]: """ Converts the RunManifest instance into a dictionary representation. This is needed to allow a RunManifest instance to be serialized into a @@ -719,7 +719,7 @@ def runmanifest_to_dict(self) -> dict[str, Any]: } @classmethod - def runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: + def _runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: """ Converts a dictionary representation of a RunManifest instance back into a RunManifest instance. Allows for RunManifest instances to be created from diff --git a/tests/test_models.py b/tests/test_models.py index 0f5f0ff..b397497 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -347,7 +347,7 @@ def test_runmanifest_to_dict(self, runmanifest) -> None: "user": None, "config":None } - assert runmanifest.runmanifest_to_dict() == expected_dict + assert runmanifest._runmanifest_to_dict() == expected_dict def test_runmanifest_from_dict(self, runmanifest) -> None: """ @@ -373,7 +373,7 @@ def test_runmanifest_from_dict(self, runmanifest) -> None: "user": None, "config":None } - new_runmanifest = RunManifest.runmanifest_from_dict(runmanifest_dict) + new_runmanifest = RunManifest._runmanifest_from_dict(runmanifest_dict) assert new_runmanifest == runmanifest def test_stageresult_to_dict(self, stageresult) -> None: From 42227d462fd43f64c9f96907327468d6c879ddbe Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:15:37 +0100 Subject: [PATCH 260/332] tweak: move run_output derivation into Pipeline init phase --- onsrap/pipeline.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 1e32b40..2fe6e33 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -108,6 +108,8 @@ def __init__( self.manifest: RunManifest | None = None self.last_run: PipelineRun | None = None + self.run_output = self._set_run_output() + self.logger.event( "Pipeline initialized", name=self.name, @@ -419,7 +421,31 @@ def add_dependencies(self, self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + def _set_run_output(self) -> Path: + """ + Private method that sets the run output directory for the Pipeline. + + Returns + ------- + ``Path`` + The path to the run output directory for the Pipeline. + Raises + ------ + ``StageConfigurationWarning`` + If the output_dir is not specified in the PipelineConfig, a warning is + raised to show that the project root or work directory will be used as + the directory for the run outputs. + """ + if self.config.output_dir is not None: + run_output = Path(self.config.output_dir) + else: + warnings.warn( + "Output directory is not specified. Using project root or work directory as the run output.", + StageConfigurationWarning + ) # TODO: fill with warnings from Pipeline branch + run_output = Path(self.config.project_root or self.config.work_dir) + return run_output / "runs" def _assign_dependencies(self, dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, From 12a99c0ce761e081f7895e89c01017d45bf96c60 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:15:57 +0100 Subject: [PATCH 261/332] tweak: add an error message to cover where historical pipeline runs load incorrectly --- onsrap/errors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/onsrap/errors.py b/onsrap/errors.py index bd29f9f..2377f87 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -83,4 +83,10 @@ class PipelineConfigurationError(OnsrapError): """ Raised when there has been an issue with the PipelineConfig instance. + """ + +class HistoricalPipelineLoadError(OnsrapError): + """ + Raised when there is an issue loading a previous PipelineRun + instance. """ \ No newline at end of file From 607d73ac8cbd082b83ce401d176699aaac6d941c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:33:31 +0100 Subject: [PATCH 262/332] feat: add a function that reviews the log file and extracts all run_ids and timestamps for every run where a log entry contains "Pipeline started" --- onsrap/logger.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index ff065f1..4fe13da 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any +from .errors import HistoricalPipelineLoadError + @dataclass class LogConfig: @@ -142,4 +144,61 @@ def warning(self, message: str, **kwargs: Any) -> None: if kwargs: self._logger.warning("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) else: - self._logger.warning(message) \ No newline at end of file + self._logger.warning(message) + + def extract_historical_run_ids(self, run_root: Path) -> list[str]: + """ + + """ + + #ensure that logger is writing to a file and extract filepath + if not self._logger.root.hasHandlers(): + raise HistoricalPipelineLoadError("The logger does not write to a" \ + "filepath. Please ensure that your logger writes to a file path so that" \ + "we can extract the run_id for historical runs.") + + logfile_path = self._logger.root.handlers[0].baseFilename + + if not Path(logfile_path).exists(): + raise HistoricalPipelineLoadError("The log file does not exist at this" \ + " location.") + + print(logfile_path) + + matches: list[dict[str, Any]] = [] + + for raw_line in reversed(Path(logfile_path).read_text(encoding="utf-8").splitlines()): + if "Pipeline started" not in raw_line or " | " not in raw_line: + continue + + # left: timestamp + message, right: JSON context + left, right = raw_line.split(" | ", 1) + + try: + payload = json.loads(right) + except json.JSONDecodeError: + continue + + run_id = payload.get("run_id") + if not run_id: + continue + + # timestamp is the first two space-separated tokens: YYYY-MM-DD HH:MM:SS,mmm + parts = left.split(" ", 2) + if len(parts) < 2: + continue + timestamp = f"{parts[0]} {parts[1]}" + + run_dir = run_root / run_id + if run_dir.exists(): + matches.append({ + "run_id": run_id, + "timestamp": timestamp, + "run_dir": run_dir, + }) + + return matches + + + + From 72224d79fd00be1191ceef207d80a1fd8436c7bc Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:35:14 +0100 Subject: [PATCH 263/332] feat: add method that allows historical runs to be loaded as PipelineRun instances --- onsrap/loader.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/onsrap/loader.py b/onsrap/loader.py index 69098e7..1b9d348 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -8,9 +8,8 @@ from types import ModuleType from typing import Any -from onsrap.models import PipelineRun - from .errors import StageConfigurationError, StageLoadError +from .models import PipelineRun PREFERRED_ENTRYPOINTS = ("run", "main", "execute") @@ -161,3 +160,23 @@ def load_python_module(path: Path) -> ModuleType: return module +def load_historical_run(run_dir: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + Returns + ------- + ``PipelineRun`` + An instance of ``PipelineRun`` representing the historical run. + """ + import glob + files = glob.glob(str(run_dir / "pipeline_attributes_for_*.yaml")) + if not files: + raise StageLoadError("Historical run file does not exist in: {0}".format(run_dir)) + file_path = Path(files[0]) + + import yaml + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file From 9d86438fcaa1c39b6c5a7deb089083c03ff040fb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Fri, 7 Aug 2026 10:59:08 +0100 Subject: [PATCH 264/332] feat: added in methods to load the latest run from a cold run log into the last_run attribute of a Pipeline --- onsrap/logger.py | 25 +++++++++++++++++-------- onsrap/pipeline.py | 30 +++++++++++++++++++++++++++++- onsrap/runner.py | 10 +--------- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 4fe13da..48dee4c 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -146,24 +146,31 @@ def warning(self, message: str, **kwargs: Any) -> None: else: self._logger.warning(message) - def extract_historical_run_ids(self, run_root: Path) -> list[str]: + def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: """ - + Extracts historical run IDs from the log files. + + Returns + ------- + list[dict[str, Any]] + A list of dictionaries containing run_id, timestamp, and run_dir for each historical run. """ #ensure that logger is writing to a file and extract filepath - if not self._logger.root.hasHandlers(): + if not self._logger.hasHandlers(): raise HistoricalPipelineLoadError("The logger does not write to a" \ "filepath. Please ensure that your logger writes to a file path so that" \ "we can extract the run_id for historical runs.") - logfile_path = self._logger.root.handlers[0].baseFilename + logfile_handler = next((h for h in self._logger.handlers if isinstance(h, logging.FileHandler)), None) + if logfile_handler is None: + raise HistoricalPipelineLoadError("The logger does not have a FileHandler. " \ + "Please ensure that your logger writes to a file path so that we can extract the run_id " + "for historical runs.") + logfile_path = logfile_handler.baseFilename if not Path(logfile_path).exists(): - raise HistoricalPipelineLoadError("The log file does not exist at this" \ - " location.") - - print(logfile_path) + raise HistoricalPipelineLoadError("The log file does not exist at this location.") matches: list[dict[str, Any]] = [] @@ -190,6 +197,8 @@ def extract_historical_run_ids(self, run_root: Path) -> list[str]: timestamp = f"{parts[0]} {parts[1]}" run_dir = run_root / run_id + + #only returns run_ids for runs where a run_directory is still present. if run_dir.exists(): matches.append({ "run_id": run_id, diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 2fe6e33..f7d1379 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -17,6 +17,7 @@ from .logger import Logger from .models import GlobalConfig, PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now from .stage import Stage, _normalize_dependencies +from .loader import load_historical_run ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") @@ -106,10 +107,12 @@ def __init__( self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None - self.last_run: PipelineRun | None = None self.run_output = self._set_run_output() + self.last_run: PipelineRun | None = None + self.last_run = self._load_latest_run() + self.logger.event( "Pipeline initialized", name=self.name, @@ -118,6 +121,31 @@ def __init__( enabled_stages=[stage.name for stage in self.graph.stages], ) + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + if previous_run_logs == []: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + + def __str__(self) -> str: """ String method that returns a human-readable representation of the ``Pipeline`` class. diff --git a/onsrap/runner.py b/onsrap/runner.py index 8658a38..8e769cd 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -87,15 +87,7 @@ def run(self, pipeline: Pipeline) -> PipelineRun: runtime_id = pipeline._create_runtime_id() pipeline.id = runtime_id - if pipeline.config.output_dir is not None: - run_output = Path(pipeline.config.output_dir) - else: - warnings.warn( - "Output directory is not specified. Using project root or work directory as the run output.", - StageConfigurationWarning - ) # TODO: fill with warnings from Pipeline branch - run_output = Path(pipeline.config.project_root or pipeline.config.work_dir) - run_dir = run_output / "runs" / runtime_id.get_id() + run_dir = pipeline.run_output / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) # Initialise the ExecutionContext which will be passed to each stage as it runs. This From 2577f9778c3e84f2af297660f12d32c85b46f123 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 08:47:54 +0100 Subject: [PATCH 265/332] tweak: reorder _load_latest_run method in pipeline.py to sit with rest of private methods --- onsrap/pipeline.py | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index f7d1379..92ccb82 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -121,30 +121,6 @@ def __init__( enabled_stages=[stage.name for stage in self.graph.stages], ) - def _load_latest_run(self) -> PipelineRun | None: - """ - Load the most recent run of the Pipeline as a PipelineRun instance. - - Returns - ------- - ``PipelineRun`` or None - An instance of ``PipelineRun`` representing the most recent run of the - Pipeline, or None if no previous runs are found. - """ - previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) - if previous_run_logs == []: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) - return None - latest_run_log = previous_run_logs[0] - latest_run_id = latest_run_log["run_id"] - if latest_run_id is None: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) - return None - - return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) - def __str__(self) -> str: """ @@ -447,7 +423,31 @@ def add_dependencies(self, self.graph = StageGraph.from_stages(self.stages) self.graph.validate() - self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + if previous_run_logs == []: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ + "will be None.", PipelineConfigurationWarning) + return None + + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) def _set_run_output(self) -> Path: """ From f9887facbd850e974b3da2f248bef864464a464b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 12:13:02 +0100 Subject: [PATCH 266/332] tweak: correct error message in _load_latest_run --- onsrap/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 92ccb82..308f3bd 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -438,13 +438,13 @@ def _load_latest_run(self) -> PipelineRun | None: previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) if previous_run_logs == []: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) + " will be None.", PipelineConfigurationWarning) return None latest_run_log = previous_run_logs[0] latest_run_id = latest_run_log["run_id"] if latest_run_id is None: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - "will be None.", PipelineConfigurationWarning) + " will be None.", PipelineConfigurationWarning) return None return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) From 66d1932eee6a25c33ec4a0549d157cf7764f074d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 12:13:20 +0100 Subject: [PATCH 267/332] tests: add testing for load_latest_run method with fully mocked data --- tests/test_pipeline.py | 274 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 273 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index f66f8d5..4e91155 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,7 +1,12 @@ +from unittest import mock +import warnings + from onsrap.pipeline import Pipeline, PipelineConfig from onsrap.execution import PythonStageExecutor from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError from onsrap.models import StageConfig +from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError +from onsrap.models import PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning from pathlib import Path @@ -331,4 +336,271 @@ def test_validate_stage_backends_errors() -> None: backend="python", ), ], - ) \ No newline at end of file + ) + +class TestLoadLatestRunIntegration: + @pytest.fixture + def pipeline_log_line(self): + def _make(run_id: str, timestamp: str) -> str: + """ + Returns a false log file line to simulate a historical run in the log file. + The line is formatted to match the expected log output. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + ``timestamp`` : str + The timestamp of when the historical run was initiated. + """ + return f"{timestamp} Pipeline started | " \ + f"{{\"run_id\": \"{run_id}\", \"run_dir\": \"/path/to/run\"}}" + return _make + + @pytest.fixture + def minimal_pipeline_yaml(self): + def _make(run_id: str) -> str: + """ + Returns a minimal YAML configuration for a historical run. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + """ + return f""" + manifest: + run_id: {run_id} + status: succeeded + started_at: '2026-08-06T17:03:30.000077' + completed_at: '2026-08-06T17:03:30.031654' + stage_results: [] + stage_outputs: {{}} + """ + return _make + +class TestLoadLatestRun(TestLoadLatestRunIntegration): + @pytest.fixture + def pipeline_no_history(self, tmp_path: Path) -> Pipeline: + """ + Sets up a blank pipeline instance for testing that accounts for warnings + in init phase rather than dealing with these in the tests. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline(config=PipelineConfig( + output_dir = tmp_path/"outputs", + )) + pipeline.run_output = tmp_path/"runs" + return pipeline + + def test_blank_historical_run_ids(self, + pipeline_no_history: Pipeline, + monkeypatch) -> None: + """ + Tests that if extract_historical_run_ids returns a blank list, the + _load_latest_run method will return None and raise a warning. Assert + that it will also store None in the last_run attribute of the Pipeline + instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: []) + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) == [] + with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ + "found for this Pipeline. Last_run attribute will be None."): + assert pipeline_no_history._load_latest_run() == None + assert pipeline_no_history.last_run == None + + def test_blank_run_ids(self, + pipeline_no_history: Pipeline, + monkeypatch) -> None: + """ + Tests that if the found log record does not have a run_id, _load_latest_run + will return None and raise a warning. Assert that it will also store None + in the last_run attribute of the Pipeline instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [{ + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/")}] + ) + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) == [{ + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/") + }] + with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ + "found for this Pipeline. Last_run attribute will be None."): + assert pipeline_no_history._load_latest_run() == None + assert pipeline_no_history.last_run == None + + def test_load_latest_run_success(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + + """ + Tests that load_latest_run works successfully with fully mocked data. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [{ + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run") + }] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + result = pipeline_no_history._load_latest_run() + + assert result is expected_run + + expected_path = pipeline_no_history.run_output / run_id + mock_load_historical_run.assert_called_once_with(run_dir = expected_path) + + def test_which_run_is_selected_load_latest_run(self, + monkeypatch, + pipeline_no_history: Pipeline + ) -> None: + + """ + Checks that the first item is selected from the list of historical runs + returned by extract_historical_run_ids. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + expected_run = mock.MagicMock(spec=PipelineRun) + run_id_1 = "2026-08-10_100000_abc12345" + run_id_2 = "2026-08-10_100000_def67890" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [ + { + "run_id": run_id_1, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_A" + }, + { + "run_id": run_id_2, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_B" + } + ] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + pipeline_no_history._load_latest_run() + + expected_path = pipeline_no_history.run_output / run_id_1 + mock_load_historical_run.assert_called_once_with(run_dir = expected_path) + + #does not refer to run_dir in the extract_historical_run_ids list but the + #parameter required in load_historical_run. + assert mock_load_historical_run.call_args.kwargs["run_dir"].name == run_id_1 + + def test_no_errors_raised_success_load_latest_run(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + + """ + Tests that no errors are raised when load_latest_run is successful. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda _: [{ + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run") + }] + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock_load_historical_run) + + with warnings.catch_warnings(record=True) as w: + pipeline_no_history._load_latest_run() + + assert not any(issubclass(warning.category, PipelineConfigurationWarning) + for warning in w) + + + + From 0994a2c8473aaeac7011da76f4a571e2358cb040 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 13:13:55 +0100 Subject: [PATCH 268/332] docs: edit documentation to clarify what the parameters refer to --- onsrap/logger.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 48dee4c..7fad6fa 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -150,6 +150,11 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: """ Extracts historical run IDs from the log files. + Parameters + ---------- + ``run_root`` : Path + The root directory where the historical runs are stored. + Returns ------- list[dict[str, Any]] From 34a436a4835cf3f6ed309747bb8596e6f8445feb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:17:55 +0100 Subject: [PATCH 269/332] tweak: adds potential issue for review in PR --- onsrap/logger.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 7fad6fa..658f8d1 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -179,6 +179,9 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: matches: list[dict[str, Any]] = [] + #TODO: This method works if the logs are recorded in chronological order. Would there + #ever be a case where a record would appear below another and not be chronological? + #If so, we may need to sort based on the timestamp rather than the ordering. for raw_line in reversed(Path(logfile_path).read_text(encoding="utf-8").splitlines()): if "Pipeline started" not in raw_line or " | " not in raw_line: continue From 1adb4412e8e3a06ed65a1652f17d012068ebdc36 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:32:05 +0100 Subject: [PATCH 270/332] tweak: extract_historical_run_ids to exclude if the timestamp is incorrect --- onsrap/logger.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/onsrap/logger.py b/onsrap/logger.py index 658f8d1..fb384b7 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json import logging from dataclasses import dataclass @@ -189,6 +190,17 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: # left: timestamp + message, right: JSON context left, right = raw_line.split(" | ", 1) + #catches where Pipeline started is not recorded in the correct place. + if not left.endswith(" Pipeline started"): + continue + + #checks that datetime is valid + timestamp = left[:-len(" Pipeline started")].strip() + try: + datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S,%f") + except ValueError: + continue + try: payload = json.loads(right) except json.JSONDecodeError: From dad0bb6d96a9128dadc31b04eee31abb4f093e02 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 16:47:48 +0100 Subject: [PATCH 271/332] tests: implement tests for extract_historical_run_ids --- tests/test_pipeline.py | 289 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 288 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 4e91155..173b130 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,3 +1,4 @@ +import logging from unittest import mock import warnings @@ -7,8 +8,12 @@ from onsrap.models import StageConfig from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError from onsrap.models import PipelineRun, StageConfig +from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError +from onsrap.models import PipelineRun, PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning +from onsrap.logger import Logger + from pathlib import Path import pytest @@ -601,6 +606,288 @@ def test_no_errors_raised_success_load_latest_run(self, assert not any(issubclass(warning.category, PipelineConfigurationWarning) for warning in w) +class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): + def test_logger_no_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. - + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers.clear() # Remove all handlers to simulate no file logging + logger._logger.propagate = False # Prevent checking root logger handlers + with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_no_file_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no file handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers = [logging.StreamHandler()] + with pytest.raises(HistoricalPipelineLoadError, match="does not have a FileHandler"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_does_not_exist(self, + tmp_path:Path) -> None: + """ + Checks that the method raises an error if the log doesn't exist at the + location specified. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + file_handler.close() + log_path.unlink(missing_ok=True) # Remove the log file to simulate non-existence + + with pytest.raises(HistoricalPipelineLoadError, + match="does not exist at this location"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_return_blank_list_no_matches_in_log(self, + tmp_path) -> None: + """ + Tests that a log file that does not have a record covering "Pipeline started" + will return a blank list from extract_historical_run_ids. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Some unrelated log entry\n" \ + "2026-08-10 10:00:01,000 Another unrelated log entry\n") + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] + + def test_skips_poor_json_in_log(self, + tmp_path: Path) -> None: + """ + Tests that if a JSON record in the log file is not valid, it will e skipped + and the next valid entry will be extracted. Assert that the returned list + contains only the valid entry. Confirms that only the incorrect record is + skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" \ + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_abc1234", + "timestamp": "2026-08-10 10:00:02,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_abc1234" + }, + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + @pytest.mark.parametrize("string, expected", + [('{"some_key":"some_value"}', []), + ('{"run_id":""}',[])]) + + def test_run_id_absent_falsy(self, + tmp_path: Path, + string: str, + expected: list) -> None: + """ + Tests that if the JSON record in the log file does not have a run_id, it will be skipped + and the returned list will be empty. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``string`` : str + A dictionary representing a valid JSON record in the log file that excludes + run_id. + ``expected`` : list + The expected output from extract_historical_run_ids when the log file + contains a record without a run_id. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") + + #creates directory for runs to avoid removal given the directory doesn't exist + (tmp_path/"runs").mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == expected + + def test_records_only_if_directory_exists(self, + tmp_path) -> None: + """ + Checks that a record is only output if the run directory exists. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + def test_reverse_chronological_order(self, + tmp_path) -> None: + """ + Checks that the run_ids are output in reverse chronological order + based on their positioning in the log file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result[0]["run_id"] == "2026-06-23_101719_abc1234" + assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" + + def test_skip_poor_timestamps(self, + tmp_path) -> None: + """ + Checks that entries with poor timestamps are skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "BADTIMESTAMP Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] From 50fa7888306415ba407aeaceb519802aecce0b73 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 17:10:29 +0100 Subject: [PATCH 272/332] tweak: tidy up test_pipeline.py imports --- tests/test_pipeline.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 173b130..be4b3ba 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,12 +4,8 @@ from onsrap.pipeline import Pipeline, PipelineConfig from onsrap.execution import PythonStageExecutor -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError -from onsrap.models import StageConfig -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError -from onsrap.models import PipelineRun, StageConfig from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError -from onsrap.models import PipelineRun, PipelineRun, StageConfig +from onsrap.models import PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning from onsrap.logger import Logger From 70168d01ec49a526b5607cb94559fafb3003c95b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 17:41:29 +0100 Subject: [PATCH 273/332] test: added stages to the fixture Pipeline so that it passes the generate_execution_context function --- tests/test_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index be4b3ba..76a7b34 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -396,7 +396,8 @@ def pipeline_no_history(self, tmp_path: Path) -> Pipeline: with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline(config=PipelineConfig( output_dir = tmp_path/"outputs", - )) + ), + stages = [Stage("Stage_0", source=tmp_path/"Stage_0.py", dependencies=())]) pipeline.run_output = tmp_path/"runs" return pipeline From ec89e819e9be6c8f49a209ac2cda586fb8cfd151 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 10 Aug 2026 17:42:18 +0100 Subject: [PATCH 274/332] tweak: adjust load_latest_run to account for errors in importing historical runs and returns None if any of these errors are reached --- onsrap/pipeline.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 308f3bd..6853afd 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -10,7 +10,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence import re -from .errors import StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError +from .errors import HistoricalPipelineLoadError, StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError from .warnings import StageConfigurationWarning, PipelineConfigurationWarning from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph @@ -435,11 +435,20 @@ def _load_latest_run(self) -> PipelineRun | None: An instance of ``PipelineRun`` representing the most recent run of the Pipeline, or None if no previous runs are found. """ - previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + try: + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output + ) + except HistoricalPipelineLoadError: + warnings.warn("Unable to load previous runs for this Pipeline. Last_run" \ + " attribute will be None.", PipelineConfigurationWarning) + return None + if previous_run_logs == []: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ " will be None.", PipelineConfigurationWarning) return None + latest_run_log = previous_run_logs[0] latest_run_id = latest_run_log["run_id"] if latest_run_id is None: From 7a5885a1c27fc24531730fe1b9c4005c21340751 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 09:48:32 +0100 Subject: [PATCH 275/332] tests: produce tests for load_historical_run method and minor tweak on method call for _runmanifest_from_dict --- onsrap/models.py | 2 +- tests/test_pipeline.py | 85 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 67f2f32..c564688 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -963,7 +963,7 @@ def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: A PipelineRun instance created from the dictionary representation. """ return cls( - manifest=RunManifest.runmanifest_from_dict(data["manifest"]), + manifest=RunManifest._runmanifest_from_dict(data["manifest"]), status=PipelineStatus(data["status"]), started_at=datetime.fromisoformat(data["started_at"]), completed_at=datetime.fromisoformat(data["completed_at"]), diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 76a7b34..d10825e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,9 +2,10 @@ from unittest import mock import warnings +from onsrap.loader import load_historical_run from onsrap.pipeline import Pipeline, PipelineConfig from onsrap.execution import PythonStageExecutor -from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError +from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError, StageLoadError from onsrap.models import PipelineRun, StageConfig from onsrap.stage import Stage from onsrap.warnings import StageConfigurationWarning @@ -375,7 +376,7 @@ def _make(run_id: str) -> str: status: succeeded started_at: '2026-08-06T17:03:30.000077' completed_at: '2026-08-06T17:03:30.031654' - stage_results: [] + stage_results: {{}} stage_outputs: {{}} """ return _make @@ -888,3 +889,83 @@ def test_skip_poor_timestamps(self, result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") assert result == [] + +class TestLoadHistoricalRun(TestLoadLatestRunIntegration): + def test_raises_stageloaderror_no_file(self, + tmp_path: Path) -> None: + """ + Checks that load_historical_run raises a StageLoadError when the specified + directory does not contain any files matching the expected pattern. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``StageLoadError`` + Raised when the specified directory does not contain any files matching + the expected pattern for historical run YAML files. + """ + run_dir = tmp_path/"empty_run" + run_dir.mkdir(parents=True, exist_ok=True) + with pytest.raises(StageLoadError, match="Historical run file does not exist"): + load_historical_run(run_dir=run_dir) + + def test_returns_valid_pipeline_run_from_yaml(self, + tmp_path:Path, + minimal_pipeline_yaml)-> None: + """ + Checks that load_historical_run successfully returns a PipelineRun instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + + run_dir = tmp_path/"valid_run" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir/"pipeline_attributes_for_test.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id"), encoding="utf-8") + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) + assert result.manifest.run_id == "test_id" + + def test_correct_yaml_file_chosen(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: + """ + Checks that if there are multiple files within the same run directory, + the method will pass successfully and return a PipelineRun. This does + not assert which file is chosen, only that the method does not raise + an error and returns a PipelineRun instance. + + Logically, this should be suitable as we would only ever expect one file + to be present in each run directory. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + run_dir = tmp_path/"multiple_runs" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir/"pipeline_attributes_for_test1.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id_1"), encoding="utf-8") + (run_dir/"pipeline_attributes_for_test2.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id_2"), encoding="utf-8") + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) + + \ No newline at end of file From 339150b38393bfe31a30ddfb47ac40c8fe28cc3d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 09:40:05 +0100 Subject: [PATCH 276/332] test: correct errors in test_pipeline and test_pipeline_architecture caused by adding global_config requirement in configuration files. --- tests/test_pipeline.py | 52 ++++++++++++++++++----------- tests/test_pipeline_architecture.py | 21 ++++++++---- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index f66f8d5..d6c4696 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -112,34 +112,40 @@ def test_add_dependencies_single_dict(tmp_path): def test_add_stage_parses_stage_configs_keyword() -> None: - pipeline = Pipeline() - stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline() + stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) - pipeline.add_stage(stage, stage_configs=[stage_config]) + pipeline.add_stage(stage, stage_configs=[stage_config]) - assert pipeline.stages[-1].name == "Stage_1" - assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 def test_add_stage_warns_when_stage_config_count_mismatches() -> None: - pipeline = Pipeline() - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline() + stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) + stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) - assert any( - "does not match the number of stages" in str(recorded_warning.message) - for recorded_warning in recorded_warnings - ) - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - assert pipeline.stage_configs["Stage_1"].to_dict() == {} + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} def test_add_stage_config_coerces_mapping_payload_for_named_stage() -> None: - pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") @@ -155,6 +161,7 @@ def test_resolve_stages_to_run_includes_transitive_dependencies() -> None: stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) stage_2 = Stage("Stage_2", source=Path("Stage_2.py"), dependencies=("Stage_1",)) + pipeline = Pipeline( stages=[stage_0, stage_1, stage_2], config=PipelineConfig(stages_to_run={"Stage_2": True}), @@ -179,7 +186,10 @@ def test_self_stages_is_full_registry_after_disable() -> None: """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline(stages=[stage_0, stage_1]) + + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -191,7 +201,9 @@ def test_self_stages_is_full_registry_after_disable() -> None: def test_disable_stage_in_implicit_mode_creates_explicit_selection() -> None: stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns(PipelineConfigurationWarning, + match = "No stages specified to run. All stages running by default."): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 7e1fa9b..02e9892 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -49,7 +49,8 @@ def main(context): config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {} + "stage_configuration": {}, + "global_config":{} }, ) @@ -86,7 +87,8 @@ def main(context): config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, + "stage_configuration": {}, + "global_config":{}}, ) with pytest.warns(StageConfigurationWarning, @@ -120,7 +122,8 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Pa config={"pipeline_config":{"work_dir": tmp_path, "project_root": tmp_path, "log_dir": tmp_path / "logs"}, - "stage_configuration": {}}, + "stage_configuration": {}, + "global_config":{}}, ) with pytest.warns(StageConfigurationWarning, @@ -185,6 +188,8 @@ def run(context): 0_data_validation: years_to_run: 2017 target_variable: "classification" + global_configuration: + dry_run: true """ ).strip() + "\n", @@ -232,6 +237,7 @@ def run(context): "stage_configuration": { "missing_stage": {"years_to_run": 2017}, }, + "global_config":{} }, ) @@ -303,6 +309,7 @@ def run(context): }, }, }, + "global_config":{} } config_file = tmp_path / "conf.yaml" @@ -320,10 +327,9 @@ def run(context): assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() assert pipeline.stages[0].metadata["owner"] == "analytics" assert pipeline.stages[1].dependencies == ("0_extract",) - assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017} - assert pipeline.stage_configs["0_extract"].datasets == {"orders": {"path": "data/orders.csv"}} + assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017, "datasets": {"orders": {"path": "data/orders.csv"}}} assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.create_stage_config(config_file, name="1_transform").require("target_variable") == "classification" + assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" def test_pipeline_from_config_scales_stage_configuration_to_many_stages(tmp_path: Path) -> None: @@ -388,6 +394,9 @@ def run(context): "stages": stage_definitions, }, "stage_configuration": stage_configuration, + "global_config": { + "dry_run": True, + }, }, sort_keys=False, ), From 2ad8472cfaf794783e921a3abb0b8cc241d6c729 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 10:30:14 +0100 Subject: [PATCH 277/332] tweak: CoPilot refactor (human review) tests into class structure --- tests/test_pipeline.py | 492 +++++++++++++++++++++-------------------- 1 file changed, 249 insertions(+), 243 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d6c4696..9786bb1 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -9,282 +9,288 @@ from onsrap.warnings import PipelineConfigurationWarning -def test_pipeline_name(): - """ - Test to confirm that Pipeline instance uses either defined name from - instance creation (shown in pipeline_named), utilises name from PipelineConfig - if no name was given (shown in pipeline_config), or defaults to "pipeline" if - no name is provided through Pipeline instance creation or through the - PipelineConfig (shown through pipeline_no_name) - """ - pipeline_config = PipelineConfig(name = "test_pipeline_config") - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_named = Pipeline(name = "test_pipeline_name") - pipeline_config = Pipeline(name = None, config = pipeline_config) - pipeline_no_name = Pipeline() - - assert pipeline_named.name == "test_pipeline_name" - assert pipeline_config.name == "test_pipeline_config" - assert pipeline_no_name.name == "pipeline" - - -def test_assign_dependencies(tmp_path): - """ - Test to ensure that different formats of dependencies can be parsed to the - Pipeline creation and appropriately assigned to each stage within the - Pipeline. Will also check for error raise if the dependencies are defined - but there are no defined stages. - """ - def example_function(): - pass - - path_1 = tmp_path/"Stage_1.py" - path_0 = tmp_path/"Stage_0.py" - - dependencies_single = {"Stage_2":("Stage_1",)} - dependencies_multiple = {"Stage_1":["Stage_0"], - "Stage_2":("Stage_1", "Stage_0")} - dependencies_non_stage_name = {"Stage_1.py":("Stage_0",), - "example_function":("Stage_1.py",)} - - with pytest.raises(PipelineInitialisationError): - Pipeline(stages = None, - dependencies = dependencies_single) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_1 = Pipeline(name = "pipeline_1", - stages = [Stage("Stage_1", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_multiple) - - pipeline_2 = Pipeline(name = "pipeline_2", - stages = [Stage("Stage_1.py", path_1, None,{}), - Stage("Stage_2", example_function, None,{}), - Stage("Stage_0", path_0, None,{}),], - dependencies = dependencies_non_stage_name) - - assert pipeline_1.stages[0].dependencies == ("Stage_0",) - assert pipeline_1.stages[1].dependencies == ("Stage_1","Stage_0") - - assert pipeline_2.stages[0].dependencies == ("Stage_0",) - assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) - - -def test_add_dependencies_single_dict(tmp_path): - """ - Tests that a dictionary correctly assigns dependencies to - individual stages and the Pipeline instance. - """ - - path_1 = tmp_path/"Stage_1.py" - path_2 = tmp_path/"Stage_2.py" - path_0 = tmp_path/"Stage_0.py" - - dependencies_multiple = {"Stage_0":(), - "Stage_1":(), - "Stage_2":("Stage_1",)} - dep_dict = {"Stage_1":("Stage_0",), - "Stage_2":("Stage_0","Stage_1")} - dep_tuple = ("Stage_0.25",) - stage_1 = Stage("Stage_1", source = path_1, dependencies = {}) - stage_2 = Stage("Stage_2", source = path_2, dependencies = {}) - stage_0 = Stage("Stage_0", source = path_0, dependencies = {}) +NO_STAGES_WARNING = "No stages specified to run. All stages running by default." + + +@pytest.fixture +def stage_factory(): + def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: + resolved_source = source if source is not None else Path(f"{name}.py") + return Stage(name, source=resolved_source, dependencies=dependencies) + + return _build_stage + + +class TestPipelineNamingAndInit: + def test_pipeline_name(self): + """ + Test to confirm that Pipeline instance uses either defined name from + instance creation (shown in pipeline_named), utilises name from PipelineConfig + if no name was given (shown in pipeline_config), or defaults to "pipeline" if + no name is provided through Pipeline instance creation or through the + PipelineConfig (shown through pipeline_no_name) + """ + pipeline_config = PipelineConfig(name="test_pipeline_config") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_named = Pipeline(name="test_pipeline_name") + pipeline_config = Pipeline(name=None, config=pipeline_config) + pipeline_no_name = Pipeline() + + assert pipeline_named.name == "test_pipeline_name" + assert pipeline_config.name == "test_pipeline_config" + assert pipeline_no_name.name == "pipeline" + + def test_assign_dependencies(self, tmp_path): + """ + Test to ensure that different formats of dependencies can be parsed to the + Pipeline creation and appropriately assigned to each stage within the + Pipeline. Will also check for error raise if the dependencies are defined + but there are no defined stages. + """ + + def example_function(): + pass + + path_1 = tmp_path / "Stage_1.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_single = {"Stage_2": ("Stage_1",)} + dependencies_multiple = { + "Stage_1": ["Stage_0"], + "Stage_2": ("Stage_1", "Stage_0"), + } + dependencies_non_stage_name = { + "Stage_1.py": ("Stage_0",), + "example_function": ("Stage_1.py",), + } + + with pytest.raises(PipelineInitialisationError): + Pipeline(stages=None, dependencies=dependencies_single) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_1 = Pipeline( + name="pipeline_1", + stages=[ + Stage("Stage_1", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_multiple, + ) + + pipeline_2 = Pipeline( + name="pipeline_2", + stages=[ + Stage("Stage_1.py", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_non_stage_name, + ) + + assert pipeline_1.stages[0].dependencies == ("Stage_0",) + assert pipeline_1.stages[1].dependencies == ("Stage_1", "Stage_0") + + assert pipeline_2.stages[0].dependencies == ("Stage_0",) + assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + + def test_add_dependencies_single_dict(self, tmp_path): + """ + Tests that a dictionary correctly assigns dependencies to + individual stages and the Pipeline instance. + """ + + path_1 = tmp_path / "Stage_1.py" + path_2 = tmp_path / "Stage_2.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_multiple = {"Stage_0": (), "Stage_1": (), "Stage_2": ("Stage_1",)} + dep_dict = {"Stage_1": ("Stage_0",), "Stage_2": ("Stage_0", "Stage_1")} + dep_tuple = ("Stage_0.25",) + stage_1 = Stage("Stage_1", source=path_1, dependencies={}) + stage_2 = Stage("Stage_2", source=path_2, dependencies={}) + stage_0 = Stage("Stage_0", source=path_0, dependencies={}) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline_dict = Pipeline(stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple) + + with pytest.raises(PipelineInitialisationError): + pipeline_dict.add_dependencies(dep_tuple) + + pipeline_dict.add_dependencies(dep_dict) + assert stage_1.dependencies == ("Stage_0",) + assert stage_2.dependencies == ("Stage_1", "Stage_0",) + assert stage_0.dependencies == () + assert pipeline_dict.dependencies == { + "Stage_0": (), + "Stage_1": ("Stage_0",), + "Stage_2": ("Stage_1", "Stage_0",), + } + + +class TestPipelineStageConfigHandling: + def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline() + stage = stage_factory("Stage_1") + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + + pipeline.add_stage(stage, stage_configs=[stage_config]) + + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + + def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline() + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} + + def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_factory) -> None: + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_factory("Stage_0")]) + + pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline_dict = Pipeline(stages = [stage_0, stage_1, stage_2], - dependencies = dependencies_multiple) - - with pytest.raises(PipelineInitialisationError): - pipeline_dict.add_dependencies(dep_tuple) - - pipeline_dict.add_dependencies(dep_dict) - assert stage_1.dependencies == ("Stage_0",) - assert stage_2.dependencies == ("Stage_1","Stage_0",) - assert stage_0.dependencies == () - assert pipeline_dict.dependencies == {"Stage_0":(), - "Stage_1":("Stage_0",), - "Stage_2":("Stage_1","Stage_0",)} - - -def test_add_stage_parses_stage_configs_keyword() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline() - stage = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) - - pipeline.add_stage(stage, stage_configs=[stage_config]) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - assert pipeline.stages[-1].name == "Stage_1" - assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 +class TestPipelineStageSelectionAndGraph: + def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) -def test_add_stage_warns_when_stage_config_count_mismatches() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline() - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - - with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) - - assert any( - "does not match the number of stages" in str(recorded_warning.message) - for recorded_warning in recorded_warnings + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), ) - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 - assert pipeline.stage_configs["Stage_1"].to_dict() == {} + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] -def test_add_stage_config_coerces_mapping_payload_for_named_stage() -> None: - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + def test_resolve_stages_to_run_rejects_disabled_dependencies(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) - pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") + with pytest.raises(PipelineConfigurationError): + Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + ) - assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: + """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_0, stage_1]) -# --------------------------------------------------------------------------- -# Stage graph and stage-selection tests -# --------------------------------------------------------------------------- + pipeline.disable_stage("Stage_1") -def test_resolve_stages_to_run_includes_transitive_dependencies() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) - stage_2 = Stage("Stage_2", source=Path("Stage_2.py"), dependencies=("Stage_1",)) + assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] - - pipeline = Pipeline( - stages=[stage_0, stage_1, stage_2], - config=PipelineConfig(stages_to_run={"Stage_2": True}), - ) + def test_disable_stage_in_implicit_mode_creates_explicit_selection(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): + pipeline = Pipeline(stages=[stage_0, stage_1]) - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] + pipeline.disable_stage("Stage_1") + assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} -def test_resolve_stages_to_run_rejects_disabled_dependencies() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) - - with pytest.raises(PipelineConfigurationError): - Pipeline( + def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + pipeline = Pipeline( stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), ) + pipeline.enable_stage("Stage_1") -def test_self_stages_is_full_registry_after_disable() -> None: - """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[stage_0, stage_1]) - - pipeline.disable_stage("Stage_1") - - assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] - - -def test_disable_stage_in_implicit_mode_creates_explicit_selection() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(stages=[stage_0, stage_1]) - - pipeline.disable_stage("Stage_1") - - assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} - + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} -def test_enable_stage_restores_stage_in_explicit_mode() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) - - pipeline.enable_stage("Stage_1") - - assert pipeline.config.stages_to_run["Stage_1"] is True - assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} - - -def test_add_stage_keeps_new_stage_out_of_explicit_selection() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) - - pipeline.add_stage( - Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), - stage_configs=[StageConfig(name="Stage_1")], - ) - - assert pipeline.config.stages_to_run["Stage_1"] is False - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + def test_add_stage_keeps_new_stage_out_of_explicit_selection(self, stage_factory) -> None: + stage_0 = stage_factory("Stage_0") + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) -def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true() -> None: - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + ) - pipeline.add_stage( - Stage("Stage_1", source=Path("Stage_1.py"), dependencies=()), - stage_configs=[StageConfig(name="Stage_1")], - enable_stages=True, - ) + assert pipeline.config.stages_to_run["Stage_1"] is False + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + + def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true( + self, + stage_factory, + ) -> None: + stage_0 = stage_factory("Stage_0") + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - assert pipeline.config.stages_to_run["Stage_1"] is True - assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} -def test_validate_skips_source_check_for_disabled_stages(tmp_path: Path) -> None: - """Disabled stages' source files need not exist — validate() only checks the effective run set.""" - enabled_file = tmp_path / "Stage_0.py" - enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") - stage_0 = Stage("Stage_0", source=enabled_file) - stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent +class TestPipelineValidationAndManifest: + def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) -> None: + """Disabled stages' source files need not exist - validate() only checks the effective run set.""" + enabled_file = tmp_path / "Stage_0.py" + enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + stage_0 = Stage("Stage_0", source=enabled_file) + stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent - pipeline.validate() # must not raise + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) + pipeline.validate() # must not raise -def test_construct_manifest_inputs_contains_only_effective_stages() -> None: - """Manifest inputs should list only the stages that are part of the execution graph.""" - stage_0 = Stage("Stage_0", source=Path("Stage_0.py"), dependencies=()) - stage_1 = Stage("Stage_1", source=Path("Stage_1.py"), dependencies=("Stage_0",)) - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + def test_construct_manifest_inputs_contains_only_effective_stages(self, stage_factory) -> None: + """Manifest inputs should list only the stages that are part of the execution graph.""" + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) - runtime_id = pipeline._create_runtime_id() - manifest = pipeline._construct_manifest(runtime_id=runtime_id) + runtime_id = pipeline._create_runtime_id() + manifest = pipeline._construct_manifest(runtime_id=runtime_id) - assert list(manifest.inputs.keys()) == ["Stage_0"] + assert list(manifest.inputs.keys()) == ["Stage_0"] def test_generate_context_correctly_assigns_executor() -> None: """ From 374bf88289318c679d692c1baa778feb47d6f1b3 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 10:40:37 +0100 Subject: [PATCH 278/332] tweak: ruff formatting and linting applied --- tests/test_pipeline.py | 105 ++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9786bb1..ffc7c24 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,13 +1,13 @@ -from onsrap.pipeline import Pipeline, PipelineConfig -from onsrap.execution import PythonStageExecutor -from onsrap.errors import PipelineInitialisationError, PipelineConfigurationError -from onsrap.models import StageConfig -from onsrap.stage import Stage -from onsrap.warnings import StageConfigurationWarning from pathlib import Path + import pytest -from onsrap.warnings import PipelineConfigurationWarning +from onsrap.errors import PipelineConfigurationError, PipelineInitialisationError +from onsrap.models import StageConfig +from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning +from onsrap.execution import PythonStageExecutor NO_STAGES_WARNING = "No stages specified to run. All stages running by default." @@ -113,19 +113,27 @@ def test_add_dependencies_single_dict(self, tmp_path): stage_0 = Stage("Stage_0", source=path_0, dependencies={}) with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline_dict = Pipeline(stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple) + pipeline_dict = Pipeline( + stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple + ) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) pipeline_dict.add_dependencies(dep_dict) assert stage_1.dependencies == ("Stage_0",) - assert stage_2.dependencies == ("Stage_1", "Stage_0",) + assert stage_2.dependencies == ( + "Stage_1", + "Stage_0", + ) assert stage_0.dependencies == () assert pipeline_dict.dependencies == { "Stage_0": (), "Stage_1": ("Stage_0",), - "Stage_2": ("Stage_1", "Stage_0",), + "Stage_2": ( + "Stage_1", + "Stage_0", + ), } @@ -134,21 +142,27 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage = stage_factory("Stage_1") - stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + stage_config = StageConfig( + name="Stage_1", _variables={"years_to_run": 2017} + ) pipeline.add_stage(stage, stage_configs=[stage_config]) assert pipeline.stages[-1].name == "Stage_1" assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 - def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) -> None: + def test_add_stage_warns_when_stage_config_count_mismatches( + self, stage_factory + ) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(StageConfigurationWarning) as recorded_warnings: - pipeline.add_stage(stage_0, stage_1, stage_configs=[{"years_to_run": 2017}]) + pipeline.add_stage( + stage_0, stage_1, stage_configs=[{"years_to_run": 2017}] + ) assert any( "does not match the number of stages" in str(recorded_warning.message) @@ -157,7 +171,9 @@ def test_add_stage_warns_when_stage_config_count_mismatches(self, stage_factory) assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 assert pipeline.stage_configs["Stage_1"].to_dict() == {} - def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_factory) -> None: + def test_add_stage_config_coerces_mapping_payload_for_named_stage( + self, stage_factory + ) -> None: with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -167,7 +183,9 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage(self, stage_fa class TestPipelineStageSelectionAndGraph: - def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_factory) -> None: + def test_resolve_stages_to_run_includes_transitive_dependencies( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) @@ -177,21 +195,36 @@ def test_resolve_stages_to_run_includes_transitive_dependencies(self, stage_fact config=PipelineConfig(stages_to_run={"Stage_2": True}), ) - assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0", "Stage_1", "Stage_2"] - assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0", "Stage_1", "Stage_2"] - - def test_resolve_stages_to_run_rejects_disabled_dependencies(self, stage_factory) -> None: + assert [stage.name for stage in pipeline.graph.stages] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + assert [stage.name for stage in pipeline.ordered_stages()] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + + def test_resolve_stages_to_run_rejects_disabled_dependencies( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) with pytest.raises(PipelineConfigurationError): Pipeline( stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": False, "Stage_1": True}), + config=PipelineConfig( + stages_to_run={"Stage_0": False, "Stage_1": True} + ), ) def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: - """Pipeline.stages always holds all stages; only graph.stages is the effective run set.""" + """ + Pipeline.stages always holds all stages; only graph.stages is the effective run + set. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -204,7 +237,9 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] - def test_disable_stage_in_implicit_mode_creates_explicit_selection(self, stage_factory) -> None: + def test_disable_stage_in_implicit_mode_creates_explicit_selection( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): @@ -227,7 +262,9 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No assert pipeline.config.stages_to_run["Stage_1"] is True assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} - def test_add_stage_keeps_new_stage_out_of_explicit_selection(self, stage_factory) -> None: + def test_add_stage_keeps_new_stage_out_of_explicit_selection( + self, stage_factory + ) -> None: stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], @@ -263,13 +300,20 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr class TestPipelineValidationAndManifest: - def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) -> None: - """Disabled stages' source files need not exist - validate() only checks the effective run set.""" + def test_validate_skips_source_check_for_disabled_stages( + self, tmp_path: Path + ) -> None: + """ + Disabled stages' source files need not exist - validate() only checks the + effective run set. + """ enabled_file = tmp_path / "Stage_0.py" enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") stage_0 = Stage("Stage_0", source=enabled_file) - stage_1 = Stage("Stage_1", source=tmp_path / "missing.py") # file intentionally absent + stage_1 = Stage( + "Stage_1", source=tmp_path / "missing.py" + ) # file intentionally absent pipeline = Pipeline( stages=[stage_0, stage_1], @@ -278,8 +322,13 @@ def test_validate_skips_source_check_for_disabled_stages(self, tmp_path: Path) - pipeline.validate() # must not raise - def test_construct_manifest_inputs_contains_only_effective_stages(self, stage_factory) -> None: - """Manifest inputs should list only the stages that are part of the execution graph.""" + def test_construct_manifest_inputs_contains_only_effective_stages( + self, stage_factory + ) -> None: + """ + Manifest inputs should list only the stages that are part of the execution + graph. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) pipeline = Pipeline( From a97c64f330972350bb798d5d9bc70ba23bb95e5c Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 11:27:58 +0100 Subject: [PATCH 279/332] doc: add doc strings for each test in test_pipeline --- tests/test_pipeline.py | 52 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ffc7c24..52f3021 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -15,6 +15,11 @@ @pytest.fixture def stage_factory(): def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: + """ + Function that builds a Stage object with a given name, dependencies, and a + source file path that's built out of the name if it is not provided. This + standardises the creation of Stage objects for testing. + """ resolved_source = source if source is not None else Path(f"{name}.py") return Stage(name, source=resolved_source, dependencies=dependencies) @@ -139,6 +144,12 @@ def test_add_dependencies_single_dict(self, tmp_path): class TestPipelineStageConfigHandling: def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: + """ + Tests that when a stage is added after a Pipeline has been initialised, + the stage and the stage_configurations are correctly added to the + Pipeline instance and the stage_configurations are correctly associated + with the stage. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage = stage_factory("Stage_1") @@ -154,6 +165,11 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: def test_add_stage_warns_when_stage_config_count_mismatches( self, stage_factory ) -> None: + """ + Tests that when a stage is added but there is not the correct number of + stage_configs provided, a warning is raised and the stage_configuration + for that stage is added as a blank StageConfig object. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() stage_0 = stage_factory("Stage_0") @@ -174,6 +190,11 @@ def test_add_stage_warns_when_stage_config_count_mismatches( def test_add_stage_config_coerces_mapping_payload_for_named_stage( self, stage_factory ) -> None: + """ + Tests that when a stage_configuration is added to a Pipeline instance, + the configuration is correctly associated with the named stage and that + the configuration is coerced into a StageConfig object if it is provided. + """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -186,6 +207,11 @@ class TestPipelineStageSelectionAndGraph: def test_resolve_stages_to_run_includes_transitive_dependencies( self, stage_factory ) -> None: + """ + Tests that when resolving stages_to_run, the Pipeline instance correctly + includes all dependent stages required in the StageGraph even if these + are not explicitly called out in the configuration. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) @@ -209,6 +235,10 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( def test_resolve_stages_to_run_rejects_disabled_dependencies( self, stage_factory ) -> None: + """ + Checks that when resolving stages_to_run, the Pipeline init raises an + error if a stage is enabled but one of its dependencies is disabled. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -240,6 +270,10 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None def test_disable_stage_in_implicit_mode_creates_explicit_selection( self, stage_factory ) -> None: + """ + Tests that when a stage is manually disabled in a Pipeline instance, it + is initialised in the stages_to_run configuration. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): @@ -250,6 +284,11 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> None: + """ + Tests that when a stage is manually enabled in a Pipeline instance, it + is correctly reflected in the stages_to_run configuration and the stage + is included in the execution graph. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") pipeline = Pipeline( @@ -265,6 +304,10 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No def test_add_stage_keeps_new_stage_out_of_explicit_selection( self, stage_factory ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance, it is + kept out of the explicit selection. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], @@ -273,8 +316,9 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( pipeline.add_stage( stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")], - ) + stage_configs=[StageConfig(name="Stage_1")] + ) + assert pipeline.config.stages_to_run["Stage_1"] is False assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] @@ -283,6 +327,10 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr self, stage_factory, ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance with + enable_stages=True, it is included in the explicit selection. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( stages=[stage_0], From a5f1f6586b1082f8f8fd8602f586b360b7125846 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 11:53:49 +0100 Subject: [PATCH 280/332] tweak: refactor test_stage.py to class structure --- tests/test_stage.py | 342 ++++++++++++++++++++++---------------------- 1 file changed, 172 insertions(+), 170 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 759bfc1..15f3f3d 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -3,22 +3,23 @@ from pathlib import Path from textwrap import dedent -def test_normalize_dependencies_none() -> None: - """ - Tests that None values return empty tuple. - """ - assert _normalize_dependencies(None) == () - -def test_normalize_dependencies_str() -> None: - """ - Tests single string and list of string values including - where whitespace appears before and after main text body - """ - assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) - assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) - assert _normalize_dependencies( - ["Stage_1.py"," Stage_2.py", "Stage_3.py "] - ) == ("Stage_1.py","Stage_2.py", "Stage_3.py") +class TestNormalizeDependencies: + def test_normalize_dependencies_none(self) -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + + def test_normalize_dependencies_str(self) -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py", " Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") @pytest.fixture def example_function(): @@ -34,163 +35,164 @@ def stage_test() -> Stage: """ return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) -def test_stage_creation_callable(stage_test) -> None: - """ - Tests that attributes have been appropriately assigned to Stage class. - """ - assert stage_test.name == "callable_stage" - assert stage_test.source == example_function - assert stage_test.dependencies == ("stage_1",) - assert stage_test.metadata == {"info":"example"} - assert stage_test.entrypoint == None - assert stage_test.backend == "python" - -def test_stage_name_error(example_function) -> None: - """ - Tests that a StageConfigurationError is raised if the name is left blank - in a Stage class instance. - """ - with pytest.raises(StageConfigurationError): - stage = Stage("",example_function,["stage_1"],{"info":"example"}) - -def test_stage_source_type() -> None: - """ - Tests that a non-valid source type returns a StageConfigurationError. - """ - with pytest.raises(StageConfigurationError): - stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) - -def test_stage_backend(example_function) -> None: - """ - Tests that backend can be any string, None, and corrects for whitespace. - """ - stage_diff = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "java") - stage = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "") - stage_white_space = Stage("callable_stage", example_function,["stage_1"], - {"info":"example"}, backend = "python ") - assert stage_diff.backend == "java" - assert stage.backend == "python" - assert stage_white_space.backend == "python" - -def test_stage_from_files_error(tmp_path: Path) -> None: - """ - Tests that if the file doesn't exist, a StageConfigurationError is raised. - """ - source_file = tmp_path / "not_an_actual_file.py" - with pytest.raises(StageConfigurationError): - stage = Stage.from_file(source_file) - -def test_stage_from_callable_name() -> None: - """ - Tests that a stage name is extracted from a callable object stage. - """ - def example_function(): - pass - test = Stage.from_callable(example_function) - assert test.name == "example_function" - -def test_from_dict_norm() -> None: - """ - Tests that a stage instance is created from a dictionary item. - """ - def example_function(): - pass - data = {"name":"test_Stage", - "callable" : example_function} - stage = Stage.from_dict(data) - assert stage.source == example_function - -def test_with_dependencies_list(stage_test) -> None: - """ - Tests adding different types of dependencies when the original dependency is - a list. - """ - new_deps = ["stage2","stage3"] - new_deps_blank = [] - stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') - assert stage_test_blank.dependencies == ("stage_1", ) - stage_test = stage_test.with_dependencies("stage2","stage3") - assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') - - -def test_validate(stage_test, tmp_path) -> None: - """ - Tests whether an error is raised if the source file isn't suitable. - """ - stage_test.source = None - with pytest.raises(StageConfigurationError): - stage_test.validate() - not_file_path = tmp_path - stage_test.source = not_file_path - with pytest.raises(StageConfigurationError): - stage_test.validate() - stage_test.source = "" - with pytest.raises(StageConfigurationError): - stage_test.validate() - -def test_source_path(stage_test, tmp_path) -> None: - """ - Tests whether source_path detects a path vs other valid and invalid source types. - """ - stage_test.source = tmp_path/"fake_file.py" - assert stage_test.source_path == tmp_path/"fake_file.py" - stage_test.source = 11 - assert stage_test.source_path == None - stage_test.source = "not a file path" - assert stage_test.source_path == None - - def example_function(): - pass - stage_test.source = example_function - assert stage_test.source_path == None - -def test_source_label(stage_test, tmp_path) -> None: - """ - Tests that source_label is created if the source is a Path or a callable and is None if it is - another type. - """ - stage_test.source = tmp_path/"fake_file.py" - temp_path_str = str(tmp_path/"fake_file.py") - assert stage_test.source_label == temp_path_str - - def example_function(): - pass - stage_test.source = example_function - assert stage_test.source_label == "tests.test_stage.example_function" - - stage_test.source = 11 - assert stage_test.source_label == None +class TestStage: + def test_stage_creation_callable(self, stage_test) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info":"example"} + assert stage_test.entrypoint == None + assert stage_test.backend == "python" + + def test_stage_name_error(self, example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("",example_function,["stage_1"],{"info":"example"}) + + def test_stage_source_type(self) -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + """ + with pytest.raises(StageConfigurationError): + stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + + def test_stage_backend(self, example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + """ + stage_diff = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "java") + stage = Stage("callable_stage",example_function,["stage_1"], + {"info":"example"}, backend = "") + stage_white_space = Stage("callable_stage", example_function,["stage_1"], + {"info":"example"}, backend = "python ") + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + + def test_stage_from_files_error(self, tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + stage = Stage.from_file(source_file) + + def test_stage_from_callable_name(self) -> None: + """ + Tests that a stage name is extracted from a callable object stage. + """ + def example_function(): + pass + test = Stage.from_callable(example_function) + assert test.name == "example_function" + + def test_from_dict_norm(self) -> None: + """ + Tests that a stage instance is created from a dictionary item. + """ + def example_function(): + pass + data = {"name":"test_Stage", + "callable" : example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + + def test_with_dependencies_list(self, stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. + """ + new_deps = ["stage2","stage3"] + new_deps_blank = [] + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_blank.dependencies == ("stage_1", ) + stage_test = stage_test.with_dependencies("stage2","stage3") + assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + + def test_validate(self, stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + + def test_source_path(self, stage_test, tmp_path) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source types. + """ + stage_test.source = tmp_path/"fake_file.py" + assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = 11 + assert stage_test.source_path == None + stage_test.source = "not a file path" + assert stage_test.source_path == None + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_path == None + + def test_source_label(self, stage_test, tmp_path) -> None: + """ + Tests that source_label is created if the source is a Path or a callable and is None if it is + another type. + """ + stage_test.source = tmp_path/"fake_file.py" + temp_path_str = str(tmp_path/"fake_file.py") + assert stage_test.source_label == temp_path_str + + def example_function(): + pass + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label == None """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ -def test_stage_instance_from_file(tmp_path) -> None: - """ - Tests that a Stage instance is created from a filepath. - """ - test_stage = tmp_path / "test_stage.py" - test_stage.write_text( - dedent( - """ - def main(): - variable = "Hello world" - return variable - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - assert Stage.from_file(test_stage, - entrypoint = "main") == Stage("test_stage", - test_stage.resolve(), - (), - {}, - "main", - "python") +class TestStageFactories: + def test_stage_instance_from_file(self, tmp_path) -> None: + """ + Tests that a Stage instance is created from a filepath. + """ + test_stage = tmp_path / "test_stage.py" + test_stage.write_text( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + assert Stage.from_file(test_stage, + entrypoint = "main") == Stage("test_stage", + test_stage.resolve(), + (), + {}, + "main", + "python") From 0cc1882a647ff3385f07853549756f7d1e2037af Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 12:08:33 +0100 Subject: [PATCH 281/332] tweak: ruff format and check test_stage.py --- tests/test_stage.py | 106 +++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 15f3f3d..b29a182 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -1,8 +1,11 @@ -import pytest -from onsrap.stage import _normalize_dependencies, Stage, StageConfigurationError from pathlib import Path from textwrap import dedent +import pytest + +from onsrap.stage import Stage, StageConfigurationError, _normalize_dependencies + + class TestNormalizeDependencies: def test_normalize_dependencies_none(self) -> None: """ @@ -21,6 +24,7 @@ def test_normalize_dependencies_str(self) -> None: ["Stage_1.py", " Stage_2.py", "Stage_3.py "] ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") + @pytest.fixture def example_function(): """ @@ -28,12 +32,14 @@ def example_function(): """ pass + @pytest.fixture def stage_test() -> Stage: """ - Stage object for testing Stage class methods and construction. + Stage object for testing Stage class methods and construction. """ - return Stage("callable_stage",example_function,["stage_1"],{"info":"example"}) + return Stage("callable_stage", example_function, ["stage_1"], {"info": "example"}) + class TestStage: def test_stage_creation_callable(self, stage_test) -> None: @@ -43,8 +49,8 @@ def test_stage_creation_callable(self, stage_test) -> None: assert stage_test.name == "callable_stage" assert stage_test.source == example_function assert stage_test.dependencies == ("stage_1",) - assert stage_test.metadata == {"info":"example"} - assert stage_test.entrypoint == None + assert stage_test.metadata == {"info": "example"} + assert stage_test.entrypoint is None assert stage_test.backend == "python" def test_stage_name_error(self, example_function) -> None: @@ -53,25 +59,40 @@ def test_stage_name_error(self, example_function) -> None: in a Stage class instance. """ with pytest.raises(StageConfigurationError): - stage = Stage("",example_function,["stage_1"],{"info":"example"}) + Stage("", example_function, ["stage_1"], {"info": "example"}) def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. """ with pytest.raises(StageConfigurationError): - stage = Stage("callable_stage",11,["stage_1"],{"info":"example"}) + Stage("callable_stage", 11, ["stage_1"], {"info": "example"}) def test_stage_backend(self, example_function) -> None: """ Tests that backend can be any string, None, and corrects for whitespace. """ - stage_diff = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "java") - stage = Stage("callable_stage",example_function,["stage_1"], - {"info":"example"}, backend = "") - stage_white_space = Stage("callable_stage", example_function,["stage_1"], - {"info":"example"}, backend = "python ") + stage_diff = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="java", + ) + stage = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="", + ) + stage_white_space = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="python ", + ) assert stage_diff.backend == "java" assert stage.backend == "python" assert stage_white_space.backend == "python" @@ -82,14 +103,16 @@ def test_stage_from_files_error(self, tmp_path: Path) -> None: """ source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): - stage = Stage.from_file(source_file) + Stage.from_file(source_file) def test_stage_from_callable_name(self) -> None: """ Tests that a stage name is extracted from a callable object stage. """ + def example_function(): pass + test = Stage.from_callable(example_function) assert test.name == "example_function" @@ -97,10 +120,11 @@ def test_from_dict_norm(self) -> None: """ Tests that a stage instance is created from a dictionary item. """ + def example_function(): pass - data = {"name":"test_Stage", - "callable" : example_function} + + data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) assert stage.source == example_function @@ -109,14 +133,14 @@ def test_with_dependencies_list(self, stage_test) -> None: Tests adding different types of dependencies when the original dependency is a list. """ - new_deps = ["stage2","stage3"] + new_deps = ["stage2", "stage3"] new_deps_blank = [] stage_test_list = stage_test.with_dependencies(new_deps) stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1",'stage2', 'stage3') - assert stage_test_blank.dependencies == ("stage_1", ) - stage_test = stage_test.with_dependencies("stage2","stage3") - assert stage_test.dependencies == ("stage_1",'stage2', 'stage3') + assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") + assert stage_test_blank.dependencies == ("stage_1",) + stage_test = stage_test.with_dependencies("stage2", "stage3") + assert stage_test.dependencies == ("stage_1", "stage2", "stage3") def test_validate(self, stage_test, tmp_path) -> None: """ @@ -135,41 +159,46 @@ def test_validate(self, stage_test, tmp_path) -> None: def test_source_path(self, stage_test, tmp_path) -> None: """ - Tests whether source_path detects a path vs other valid and invalid source types. + Tests whether source_path detects a path vs other valid and invalid source + types. """ - stage_test.source = tmp_path/"fake_file.py" - assert stage_test.source_path == tmp_path/"fake_file.py" + stage_test.source = tmp_path / "fake_file.py" + assert stage_test.source_path == tmp_path / "fake_file.py" stage_test.source = 11 - assert stage_test.source_path == None + assert stage_test.source_path is None stage_test.source = "not a file path" - assert stage_test.source_path == None + assert stage_test.source_path is None def example_function(): pass + stage_test.source = example_function - assert stage_test.source_path == None + assert stage_test.source_path is None def test_source_label(self, stage_test, tmp_path) -> None: """ - Tests that source_label is created if the source is a Path or a callable and is None if it is - another type. + Tests that source_label is created if the source is a Path or a callable + and is None if it is another type. """ - stage_test.source = tmp_path/"fake_file.py" - temp_path_str = str(tmp_path/"fake_file.py") + stage_test.source = tmp_path / "fake_file.py" + temp_path_str = str(tmp_path / "fake_file.py") assert stage_test.source_label == temp_path_str def example_function(): pass + stage_test.source = example_function assert stage_test.source_label == "tests.test_stage.example_function" stage_test.source = 11 - assert stage_test.source_label == None + assert stage_test.source_label is None + """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ + class TestStageFactories: def test_stage_instance_from_file(self, tmp_path) -> None: """ @@ -188,11 +217,6 @@ def main(): encoding="utf-8", ) - assert Stage.from_file(test_stage, - entrypoint = "main") == Stage("test_stage", - test_stage.resolve(), - (), - {}, - "main", - "python") - + assert Stage.from_file(test_stage, entrypoint="main") == Stage( + "test_stage", test_stage.resolve(), (), {}, "main", "python" + ) From 05d535cd8f4c47c771706018606cfc0f462bb2e1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 13:17:13 +0100 Subject: [PATCH 282/332] tweak: CoPilot refactor (human review) of test_execution to add class structure. Ruff formatting and checking completed --- tests/test_execution.py | 673 +++++++++++++++++++++------------------- 1 file changed, 351 insertions(+), 322 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 05e0f5a..496ee37 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,12 +1,20 @@ -from onsrap.execution import ExecutionContext, PythonStageExecutor -from onsrap.models import GlobalConfig, PipelineConfig, StageConfig, StageResult, StageStatus -from onsrap.logger import Logger from pathlib import Path + import pytest -import onsrap.execution as execution_module + from onsrap.errors import PipelineConfigurationError +from onsrap.execution import ExecutionContext, PythonStageExecutor +from onsrap.logger import Logger +from onsrap.models import ( + GlobalConfig, + PipelineConfig, + StageConfig, + StageResult, + StageStatus, +) from onsrap.warnings import StageConfigurationWarning + @pytest.fixture def logger() -> Logger: """ @@ -14,51 +22,66 @@ def logger() -> Logger: """ return Logger() + @pytest.fixture def config() -> PipelineConfig: """ Return a PipelineConfig object for testing. """ - work_dir = Path('tmp/work_dir') - project_root = Path('tmp/project') - log_dir = Path('tmp/log') + work_dir = Path("tmp/work_dir") + project_root = Path("tmp/project") + log_dir = Path("tmp/log") data_dir = Path("tmp/config_data") return PipelineConfig( "test_pipeline", - {"stage_test":True}, + {"stage_test": True}, "python", - work_dir, + work_dir, project_root, None, - log_dir, - data_dir, - True, + log_dir, + data_dir, + True, None, - {} + {}, ) + +@pytest.fixture +def stage_config() -> StageConfig: + """ + Return a StageConfig object for testing. + """ + return StageConfig( + name="stage_test", + _variables={"sex": "gender", "dob": "date_of_birth"}, + metadata={}, + ) + + @pytest.fixture def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. """ run_dir = Path("tmp/run") - work_dir = Path('tmp/work_dir') + work_dir = Path("tmp/work_dir") return ExecutionContext( "test_pipeline", "run_id_1234", - config, + config, logger, run_dir, - '2024-05-06 15:45:30', + "2024-05-06 15:45:30", work_dir, - {"stage_test":stageresult}, - {"stage_test":stage_config}, + {"stage_test": stageresult}, + {"stage_test": stage_config}, {}, - None + None, ) + @pytest.fixture def stageresult() -> StageResult: """ @@ -67,338 +90,344 @@ def stageresult() -> StageResult: return StageResult( "stage_test", StageStatus.PENDING, - '2024-05-06 15:45:30', - '2024-05-07 15:45:30', + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", metadata={}, - outputs = "example output" + outputs="example output", ) -def test_executioncontext_creation(execution, logger, config, stageresult) -> None: - """ - Test that the ExecutionContext creates the right attributes. - """ - assert execution.pipeline_name == "test_pipeline" - assert execution.run_id == "run_id_1234" - assert execution.config == config - assert execution.logger == logger - assert execution.run_dir == Path("tmp/run") - assert execution.started_at == '2024-05-06 15:45:30' - assert execution.working_directory == Path('tmp/work_dir') - assert execution.stage_results == {"stage_test":stageresult} - assert execution.variables == {} - -def test_record(stageresult, execution) -> None: - """ - Tests that StageResult attributes are attached to stage_results and variables - attributes in the ExecutionContext instance. - """ - execution.record(stageresult) - assert execution.stage_results == {'stage_test':StageResult(name='stage_test', - status='pending', - started_at='2024-05-06 15:45:30', - finished_at='2024-05-07 15:45:30', - outputs="example output", - stdout='', - stderr='', - return_code=None, - metadata={}, - error=None, - source=None)} - assert execution.variables == {'stage_test':"example output"} - -def test_result_for(execution, stageresult) -> None: - """ - Tests that result_for correctly extracts the results of a requested stage. - """ - execution.record(stageresult) - assert execution.result_for("stage_test") == StageResult(name='stage_test', - status='pending', - started_at='2024-05-06 15:45:30', - finished_at='2024-05-07 15:45:30', - outputs="example output", - stdout='', - stderr='', - return_code=None, - metadata={}, - error=None, - source=None) - -def test_stage_outputs(execution, stageresult) -> None: +@pytest.fixture +def expected_recorded_stage_result() -> StageResult: """ - Tests that stage_outputs shows the outputs attribute of the StageResult - instance for a requested stage is extracted. + Expected StageResult after recording for assertions. """ - execution.record(stageresult) - assert execution.stage_outputs == {"stage_test":"example output"} + return StageResult( + name="stage_test", + status="pending", + started_at="2024-05-06 15:45:30", + finished_at="2024-05-07 15:45:30", + outputs="example output", + stdout="", + stderr="", + return_code=None, + metadata={}, + error=None, + source=None, + ) -def test_get_data_dir(execution, stageresult) -> None: - """ - Tests that get_data_dir method extracts the path from the execution context - or, if the context is None, returns an error to indicate that additional input is - required. - """ - assert execution.get_data_dir() == Path("tmp/config_data") - - run_dir = Path("tmp/run") - work_dir = Path('tmp/work_dir') - execution_blank_config = ExecutionContext("test_pipeline", - "run_id_1234", - None, - Logger(), - run_dir, - '2024-05-06 15:45:30', - work_dir, - {"stage_test":stageresult}, - {} ) - - with pytest.raises(PipelineConfigurationError): - execution_blank_config.get_data_dir() -def test_resolve_output_root(execution) -> None: +class TestExecutionContext: + def test_executioncontext_creation( + self, execution, logger, config, stageresult + ) -> None: + """ + Test that the ExecutionContext creates the right attributes. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == Path("tmp/run") + assert execution.started_at == "2024-05-06 15:45:30" + assert execution.working_directory == Path("tmp/work_dir") + assert execution.stage_results == {"stage_test": stageresult} + assert execution.variables == {} + + def test_record( + self, stageresult, execution, expected_recorded_stage_result + ) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + """ + execution.record(stageresult) + assert execution.stage_results == {"stage_test": expected_recorded_stage_result} + assert execution.variables == {"stage_test": "example output"} + + def test_result_for( + self, execution, stageresult, expected_recorded_stage_result + ) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + """ + execution.record(stageresult) + assert execution.result_for("stage_test") == expected_recorded_stage_result + + def test_stage_outputs(self, execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + """ + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test": "example output"} + + @pytest.fixture + def blank_context_with_config_none(self, stageresult) -> ExecutionContext: + run_dir = Path("tmp/run") + work_dir = Path("tmp/work_dir") + return ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + run_dir, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: + """ + Tests that get_data_dir method extracts the path from the execution context + or, if the context is None, returns an error to indicate that additional input + is required. + """ + assert execution.get_data_dir() == Path("tmp/config_data") + + with pytest.raises(PipelineConfigurationError): + blank_context_with_config_none.get_data_dir() + + def test_resolve_output_root(self, execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, raises an error to indicate additional input + is required.. + """ + work_dir = Path("tmp/work_dir") + assert execution.resolve_output_root() == Path("tmp/run") + + execution_blank_config = ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + None, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + with pytest.raises(PipelineConfigurationError): + execution_blank_config.resolve_output_root() + + def test_stage_config_accessors_return_named_and_active_configs( + self, config, logger + ) -> None: + stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) + context = ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + Path("tmp/run"), + stage_configs={"stage_test": stage_config}, + active_stage_name="stage_test", + ) + + assert context.stage_config_for("stage_test") == stage_config + assert context.get_stage_config("stage_test") == {"years_to_run": 2017} + assert context.get_stage_config() == {"years_to_run": 2017} + with pytest.raises(PipelineConfigurationError): + context.get_stage_config(vars_only=False) + assert ( + context.get_stage_config(with_global=False, vars_only=False) == stage_config + ) + + def test_set_active_stage(self, execution, stage_config) -> None: + """ + Tests that set_active_stage correctly sets the active_stage attribute in the + ExecutionContext instance. + """ + + execution.set_active_stage(stage_config.name) + assert execution.active_stage_name == stage_config.name + execution.set_active_stage(None) + assert execution.active_stage_name is None + + def test_stage_config_for(self, execution, stage_config) -> None: + """ + Tests that stage_config_for returns the StageConfig for a named stage. + """ + assert execution.stage_config_for(stage_config.name) == stage_config + assert execution.stage_config_for("missing_stage") is None + + def test_stage_config(self, execution, stage_config) -> None: + """ + Tests that stage_config exposes the currently active stage configuration. + """ + assert execution.stage_config is None + execution.set_active_stage(stage_config.name) + assert execution.stage_config == stage_config + + def test_get_stage_config(self, execution, stage_config) -> None: + """ + Tests that get_stage_config returns variables by default and the full + StageConfig object when requested. + """ + assert execution.get_stage_config() == {} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert execution.get_stage_config(with_global=False, vars_only=False) is None + + execution.set_active_stage(stage_config.name) + assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert ( + execution.get_stage_config(with_global=False, vars_only=False) + == stage_config + ) + + +class TestResolveGivenPath: """ - Tests that resolve_output_root method extracts the path from the given run - directory or, if None are given, raises an error to indicate additional input - is required.. + Parameters for testing multiple add_folder options in + test_resolve_given_path_add_folders function. """ - work_dir = Path('tmp/work_dir') - assert execution.resolve_output_root() == Path("tmp/run") - execution_blank_config = ExecutionContext("test_pipeline", - "run_id_1234", - None, - Logger(), - None, - '2024-05-06 15:45:30', - work_dir, - {"stage_test":stageresult}, - {} ) - - with pytest.raises(PipelineConfigurationError): - execution_blank_config.resolve_output_root() - -def test_stage_config_accessors_return_named_and_active_configs(config, logger) -> None: - stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) - context = ExecutionContext( - "test_pipeline", - "run_id_1234", - config, - logger, - Path("tmp/run"), - stage_configs={"stage_test": stage_config}, - active_stage_name="stage_test", - ) - - assert context.stage_config_for("stage_test") == stage_config - assert context.get_stage_config("stage_test") == {"years_to_run": 2017} - assert context.get_stage_config() == {"years_to_run": 2017} - with pytest.raises(PipelineConfigurationError): - context.get_stage_config(vars_only=False) - assert context.get_stage_config(with_global=False, vars_only=False) == stage_config - -""" -Parameters for testing multiple add_folder options in -test_resolve_given_path_add_folders function. -""" -@pytest.mark.parametrize( + @pytest.mark.parametrize( "add_folder,file_name,expected", [ ( - ["interim","testing_files"], + ["interim", "testing_files"], "clean.py", - Path("tmp/data/interim/testing_files/clean.py") + Path("tmp/data/interim/testing_files/clean.py"), ), + ("interim", "clean.py", Path("tmp/data/interim/clean.py")), + (None, "clean.py", Path("tmp/data/clean.py")), ( - "interim", - "clean.py", - Path("tmp/data/interim/clean.py") - ), - ( - None, - "clean.py", - Path("tmp/data/clean.py") - ), - ( - ["interim","testing_files"], + ["interim", "testing_files"], None, - Path("tmp/data/interim/testing_files") + Path("tmp/data/interim/testing_files"), ), - ( - "interim", - None, - Path("tmp/data/interim") - ), - ( - None, - None, - Path("tmp/data") - ) + ("interim", None, Path("tmp/data/interim")), + (None, None, Path("tmp/data")), ], -) + ) + def test_resolve_given_path_add_folders( + self, execution, add_folder, file_name, expected + ) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid + string or None type. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert ( + execution.resolve_given_path(None, path_name, file_name, root, add_folder) + == expected + ) + def test_resolve_given_path_norm(self, execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + """ + execution.record( + StageResult( + "stage_test2", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs={"data_path": "clean.py"}, + ) + ) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") -def test_resolve_given_path_add_folders(execution, add_folder, file_name, expected) -> None: - """ - Tests the add_folder functionality for lists, single strings, or None type in - the resolve_given_path class method as well as when the file_name is a valid string - or None type. - """ - path_name = "data_path" - root = Path("tmp/data") + assert execution.resolve_given_path( + stage_name, path_name, None, root, None + ) == Path("clean.py") - assert execution.resolve_given_path(None, - path_name, - file_name, - root, - add_folder) == expected -def test_resolve_given_path_norm(execution) -> None: - """ - Tests that resolve_given_path returns a file path that has been output in a - StageResult instance. - """ - execution.record(StageResult("stage_test2", - StageStatus.PENDING, - '2024-05-06 15:45:30', - '2024-05-07 15:45:30', - metadata={}, - outputs = {"data_path":"clean.py"} )) - stage_name = "stage_test2" - path_name = "data_path" - root = Path("tmp/data") - - assert execution.resolve_given_path(stage_name, - path_name, - None, - root, - None) == Path("clean.py") - """TEST NOT RUN FOR StageExecutor AS COVERED UNDER PythonStageExecutor""" + @pytest.fixture def pythonstageexecutor() -> PythonStageExecutor: - return PythonStageExecutor(("main.py","run.py")) - -def test_pythonstageexecutor_setup(pythonstageexecutor) -> None: - assert pythonstageexecutor.preferred_entrypoints == ("main.py","run.py") - - -def test_combine_vars(execution) -> None: - """ - Test that checks that a dictionary is returned, combining values from a global - configuration and a stage configuration whilst removing any stage specific - exclusions. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {"stage_1": ["global_var2"]} - stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - combined_vars = execution._combine_vars() - assert combined_vars == { - "stage_var1": "value3", - "stage_var2": "value4", - "global_var1": "value1" - } - -def test_combine_vars_errors(execution) -> None: - """ - Test that confirms that a warning is raised if there is a variable defined in both - the global and the stage configurations as well as asserting the correct values. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {"stage_1": ["global_var2"]} - stage_vars = {"stage_var1": "value3", "global_var1": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - - with pytest.warns(StageConfigurationWarning, - match="Stage defines variable\\(s\\) that are also defined in global " - "variables: global_var1\\. Stage variables will take precedence."): - combined_vars = execution._combine_vars() - assert combined_vars == { + return PythonStageExecutor(("main.py", "run.py")) + + +class TestPythonStageExecutor: + def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: + assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") + + +class TestCombineVars: + def test_combine_vars(self, execution) -> None: + """ + Test that checks that a dictionary is returned, combining values from a global + configuration and a stage configuration whilst removing any stage specific + exclusions. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { "stage_var1": "value3", - "global_var1": "value4" + "stage_var2": "value4", + "global_var1": "value1", } -def test_combine_vars_no_exclusion(execution) -> None: - """ - Test confirming that a dictionary is returned, combining values from a global configuration - and a stage configuration when there are no exclusions defined. - """ - global_vars = {"global_var1": "value1", "global_var2": "value2"} - exclusions = {} - stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} - execution.global_config = GlobalConfig(_variables=global_vars, exclusion=exclusions) - execution.stage_configs = { - "stage_1": StageConfig(name="stage_1", _variables=stage_vars), - } - execution.active_stage_name = "stage_1" - combined_vars = execution._combine_vars() - assert combined_vars == { - "stage_var1": "value3", - "stage_var2": "value4", - "global_var1": "value1", - "global_var2": "value2" - } -"""CONTINUE FROM EXECUTE CLASS METHOD""" -@pytest.fixture -def stage_config() -> StageConfig: - """ - Return a StageConfig object for testing. - """ - return StageConfig( - name="stage_test", - _variables={"sex":"gender", - "dob":"date_of_birth"}, - metadata={} + def test_combine_vars_errors(self, execution) -> None: + """ + Test that confirms that a warning is raised if there is a variable defined in + both the global and the stage configurations as well as asserting the correct + values. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "global_var1": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions ) - -def test_set_active_stage(execution, stage_config) -> None: - """ - Tests that set_active_stage correctly sets the active_stage attribute in the - ExecutionContext instance. - """ - - execution.set_active_stage(stage_config.name) - assert execution.active_stage_name == stage_config.name - execution.set_active_stage(None) - assert execution.active_stage_name == None - -def test_stage_config_for(execution, stage_config) -> None: - """ - Tests that stage_config_for returns the StageConfig for a named stage. - """ - assert execution.stage_config_for(stage_config.name) == stage_config - assert execution.stage_config_for("missing_stage") is None - -def test_stage_config(execution, stage_config) -> None: - """ - Tests that stage_config exposes the currently active stage configuration. - """ - assert execution.stage_config is None - execution.set_active_stage(stage_config.name) - assert execution.stage_config == stage_config - -def test_get_stage_config(execution, stage_config) -> None: - """ - Tests that get_stage_config returns variables by default and the full - StageConfig object when requested. - """ - assert execution.get_stage_config() == {} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) - assert execution.get_stage_config(with_global=False, vars_only=False) is None - - execution.set_active_stage(stage_config.name) - assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) - assert execution.get_stage_config(with_global=False, vars_only=False) == stage_config - + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + + with pytest.warns( + StageConfigurationWarning, + match="Stage defines variable\\(s\\) that are also defined in global " + "variables: global_var1\\. Stage variables will take precedence.", + ): + combined_vars = execution._combine_vars() + assert combined_vars == {"stage_var1": "value3", "global_var1": "value4"} + + def test_combine_vars_no_exclusion(self, execution) -> None: + """ + Test confirming that a dictionary is returned, combining values from a global + configuration and a stage configuration when there are no exclusions defined. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1", + "global_var2": "value2", + } From f1fba224d6ec118088b6027750e722892f512b5b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 13:22:44 +0100 Subject: [PATCH 283/332] docs: added docstrings to tests in test_execution.py --- tests/test_execution.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_execution.py b/tests/test_execution.py index 496ee37..618e112 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -216,6 +216,10 @@ def test_resolve_output_root(self, execution) -> None: def test_stage_config_accessors_return_named_and_active_configs( self, config, logger ) -> None: + """ + Tests that getter methods to return the stage_config for a named stage + returns correct attributes based on given parameters. + """ stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) context = ExecutionContext( "test_pipeline", @@ -357,6 +361,10 @@ def pythonstageexecutor() -> PythonStageExecutor: class TestPythonStageExecutor: def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: + """ + Checks that entrypoints are set correctly in the PythonStageExecutor + instance. + """ assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") From 14269266813b7d0097cc58cce2cb185d48533a91 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:03:21 +0100 Subject: [PATCH 284/332] tweal: test_models.py refactor tests into classes with CoPilot, utilises Ruff formatting and checking --- tests/test_models.py | 515 ++++++++++++++++++++++--------------------- 1 file changed, 269 insertions(+), 246 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 7e7d0b4..21f45f4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,282 +1,305 @@ -from onsrap.models import StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig -import pytest import datetime from pathlib import Path from textwrap import dedent -from tests.test_execution import stageresult -def test_stagestatus() -> None: - """ - Test that stagestatus outputs the correct values. - """ - assert StageStatus.PENDING == "pending" - assert StageStatus.RUNNING == "running" - assert StageStatus.SUCCEEDED == "succeeded" - assert StageStatus.FAILED == "failed" - assert StageStatus.SKIPPED == "skipped" +import pytest + +from onsrap.models import ( + PipelineConfig, + PipelineRun, + PipelineStatus, + RunManifest, + RuntimeID, + StageStatus, +) + +STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) +FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) + + +class TestStatuses: + def test_stagestatus(self) -> None: + """ + Test that stagestatus outputs the correct values. + """ + assert StageStatus.PENDING == "pending" + assert StageStatus.RUNNING == "running" + assert StageStatus.SUCCEEDED == "succeeded" + assert StageStatus.FAILED == "failed" + assert StageStatus.SKIPPED == "skipped" + + def test_pipeline_status(self) -> None: + """ + Test that pipeline status outputs the correct values. + """ + assert PipelineStatus.PENDING == "pending" + assert PipelineStatus.RUNNING == "running" + assert PipelineStatus.SUCCEEDED == "succeeded" + assert PipelineStatus.FAILED == "failed" -def test_pipeline_status() -> None: - """ - Test that pipeline status outputs the correct values. - """ - assert PipelineStatus.PENDING == "pending" - assert PipelineStatus.RUNNING == "running" - assert PipelineStatus.SUCCEEDED == "succeeded" - assert PipelineStatus.FAILED == "failed" @pytest.fixture def runtimeID() -> RuntimeID: """ - Example RuntimeID instance for testing of other methods. + Example RuntimeID instance for testing of other methods. """ - return RuntimeID(id = "abc123", - timestamp = datetime.datetime(2026, 7, 7, 13, 5, 46), - hash = "fnruw9574893ghkwq234h5kg", - short_hash = "4h5kg") + return RuntimeID( + id="abc123", + timestamp=datetime.datetime(2026, 7, 7, 13, 5, 46), + hash="fnruw9574893ghkwq234h5kg", + short_hash="4h5kg", + ) -def test_runtimeID_creation(runtimeID) -> None: - """ - Test that a RuntimeID is correctly created. - """ - assert runtimeID.id == "abc123" - assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) - assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" - assert runtimeID.short_hash == "4h5kg" -def test_getter_functions_runtimeID(runtimeID) -> None: - """ - Tests all the getter functions for the RuntimeID instance. - """ - assert runtimeID.get_id() == "abc123" - assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) - assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" - assert runtimeID.get_short_hash() == "4h5kg" +class TestRuntimeID: + def test_runtimeID_creation(self, runtimeID) -> None: + """ + Test that a RuntimeID is correctly created. + """ + assert runtimeID.id == "abc123" + assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" + assert runtimeID.short_hash == "4h5kg" + + def test_getter_functions_runtimeID(self, runtimeID) -> None: + """ + Tests all the getter functions for the RuntimeID instance. + """ + assert runtimeID.get_id() == "abc123" + assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" + assert runtimeID.get_short_hash() == "4h5kg" + @pytest.fixture def blankpipelineconfig() -> PipelineConfig: """ - Blank PipelineConfig instance for class method testing. + Blank PipelineConfig instance for class method testing. """ return PipelineConfig() + @pytest.fixture -def pipelineconfig() -> PipelineConfig: +def expected_pipeline_config() -> PipelineConfig: """ - Example PipelineConfig completed class instance for method testing. + Example PipelineConfig completed class instance for method testing. """ - return PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) + return PipelineConfig( + name="test_rap", + backend="python", + work_dir=Path("tmp/work"), + project_root=Path("project"), + log_dir=Path("tmp/logs"), + data_dir=Path("tmp/data"), + allow_subprocess_fallback=True, + python_executable=None, + metadata={"variables": ["name", "age"], "num_stages": 6}, + ) + + +@pytest.fixture +def pipelineconfig(expected_pipeline_config) -> PipelineConfig: + return expected_pipeline_config + @pytest.fixture def mapping() -> dict: """ - Example mapping dictionary for use in testing from_mapping() method. - """ - return {"name":"test_rap", - "backend":"python", - "work_dir":Path("tmp/work"), - "project_root":Path("project"), - "log_dir":Path("tmp/logs"), - "data_dir":Path("tmp/data"), - "allow_subprocess_fallback":True, - "python_executable":None, - "metadata":{"variables":["name","age"], - "num_stages":6}} - - -def test_from_any(mapping, pipelineconfig, blankpipelineconfig) -> None: - """ - Test derivation for a PipelineConfig instance using the from_any() method. This test - checks all methods EXCEPT from_file as this will be covered in another test due to - creation of a mock file being required. - """ - assert blankpipelineconfig.from_any(None) == PipelineConfig() - assert blankpipelineconfig.from_any(pipelineconfig) == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - assert blankpipelineconfig.from_any(mapping) == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - - with pytest.raises(TypeError): - blankpipelineconfig.from_any(11) - -def test_from_file(tmp_path,) -> PipelineConfig: - pipeline_config = tmp_path / "configuration.py" - pipeline_config.write_text( - dedent( - """ - {"name":"test_rap", - "backend":"python", - "work_dir":"tmp/work", - "project_root":"project", - "log_dir":"tmp/logs", - "data_dir":"tmp/data", - "allow_subprocess_fallback":True, - "python_executable": , - "metadata":{"variables":["name","age"], - "num_stages":6} - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) - no_map_pipeline_config = tmp_path / "not_valid.py" - no_map_pipeline_config.write_text( - dedent( - """ - variable = "Hello world" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - configuration = PipelineConfig.from_file(pipeline_config) - assert configuration == PipelineConfig(name = "test_rap", - backend = "python", - work_dir = Path("tmp/work"), - project_root = Path("project"), - log_dir = Path("tmp/logs"), - data_dir = Path("tmp/data"), - allow_subprocess_fallback = True, - python_executable = None, - metadata = {"variables":["name","age"], - "num_stages":6}) - - fake_file = "path_not_real" - with pytest.raises(FileNotFoundError): - PipelineConfig.from_file(fake_file) - with pytest.raises(TypeError): - PipelineConfig.from_file(no_map_pipeline_config) - -def test_to_dict(pipelineconfig) -> None: - """ - Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values - as a dictionary. + Example mapping dictionary for use in testing from_mapping() method. """ + return { + "name": "test_rap", + "backend": "python", + "work_dir": Path("tmp/work"), + "project_root": Path("project"), + "log_dir": Path("tmp/logs"), + "data_dir": Path("tmp/data"), + "allow_subprocess_fallback": True, + "python_executable": None, + "metadata": {"variables": ["name", "age"], "num_stages": 6}, + } + + +class TestPipelineConfig: + def test_from_any( + self, mapping, pipelineconfig, blankpipelineconfig, expected_pipeline_config + ) -> None: + """ + Test derivation for a PipelineConfig instance using the from_any() method. This + test checks all methods EXCEPT from_file as this will be covered in another + test due to creation of a mock file being required. + """ + assert blankpipelineconfig.from_any(None) == PipelineConfig() + assert blankpipelineconfig.from_any(pipelineconfig) == expected_pipeline_config + assert blankpipelineconfig.from_any(mapping) == expected_pipeline_config + + with pytest.raises(TypeError): + blankpipelineconfig.from_any(11) + + def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config + + fake_file = "path_not_real" + with pytest.raises(FileNotFoundError): + PipelineConfig.from_file(fake_file) + with pytest.raises(TypeError): + PipelineConfig.from_file(no_map_pipeline_config) + + def test_to_dict(self, pipelineconfig) -> None: + """ + Test of to_dict() class method for PipelineConfig that it outputs the + PipelineConfig values as a dictionary. + """ + + assert pipelineconfig.to_dict() == { + "name": "test_rap", + "backend": "python", + "work_dir": "tmp\\work", + "project_root": "project", + "output_dir": None, + "log_dir": "tmp\\logs", + "data_dir": "tmp\\data", + "allow_subprocess_fallback": True, + "python_executable": None, + "variables": ["name", "age"], + "num_stages": 6, + } + - assert pipelineconfig.to_dict() == {"name":"test_rap", - "backend":"python", - "work_dir":"tmp\\work", - "project_root":"project", - "output_dir":None, - "log_dir":"tmp\\logs", - "data_dir":"tmp\\data", - "allow_subprocess_fallback":True, - "python_executable":None, - "variables":["name","age"], - "num_stages":6} - - @pytest.fixture def runmanifest() -> RunManifest: """ - Example RunManifest class instance for testing of class method. - """ - return RunManifest("pipeline", - "1", - None, - ["stage1","stage2"], - {"uniqueID":"example"}, - {"input_path":"input/data/example.csv"}, - {"output_path":"output/data/example.csv"}, - "python", - ["1.3.2"], - "", - None, - None) - -def test_stage_result(stageresult) -> None: - """ - Uses a StageResult instance created in test_execution to ensure that - the class instance is created suitably with required defaults. - """ - assert stageresult.name == "stage_test" - assert stageresult.status == "pending" - assert stageresult.started_at == '2024-05-06 15:45:30' - assert stageresult.finished_at == '2024-05-07 15:45:30' - assert stageresult.outputs == "example output" - assert stageresult.stdout == "" - assert stageresult.stderr == "" - assert stageresult.return_code == None - assert stageresult.metadata == {} - assert stageresult.error == None - assert stageresult.source == None - -@pytest.mark.parametrize("status_stage,expected_stage", - [(StageStatus.PENDING, False), - (StageStatus.RUNNING, False), - (StageStatus.FAILED, False), - (StageStatus.SUCCEEDED, True), - (StageStatus.SKIPPED, False)]) - -def test_succeeded(stageresult, status_stage, expected_stage) -> None: - """ - Tests succeeded() method for StageResult which outputs True or False depending on - the status of the StageResult. + Example RunManifest class instance for testing of class method. """ - stageresult.status = status_stage - assert stageresult.succeeded == expected_stage + return RunManifest( + "pipeline", + "1", + None, + ["stage1", "stage2"], + {"uniqueID": "example"}, + {"input_path": "input/data/example.csv"}, + {"output_path": "output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None, + ) + + +class TestStageResult: + def test_stage_result(self, stageresult) -> None: + """ + Uses a StageResult instance created in test_execution to ensure that + the class instance is created suitably with required defaults. + """ + assert stageresult.name == "stage_test" + assert stageresult.status == "pending" + assert stageresult.started_at == "2024-05-06 15:45:30" + assert stageresult.finished_at == "2024-05-07 15:45:30" + assert stageresult.outputs == "example output" + assert stageresult.stdout == "" + assert stageresult.stderr == "" + assert stageresult.return_code is None + assert stageresult.metadata == {} + assert stageresult.error is None + assert stageresult.source is None + + @pytest.mark.parametrize( + "status_stage,expected_stage", + [ + (StageStatus.PENDING, False), + (StageStatus.RUNNING, False), + (StageStatus.FAILED, False), + (StageStatus.SUCCEEDED, True), + (StageStatus.SKIPPED, False), + ], + ) + def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: + """ + Tests succeeded() method for StageResult which outputs True or False depending + on the status of the StageResult. + """ + stageresult.status = status_stage + assert stageresult.succeeded == expected_stage + + def test_duration_seconds(self, stageresult) -> None: + stageresult.started_at = STARTED_AT + stageresult.finished_at = FINISHED_AT + seconds_value = (FINISHED_AT - STARTED_AT).total_seconds() + assert stageresult.duration_seconds == seconds_value -def test_duration_seconds(stageresult) -> None: - stageresult.started_at = datetime.datetime(2024,5,6,15,45,30) - stageresult.finished_at = datetime.datetime(2024,5,7,15,45,30) - seconds_value = (datetime.datetime(2024,5,7,15,45,30) - datetime.datetime(2024,5,6,15,45,30)).total_seconds() - assert stageresult.duration_seconds == seconds_value @pytest.fixture def pipelinerun(stageresult, runmanifest) -> PipelineRun: - return PipelineRun(runmanifest, - PipelineStatus.SUCCEEDED, - datetime.datetime(2024,5,6,15,45,30), - datetime.datetime(2024,5,7,15,45,30), - [stageresult], - {"stage_test":"example output"}) - -def test_pipelinerun_configuration(pipelinerun, runmanifest, stageresult) -> None: - assert pipelinerun.manifest == runmanifest - assert pipelinerun.status == PipelineStatus.SUCCEEDED - assert pipelinerun.started_at == datetime.datetime(2024,5,6,15,45,30) - assert pipelinerun.completed_at == datetime.datetime(2024,5,7,15,45,30) - assert pipelinerun.stage_results == [stageresult] - assert pipelinerun.stage_outputs == {"stage_test":"example output"} - -def test_result_for(pipelinerun, stageresult) -> None: - assert pipelinerun.result_for("stage_test") == stageresult - assert pipelinerun.result_for("not_a_stage") == None - -@pytest.mark.parametrize("status,expected", - [(PipelineStatus.PENDING, False), - (PipelineStatus.RUNNING, False), - (PipelineStatus.FAILED, False), - (PipelineStatus.SUCCEEDED, True)]) - -def test_succeeded_pipeline(pipelinerun, status, expected) -> None: - pipelinerun.status = status - assert pipelinerun.succeeded == expected - - -#TODO: Test _extract_stages_run and all methods in StageConfig class \ No newline at end of file + return PipelineRun( + runmanifest, + PipelineStatus.SUCCEEDED, + STARTED_AT, + FINISHED_AT, + [stageresult], + {"stage_test": "example output"}, + ) + + +class TestPipelineRun: + def test_pipelinerun_configuration( + self, pipelinerun, runmanifest, stageresult + ) -> None: + assert pipelinerun.manifest == runmanifest + assert pipelinerun.status == PipelineStatus.SUCCEEDED + assert pipelinerun.started_at == STARTED_AT + assert pipelinerun.completed_at == FINISHED_AT + assert pipelinerun.stage_results == [stageresult] + assert pipelinerun.stage_outputs == {"stage_test": "example output"} + + def test_result_for(self, pipelinerun, stageresult) -> None: + assert pipelinerun.result_for("stage_test") == stageresult + assert pipelinerun.result_for("not_a_stage") is None + + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.PENDING, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.FAILED, False), + (PipelineStatus.SUCCEEDED, True), + ], + ) + def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: + pipelinerun.status = status + assert pipelinerun.succeeded == expected + + +# TODO: Test _extract_stages_run and all methods in StageConfig class From f482133e5fb4fcca5e906dc52ac5547f1087f5eb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:27:24 +0100 Subject: [PATCH 285/332] tweak: split test_from_file into two methods to act as singular unit tests rather than doing multiple tasks in one function --- tests/test_models.py | 72 ++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 21f45f4..b831773 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -13,6 +13,8 @@ StageStatus, ) +from tests.test_execution import stageresult + STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) @@ -136,27 +138,12 @@ def test_from_any( with pytest.raises(TypeError): blankpipelineconfig.from_any(11) - def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: - pipeline_config = tmp_path / "configuration.py" - pipeline_config.write_text( - dedent( - """ - {"name":"test_rap", - "backend":"python", - "work_dir":"tmp/work", - "project_root":"project", - "log_dir":"tmp/logs", - "data_dir":"tmp/data", - "allow_subprocess_fallback":True, - "python_executable": , - "metadata":{"variables":["name","age"], - "num_stages":6} - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) + def test_from_file_errors(self, tmp_path) -> PipelineConfig: + """ + Checks that a PipelineConfig instance raises the correct exceptions when a + file is not found or the file does not contain a dictionary mapping. + """ + no_map_pipeline_config = tmp_path / "not_valid.py" no_map_pipeline_config.write_text( dedent( @@ -167,8 +154,6 @@ def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: + "\n", encoding="utf-8", ) - configuration = PipelineConfig.from_file(pipeline_config) - assert configuration == expected_pipeline_config fake_file = "path_not_real" with pytest.raises(FileNotFoundError): @@ -176,6 +161,47 @@ def test_from_file(self, tmp_path, expected_pipeline_config) -> PipelineConfig: with pytest.raises(TypeError): PipelineConfig.from_file(no_map_pipeline_config) + def test_from_file_success( + self, tmp_path, expected_pipeline_config + ) -> PipelineConfig: + """ + Checks that a PipelineConfig instance is created successfully from a mock + file. + """ + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": , + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config + + def test_to_dict(self, pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the From 134d1abd078248adaefe48b0cac4cd7e2b1e5b26 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 14:31:26 +0100 Subject: [PATCH 286/332] docs: complete docstrings for test_models.py --- tests/test_models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index b831773..7486f47 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -281,6 +281,10 @@ def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: assert stageresult.succeeded == expected_stage def test_duration_seconds(self, stageresult) -> None: + """ + Tests that duration_seconds() method calculates the correct duration in seconds + between the started_at and finished_at attributes of the StageResult instance. + """ stageresult.started_at = STARTED_AT stageresult.finished_at = FINISHED_AT seconds_value = (FINISHED_AT - STARTED_AT).total_seconds() @@ -303,6 +307,10 @@ class TestPipelineRun: def test_pipelinerun_configuration( self, pipelinerun, runmanifest, stageresult ) -> None: + """ + Checks that the PipelineRun instance is created successfully with the correct + attributes and values. + """ assert pipelinerun.manifest == runmanifest assert pipelinerun.status == PipelineStatus.SUCCEEDED assert pipelinerun.started_at == STARTED_AT @@ -324,6 +332,10 @@ def test_result_for(self, pipelinerun, stageresult) -> None: ], ) def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: + """ + Checks that the succeeded() method of the PipelineRun instance returns the + correct boolean value based on its status. + """ pipelinerun.status = status assert pipelinerun.succeeded == expected From 2e5f42579e847878f815f467095709fcccad743a Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 15:07:52 +0100 Subject: [PATCH 287/332] tweak: refactor with CoPilot to structure tests as classes --- tests/test_pipeline_architecture.py | 878 ++++++++++++++-------------- 1 file changed, 438 insertions(+), 440 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 02e9892..15add2d 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -1,12 +1,12 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from textwrap import dedent import pytest import yaml -import subprocess -import sys from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph @@ -14,418 +14,415 @@ from onsrap.stage import Stage from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning +NO_STAGES_SPECIFIED_WARNING = "No stages specified to run. All stages running by default." +OUTPUT_DIRECTORY_WARNING = ( + "Output directory is not specified. Using project root or work directory as the run output." +) -def test_pipeline_from_files_executes_python_entrypoints(tmp_path: Path) -> None: - first_stage = tmp_path / "first_stage.py" - first_stage.write_text( - dedent( - """ - def run(context): - return "alpha" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - second_stage = tmp_path / "second_stage.py" - second_stage.write_text( - dedent( - """ - def main(context): - return context.result_for("first_stage").outputs + "-beta" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{} - }, - ) +def _base_pipeline_config(tmp_path: Path) -> dict: + return { + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_config": {}, + } - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.succeeded is True - assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] - assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} - - -def test_pipeline_uses_run_specific_output_directory(tmp_path: Path) -> None: - writer_stage = tmp_path / "writer_stage.py" - writer_stage.write_text( - dedent( - """ - from pathlib import Path - - def main(context): - output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(context.run_id, encoding="utf-8") - return {"output_path": str(output_path), "run_id": context.run_id} - """ - ).strip() - + "\n", - encoding="utf-8", - ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [writer_stage], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{}}, - ) - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - - #TODO: This warning functions however from the name of the test, I would assume that the output - #directory has been set so this needs to be reviewed. - first_run = pipeline.run() - second_run = pipeline.run() - - first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) - second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) - - assert first_run.manifest.run_id != second_run.manifest.run_id - assert first_output != second_output - assert first_output.exists() - assert second_output.exists() - assert first_output.parents[2].name == first_run.manifest.run_id - assert second_output.parents[2].name == second_run.manifest.run_id - - -def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Path) -> None: - script_stage = tmp_path / "script_stage.py" - script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": {}, - "global_config":{}}, +class TestPipelineFromFiles: + def test_pipeline_from_files_executes_python_entrypoints( + self, + tmp_path: Path + ) -> None: + """ + + """ + first_stage = tmp_path / "first_stage.py" + first_stage.write_text( + dedent( + """ + def run(context): + return "alpha" + """ + ).strip() + + "\n", + encoding="utf-8", ) - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.stage_results[0].outputs.strip() == "script fallback works" - assert run.stage_results[0].stdout.strip() == "script fallback works" - - -def test_stage_graph_detects_cycles() -> None: - first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) - second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) - - graph = StageGraph.from_stages([first_stage, second_stage]) + second_stage = tmp_path / "second_stage.py" + second_stage.write_text( + dedent( + """ + def main(context): + return context.result_for("first_stage").outputs + "-beta" + """ + ).strip() + + "\n", + encoding="utf-8", + ) - try: - graph.topological_order() - except Exception as exc: # noqa: BLE001 - assert exc.__class__.__name__ == "DependencyCycleError" - else: - raise AssertionError("Expected a dependency cycle error") + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config=_base_pipeline_config(tmp_path), + ) + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() -def test_pipeline_from_config_builds_stages_and_injects_stage_config(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() - - stage_file = scripts_dir / "0_data_validation.py" - stage_file.write_text( - dedent( - """ - def run(context): - return { - "stage_name": context.stage_config.name, - "years_to_run": context.stage_config.get("years_to_run"), - "target_variable": context.stage_config.require("target_variable"), - } - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - config_file = tmp_path / "conf.yaml" - config_file.write_text( - dedent( - f""" - pipeline_variables: - name: "configured-pipeline" - backend: python - working_dir: "{tmp_path.as_posix()}" - project_root: "{tmp_path.as_posix()}" - log_dir: "{(tmp_path / 'logs').as_posix()}" - stages: - - 0_data_validation: - location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" - run: true - dependencies: [] - - stage_configuration: - 0_data_validation: - years_to_run: 2017 - target_variable: "classification" - global_configuration: - dry_run: true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) - - assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] - assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 - - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() - - assert run.stage_outputs["0_data_validation"] == { - "stage_name": "0_data_validation", - "years_to_run": 2017, - "target_variable": "classification", - } + assert run.succeeded is True + assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] + assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} + def test_pipeline_uses_run_specific_output_directory(self, tmp_path: Path) -> None: + writer_stage = tmp_path / "writer_stage.py" + writer_stage.write_text( + dedent( + """ + from pathlib import Path -def test_pipeline_rejects_unknown_stage_configuration(tmp_path: Path) -> None: - stage_file = tmp_path / "single_stage.py" - stage_file.write_text( - dedent( - """ - def run(context): - return "ok" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [stage_file], - config={"pipeline_config":{"work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs"}, - "stage_configuration": { - "missing_stage": {"years_to_run": 2017}, - }, - "global_config":{} - }, + def main(context): + output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(context.run_id, encoding="utf-8") + return {"output_path": str(output_path), "run_id": context.run_id} + """ + ).strip() + + "\n", + encoding="utf-8", ) - - with pytest.raises(StageConfigurationError, match="unknown stages"): - pipeline.validate() - - -def test_pipeline_from_config_parses_stage_configuration_payloads(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() - - for stage_name in ("0_extract", "1_transform"): - (scripts_dir / f"{stage_name}.py").write_text( + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [writer_stage], + config=_base_pipeline_config(tmp_path), + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + # TODO: This warning functions however from the name of the test, I would assume that the output + # directory has been set so this needs to be reviewed. + first_run = pipeline.run() + second_run = pipeline.run() + + first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) + second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) + + assert first_run.manifest.run_id != second_run.manifest.run_id + assert first_output != second_output + assert first_output.exists() + assert second_output.exists() + assert first_output.parents[2].name == first_run.manifest.run_id + assert second_output.parents[2].name == second_run.manifest.run_id + + def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + script_stage = tmp_path / "script_stage.py" + script_stage.write_text("print('script fallback works')\n", encoding="utf-8") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config=_base_pipeline_config(tmp_path), + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() + + assert run.stage_results[0].outputs.strip() == "script fallback works" + assert run.stage_results[0].stdout.strip() == "script fallback works" + + +class TestStageGraph: + def test_stage_graph_detects_cycles(self) -> None: + first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) + second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) + + graph = StageGraph.from_stages([first_stage, second_stage]) + + try: + graph.topological_order() + except Exception as exc: # noqa: BLE001 + assert exc.__class__.__name__ == "DependencyCycleError" + else: + raise AssertionError("Expected a dependency cycle error") + + +class TestPipelineFromConfig: + def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_file = scripts_dir / "0_data_validation.py" + stage_file.write_text( dedent( """ def run(context): - return context.stage_config.to_dict() + return { + "stage_name": context.stage_config.name, + "years_to_run": context.stage_config.get("years_to_run"), + "target_variable": context.stage_config.require("target_variable"), + } """ ).strip() + "\n", encoding="utf-8", ) - config_payload = { - "pipeline_variables": { - "name": "parse-test", - "backend": "python", - "working_dir": tmp_path.as_posix(), - "project_root": tmp_path.as_posix(), - "data_dir": (tmp_path / "data").as_posix(), - "log_dir": (tmp_path / "logs").as_posix(), - "metadata": { - "description": "configuration parsing test", - }, - "stages": [ - { - "0_extract": { - "location": "", - "run": True, - "dependencies": [], - "owner": "analytics", - } - }, - { - "1_transform": { - "location": "", - "run": True, - "dependencies": ["0_extract"], - } - }, - ], - }, - "stage_configuration": { - "0_extract": { - "years_to_run": 2017, - "datasets": { - "orders": { - "path": "data/orders.csv", - } - }, - "metadata": { - "purpose": "extract", - }, - }, - "1_transform": { - "target_variable": "classification", - "metadata": { - "purpose": "transform", - }, - }, - }, - "global_config":{} - } - - config_file = tmp_path / "conf.yaml" - config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + config_file = tmp_path / "conf.yaml" + config_file.write_text( + dedent( + f""" + pipeline_variables: + name: "configured-pipeline" + backend: python + working_dir: "{tmp_path.as_posix()}" + project_root: "{tmp_path.as_posix()}" + log_dir: "{(tmp_path / 'logs').as_posix()}" + stages: + - 0_data_validation: + location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" + run: true + dependencies: [] + + stage_configuration: + 0_data_validation: + years_to_run: 2017 + target_variable: "classification" + global_configuration: + dry_run: true + """ + ).strip() + + "\n", + encoding="utf-8", + ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) - assert pipeline.name == "parse-test" - assert pipeline.config.work_dir == tmp_path - assert pipeline.config.project_root == tmp_path - assert pipeline.config.log_dir == tmp_path / "logs" - assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] - assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() - assert pipeline.stages[0].metadata["owner"] == "analytics" - assert pipeline.stages[1].dependencies == ("0_extract",) - assert pipeline.stage_configs["0_extract"].variables == {"years_to_run": 2017, "datasets": {"orders": {"path": "data/orders.csv"}}} - assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" + assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] + assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() -def test_pipeline_from_config_scales_stage_configuration_to_many_stages(tmp_path: Path) -> None: - scripts_dir = tmp_path / "scripts" - scripts_dir.mkdir() + assert run.stage_outputs["0_data_validation"] == { + "stage_name": "0_data_validation", + "years_to_run": 2017, + "target_variable": "classification", + } - stage_count = 6 - stage_names = [f"{index}_stage" for index in range(stage_count)] - for index, stage_name in enumerate(stage_names): - stage_file = scripts_dir / f"{stage_name}.py" - previous_stage_name = stage_names[index - 1] if index > 0 else None + def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: + stage_file = tmp_path / "single_stage.py" stage_file.write_text( dedent( - f""" + """ def run(context): - previous_ordinal = None - if {index} > 0: - previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] - return {{ - "stage_name": context.stage_config.name, - "ordinal": context.stage_config.require("ordinal"), - "label": context.stage_config.require("label"), - "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), - "known_stage_configs": sorted(context.stage_configs), - "previous_ordinal": previous_ordinal, - }} + return "ok" """ ).strip() + "\n", encoding="utf-8", ) - stage_definitions = [] - stage_configuration = {} - for index, stage_name in enumerate(stage_names): - dependencies = [stage_names[index - 1]] if index > 0 else [] - stage_definitions.append( - { - stage_name: { - "location": "", - "run": True, - "dependencies": dependencies, - } - } - ) - stage_configuration[stage_name] = { - "ordinal": index, - "label": f"label-{index}", + config = _base_pipeline_config(tmp_path) + config["stage_configuration"] = { + "missing_stage": {"years_to_run": 2017}, } - - config_file = tmp_path / "conf.yaml" - config_file.write_text( - yaml.safe_dump( - { - "pipeline_variables": { - "name": "many-stage-pipeline", - "backend": "python", - "working_dir": tmp_path.as_posix(), - "project_root": tmp_path.as_posix(), - "log_dir": (tmp_path / "logs").as_posix(), - "stages": stage_definitions, + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [stage_file], + config=config, + ) + + with pytest.raises(StageConfigurationError, match="unknown stages"): + pipeline.validate() + + + def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + for stage_name in ("0_extract", "1_transform"): + (scripts_dir / f"{stage_name}.py").write_text( + dedent( + """ + def run(context): + return context.stage_config.to_dict() + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_payload = { + "pipeline_variables": { + "name": "parse-test", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "data_dir": (tmp_path / "data").as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "metadata": { + "description": "configuration parsing test", }, - "stage_configuration": stage_configuration, - "global_config": { - "dry_run": True, + "stages": [ + { + "0_extract": { + "location": "", + "run": True, + "dependencies": [], + "owner": "analytics", + } + }, + { + "1_transform": { + "location": "", + "run": True, + "dependencies": ["0_extract"], + } + } + ], + }, + "stage_configuration": { + "0_extract": { + "years_to_run": 2017, + "datasets": { + "orders": { + "path": "data/orders.csv", + } + }, + "metadata": { + "purpose": "extract", + }, + }, + "1_transform": { + "target_variable": "classification", + "metadata": { + "purpose": "transform", + }, }, }, - sort_keys=False, - ), - encoding="utf-8", - ) + "global_config": {}, + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) + + assert pipeline.name == "parse-test" + assert pipeline.config.work_dir == tmp_path + assert pipeline.config.project_root == tmp_path + assert pipeline.config.log_dir == tmp_path / "logs" + assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] + assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + assert pipeline.stages[0].metadata["owner"] == "analytics" + assert pipeline.stages[1].dependencies == ("0_extract",) + assert pipeline.stage_configs["0_extract"].variables == { + "years_to_run": 2017, + "datasets": {"orders": {"path": "data/orders.csv"}}, + } + assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} + assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" + + + def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_count = 6 + stage_names = [f"{index}_stage" for index in range(stage_count)] + + for index, stage_name in enumerate(stage_names): + stage_file = scripts_dir / f"{stage_name}.py" + previous_stage_name = stage_names[index - 1] if index > 0 else None + stage_file.write_text( + dedent( + f""" + def run(context): + previous_ordinal = None + if {index} > 0: + previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] + return {{ + "stage_name": context.stage_config.name, + "ordinal": context.stage_config.require("ordinal"), + "label": context.stage_config.require("label"), + "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), + "known_stage_configs": sorted(context.stage_configs), + "previous_ordinal": previous_ordinal, + }} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + stage_definitions = [] + stage_configuration = {} + for index, stage_name in enumerate(stage_names): + dependencies = [stage_names[index - 1]] if index > 0 else [] + stage_definitions.append( + { + stage_name: { + "location": "", + "run": True, + "dependencies": dependencies, + } + } + ) + stage_configuration[stage_name] = { + "ordinal": index, + "label": f"label-{index}", + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + yaml.safe_dump( + { + "pipeline_variables": { + "name": "many-stage-pipeline", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "stages": stage_definitions, + }, + "stage_configuration": stage_configuration, + "global_config": { + "dry_run": True, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_config(config_file) + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_config(config_file) - assert [stage.name for stage in pipeline.stages] == stage_names - assert sorted(pipeline.stage_configs) == stage_names + assert [stage.name for stage in pipeline.stages] == stage_names + assert sorted(pipeline.stage_configs) == stage_names - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() - assert run.manifest.stages_run == stage_names - assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names + assert run.manifest.stages_run == stage_names + assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names - for index, stage_name in enumerate(stage_names): - output = run.stage_outputs[stage_name] - assert output["stage_name"] == stage_name - assert output["ordinal"] == index - assert output["label"] == f"label-{index}" - assert output["first_stage_ordinal"] == 0 - assert output["known_stage_configs"] == stage_names - expected_previous = None if index == 0 else index - 1 - assert output["previous_ordinal"] == expected_previous + for index, stage_name in enumerate(stage_names): + output = run.stage_outputs[stage_name] + assert output["stage_name"] == stage_name + assert output["ordinal"] == index + assert output["label"] == f"label-{index}" + assert output["first_stage_ordinal"] == 0 + assert output["known_stage_configs"] == stage_names + expected_previous = None if index == 0 else index - 1 + assert output["previous_ordinal"] == expected_previous REPO_ROOT = Path(__file__).resolve().parents[1] @@ -435,80 +432,81 @@ def run(context): REPO_ROOT / "examples" / "pipeline_2" / "main.py", ] -@pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) -def test_example_main_scripts_run_successfully(script_path: Path) -> None: - result = subprocess.run( - [sys.executable, str(script_path)], - cwd=REPO_ROOT, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, ( - f"Script failed: {script_path}\n" - f"stdout:\n{result.stdout}\n" - f"stderr:\n{result.stderr}" - ) - assert "completed with" in result.stdout.lower() - -def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(tmp_path: Path) -> None: - """ - Integration test that checks that the _log_config method is correctly called within - PipelineRunner.run() and that the information is parsed in a suitable format to a YAML - file in the run directory. - - This test also captures that _combine_configs() correctly converts all configuration - information into a single dictionary that can be serialized to YAML. - """ - stage_file = tmp_path / "single_stage.py" - stage_file.write_text( - dedent( - """ - def run(context): - return {"status": "ok"} - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline.from_files( - [stage_file], - name="config-export-pipeline", - config={ - "pipeline_config": { - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", - }, - "stage_configuration": {}, - "global_configuration": { - "dry_run": True, - } - }, +class TestExamples: + @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) + def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"Script failed: {script_path}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" ) + assert "completed with" in result.stdout.lower() + - with pytest.warns(StageConfigurationWarning, - match = "Output directory is not specified. Using project root or work directory as the run output."): - run = pipeline.run() +class TestPipelineRunConfigurationLogging: + def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(self, tmp_path: Path) -> None: + """ + Integration test that checks that the _log_config method is correctly called within + PipelineRunner.run() and that the information is parsed in a suitable format to a YAML + file in the run directory. - run_dir = tmp_path / "runs" / run.manifest.run_id - config_file = run_dir / ( - f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" - ) + This test also captures that _combine_configs() correctly converts all configuration + information into a single dictionary that can be serialized to YAML. + """ + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return {"status": "ok"} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + }, + }, + ) + + with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): + run = pipeline.run() + + run_dir = tmp_path / "runs" / run.manifest.run_id + config_file = run_dir / ( + f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" + ) - assert config_file.exists() + assert config_file.exists() - file_text = config_file.read_text(encoding="utf-8") - parsed_yaml = yaml.safe_load(file_text) + file_text = config_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) - assert parsed_yaml == run.manifest.config - assert "pipeline_config:\n" in file_text - assert "stage_configs:\n" in file_text - assert "global_config:\n" in file_text - assert "pipeline_config: {" not in file_text - assert "stage_configs: {" not in file_text - assert "global_config: {" not in file_text - assert " dry_run: true" in file_text + assert parsed_yaml == run.manifest.config + assert "pipeline_config:\n" in file_text + assert "stage_configs:\n" in file_text + assert "global_config:\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + assert " dry_run: true" in file_text From d330fbbf36390ec86dd57c6ce372f30307b778d4 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:20:58 +0100 Subject: [PATCH 288/332] docs: add docstrings for all tests in test_pipeline_architecture.py --- tests/test_pipeline_architecture.py | 55 ++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 15add2d..ffd9eb8 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -38,7 +38,8 @@ def test_pipeline_from_files_executes_python_entrypoints( tmp_path: Path ) -> None: """ - + Checks that the pipeline entrypoints are run successfully by reviewing + the outputs of the stages. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -78,7 +79,24 @@ def main(context): assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} - def test_pipeline_uses_run_specific_output_directory(self, tmp_path: Path) -> None: + def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> None: + """ + Checks that the pipelines produce outputs in unique locations based on runs. + As each run produces a unique run_id, the outputs should be written to unique + directories. This test checks that when the same pipeline is run twice, the + outputs are saved into two locations. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage specification in the Pipeline + configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the parent directory for the run + output. This does not affect the test capability. + """ writer_stage = tmp_path / "writer_stage.py" writer_stage.write_text( dedent( @@ -102,11 +120,10 @@ def main(context): ) with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - # TODO: This warning functions however from the name of the test, I would assume that the output - # directory has been set so this needs to be reviewed. first_run = pipeline.run() second_run = pipeline.run() + first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -118,6 +135,10 @@ def main(context): assert second_output.parents[2].name == second_run.manifest.run_id def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + """ + Checks that a pipeline will run with a non-module based Python script by running the + entire script. + """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") @@ -137,6 +158,10 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_pa class TestStageGraph: def test_stage_graph_detects_cycles(self) -> None: + """ + Checks that the stage graph appropriately detects required orders + based on dependencies in the stages. + """ first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) @@ -152,6 +177,10 @@ def test_stage_graph_detects_cycles(self) -> None: class TestPipelineFromConfig: def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + """ + Checks that from_config() method appropriately builds the configurations + for the pipeline and uses the configurations to run the Pipeline. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -191,6 +220,7 @@ def run(context): 0_data_validation: years_to_run: 2017 target_variable: "classification" + global_configuration: dry_run: true """ @@ -216,6 +246,10 @@ def run(context): def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: + """ + Checks that the pipeline raises an error when a stage configuration is provided + for a stage that is not within the pipeline. + """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( dedent( @@ -243,6 +277,12 @@ def run(context): def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + """ + Checks that the correct information from a configuration file is parsed into + the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig + instance. Also covers that the stage configuration is correctly injected into + the stage. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -332,6 +372,10 @@ def run(context): def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + """ + Checks that multiple stage configurations can be parsed from a configuration + file and input in the correct order into the pipeline. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -435,6 +479,9 @@ def run(context): class TestExamples: @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: + """ + Checks that a main script in a pipeline is successfully run. + """ result = subprocess.run( [sys.executable, str(script_path)], cwd=REPO_ROOT, From b013120ea50e7065731dd2608cc7a8784dc8c3cd Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:33:16 +0100 Subject: [PATCH 289/332] tweak: ruff format and check test_pipeline_architecture.py --- tests/test_pipeline_architecture.py | 163 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 54 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index ffd9eb8..adaf028 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -14,10 +14,11 @@ from onsrap.stage import Stage from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning -NO_STAGES_SPECIFIED_WARNING = "No stages specified to run. All stages running by default." -OUTPUT_DIRECTORY_WARNING = ( - "Output directory is not specified. Using project root or work directory as the run output." +NO_STAGES_SPECIFIED_WARNING = ( + "No stages specified to run. All stages running by default." ) +OUTPUT_DIRECTORY_WARNING = "Output directory is not specified. Using project root or " \ + "work directory as the run output." def _base_pipeline_config(tmp_path: Path) -> dict: @@ -34,12 +35,11 @@ def _base_pipeline_config(tmp_path: Path) -> dict: class TestPipelineFromFiles: def test_pipeline_from_files_executes_python_entrypoints( - self, - tmp_path: Path - ) -> None: + self, tmp_path: Path + ) -> None: """ Checks that the pipeline entrypoints are run successfully by reviewing - the outputs of the stages. + the outputs of the stages. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -65,7 +65,9 @@ def main(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [first_stage, second_stage], dependencies={"second_stage": ("first_stage",)}, @@ -76,24 +78,30 @@ def main(context): run = pipeline.run() assert run.succeeded is True - assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] - assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} + assert [result.name for result in run.stage_results] == [ + "first_stage", + "second_stage", + ] + assert run.stage_outputs == { + "first_stage": "alpha", + "second_stage": "alpha-beta", + } def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> None: """ Checks that the pipelines produce outputs in unique locations based on runs. As each run produces a unique run_id, the outputs should be written to unique directories. This test checks that when the same pipeline is run twice, the - outputs are saved into two locations. + outputs are saved into two locations. Raises ------ 'PipelineConfigurationWarning' Expected and asserted as there is no stage specification in the Pipeline - configuration. This does not affect the test capability. + configuration. This does not affect the test capability. 'StageConfigurationWarning' - Expected and asserted as there is no output directory specified - in the configuration. This means that the Pipeline defaults to using + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using the project root or working directory as the parent directory for the run output. This does not affect the test capability. """ @@ -104,7 +112,9 @@ def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> Non from pathlib import Path def main(context): - output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" + output_path = Path( + context.run_dir + ) / "data" / "interim" / "artifact.txt" output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(context.run_id, encoding="utf-8") return {"output_path": str(output_path), "run_id": context.run_id} @@ -113,7 +123,9 @@ def main(context): + "\n", encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [writer_stage], config=_base_pipeline_config(tmp_path), @@ -123,7 +135,6 @@ def main(context): first_run = pipeline.run() second_run = pipeline.run() - first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -134,15 +145,19 @@ def main(context): assert first_output.parents[2].name == first_run.manifest.run_id assert second_output.parents[2].name == second_run.manifest.run_id - def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(self, tmp_path: Path) -> None: + def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( + self, tmp_path: Path + ) -> None: """ - Checks that a pipeline will run with a non-module based Python script by running the - entire script. + Checks that a pipeline will run with a non-module based Python script by + running the entire script. """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [script_stage], name="script-pipeline", @@ -160,10 +175,18 @@ class TestStageGraph: def test_stage_graph_detects_cycles(self) -> None: """ Checks that the stage graph appropriately detects required orders - based on dependencies in the stages. + based on dependencies in the stages. """ - first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) - second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) + first_stage = Stage( + name="first_stage", + source=lambda context: None, + dependencies=("second_stage",), + ) + second_stage = Stage( + name="second_stage", + source=lambda context: None, + dependencies=("first_stage",), + ) graph = StageGraph.from_stages([first_stage, second_stage]) @@ -176,7 +199,9 @@ def test_stage_graph_detects_cycles(self) -> None: class TestPipelineFromConfig: - def test_pipeline_from_config_builds_stages_and_injects_stage_config(self, tmp_path: Path) -> None: + def test_pipeline_from_config_builds_stages_and_injects_stage_config( + self, tmp_path: Path + ) -> None: """ Checks that from_config() method appropriately builds the configurations for the pipeline and uses the configurations to run the Pipeline. @@ -192,7 +217,9 @@ def run(context): return { "stage_name": context.stage_config.name, "years_to_run": context.stage_config.get("years_to_run"), - "target_variable": context.stage_config.require("target_variable"), + "target_variable": context.stage_config.require( + "target_variable" + ), } """ ).strip() @@ -209,10 +236,12 @@ def run(context): backend: python working_dir: "{tmp_path.as_posix()}" project_root: "{tmp_path.as_posix()}" - log_dir: "{(tmp_path / 'logs').as_posix()}" + log_dir: "{(tmp_path / "logs").as_posix()}" stages: - 0_data_validation: - location: "{(tmp_path / 'scripts' / '0_data_validation.py').as_posix()}" + location: "{ + (tmp_path / "scripts" / "0_data_validation.py").as_posix() + }" run: true dependencies: [] @@ -229,7 +258,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] @@ -244,11 +275,10 @@ def run(context): "target_variable": "classification", } - def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: """ Checks that the pipeline raises an error when a stage configuration is provided - for a stage that is not within the pipeline. + for a stage that is not within the pipeline. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -266,7 +296,9 @@ def run(context): config["stage_configuration"] = { "missing_stage": {"years_to_run": 2017}, } - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [stage_file], config=config, @@ -275,11 +307,12 @@ def run(context): with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() - - def test_pipeline_from_config_parses_stage_configuration_payloads(self, tmp_path: Path) -> None: + def test_pipeline_from_config_parses_stage_configuration_payloads( + self, tmp_path: Path + ) -> None: """ - Checks that the correct information from a configuration file is parsed into - the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig + Checks that the correct information from a configuration file is parsed into + the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig instance. Also covers that the stage configuration is correctly injected into the stage. """ @@ -324,7 +357,7 @@ def run(context): "run": True, "dependencies": ["0_extract"], } - } + }, ], }, "stage_configuration": { @@ -350,9 +383,13 @@ def run(context): } config_file = tmp_path / "conf.yaml" - config_file.write_text(yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8") + config_file.write_text( + yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8" + ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" @@ -360,7 +397,9 @@ def run(context): assert pipeline.config.project_root == tmp_path assert pipeline.config.log_dir == tmp_path / "logs" assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] - assert pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + assert ( + pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + ) assert pipeline.stages[0].metadata["owner"] == "analytics" assert pipeline.stages[1].dependencies == ("0_extract",) assert pipeline.stage_configs["0_extract"].variables == { @@ -368,12 +407,16 @@ def run(context): "datasets": {"orders": {"path": "data/orders.csv"}}, } assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} - assert pipeline.stage_configs["1_transform"].require("target_variable") == "classification" - + assert ( + pipeline.stage_configs["1_transform"].require("target_variable") + == "classification" + ) - def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tmp_path: Path) -> None: + def test_pipeline_from_config_scales_stage_configuration_to_many_stages( + self, tmp_path: Path + ) -> None: """ - Checks that multiple stage configurations can be parsed from a configuration + Checks that multiple stage configurations can be parsed from a configuration file and input in the correct order into the pipeline. """ scripts_dir = tmp_path / "scripts" @@ -391,12 +434,16 @@ def test_pipeline_from_config_scales_stage_configuration_to_many_stages(self, tm def run(context): previous_ordinal = None if {index} > 0: - previous_ordinal = context.result_for("{previous_stage_name}").outputs["ordinal"] + previous_ordinal = context.result_for( + "{previous_stage_name}" + ).outputs["ordinal"] return {{ "stage_name": context.stage_config.name, "ordinal": context.stage_config.require("ordinal"), "label": context.stage_config.require("label"), - "first_stage_ordinal": context.stage_config_for("{stage_names[0]}").require("ordinal"), + "first_stage_ordinal": context.stage_config_for( + "{stage_names[0]}" + ).require("ordinal"), "known_stage_configs": sorted(context.stage_configs), "previous_ordinal": previous_ordinal, }} @@ -446,7 +493,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names @@ -476,6 +525,7 @@ def run(context): REPO_ROOT / "examples" / "pipeline_2" / "main.py", ] + class TestExamples: @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: @@ -498,14 +548,17 @@ def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: class TestPipelineRunConfigurationLogging: - def test_pipeline_run_writes_manifest_config_yaml_to_run_directory(self, tmp_path: Path) -> None: + def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( + self, tmp_path: Path + ) -> None: """ - Integration test that checks that the _log_config method is correctly called within - PipelineRunner.run() and that the information is parsed in a suitable format to a YAML - file in the run directory. + Integration test that checks that the _log_config method is correctly called + within PipelineRunner.run() and that the information is parsed in a suitable + format to a YAML file in the run directory. - This test also captures that _combine_configs() correctly converts all configuration - information into a single dictionary that can be serialized to YAML. + This test also captures that _combine_configs() correctly converts all + configuration information into a single dictionary that can be serialized + to YAML. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -519,7 +572,9 @@ def run(context): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING): + with pytest.warns( + PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + ): pipeline = Pipeline.from_files( [stage_file], name="config-export-pipeline", From 09e9fd60828e431dc6d35d4eb8494f3e37b17129 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:41:40 +0100 Subject: [PATCH 290/332] tweak: refactor test_runner.py with CoPilot (human review) to be structured as classes --- tests/test_runner.py | 272 ++++++++++++++++++++++--------------------- 1 file changed, 142 insertions(+), 130 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 225413f..6333af2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,9 +1,7 @@ +from pathlib import Path from textwrap import dedent import pytest - -from pathlib import Path - import yaml from onsrap.execution import ExecutionContext @@ -12,131 +10,145 @@ from onsrap.runner import _log_config, print_config_diffs -def test_log_config_writes_manifest_config_as_block_style_yaml(tmp_path: Path) -> None: - """ - Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. - """ - run_dir = tmp_path / "runs" / "synthetic_run" - run_dir.mkdir(parents=True) - - config = PipelineConfig( - name="synthetic_pipeline", - stages_to_run={"stage_a": True}, - backend="python", - work_dir=tmp_path / "work", - project_root=tmp_path, - output_dir=tmp_path / "outputs", - log_dir=tmp_path / "logs", - data_dir=tmp_path / "data", - allow_subprocess_fallback=True, - python_executable=None, - metadata={"reason": "unit test"}, - ) - - context = ExecutionContext( - pipeline_name="synthetic_pipeline", - run_id="run_1234", - config=config, - logger=Logger(), - run_dir=run_dir, - working_directory=tmp_path, - stage_configs={}, - global_config=None, - ) - - manifest_config = { - "pipeline_config": { - "name": "synthetic_pipeline", - "backend": "python", - "output_dir": str(tmp_path / "outputs"), - }, - "stage_configs": { - "stage_a": { - "years_to_run": 2026, - "target_variable": "classification", - } - }, - "global_config": { - "dry_run": True, - }, - } - - manifest = RunManifest( - rap_name="synthetic_pipeline", - run_id="run_1234", - config=manifest_config, - ) - - _log_config(run_dir, context, manifest) - - expected_file = run_dir / ( - "configuration_for_" - f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" - ) - - assert expected_file.exists() - - file_text = expected_file.read_text(encoding="utf-8") - parsed_yaml = yaml.safe_load(file_text) - - assert parsed_yaml == manifest_config - assert "stage_configs:\n" in file_text - assert " stage_a:\n" in file_text - assert " years_to_run: 2026\n" in file_text - assert "pipeline_config: {" not in file_text - assert "stage_configs: {" not in file_text - assert "global_config: {" not in file_text - -def test_print_config_diffs(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """ - Tests that two configuration files are correctly compared and the differences are - both returned in a computer-readable format and printed to the console. One change - for each category (changed, added, removed) is included in the test to ensure that - all cases are handled correctly. - """ - test_file_a = tmp_path / "config_a.yaml" - test_file_b = tmp_path / "config_b.yaml" - - test_file_a.write_text( - dedent(""" - pipeline_config: - name: synthetic_pipeline - output_dir: outputs - stage_configs: - stage_a: - years_to_run: 2026 - target_variable: classification - global_config: - dry_run: True - """).strip() - + "\n", - encoding="utf-8", +class TestLogConfig: + def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + """ + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True) + + config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + manifest_config = { + "pipeline_config": { + "name": "synthetic_pipeline", + "backend": "python", + "output_dir": str(tmp_path / "outputs"), + }, + "stage_configs": { + "stage_a": { + "years_to_run": 2026, + "target_variable": "classification", + } + }, + "global_config": { + "dry_run": True, + }, + } + + manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + config=manifest_config, + ) + + _log_config(run_dir, context, manifest) + + expected_file = run_dir / ( + "configuration_for_" + f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" + ) + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + + assert parsed_yaml == manifest_config + assert "stage_configs:\n" in file_text + assert " stage_a:\n" in file_text + assert " years_to_run: 2026\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + + +class TestPrintConfigDiffs: + @staticmethod + def _write_yaml(path: Path, content: str) -> None: + path.write_text(dedent(content).strip() + "\n", encoding="utf-8") + + def test_returns_and_prints_differences( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + """ + Tests that two configuration files are correctly compared and the differences are + both returned in a computer-readable format and printed to the console. One change + for each category (changed, added, removed) is included in the test to ensure that + all cases are handled correctly. + """ + test_file_a = tmp_path / "config_a.yaml" + test_file_b = tmp_path / "config_b.yaml" + + self._write_yaml( + test_file_a, + """ + pipeline_config: + name: synthetic_pipeline + output_dir: outputs + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: classification + global_config: + dry_run: True + """, + ) + + self._write_yaml( + test_file_b, + """ + pipeline_config: + name: synthetic_pipeline + backend: python + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: identification + global_config: + dry_run: True + """, + ) + + assert print_config_diffs(test_file_a, test_file_b) == { + "changed": { + "stage_configs.stage_a.target_variable": ( + "classification", + "identification", ) - - test_file_b.write_text( - dedent(""" - pipeline_config: - name: synthetic_pipeline - backend: python - stage_configs: - stage_a: - years_to_run: 2026 - target_variable: identification - global_config: - dry_run: True - """).strip() - + "\n", - encoding="utf-8", - ) - - assert print_config_diffs(test_file_a, test_file_b) == { - "changed": {"stage_configs.stage_a.target_variable": ("classification", "identification")}, - "added": {"pipeline_config.backend": "python"}, - "removed": {"pipeline_config.output_dir": "outputs"} - } - - captured = capsys.readouterr() - assert "CHANGED (1)" in captured.out - assert "ADDED in second configuration (1)" in captured.out - assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + }, + "added": {"pipeline_config.backend": "python"}, + "removed": {"pipeline_config.output_dir": "outputs"}, + } + + captured = capsys.readouterr() + assert "CHANGED (1)" in captured.out + assert "ADDED in second configuration (1)" in captured.out + assert "REMOVED in second configuration (1)" in captured.out + assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file From 83b72f3396da416e06f6a0037a849fbf76c07000 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:45:04 +0100 Subject: [PATCH 291/332] docs: add doc strings to test_runner.py methods --- tests/test_runner.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index 6333af2..e48f74c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,9 +11,13 @@ class TestLogConfig: - def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: + def test_writes_manifest_config_as_block_style_yaml( + self, + tmp_path: Path + ) -> None: """ - Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + Tests that the ``_log_config`` function correctly writes the manifest + configuration to a YAML file in block style format. """ run_dir = tmp_path / "runs" / "synthetic_run" run_dir.mkdir(parents=True) @@ -70,7 +74,8 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non expected_file = run_dir / ( "configuration_for_" - f"{context.pipeline_name}_{context.started_at.date()}_{context.run_id[-8:]}.yaml" + f"{context.pipeline_name}_{context.started_at.date()}_" + f"{context.run_id[-8:]}.yaml" ) assert expected_file.exists() @@ -90,6 +95,12 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non class TestPrintConfigDiffs: @staticmethod def _write_yaml(path: Path, content: str) -> None: + """ + Helper function that writes a YAML file to the specified path with + the provided content. The content is dedented and stripped of + leading/trailing whitespace before being written to the file. A newline is + added at the end of the file. + """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") def test_returns_and_prints_differences( @@ -98,10 +109,10 @@ def test_returns_and_prints_differences( capsys: pytest.CaptureFixture[str], ) -> None: """ - Tests that two configuration files are correctly compared and the differences are - both returned in a computer-readable format and printed to the console. One change - for each category (changed, added, removed) is included in the test to ensure that - all cases are handled correctly. + Tests that two configuration files are correctly compared and the differences + are both returned in a computer-readable format and printed to the console. + One change for each category (changed, added, removed) is included in the test + to ensure that all cases are handled correctly. """ test_file_a = tmp_path / "config_a.yaml" test_file_b = tmp_path / "config_b.yaml" From 30171a880052bfffeccb450bfbd56b76dbd98afc Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 16:46:03 +0100 Subject: [PATCH 292/332] tweak: Ruff formatting and checking on test_runner.py --- tests/test_runner.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_runner.py b/tests/test_runner.py index e48f74c..9265465 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,12 +11,9 @@ class TestLogConfig: - def test_writes_manifest_config_as_block_style_yaml( - self, - tmp_path: Path - ) -> None: + def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: """ - Tests that the ``_log_config`` function correctly writes the manifest + Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. """ run_dir = tmp_path / "runs" / "synthetic_run" @@ -96,9 +93,9 @@ class TestPrintConfigDiffs: @staticmethod def _write_yaml(path: Path, content: str) -> None: """ - Helper function that writes a YAML file to the specified path with - the provided content. The content is dedented and stripped of - leading/trailing whitespace before being written to the file. A newline is + Helper function that writes a YAML file to the specified path with + the provided content. The content is dedented and stripped of + leading/trailing whitespace before being written to the file. A newline is added at the end of the file. """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") @@ -162,4 +159,4 @@ def test_returns_and_prints_differences( assert "CHANGED (1)" in captured.out assert "ADDED in second configuration (1)" in captured.out assert "REMOVED in second configuration (1)" in captured.out - assert "stage_configs.stage_a.target_variable" in captured.out \ No newline at end of file + assert "stage_configs.stage_a.target_variable" in captured.out From 5fa1db246198341f445366d32fe3b25da104158d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Wed, 5 Aug 2026 17:40:03 +0100 Subject: [PATCH 293/332] docs: add parameters and raises sections to docstrings of all test files --- tests/test_execution.py | 178 +++++++++++++++++++++++++++- tests/test_models.py | 109 +++++++++++++++++ tests/test_pipeline.py | 143 ++++++++++++++++++++++ tests/test_pipeline_architecture.py | 111 ++++++++++++++++- tests/test_runner.py | 19 +++ tests/test_stage.py | 116 +++++++++++++++--- 6 files changed, 652 insertions(+), 24 deletions(-) diff --git a/tests/test_execution.py b/tests/test_execution.py index 618e112..19e191f 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -63,6 +63,17 @@ def stage_config() -> StageConfig: def execution(config, logger, stageresult, stage_config) -> ExecutionContext: """ Create an ExecutionContext object for testing. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ run_dir = Path("tmp/run") work_dir = Path("tmp/work_dir") @@ -123,6 +134,17 @@ def test_executioncontext_creation( ) -> None: """ Test that the ExecutionContext creates the right attributes. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. """ assert execution.pipeline_name == "test_pipeline" assert execution.run_id == "run_id_1234" @@ -140,6 +162,15 @@ def test_record( """ Tests that StageResult attributes are attached to stage_results and variables attributes in the ExecutionContext instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. """ execution.record(stageresult) assert execution.stage_results == {"stage_test": expected_recorded_stage_result} @@ -150,6 +181,15 @@ def test_result_for( ) -> None: """ Tests that result_for correctly extracts the results of a requested stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. """ execution.record(stageresult) assert execution.result_for("stage_test") == expected_recorded_stage_result @@ -158,12 +198,28 @@ def test_stage_outputs(self, execution, stageresult) -> None: """ Tests that stage_outputs shows the outputs attribute of the StageResult instance for a requested stage is extracted. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. """ execution.record(stageresult) assert execution.stage_outputs == {"stage_test": "example output"} @pytest.fixture def blank_context_with_config_none(self, stageresult) -> ExecutionContext: + """ + Fixture that returns a test ExecutionContext instance with a None config for + testing error handling. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + """ run_dir = Path("tmp/run") work_dir = Path("tmp/work_dir") return ExecutionContext( @@ -183,6 +239,19 @@ def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: Tests that get_data_dir method extracts the path from the execution context or, if the context is None, returns an error to indicate that additional input is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``blank_context_with_config_none`` : ExecutionContext + An ``ExecutionContext`` object with a None config for testing error + handling. + + Raises + ------ + ``PipelineConfigurationError`` + If the config attribute of the ExecutionContext instance is None. """ assert execution.get_data_dir() == Path("tmp/config_data") @@ -193,7 +262,17 @@ def test_resolve_output_root(self, execution) -> None: """ Tests that resolve_output_root method extracts the path from the given run directory or, if None are given, raises an error to indicate additional input - is required.. + is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + If the run_dir attribute of the ExecutionContext instance is None. """ work_dir = Path("tmp/work_dir") assert execution.resolve_output_root() == Path("tmp/run") @@ -219,6 +298,20 @@ def test_stage_config_accessors_return_named_and_active_configs( """ Tests that getter methods to return the stage_config for a named stage returns correct attributes based on given parameters. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. """ stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) context = ExecutionContext( @@ -244,8 +337,14 @@ def test_set_active_stage(self, execution, stage_config) -> None: """ Tests that set_active_stage correctly sets the active_stage attribute in the ExecutionContext instance. - """ + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ execution.set_active_stage(stage_config.name) assert execution.active_stage_name == stage_config.name execution.set_active_stage(None) @@ -254,6 +353,13 @@ def test_set_active_stage(self, execution, stage_config) -> None: def test_stage_config_for(self, execution, stage_config) -> None: """ Tests that stage_config_for returns the StageConfig for a named stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ assert execution.stage_config_for(stage_config.name) == stage_config assert execution.stage_config_for("missing_stage") is None @@ -261,6 +367,13 @@ def test_stage_config_for(self, execution, stage_config) -> None: def test_stage_config(self, execution, stage_config) -> None: """ Tests that stage_config exposes the currently active stage configuration. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. """ assert execution.stage_config is None execution.set_active_stage(stage_config.name) @@ -270,6 +383,21 @@ def test_get_stage_config(self, execution, stage_config) -> None: """ Tests that get_stage_config returns variables by default and the full StageConfig object when requested. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. + """ assert execution.get_stage_config() == {} with pytest.raises(PipelineConfigurationError): @@ -278,8 +406,7 @@ def test_get_stage_config(self, execution, stage_config) -> None: execution.set_active_stage(stage_config.name) assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} - with pytest.raises(PipelineConfigurationError): - execution.get_stage_config(vars_only=False) + assert ( execution.get_stage_config(with_global=False, vars_only=False) == stage_config @@ -318,6 +445,18 @@ def test_resolve_given_path_add_folders( Tests the add_folder functionality for lists, single strings, or None type in the resolve_given_path class method as well as when the file_name is a valid string or None type. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``add_folder`` : Union[str, List[str], None] + A string, list of strings, or None type to specify additional folders to + add to the path. + ``file_name`` : Union[str, None] + A string or None type to specify the file name to append to the path. + ``expected`` : Path + The expected Path object that should be returned by the method. """ path_name = "data_path" root = Path("tmp/data") @@ -331,6 +470,11 @@ def test_resolve_given_path_norm(self, execution) -> None: """ Tests that resolve_given_path returns a file path that has been output in a StageResult instance. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ execution.record( StageResult( @@ -364,6 +508,11 @@ def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: """ Checks that entrypoints are set correctly in the PythonStageExecutor instance. + + Parameters + ---------- + ``pythonstageexecutor`` : PythonStageExecutor + A ``PythonStageExecutor`` object for testing. """ assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") @@ -374,6 +523,11 @@ def test_combine_vars(self, execution) -> None: Test that checks that a dictionary is returned, combining values from a global configuration and a stage configuration whilst removing any stage specific exclusions. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} @@ -397,6 +551,17 @@ def test_combine_vars_errors(self, execution) -> None: Test that confirms that a warning is raised if there is a variable defined in both the global and the stage configurations as well as asserting the correct values. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``StageConfigurationWarning`` + If a variable is defined in both the global and stage configurations, a + warning is raised to indicate that the stage variable will take precedence. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {"stage_1": ["global_var2"]} @@ -421,6 +586,11 @@ def test_combine_vars_no_exclusion(self, execution) -> None: """ Test confirming that a dictionary is returned, combining values from a global configuration and a stage configuration when there are no exclusions defined. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. """ global_vars = {"global_var1": "value1", "global_var2": "value2"} exclusions = {} diff --git a/tests/test_models.py b/tests/test_models.py index 7486f47..f2aa837 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -57,6 +57,11 @@ class TestRuntimeID: def test_runtimeID_creation(self, runtimeID) -> None: """ Test that a RuntimeID is correctly created. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. """ assert runtimeID.id == "abc123" assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) @@ -66,6 +71,11 @@ def test_runtimeID_creation(self, runtimeID) -> None: def test_getter_functions_runtimeID(self, runtimeID) -> None: """ Tests all the getter functions for the RuntimeID instance. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. """ assert runtimeID.get_id() == "abc123" assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) @@ -101,6 +111,12 @@ def expected_pipeline_config() -> PipelineConfig: @pytest.fixture def pipelineconfig(expected_pipeline_config) -> PipelineConfig: + """ + Returns a PipelineConfig instance for testing that is derived + fromthe expected_pipeline_config fixture. Used as a separate + fixture to ensure that the behaviour of the from_any() method is + tested correctly in the TestPipelineConfig class. + """ return expected_pipeline_config @@ -130,6 +146,16 @@ def test_from_any( Test derivation for a PipelineConfig instance using the from_any() method. This test checks all methods EXCEPT from_file as this will be covered in another test due to creation of a mock file being required. + + Parameters + ---------- + ``mapping`` : dict + A dictionary mapping of values for a PipelineConfig instance. + + Raises + ------ + TypeError + If the input to from_any() is not of a supported type. """ assert blankpipelineconfig.from_any(None) == PipelineConfig() assert blankpipelineconfig.from_any(pipelineconfig) == expected_pipeline_config @@ -142,6 +168,20 @@ def test_from_file_errors(self, tmp_path) -> PipelineConfig: """ Checks that a PipelineConfig instance raises the correct exceptions when a file is not found or the file does not contain a dictionary mapping. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``FileNotFoundError`` + If the file path provided does not exist. + + ``TypeError`` + If the file does not contain a dictionary mapping. """ no_map_pipeline_config = tmp_path / "not_valid.py" @@ -167,6 +207,15 @@ def test_from_file_success( """ Checks that a PipelineConfig instance is created successfully from a mock file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``expected_pipeline_config`` : PipelineConfig + A PipelineConfig instance that is expected to be created from the mock + file. """ pipeline_config = tmp_path / "configuration.py" pipeline_config.write_text( @@ -206,6 +255,11 @@ def test_to_dict(self, pipelineconfig) -> None: """ Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values as a dictionary. + + Parameters + ---------- + ``pipelineconfig`` : PipelineConfig + A PipelineConfig instance for testing. """ assert pipelineconfig.to_dict() == { @@ -249,6 +303,11 @@ def test_stage_result(self, stageresult) -> None: """ Uses a StageResult instance created in test_execution to ensure that the class instance is created suitably with required defaults. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. """ assert stageresult.name == "stage_test" assert stageresult.status == "pending" @@ -276,6 +335,16 @@ def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: """ Tests succeeded() method for StageResult which outputs True or False depending on the status of the StageResult. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``status_stage`` : StageStatus + A StageStatus value to set the status of the StageResult instance. + ``expected_stage`` : bool + The expected boolean output from the succeeded() method based on the + status of the StageResult instance. """ stageresult.status = status_stage assert stageresult.succeeded == expected_stage @@ -284,6 +353,11 @@ def test_duration_seconds(self, stageresult) -> None: """ Tests that duration_seconds() method calculates the correct duration in seconds between the started_at and finished_at attributes of the StageResult instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. """ stageresult.started_at = STARTED_AT stageresult.finished_at = FINISHED_AT @@ -293,6 +367,17 @@ def test_duration_seconds(self, stageresult) -> None: @pytest.fixture def pipelinerun(stageresult, runmanifest) -> PipelineRun: + """ + Creates a PipelineRun instance for testing that is used in the TestPipelineRun + class. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + """ return PipelineRun( runmanifest, PipelineStatus.SUCCEEDED, @@ -310,6 +395,15 @@ def test_pipelinerun_configuration( """ Checks that the PipelineRun instance is created successfully with the correct attributes and values. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + ``stageresult`` : StageResult + A StageResult instance for testing. """ assert pipelinerun.manifest == runmanifest assert pipelinerun.status == PipelineStatus.SUCCEEDED @@ -319,6 +413,11 @@ def test_pipelinerun_configuration( assert pipelinerun.stage_outputs == {"stage_test": "example output"} def test_result_for(self, pipelinerun, stageresult) -> None: + """ + Checks that the result_for() method of the PipelineRun instance returns the + correct StageResult instance when provided with a valid stage name, and returns + None when the stage name is not found. + """ assert pipelinerun.result_for("stage_test") == stageresult assert pipelinerun.result_for("not_a_stage") is None @@ -335,6 +434,16 @@ def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: """ Checks that the succeeded() method of the PipelineRun instance returns the correct boolean value based on its status. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``status`` : PipelineStatus + A PipelineStatus value to set the status of the PipelineRun instance. + ``expected`` : bool + The expected boolean output from the succeeded() method based on the + status of the PipelineRun instance. """ pipelinerun.status = status assert pipelinerun.succeeded == expected diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 52f3021..9558a51 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -19,6 +19,16 @@ def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stag Function that builds a Stage object with a given name, dependencies, and a source file path that's built out of the name if it is not provided. This standardises the creation of Stage objects for testing. + + Parameters + ---------- + ``name`` : str + The name of the stage to be created. + ``dependencies`` : tuple + A tuple of stage names that the created stage depends on. + ``source`` : Path | None + A Path object representing the source file for the stage. If None, a default + source file path is created based on the stage name. """ resolved_source = source if source is not None else Path(f"{name}.py") return Stage(name, source=resolved_source, dependencies=dependencies) @@ -34,6 +44,13 @@ def test_pipeline_name(self): if no name was given (shown in pipeline_config), or defaults to "pipeline" if no name is provided through Pipeline instance creation or through the PipelineConfig (shown through pipeline_no_name) + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ pipeline_config = PipelineConfig(name="test_pipeline_config") @@ -52,6 +69,20 @@ def test_assign_dependencies(self, tmp_path): Pipeline creation and appropriately assigned to each stage within the Pipeline. Will also check for error raise if the dependencies are defined but there are no defined stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as there are no stages defined in the Pipeline + but there are dependencies. """ def example_function(): @@ -104,6 +135,20 @@ def test_add_dependencies_single_dict(self, tmp_path): """ Tests that a dictionary correctly assigns dependencies to individual stages and the Pipeline instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as dependencies are specified for stages that do not + exist in the Pipeline instance. """ path_1 = tmp_path / "Stage_1.py" @@ -149,6 +194,17 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: the stage and the stage_configurations are correctly added to the Pipeline instance and the stage_configurations are correctly associated with the stage. + + Parameter + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() @@ -169,6 +225,17 @@ def test_add_stage_warns_when_stage_config_count_mismatches( Tests that when a stage is added but there is not the correct number of stage_configs provided, a warning is raised and the stage_configuration for that stage is added as a blank StageConfig object. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline() @@ -194,6 +261,17 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage( Tests that when a stage_configuration is added to a Pipeline instance, the configuration is correctly associated with the named stage and that the configuration is coerced into a StageConfig object if it is provided. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): pipeline = Pipeline(stages=[stage_factory("Stage_0")]) @@ -211,6 +289,11 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( Tests that when resolving stages_to_run, the Pipeline instance correctly includes all dependent stages required in the StageGraph even if these are not explicitly called out in the configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -238,6 +321,16 @@ def test_resolve_stages_to_run_rejects_disabled_dependencies( """ Checks that when resolving stages_to_run, the Pipeline init raises an error if a stage is enabled but one of its dependencies is disabled. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Raised when a stage is enabled but one of its dependencies is disabled. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) @@ -254,6 +347,17 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None """ Pipeline.stages always holds all stages; only graph.stages is the effective run set. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -273,6 +377,17 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( """ Tests that when a stage is manually disabled in a Pipeline instance, it is initialised in the stages_to_run configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -288,6 +403,12 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No Tests that when a stage is manually enabled in a Pipeline instance, it is correctly reflected in the stages_to_run configuration and the stage is included in the execution graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -307,6 +428,12 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( """ Tests that when a new stage is added to a Pipeline instance, it is kept out of the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( @@ -330,6 +457,12 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr """ Tests that when a new stage is added to a Pipeline instance with enable_stages=True, it is included in the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ stage_0 = stage_factory("Stage_0") pipeline = Pipeline( @@ -354,6 +487,11 @@ def test_validate_skips_source_check_for_disabled_stages( """ Disabled stages' source files need not exist - validate() only checks the effective run set. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. """ enabled_file = tmp_path / "Stage_0.py" enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") @@ -376,6 +514,11 @@ def test_construct_manifest_inputs_contains_only_effective_stages( """ Manifest inputs should list only the stages that are part of the execution graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index adaf028..4515107 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -40,6 +40,21 @@ def test_pipeline_from_files_executes_python_entrypoints( """ Checks that the pipeline entrypoints are run successfully by reviewing the outputs of the stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the run output. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -94,11 +109,16 @@ def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> Non directories. This test checks that when the same pipeline is run twice, the outputs are saved into two locations. + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage specification in the Pipeline - configuration. This does not affect the test capability. + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified in the configuration. This means that the Pipeline defaults to using @@ -151,6 +171,20 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( """ Checks that a pipeline will run with a non-module based Python script by running the entire script. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") @@ -205,6 +239,20 @@ def test_pipeline_from_config_builds_stages_and_injects_stage_config( """ Checks that from_config() method appropriately builds the configurations for the pipeline and uses the configurations to run the Pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -279,6 +327,20 @@ def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> N """ Checks that the pipeline raises an error when a stage configuration is provided for a stage that is not within the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( @@ -315,6 +377,18 @@ def test_pipeline_from_config_parses_stage_configuration_payloads( the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig instance. Also covers that the stage configuration is correctly injected into the stage. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -418,6 +492,20 @@ def test_pipeline_from_config_scales_stage_configuration_to_many_stages( """ Checks that multiple stage configurations can be parsed from a configuration file and input in the correct order into the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() @@ -531,6 +619,11 @@ class TestExamples: def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: """ Checks that a main script in a pipeline is successfully run. + + Parameters + ---------- + ``script_path`` : Path + The path to the main.py script of a pipeline example. """ result = subprocess.run( [sys.executable, str(script_path)], @@ -559,6 +652,20 @@ def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( This test also captures that _combine_configs() correctly converts all configuration information into a single dictionary that can be serialized to YAML. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. """ stage_file = tmp_path / "single_stage.py" stage_file.write_text( diff --git a/tests/test_runner.py b/tests/test_runner.py index 9265465..faab644 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -15,6 +15,11 @@ def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> Non """ Tests that the ``_log_config`` function correctly writes the manifest configuration to a YAML file in block style format. + + Parameters + ---------- + tmp_path : Path + A temporary directory provided by pytest for creating test files. """ run_dir = tmp_path / "runs" / "synthetic_run" run_dir.mkdir(parents=True) @@ -97,6 +102,13 @@ def _write_yaml(path: Path, content: str) -> None: the provided content. The content is dedented and stripped of leading/trailing whitespace before being written to the file. A newline is added at the end of the file. + + Parameters + ---------- + ``path`` : Path + The path where the YAML file will be written. + ``content`` : str + The YAML content to write to the file. """ path.write_text(dedent(content).strip() + "\n", encoding="utf-8") @@ -110,6 +122,13 @@ def test_returns_and_prints_differences( are both returned in a computer-readable format and printed to the console. One change for each category (changed, added, removed) is included in the test to ensure that all cases are handled correctly. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. + ``capsys`` : pytest.CaptureFixture[str] + A pytest fixture that captures output to stdout and stderr during the test. """ test_file_a = tmp_path / "config_a.yaml" test_file_b = tmp_path / "config_b.yaml" diff --git a/tests/test_stage.py b/tests/test_stage.py index b29a182..9f61b33 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -45,6 +45,11 @@ class TestStage: def test_stage_creation_callable(self, stage_test) -> None: """ Tests that attributes have been appropriately assigned to Stage class. + + Parameter + --------- + stage_test : Stage + A ``Stage`` object created with a callable source for testing. """ assert stage_test.name == "callable_stage" assert stage_test.source == example_function @@ -57,6 +62,16 @@ def test_stage_name_error(self, example_function) -> None: """ Tests that a StageConfigurationError is raised if the name is left blank in a Stage class instance. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the name is left blank in a ``Stage`` class instance. """ with pytest.raises(StageConfigurationError): Stage("", example_function, ["stage_1"], {"info": "example"}) @@ -64,6 +79,12 @@ def test_stage_name_error(self, example_function) -> None: def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. """ with pytest.raises(StageConfigurationError): Stage("callable_stage", 11, ["stage_1"], {"info": "example"}) @@ -71,6 +92,11 @@ def test_stage_source_type(self) -> None: def test_stage_backend(self, example_function) -> None: """ Tests that backend can be any string, None, and corrects for whitespace. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_diff = Stage( "callable_stage", @@ -100,29 +126,44 @@ def test_stage_backend(self, example_function) -> None: def test_stage_from_files_error(self, tmp_path: Path) -> None: """ Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance """ source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): Stage.from_file(source_file) - def test_stage_from_callable_name(self) -> None: + def test_stage_from_callable_name(self, example_function) -> None: """ Tests that a stage name is extracted from a callable object stage. - """ - - def example_function(): - pass + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ test = Stage.from_callable(example_function) assert test.name == "example_function" - def test_from_dict_norm(self) -> None: + def test_from_dict_norm(self, example_function) -> None: """ Tests that a stage instance is created from a dictionary item. - """ - def example_function(): - pass + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) @@ -132,6 +173,11 @@ def test_with_dependencies_list(self, stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is a list. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. """ new_deps = ["stage2", "stage3"] new_deps_blank = [] @@ -145,6 +191,21 @@ def test_with_dependencies_list(self, stage_test) -> None: def test_validate(self, stage_test, tmp_path) -> None: """ Tests whether an error is raised if the source file isn't suitable. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. In this instance, it raises if the source is None, an empty + string, or a Path object that is not a file. """ stage_test.source = None with pytest.raises(StageConfigurationError): @@ -157,10 +218,20 @@ def test_validate(self, stage_test, tmp_path) -> None: with pytest.raises(StageConfigurationError): stage_test.validate() - def test_source_path(self, stage_test, tmp_path) -> None: + def test_source_path(self, stage_test, tmp_path, example_function) -> None: """ Tests whether source_path detects a path vs other valid and invalid source types. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_test.source = tmp_path / "fake_file.py" assert stage_test.source_path == tmp_path / "fake_file.py" @@ -168,25 +239,28 @@ def test_source_path(self, stage_test, tmp_path) -> None: assert stage_test.source_path is None stage_test.source = "not a file path" assert stage_test.source_path is None - - def example_function(): - pass - stage_test.source = example_function assert stage_test.source_path is None - def test_source_label(self, stage_test, tmp_path) -> None: + def test_source_label(self, stage_test, tmp_path, example_function) -> None: """ Tests that source_label is created if the source is a Path or a callable and is None if it is another type. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ stage_test.source = tmp_path / "fake_file.py" temp_path_str = str(tmp_path / "fake_file.py") assert stage_test.source_label == temp_path_str - def example_function(): - pass - stage_test.source = example_function assert stage_test.source_label == "tests.test_stage.example_function" @@ -203,6 +277,12 @@ class TestStageFactories: def test_stage_instance_from_file(self, tmp_path) -> None: """ Tests that a Stage instance is created from a filepath. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. """ test_stage = tmp_path / "test_stage.py" test_stage.write_text( From cebb435fc6775c255ba9c6e878c52c2dac8a1b87 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 09:18:17 +0100 Subject: [PATCH 294/332] tweak: switch around tests in test_stage.py to more logically fit the class structure --- tests/test_stage.py | 92 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 9f61b33..c68ebc9 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -123,52 +123,6 @@ def test_stage_backend(self, example_function) -> None: assert stage.backend == "python" assert stage_white_space.backend == "python" - def test_stage_from_files_error(self, tmp_path: Path) -> None: - """ - Tests that if the file doesn't exist, a StageConfigurationError is raised. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and - manipulation. - - Raises - ------ - ``StageConfigurationError`` - If the source file doesn't exist when attempting to create a - ``Stage`` instance - """ - source_file = tmp_path / "not_an_actual_file.py" - with pytest.raises(StageConfigurationError): - Stage.from_file(source_file) - - def test_stage_from_callable_name(self, example_function) -> None: - """ - Tests that a stage name is extracted from a callable object stage. - - Parameters - ---------- - ``example_function`` : callable - A callable function to pass as a source for a ``Stage`` class instance. - """ - test = Stage.from_callable(example_function) - assert test.name == "example_function" - - def test_from_dict_norm(self, example_function) -> None: - """ - Tests that a stage instance is created from a dictionary item. - - Parameters - ---------- - ``example_function`` : callable - A callable function to pass as a source for a ``Stage`` class instance. - """ - - data = {"name": "test_Stage", "callable": example_function} - stage = Stage.from_dict(data) - assert stage.source == example_function - def test_with_dependencies_list(self, stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is @@ -300,3 +254,49 @@ def main(): assert Stage.from_file(test_stage, entrypoint="main") == Stage( "test_stage", test_stage.resolve(), (), {}, "main", "python" ) + + def test_stage_from_files_error(self, tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + Stage.from_file(source_file) + + def test_stage_from_callable_name(self, example_function) -> None: + """ + Tests that a stage name is extracted from a callable object stage. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function) + assert test.name == "example_function" + + def test_from_dict_norm(self, example_function) -> None: + """ + Tests that a stage instance is created from a dictionary item. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + + data = {"name": "test_Stage", "callable": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function From 02399f7eab064bf7e9c1b95390ad753bf66406ba Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Thu, 6 Aug 2026 14:54:35 +0100 Subject: [PATCH 295/332] tests: added additional tests to test_stage.py on recommendation from CoPilot. Some tests not added and these are detailed in issue #40 --- tests/test_stage.py | 618 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 572 insertions(+), 46 deletions(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index c68ebc9..9be5702 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -3,6 +3,7 @@ import pytest +from onsrap.errors import StageDependencyError from onsrap.stage import Stage, StageConfigurationError, _normalize_dependencies @@ -24,17 +25,65 @@ def test_normalize_dependencies_str(self) -> None: ["Stage_1.py", " Stage_2.py", "Stage_3.py "] ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") + def test_normalize_dependencies_dedupe(self) -> None: + """ + Tests that duplicate values are removed from the normalized dependencies + whilst preserving first seen order. + """ + assert _normalize_dependencies( + ["Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_1.py", "Stage_2.py") + + assert _normalize_dependencies( + ["Stage_2.py","Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_2.py","Stage_1.py") + + def test_normalize_dependencies_whitespace_handling(self) -> None: + """ + Tests that whitespace only or blank dependency values are removed from + the normalised dependencies. + """ + assert _normalize_dependencies( + [" ", ""] + ) == () + + def test_normalize_dependencies_type_check(self) -> None: + """ + Tests that normalize_dependencies works with other iterables such as tuples + and sets, and raises a TypeError for non-iterable types. + """ + assert _normalize_dependencies(("Stage_1.py", "Stage_2.py")) == ( + "Stage_1.py", + "Stage_2.py", + ) + + result = _normalize_dependencies({"Stage_1.py", "Stage_2.py"}) + assert set(result) == {"Stage_1.py", "Stage_2.py"} + + with pytest.raises(TypeError): + _normalize_dependencies(11) + + def test_normalize_dependencies_mixed_types(self) -> None: + """ + Tests that normalize_dependencies stringifies non-string types in a + dependency iterable. + """ + assert _normalize_dependencies(["Stage_1.py", 11, "Stage_2.py"]) == ( + "Stage_1.py", "11", "Stage_2.py" + ) + + @pytest.fixture def example_function(): """ Test function to pass as a callable stage for stage testing. """ - pass + return example_function @pytest.fixture -def stage_test() -> Stage: +def stage_test(example_function) -> Stage: """ Stage object for testing Stage class methods and construction. """ @@ -61,7 +110,7 @@ def test_stage_creation_callable(self, stage_test) -> None: def test_stage_name_error(self, example_function) -> None: """ Tests that a StageConfigurationError is raised if the name is left blank - in a Stage class instance. + in a Stage class instance. This also includes whitespace only names. Parameters ---------- @@ -71,11 +120,15 @@ def test_stage_name_error(self, example_function) -> None: Raises ------ ``StageConfigurationError`` - If the name is left blank in a ``Stage`` class instance. + If the name is left blank or entirely whitespace in a ``Stage`` class + instance. """ with pytest.raises(StageConfigurationError): Stage("", example_function, ["stage_1"], {"info": "example"}) + with pytest.raises(StageConfigurationError): + Stage(" ", example_function, ["stage_1"], {"info": "example"}) + def test_stage_source_type(self) -> None: """ Tests that a non-valid source type returns a StageConfigurationError. @@ -123,54 +176,132 @@ def test_stage_backend(self, example_function) -> None: assert stage.backend == "python" assert stage_white_space.backend == "python" - def test_with_dependencies_list(self, stage_test) -> None: + def test_stage_backend_irregular_values(self, example_function) -> None: """ - Tests adding different types of dependencies when the original dependency is - a list. + Tests that backend defaults with a None or whitespace only string to "python" + and converts any non-string type (other than None) to a string. Parameters ---------- - ``stage_test`` : Stage - A ``Stage`` object created with a callable source for testing. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. """ - new_deps = ["stage2", "stage3"] - new_deps_blank = [] - stage_test_list = stage_test.with_dependencies(new_deps) - stage_test_blank = stage_test.with_dependencies(new_deps_blank) - assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") - assert stage_test_blank.dependencies == ("stage_1",) - stage_test = stage_test.with_dependencies("stage2", "stage3") - assert stage_test.dependencies == ("stage_1", "stage2", "stage3") + stage_none = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=None, + ) - def test_validate(self, stage_test, tmp_path) -> None: + stage_blank = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=" ", + ) + stage_non_string = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=11, + ) + + assert stage_none.backend == "python" + assert stage_blank.backend == "python" + assert stage_non_string.backend == "11" + + + def test_stage_constructor_expands_string_source_with_home( + self, + monkeypatch, + tmp_path): """ - Tests whether an error is raised if the source file isn't suitable. + Check that a string source is converted to a Path and expanded with + expanduser(). This uses fake environmental variables to make sure that the + tests are not dependent on the actual user's home directory. Parameters ---------- - ``stage_test`` : Stage - A ``Stage`` object created with a callable source for testing. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. ``tmp_path`` : Path A temporary path provided by pytest for testing file creation and manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() - Raises - ------ - ``StageConfigurationError`` - If the source is not a valid callable or file path in a ``Stage`` class - instance. In this instance, it raises if the source is None, an empty - string, or a Path object that is not a file. + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="string_source_stage", + source="~/scripts/my_stage.py", + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + + def test_stage_constructor_expands_path_source_with_home( + self, + monkeypatch, + tmp_path): """ - stage_test.source = None - with pytest.raises(StageConfigurationError): - stage_test.validate() - not_file_path = tmp_path - stage_test.source = not_file_path - with pytest.raises(StageConfigurationError): - stage_test.validate() - stage_test.source = "" - with pytest.raises(StageConfigurationError): - stage_test.validate() + Check that a Path source is expanded with expanduser(). This uses fake + environmental variables to make sure that the tests are not dependent on the + actual user's home directory. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="path_source_stage", + source=Path("~/scripts/my_stage.py"), + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + def test_normalise_dependencies_within_stage_init(self, example_function) -> None: + """ + Thin smoke test to check that _normalize_dependencies is called within the Stage + post_init method. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage = Stage( + name="test_stage", + source=example_function, + dependencies=["dep1", "dep2", "dep1", " dep3 ", "", " "], + metadata={}, + ) + assert stage.dependencies == ("dep1", "dep2", "dep3") def test_source_path(self, stage_test, tmp_path, example_function) -> None: """ @@ -199,7 +330,8 @@ def test_source_path(self, stage_test, tmp_path, example_function) -> None: def test_source_label(self, stage_test, tmp_path, example_function) -> None: """ Tests that source_label is created if the source is a Path or a callable - and is None if it is another type. + and is None if it is another type. This also checks that if the callable + has no name attribute, the stage name is used as the source label. Parameters ---------- @@ -221,16 +353,236 @@ def test_source_label(self, stage_test, tmp_path, example_function) -> None: stage_test.source = 11 assert stage_test.source_label is None + class NoName: + def __call__(self): pass + + stage_test.source = NoName() + assert stage_test.source_label == f"tests.test_stage.{stage_test.name}" + + def test_metadata_copy_safely(self) -> None: + """ + Checks that if the original metadata dictionary is modified after the Stage + instance is created, the Stage instance's metadata remains unchanged. + """ + #TODO: do we want this to be how it works? Or would the user assume that if + #they modify the original dict, it modifies the Stage instance. + original_metadata = {"info": "example"} + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=[], + metadata=original_metadata, + ) + original_metadata["info"] = "modified" + assert stage.metadata["info"] == "example" + + def test_repr_function(self) -> None: + """ + Tests that the __repr__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_repr = ( + "Stage(name=callable_stage, " + "source=tests.test_stage., " + "dependencies=('stage_1',), " + "metadata={'info': 'example'}, " + "entrypoint=main, " + "backend=python)" + ) + assert repr(stage) == expected_repr + + def test_str_function(self) -> None: + """ + Tests that the __str__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_str = ( + " Name: callable_stage\n" + " Source: tests.test_stage. \n" + " Dependencies: ('stage_1',)\n" + " Metadata: {'info': 'example'} \n" + " Entrypoint: main \n" + " Backend: python" + ) + assert str(stage) == expected_str + +class TestValidateStage: + def test_validate(self, stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. In this instance, it raises if the source is None, an empty + string, or a Path object that is not a file. + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + + def test_validate_successes(self, stage_test, example_function, temp_script) -> None: + """ + Tests that validate successfully approves of callables and file paths as + sources for a stage instance. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + stage_test.source = example_function + assert stage_test.validate() == None + + stage_test.source = temp_script(filename="valid_script.py") + assert stage_test.validate() == None + +class TestWithDependencies: + def test_with_dependencies_list(self, stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. Also checks that the original stage_test instance is not modified when + with_dependencies is called. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + new_deps_blank = [] + original_deps = stage_test.dependencies + + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") + assert stage_test_blank.dependencies == ("stage_1",) + stage_test_2 = stage_test.with_dependencies("stage2", "stage3") + assert stage_test_2.dependencies == ("stage_1", "stage2", "stage3") + + stage_test.with_dependencies("stage2", "stage3") + assert stage_test.dependencies == original_deps + + def test_with_dependencies_errors(self, stage_test) -> None: + """ + Tests that a StageDependencyError is raised if a nested list is + provided in dependencies. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + with pytest.raises(StageDependencyError): + stage_test.with_dependencies(["stage2", ["nested_stage"]]) + + def test_with_dependencies_list_positional_args(self, stage_test) -> None: + """ + Tests that with_dependencies can accept a list and positional arguments in the + same call and combine them into a single normalized dependencies tuple. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + stage_test_combined = stage_test.with_dependencies(new_deps, "stage4") + assert stage_test_combined.dependencies == ( + "stage_1", + "stage2", + "stage3", + "stage4" + ) + + def test_with_dependencies_duplicates(self, stage_test) -> None: + """ + Tests that when the same dependency is added through with_dependencies, + it is not duplicated in the dependencies tuple of the new Stage instance. + + Caution that this only deduplicates due to Stage post_init calling + _normalize_dependencies however if that moves, + test_normalise_dependencies_within_stage_init will capture the issue. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + print(f"before: {stage_test.dependencies}") + new_stage = stage_test.with_dependencies(["stage_1"],) + print(f"after: {new_stage.dependencies}") + assert new_stage.dependencies == ("stage_1",) + """ TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST """ +@pytest.fixture +def temp_script(tmp_path): + """ + Fixture factory that creates temporary Python scripts. + + Usage: + script = temp_script("def main(): pass") + script = temp_script("def process(): return 42", "processor.py") + """ + def _create_script(content="def main(): pass\n", filename="temp_script.py"): + script = tmp_path / filename + script.write_text(content, encoding="utf-8") + return script + return _create_script class TestStageFactories: - def test_stage_instance_from_file(self, tmp_path) -> None: + """ + Parent class for tests which create Stage class instances from different methods. + """ + +class TestStageFromFile(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a file path. + """ + def test_stage_instance_from_file(self, temp_script) -> None: """ - Tests that a Stage instance is created from a filepath. + Tests that a Stage instance is created from a filepath where name is either + default value from file stem or a user defined name. Parameters ---------- @@ -238,8 +590,7 @@ def test_stage_instance_from_file(self, tmp_path) -> None: A temporary path provided by pytest for testing file creation and manipulation. """ - test_stage = tmp_path / "test_stage.py" - test_stage.write_text( + test_stage = temp_script( dedent( """ def main(): @@ -248,13 +599,17 @@ def main(): """ ).strip() + "\n", - encoding="utf-8", + "test_stage.py", ) assert Stage.from_file(test_stage, entrypoint="main") == Stage( "test_stage", test_stage.resolve(), (), {}, "main", "python" ) + assert Stage.from_file(test_stage, name="Stage_1", entrypoint="main") == Stage( + "Stage_1", test_stage.resolve(), (), {}, "main", "python" + ) + def test_stage_from_files_error(self, tmp_path: Path) -> None: """ Tests that if the file doesn't exist, a StageConfigurationError is raised. @@ -274,7 +629,64 @@ def test_stage_from_files_error(self, tmp_path: Path) -> None: source_file = tmp_path / "not_an_actual_file.py" with pytest.raises(StageConfigurationError): Stage.from_file(source_file) - + + def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch): + """ + Tests that a relative path passed to from_file is resolved to an + absolute path on the Stage source attribute. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + script = temp_script() + + monkeypatch.chdir(script.parent) + + stage = Stage.from_file(script.name) + + assert stage.source.is_absolute() + assert stage.source == script.resolve() + + def test_from_file_expands_source_path(self, tmp_path, monkeypatch): + """ + Tests that a path with a tilde (~) is expanded to the user's home directory + when passed to from_file. Uses monkeypatch to set a fake home directory for + testing purposes. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + script = fake_home / "scripts" / "my_stage.py" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("def main(): pass\n", encoding="utf-8") + + stage = Stage.from_file("~/scripts/my_stage.py") + + expected = fake_home / "scripts" / "my_stage.py" + assert stage.source == expected + + +class TestStageFromCallable(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a callable object. + """ def test_stage_from_callable_name(self, example_function) -> None: """ Tests that a stage name is extracted from a callable object stage. @@ -287,9 +699,40 @@ def test_stage_from_callable_name(self, example_function) -> None: test = Stage.from_callable(example_function) assert test.name == "example_function" - def test_from_dict_norm(self, example_function) -> None: + def test_from_callable_fallback_name(self): + """ + Tests that a fallback name is assigned to a stage instance if the callable + object does not have a name attribute. + """ + class NoName: + def __call__(self): pass + + stage = Stage.from_callable(NoName()) + assert stage.name == "stage" + + def test_from_callable_explicit_name(self, example_function) -> None: """ - Tests that a stage instance is created from a dictionary item. + Tests that an explicit name is assigned to a stage instance if provided. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function, name="explicit_name") + assert test.name == "explicit_name" + +class TestStageFromDict(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a dictionary. + """ + + def test_from_dict_callable_sources(self, example_function) -> None: + """ + Tests that callable sources are correctly used to create a stage instance + from a dictionary regardless of whether the key is source or callable. + Also validates that the name is correctly assigned from the dictionary or + derived from the callable. Parameters ---------- @@ -300,3 +743,86 @@ def test_from_dict_norm(self, example_function) -> None: data = {"name": "test_Stage", "callable": example_function} stage = Stage.from_dict(data) assert stage.source == example_function + assert stage.name == "test_Stage" + + data = {"source": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + assert stage.name == "example_function" + + def test_from_dict_aliases(self, temp_script) -> None: + """ + Tests that a stage instance is created from a dictionary item with aliases + source and path as source options. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + script = temp_script( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + "test_stage.py", + ) + + data = { + "name": "test_Stage", + "source": script, + "entrypoint": "main", + } + + data_2 = { + "name": "test_Stage2", + "path": script, + "entrypoint": "main", + } + stage = Stage.from_dict(data) + stage_2 = Stage.from_dict(data_2) + assert stage.source == script.resolve() + assert stage.name == "test_Stage" + assert stage_2.source == script.resolve() + assert stage_2.name == "test_Stage2" + + def test_from_dict_errors(self) -> None: + """ + Tests that a StageConfigurationError is raised if the dictionary does not + contain a valid source or callable key. + + Raises + ------ + ``StageConfigurationError`` + If the dictionary does not contain a valid source or callable key. + """ + data = {"name": "test_Stage"} + with pytest.raises(StageConfigurationError): + Stage.from_dict(data) + + def test_all_keys_from_dict_in_stage(self, example_function) -> None: + """ + Checks that from_dict does not change the originally parsed dictionary so + that if the dictionary is needed later, it is not permanently changed when + creating a stage instance from it. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + data = { + "name": "test_Stage", + "source": example_function, + "dependencies": ["dep1", "dep2"], + "metadata": {"info": "example"}, + "entrypoint": "main", + "backend": "python", + } + original = dict(data) + Stage.from_dict(data) + assert original == data From 4723a4ce26d73af331e9109e653167a80c13d7fb Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 10:18:45 +0100 Subject: [PATCH 296/332] test: add fixture for integration testing --- tests/test_pipeline.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d10825e..87a3972 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -968,4 +968,32 @@ def test_correct_yaml_file_chosen(self, result = load_historical_run(run_dir=run_dir) assert isinstance(result, PipelineRun) - \ No newline at end of file +class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): + @pytest.fixture + def pipeline_with_history(self, tmp_path: Path, minimal_pipeline_yaml) -> Pipeline: + """ + Sets up a Pipeline instance with a historical run for testing. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + config=PipelineConfig(output_dir=tmp_path / "outputs"), + stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + ) + pipeline.run_output = tmp_path / "runs" + + # Create a historical run directory and YAML file + historical_run_dir = pipeline.run_output / "2026-08-10_100000_abc12345" + historical_run_dir.mkdir(parents=True, exist_ok=True) + (historical_run_dir / "pipeline_attributes_for_test.yaml").write_text( + minimal_pipeline_yaml(run_id="2026-08-10_100000_abc12345"), encoding="utf-8" + ) + + return pipeline \ No newline at end of file From 54b637d422cbbcde01fe19864ff933adf356d93d Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 10:25:25 +0100 Subject: [PATCH 297/332] tweak: correct failed tests given merge with execution context work --- tests/test_pipeline.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 444db2e..b2efe7a 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -55,9 +55,11 @@ def test_pipeline_name(self): pipeline_config = PipelineConfig(name="test_pipeline_config") with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline_named = Pipeline(name="test_pipeline_name") - pipeline_config = Pipeline(name=None, config=pipeline_config) - pipeline_no_name = Pipeline() + pipeline_named = Pipeline(name="test_pipeline_name", + stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + pipeline_config = Pipeline(name=None, config=pipeline_config, + stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + pipeline_no_name = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) assert pipeline_named.name == "test_pipeline_name" assert pipeline_config.name == "test_pipeline_config" @@ -207,7 +209,7 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline() + pipeline = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) stage = stage_factory("Stage_1") stage_config = StageConfig( name="Stage_1", _variables={"years_to_run": 2017} @@ -238,7 +240,7 @@ def test_add_stage_warns_when_stage_config_count_mismatches( Pipeline configuration. This does not affect the test capability. """ with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline() + pipeline = Pipeline(stages = [Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=())]) stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") @@ -532,7 +534,7 @@ def test_construct_manifest_inputs_contains_only_effective_stages( assert list(manifest.inputs.keys()) == ["Stage_0"] - def test_generate_context_correctly_assigns_executor() -> None: + def test_generate_context_correctly_assigns_executor(self,) -> None: """ Test that the correct executor class is assigned to the Pipeline instance based on the backend specified. If the backend does not have a compatible @@ -549,7 +551,7 @@ def test_generate_context_correctly_assigns_executor() -> None: Pipeline(backend="nonexistent_backend", stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - def test_validate_stage_backends_errors() -> None: + def test_validate_stage_backends_errors(self) -> None: """ Test that the _validate_stage_backends method correctly raises an error if the backends for a stage do not match the Pipeline backend or if there are From 3aa29cf6fe03c45418fcf142fcacdb82ce3e125e Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 11:08:56 +0100 Subject: [PATCH 298/332] tests: add integration tests for load_latest_run --- tests/test_pipeline.py | 65 +++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 87a3972..1764ed8 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -969,18 +969,23 @@ def test_correct_yaml_file_chosen(self, assert isinstance(result, PipelineRun) class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): - @pytest.fixture - def pipeline_with_history(self, tmp_path: Path, minimal_pipeline_yaml) -> Pipeline: + def test_no_previous_runs_pipeline(self, tmp_path: Path) -> None: """ - Sets up a Pipeline instance with a historical run for testing. + Tests that if a Pipeline instance has no previous runs, the _load_latest_run + method will return None and raise a warning. Assert that it will also store + None in the last_run attribute of the Pipeline instance. Parameters ---------- ``tmp_path`` : Path A temporary directory provided by pytest for creating test files and directories. - ``minimal_pipeline_yaml`` : callable - A fixture that returns a minimal YAML configuration for a historical run. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. """ with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( @@ -988,12 +993,50 @@ def pipeline_with_history(self, tmp_path: Path, minimal_pipeline_yaml) -> Pipeli stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], ) pipeline.run_output = tmp_path / "runs" + assert pipeline.last_run is None + + def test_last_run_populated_one_run(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: + """ + Tests that if a Pipeline instance has one previous run, this is loaded in + last_run attribute at Pipeline creation. - # Create a historical run directory and YAML file - historical_run_dir = pipeline.run_output / "2026-08-10_100000_abc12345" - historical_run_dir.mkdir(parents=True, exist_ok=True) - (historical_run_dir / "pipeline_attributes_for_test.yaml").write_text( - minimal_pipeline_yaml(run_id="2026-08-10_100000_abc12345"), encoding="utf-8" + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. + """ + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + "{\"run_id\": \"2026-08-11_100000_abc12345\", " + " \"run_dir\": \"/path/to/run\"}\n" ) - return pipeline \ No newline at end of file + temp_attributes = (tmp_path / "outputs" / "runs" / "2026-08-11_100000_abc12345" + / "pipeline_attributes_for_test.yaml") + temp_attributes.parent.mkdir(parents=True, exist_ok=True) + temp_attributes.write_text(minimal_pipeline_yaml( + run_id = "2026-08-11_100000_abc12345" + ), encoding="utf-8") + + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + config=PipelineConfig(output_dir=tmp_path / "outputs", + log_dir=logs), + stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + ) + + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "2026-08-11_100000_abc12345" \ No newline at end of file From d7141c7a91f24f4d382e1d2d4893a9a2b2f77aae Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 11:18:33 +0100 Subject: [PATCH 299/332] tweak: add warning if no stages are defined in a Pipeline --- onsrap/pipeline.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 1e32b40..3f8f9d9 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -1024,8 +1024,16 @@ def _generate_context(self) -> None: ``PipelineInitialisationError`` If the backend for the Pipeline does not have a compatible executor class """ - # Check all stages have the same backend as Pipeline - self._validate_stage_backends() + + if len(self.stages) == 0: + warnings.warn( + "No stages have been defined in the Pipeline. The Pipeline will not run any stages.", + PipelineConfigurationWarning, + ) + + else: + # Check all stages have the same backend as Pipeline + self._validate_stage_backends() backend_key = str(self.backend).strip().lower() execution_class_name = f"{backend_key.capitalize()}StageExecutor" From b6a8b37dfcf868699bf1e128f342680a39ab148b Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 11:53:28 +0100 Subject: [PATCH 300/332] tweak: add example_function fixture into TestStage --- tests/test_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_stage.py b/tests/test_stage.py index 9be5702..04bb1c3 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -91,7 +91,7 @@ def stage_test(example_function) -> Stage: class TestStage: - def test_stage_creation_callable(self, stage_test) -> None: + def test_stage_creation_callable(self, stage_test, example_function) -> None: """ Tests that attributes have been appropriately assigned to Stage class. From 0c992338278f3074fcc1e8fbe40789d01b339d29 Mon Sep 17 00:00:00 2001 From: Sophie Pike_ONS <133784265+pikes-ons@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:55:24 +0100 Subject: [PATCH 301/332] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_models.py | 18 ++++-------------- tests/test_stage.py | 11 ++++------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index f2aa837..ec8468b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -228,7 +228,7 @@ def test_from_file_success( "log_dir":"tmp/logs", "data_dir":"tmp/data", "allow_subprocess_fallback":True, - "python_executable": , + "python_executable": null, "metadata":{"variables":["name","age"], "num_stages":6} } @@ -237,16 +237,6 @@ def test_from_file_success( + "\n", encoding="utf-8", ) - no_map_pipeline_config = tmp_path / "not_valid.py" - no_map_pipeline_config.write_text( - dedent( - """ - variable = "Hello world" - """ - ).strip() - + "\n", - encoding="utf-8", - ) configuration = PipelineConfig.from_file(pipeline_config) assert configuration == expected_pipeline_config @@ -265,11 +255,11 @@ def test_to_dict(self, pipelineconfig) -> None: assert pipelineconfig.to_dict() == { "name": "test_rap", "backend": "python", - "work_dir": "tmp\\work", + "work_dir": str(Path("tmp/work")), "project_root": "project", "output_dir": None, - "log_dir": "tmp\\logs", - "data_dir": "tmp\\data", + "log_dir": str(Path("tmp/logs")), + "data_dir": str(Path("tmp/data")), "allow_subprocess_fallback": True, "python_executable": None, "variables": ["name", "age"], diff --git a/tests/test_stage.py b/tests/test_stage.py index 04bb1c3..c4dd016 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -354,7 +354,8 @@ def test_source_label(self, stage_test, tmp_path, example_function) -> None: assert stage_test.source_label is None class NoName: - def __call__(self): pass + def __call__(self): + pass stage_test.source = NoName() assert stage_test.source_label == f"tests.test_stage.{stage_test.name}" @@ -545,15 +546,11 @@ def test_with_dependencies_duplicates(self, stage_test) -> None: ``stage_test`` : Stage A ``Stage`` object created with a callable source for testing. """ - print(f"before: {stage_test.dependencies}") - new_stage = stage_test.with_dependencies(["stage_1"],) - print(f"after: {new_stage.dependencies}") + new_stage = stage_test.with_dependencies(["stage_1"]) assert new_stage.dependencies == ("stage_1",) -""" -TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST -""" +# TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST @pytest.fixture def temp_script(tmp_path): From 9246dd7ae8d346d95c90b887a09d11b2a460e9e6 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 12:04:51 +0100 Subject: [PATCH 302/332] tweak: adjust gitignore for example scripts --- .gitignore | 12 +++-- examples/pipeline_1/logs/onsrap.log | 45 ------------------ .../0_data_validation.cpython-314.pyc | Bin 7462 -> 0 bytes .../1_preprocessing.cpython-314.pyc | Bin 8553 -> 0 bytes .../__pycache__/2_reporting.cpython-314.pyc | Bin 8141 -> 0 bytes 5 files changed, 7 insertions(+), 50 deletions(-) delete mode 100644 examples/pipeline_1/logs/onsrap.log delete mode 100644 examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc delete mode 100644 examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc delete mode 100644 examples/pipeline_1/scripts/__pycache__/2_reporting.cpython-314.pyc diff --git a/.gitignore b/.gitignore index 961976d..59f0e36 100644 --- a/.gitignore +++ b/.gitignore @@ -895,14 +895,11 @@ docs/_linkcheck/ !examples/ !examples/** -# Allow logs folders and their contents inside examples -!examples/**/logs/ -!examples/**/logs/** -!examples/**/*.log # Allow scripts folders and their contents inside examples !examples/**/scripts/ !examples/**/scripts/** +examples/**/scripts/__pycache__/ # Allow example data files !examples/**/*.csv @@ -913,4 +910,9 @@ examples/**/runs/**/ # Ignore coding playground tests/playground/ -tests/sandbox/ \ No newline at end of file +tests/sandbox/ + +# Allow logs folders and their contents inside examples +examples/**/logs/ +examples/**/logs/** +examples/**/*.log \ No newline at end of file diff --git a/examples/pipeline_1/logs/onsrap.log b/examples/pipeline_1/logs/onsrap.log deleted file mode 100644 index d9ab511..0000000 --- a/examples/pipeline_1/logs/onsrap.log +++ /dev/null @@ -1,45 +0,0 @@ -2026-06-19 17:45:43,313 Pipeline initialized | {"backend": "python", "id": "2026-06-19_164543_e9f91a60", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-19 17:45:43,313 Validating pipeline | {"name": "pipeline_1"} -2026-06-19 17:45:43,348 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-19_164543_a1816f20", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-19 17:45:43,348 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-19 17:45:43,352 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-19 17:45:43,353 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-19 17:45:43,354 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-19 17:45:43,357 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-19 17:45:43,358 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-19 17:45:43,358 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-19 17:45:43,362 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-19 17:45:43,365 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-19 17:45:43,366 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-19_164543_a1816f20", "stages": 3} -2026-06-22 13:17:50,023 Pipeline initialized | {"backend": "python", "id": "2026-06-22_121749_ae8ff387", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-22 13:17:50,024 Validating pipeline | {"name": "pipeline_1"} -2026-06-22 13:17:50,068 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-22_121749_ae8ff387", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-22 13:17:50,069 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-22 13:17:50,101 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-22 13:17:50,128 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-22 13:17:50,128 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-22 13:17:50,161 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-22 13:17:50,163 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-22 13:17:50,163 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-22 13:17:50,195 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-22 13:17:50,222 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-22 13:17:50,223 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-22_121749_ae8ff387", "stages": 3} -2026-06-23 10:17:19,051 Pipeline initialized | {"backend": "python", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:17:19,052 Validating pipeline | {"name": "pipeline_1"} -2026-06-23 10:17:19,109 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-23_101719_878fcb33", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:17:19,109 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-23 10:17:19,114 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-23 10:17:19,114 Pipeline failed | {"error": "Callable stage failed.", "name": "pipeline_1", "run_id": "2026-06-23_101719_878fcb33"} -2026-06-23 10:19:03,621 Pipeline initialized | {"backend": "python", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:19:03,621 Validating pipeline | {"name": "pipeline_1"} -2026-06-23 10:19:03,676 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-23_101903_e95b29dc", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:19:03,677 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-23 10:19:03,681 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-23 10:19:03,683 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-23 10:19:03,683 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-23 10:19:03,687 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-23 10:19:03,688 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-23 10:19:03,688 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-23 10:19:03,692 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-23 10:19:03,695 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-23 10:19:03,695 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-23_101903_e95b29dc", "stages": 3} diff --git a/examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc b/examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc deleted file mode 100644 index 5ef1c3127ab2d7ff45cfba212837264b44bcbc92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7462 zcmcIpU2GfImA=Cn4u?MysUM57eo9KRB~g;)#F2m2a-FzVWUI1eWvZ~7lr}?&BbhNp zGIxgl(e467P@qb?K;^X;W?S!uyI7$1LlL*=gCCQ(VxOo`w$!PE-J*HPLjkI`;jLeG z&$+`HQnG6U-S$en_uT(`&;9N>=bjm?uX7Oy3ug|$zptK<-(kiKl0k3}C616BIY(q} zf}G?uE`YRc!lv0+%4DJ>wvCjbe-%4+Q;a6xdG@#MtkKZpqm-(gK>92t_5>2Ps7

tVl8$5&Vo z;&gW`l}c+dEs;*CQ;-*?W7>I0`D3XobJ(xamk2<^c-+Gv5IGVi=WK_*3+sZi4)PXv zo|8G*CiAke7v{Ka|H=lJtetk6CDqhC&uND5YB^HpxFOtACCJ>6 z+oTEUAtBq8E+BV8B4e~J!coj87IcZRES{jc6phX&lE5V0K@~Nfyr}4oOpGciO}))g zTn#&sURnpF-cV^{u$-pT8f}Gg#@jc5$dL_4{pXIh2i~TY6E~)=POUc&t~L*ryhBB4 z=u5Zvy7Ysi>+Y^qcUQ^Xv+f>Tbr0S;cF#S0r{$jeU{N?oy)ek1k2vap?i5)N;;07_ z15Ir03wvZ_*db;}_)j1h5{9-T0kBU-5HS#xPF7pV5yZ(0 zFd^Hvo?FD~_ZLUVwwXfw6dZgu6$LEN23UAV)Ex^7$O4Tr$WY`K#RH~0Flxcb4^b^F znoL-vvsxyrMHw)F1B9{qG!P~*ZeA)}y5qb%QVP8BdEk@*lQUP(l)SqN7w>uZ6s0|X zeUyak6!n2F&8Zio8IW|wM)7t%ZB8as?K|<%GK>|VYEL`w#8C+x_ay43HhKzF7 zaY2g?JS1Ws@w6pNpj#!F8pZ52fVzUC$YMB19zd_#AxdPRa1RvL-gPOVMrV_;)CKw) z)Uci4A?C;fZ{y1T8>1_u`EQiGfpzchRqyVSw{OkeR}}iH+c7b2Z70Zjzu1(hu$(Q1 z4nqgV?Iq7VphT;f6ffmEqR8U9$PH`~hzc!bOm3ZGx#}{ZEen?1&qFo`0 z2bcn?63wL5gqFCdM2#l&2()M1JOxCKlx@7d`9VX=bv3{L+S0neZ`I#d@`p+d1IuDr zA}t;H*!6dp4-|z^VFsfAdE_8W2xxg_lR)zX5HIV<{^EmqW=G|doTH<<(QNI zOyVH^|6f#rrZ1^92tza2JasZ5kD~phn8CBNBzryAxJe)?whomsy|a^Lp9Kvb9=odF z2ejwNFaarT;+`4JGG|J39~u9IkW0XH+7m(6vjC;Q5erOv~_YVMh)a4($-TpNYP zG)4ExZ@%^Rg#6}M^!S-mZ=as}789a64;Gg$#xkHPg4>8JjG^WSPKb&ZtV`#*PoeZ$lE!_yy~&W{z^3ZCN7iDKKyHP6JC?qH>5 zN2%*T$$fCm{ra|+6UDY~-uFyyHj$2zatrY^=1y1dlzDHgJJrwj@~8EMY?npZAxnD& zIB{pi)T6OWHKrc&){qA$k$N2?X8tu)Ek8r6zg$AR37ypeh|ul@*`zTihZLj z{q!9p`>i;uH8xkPmm8r!6Bz%^1V&X@$ixp<%4eB-=2gh>)izlFwve^GLbA5;2ooFD z*kNJX5+N{0CCJYhf~MFoUh*h5B9^%MQ)9@sk&3`FSM@wWV`{%?OQ4vmFtCZa3%1E~ z#x>2_GL~S;*k!(E6_$>bg-D$_M%AR1?O~g_ivjS_)U>gg>ysTbHd%_Gl3^j@GbyWu zGt8T#!~qf@(7+>9`^G-a$u8Ml(+bvtt>n7eRm@Ky-`J;GSp4RjoX8{npwIQ3>Z5hF59r%ASV@^NcQ&9C*DZ z-!RTX29dd+2Q%NMyO<%4uYP4J#518q$6>k5&VYK`@GJD2kl$iFIFBqSDdlp89@PW0 z*+eoPrAj7Ewc#Vl^js{d9v!wC;8^M?L|?(~;UT_L{B4te`FK?@c}lH&*1f}z?8Mo< z%rBq1C+%F90;^JBUFs@IUB7Aa|53#o{9*SoiMt(u2t7;-;0HxL53&iWU`r-#b>Dn~s+#dEF}R_7ld&x& zdOc`^53(SE+4SmH{|h1H)I}AxuEcA2S2fh^pmCCl&L@==#S^3`RKRC5g)|dZY(4Jk zDX5Iy6mAVs5kS4Ys09oq;HhOcjYBOCr~V!yu+ke_!NbA$`_K;QLn1VZxyeT^Fw-_g z(%J!~PSY(h`Fh9BXOc}c<9VoEwsFW&ZlW zhO`4J4qW>?%=d0cZ8yi(x?U}H>;v+X3;D@g+`mrU)i$J^kQpt|e|ddFYRwOqn_M2T zOk8%c>?3b*W8AjVKmNfBvl}nI?&E{fXZ$F{TMS8L8k4xY7+YSfPWHl+H{>u>iQ52i zRhuBt5Z3ezjKtK4E})C^rMBgJ&|bykHoOM29Qp}nxqKY`2D4mo=<5dY=`N$fP#czw z2puK(u}%Ct8wETFkZupp7aZtUFm2EPx~J^h`+*8TCb2M$Gi?j#PXHG)xROm5v&@4} z%nGs~+p9CkBC>D>9vkRE0w2L;@CZcH7jE0Y$VtV)->5tBhD0wYS5%#I(H>|9&rgtG zWGZCGTZ&Fe-M)AM-IlIPG@&U`bSvw^`)WF++vC~Aj0)q=Ll3$Dks*vKo=cC>p>k9j zRKJH!jzI*5k>FhyT2_UYb)jQb=m7g@tSEG>34g;X+E#_On|i?B_|HezUJh8oGgL9y7yv>w`6BP70`+P$8}

    ;~Ch)$n0!Mz9Dhy$|zM z%u3l27EPTAR)Z84_aac8ONPc%htXx_NWk|Cnpj+FeWm6zgQ?9BeTJlF$b-)`guaW> zI~bwhhIa>g$tik7nJ~VhO$e<^}0ED zex>0B6-~t;4nsYBqoC<{7QAlXfE+!J5xT|b0f<67bQgTp$inv}{P+XtC$t|U3K4$Y z(9S0lvy~4b+La7E{3uF4rggWU6j)}2J;}D88FChvjcbChCw6!YrZlQzbG$QbTZTTC zjx54jlgd&0U8q3dsTqjCxs)RTIM9@m2p z`LPQ;f{!KeQ$213zr*8hcw0O^fJ)+t3vPdTfC!%Dr8R!%uQ@TNeQ)W*rK`KH^%fd# S#n&9K-RDL&c_Os275yLTJ9C@> diff --git a/examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc b/examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc deleted file mode 100644 index 91c1a90598fae4812a97ef8f48d3db842f1f102d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8553 zcmb_BTWk|qmbV_Z%Px{QiNRneaY7)(BqT7n8wTh=fQ8U>hit)w>F$8n*aet&Z11fy zubrN;Mk6h=v(nJhXxkqd$y#Zpu+m7;Os@n#tL@&89nF_RC7n!dTH8|3PydBMENZ^? zoLerJL%SzE>|7`3o?G`+)xGDQcMUW*`3SVXJ9FS#m!FU?a9}sdLbxYKI6{i#B2l45#~G zEWy}Y9m|RjMlz1{4ax?C|NDfbdI{+u7dH2h3&O;AY^FFp zT;J3~V$};5d8J3$x=*l%G2j?B*frnF{-fKz#0yc*loQ!(PEY9RTvi*0v6xEeD)tiz z{VMdrX!ceT@2@_Ch=9X$xhLI#6v-sH$nXCNtPierkiX`xatf#LilB(Y@UHu=FL@v{ zJ;_{FS8wXieek&MA9RoBva0E(s$QU3>H>=UxEA;29Gy<+8op|)dFc3osjG!cQ?M|a z$WN&^6Vv&Os!iq7c{P*Hs_{KjT9T&ox;C{Zo~LS_=8~$WrL$Lt^0&+l@pyG-YCKLs zmrFi^uIs;5HJyg|lnd_?Q$tS3q)wTN^WlY1(_R$uAW^0uu z&hDqVoKErejO+*?MY8N^e&h+Qw6xzn`QiBc<4b{o`M`kDGFXuY{~)*AlYTa`B=^kA zJ%-%7BoEBX1GA%x^3dF-MR{LE+(%pCAulg)6s-hp0ubeBGjx_Lv9&WHU;y9oHAyIk zEJ@IrBsl8;TGmiXnI!xU5GDzOFG>PPK8c~kKvO%}Y;{CXPX2^UDE!)Uo2uRS7nhJ1 z-V_za;ou8dlm*%j3y-=@&lMFcf=-#pP}m5en-ByMY(}sJ0Ap=cEZS|c$QAT_L60-R zhyW2|^_zg$f-!TWd}Gf0AZA34Jc_(wiOHGw&loM+%GVcLc2%TZFC`_YN9=UAv3Gs5 zRZ?7FnFP=bH$_cj0!~q9Os^x-c-C|Sg7riRxuNv{K%*jY+a4n*U0cO&?i}kLq}WAE zqUvo0o-teVKePM6wj4*90ZFM@7nrCvg|jV0o}c8^VMfU}v^}``7n;J??iMD69o5+s zY!Pgk$0nW>Y^q)|$tl8GnT(3#X6tTC@vkQ`XC%c;wF5 z|Fi^Gg5lEG2S2R%w|_1N?*3p=?yZQuj{e(cfs0|M)y-b^%@T%wFY02_WlF#c#vDtP zP^6v*GHFeJCzVd>?`S$5>;`b*!n%Ui3%6=v_UdPZ=LR1mC~aHCuF|`Y4V^^;9cy^B zY|$S9f<~*MXhA5Y@cXKwd60{91->A}Yr#aGt20N)2G%BOuFJkcNDqjOEkbr*5DFL$ zz|S^~5pYc1WPsRWcm>|vLDGDLbV7)NKLr%@pwLOu90(G5?keb)ckRE=o1UzC16{u< zso7*M1+IP^2*g}o&6*;z)f9l?rtdf~S5XrnjZvNs!1_%0)kHQ0(F1O>uPRV-CYMOX zEup3RVS@2^A0Qx0-qxRY{iys-yvuvYhueK`J$@$#!iV5iYC@Jt}S zS6~+Y3~%hYhIV8oGU|4H?3PD^-k0}d_p9>b0?jBuq?EgKO-<^|B$)yT%(~*z4=bs; zIF9RjOaPSnR%x@DTJSP~G%qweJb`eJ8wZBJ2J@y1R+-L2 z1aS~X^{%2hi$Pu{eOrxVL`aXqJX`HXK#F8VZoX6a*?}cFJTHe0xw9g6ItzU%U8D!# z!Rs;%oOwON_8^P)l-m=j(vv{NwuO!uIlHB_-H^MN9NUb6g+ty6^5Ja=x!Du#vMmpk=KVjqQYC zJ?9-$@r$GvmFsuarqf!^%6~15_d4Sj)!k>Y z8v-zuo`d_@ZhHVhC2v`hJ0HoNGuhei8C`n~IkqH^%*!K&e54{CaW?!?%(ufDk>+k_ z6m1LB^XI`ALK@3meT%3DO@-FMR{o5%bDJL{sBDa`vVQ2V*w|S#5VD5Z2-sfk$!*w1 zkqnVMH%a{D1SIKszAiWC6;hua+jbx?I98(W1bN7w>vCzE`XV2zhDLdDl52FT3O`lL z+9!F)5mBUk}e4JAj*|Y>2C{(EI6fGrmv7q>rjVDCm}pzK~+4RGMkbG zP0vlMG@eaNvy#shOd32;tVtD;x@qMMm{h2yY~95|o~VbJU(-c%g=~tVO)=exJOp|v z%LJiwW5N$IIf=6yk{LCTRa20A-he~{AbksI-UguIwjhzPQwblVWt3EfDJF)nF_5L@ z!J*eUClFA6V@s8#Y(OpranPPX9T{u$v{r9xGj`@&&UwD`7#-((?!~bumS(+pX)+ z6(H5N0zl*I=OT|0lmn~SRdyX`{ci*8s57oB5jcM2u|r`qFQ{eUquf&{~g6`zYk1dHxXap=aHXAO6}!KbNlW@mb?cS zyayj#Uug||*!7F9vU{O*R|Zhd1-Sgupo6k^AO)5xC#Cqc@J87 z-M@eLDChcwlaIEo&FfG!n80EZgDLhm)Wdm2gxZr^@eGS-IW2{gwOoNF)w&Xa!>5|s zJ#Tf21y#?wt`4GA)2!I|8S!!M#|TRLDt2?AiqKVFJ7@K8U1PJBE@Db!x0;*1%XHCk ztd7f2NRUkO{#syTI||2n-9kkqFfzONf zo)z!6eyZ==KXn*%ya2Tkd{ep(i^{CLP~`|gUV=_?Pfh@`73@usoFgY-l~BEL_yh&& zMsL1JitwrIG*orbwJYG-UlE}8!ZY26uR;_<|0r+rL%V4T&^2Wg@1ZMdR=t_0BWCxd zLOPR*YlZ3Q1idwMD3ePjGTO*cjp8!ij0NQ{VSSa(v5#X9j?Jj$W3x?*+g@Gj+E?k? z_n)z^SmE!-ksgi~fRvdI(+hSao`hgPH=A_$V3mPN;dM0&#gX>Jb(JQrs8%H>&cX-O zBG&Mu21Xnjvu~zUjXN}E>A)-{fC?%qg=wp5A!MMEfj5e8elBgir61{ar?-Ba0`*6m5H;1_dX5pdKj1wjTr5Rm!-CQCmzO(@R5hHxnncO%0l_nzchc^G5jz# zA3kCPMps*;4N%;WT<)*_xPgQZe`SV`Xz1fV-W@(V()CF=d`#H7rhbJ^n%OxkTyJdW zhCy_oWWg%y_%wPQ0s6f^^*siyx)uO{0fWpzat5Fh8eqK>-T(n>$b7bvPXMXU{hb`F zzGPWC0P#4fHx_HZqJcqhOj4~vo_EzLyUR8KdAAdtpFd&0AwryZ{CsTE zZM*oqXA+bf^vVO}29>6#Z@2BKy1lwmx`%yXqgcdY6>nCmWML~szsRHw{D}|o9sm$W z*L{T9hL=$tED=s3fv2jV8u~647T0{QIGspm=|3Pn{A(B#fB^e@$~z4I&f=++w(aFB zM%z&F^h!&l>@iyQ7RQ!bJ4&|}Tl*^mhZkE97hnH_yy@;us52Fhue5iTQbv1k@r@NR zaQE#6acHF(YVC7_M)Tp~${AC_*g^Qmk*!}4-+vSNGqU9~viUQz>6sfoLqBT<@U$7e9X;Iyb^WJ4sMbD}paO}Y z4?#bIQSPY(dGgai2&kX-fh&I&0)<)aAmWBQw-dOu^2$77rS7zki6ge=5?K**9n5;iv^B!PqwcFLNSLgH)ikT@~6_slqK za&KEI-3RvGs>|M+)ml}mTGc)fsSl{yhurp|+e%eeRUhma$L^HXuC(`M--yx_H81V= zo$=U)WRrBY2mHW|A7DJ%eUFIRLZ903$bbu%D;td(7}~ zn!_IsM^wKWjD(d*Ob6rze=MX1g7YdQ`6<781D@uQ@SS<=Cf^m|Ar9ShPun1}OuXcp zY3NrlOvrPQcexv!#7QQ}Eb)Et$_JM3O;Am<&PT$k{Gs~X4&Al=vTZUFmNkLOYK(@d z1!_FNAqfi+y69IGeAQlUX!Q8(jo92QjF0-Gv+{@j#b`)YW~0HV914bIU;nH!PlHia znLX^Iax_BKVEB4(^o~~M^W`Te`+T$tdP9ULO(2%ZHsQIUCn(}~xA3t(%~DUHl(3m% z1OqN2o9W*{1ZOI@;T&g;m4KWnoZYT4#qY|1q_GhlC9P)#qn=SZWixV$Ph(@J7|4tH z7`w*x;GFV7;(8oLoRALUF->ZGz_0o}0$XBDfK6BkUe`p-^aX-c6MepgU(vhGtBoxO1by+}!G9>&PL>Ll!b|V2$PX-V% z5abqCnokkL$ty4+nRf2mY@yvxj*wk%@|Y)K<6~jO0lYqrOm^l)__GMrbc6`j{(CCmDo~Wf`|c(`%8(Xb=$*%wENT}_k<3U%b#3M zSME#P&Qu;uiU)rbO7I-9-Py<9^}`xTvVc}H!^sdRo-_tyOtI-*K-1@18$^Ezt}x@8nG14t=o7bwUFi8C}r)Z{hE zeXOSGrMd@4zbijib;ttO~LwECbi0bUlf!s zjm*o60(w`o&xd4x*cXLsL$d}!+tc9U-I{)%o*kh9nJT^W%54v?*#R+d)A&H$YgQdK zie?7jG0=H zf6CpTJbWhY9!-j_oWuRd(Vc5Nm~A|gYCMu{JeF!a_OK`2cz)%=H4bAZj>4vUk zRoAx;=jxq|qb=Dn_>-tCkA(qI6JcH0%IT*L(f8aL3M%Rqyp69YD(z_larNqJG1&o4 zm1Z{GZDQhjo}h^D+rkHonk8SZO91CwSA%S*@*G*J%EM*@J(NvOa_qHEIYJSCz^4P49)kk5#)A+6NZG1CZuzJsPS?(_v}A1q8@7Rm z0w}h7XFi+!?PTI;+Ib*d*}ZKh_9IXV|7qL6a|Kn-efNl9`6t0K(qQ}XSDt|%o<^1w zdJwv0n(KLW5jK&zoASK+df*_lR9tLQ+oYWno}ftX>Dgk>{?jb|B68}W1w+oSVaJzC z*o~J+1n4Yh;N|7$Fm^v`D#1E^)(7s%<&LV$dp{QFVaoQo-O8B~I3h}xt z?4DN@dC&S@Ma7RJ5#g+0_6z4u?y6DtbGG>leM6Hs27O4^oC2uyr z$(xOpl^ieaga3c^QC9MH%bT=>d5bU1`!BrW8_ipk%BRbuis^Ex(#sY05*Q$G8txh? zJtI<8X$tpWsxD~*D~@fXn$lW3+w|GNm=#dQ;5N|LJy?v3LE@?1VaXv*0Uz8`*ecLc zsGCZA2S&qbcBbLcwvqF^GFdmTPdq*}!c-^P)3@Y1X!%|TyBmxx8VSVaRZaXL<`1hu^^WGy zhjH53T7DZ0;W`G zX5`t`NYv=VfpMdtY9UA!RUd%9Ha~+|X_X=Wj@zfikkbc-lYHG~*WoYTPIrO%>MV_RAN?Yh?tUl9*MC>p zykgE(J6GTTv}Hxe)zqy9KkZ$yZr0YXD)Hl={%N|lYen3wZ;qcylxJMM>H5AE$G2i* zR%}mP$%u#2VtY>9lNH+%BN_3nwAjW{?TPm@;z3r{lodM?zsQJ(Gh)Y+T2kfuuX<9~ zbiX6n+LvuTnrc0oZXHTD9edcHcD}QG>6>#i_iB@Mo!Op|RL@AJ`gHQDfBC{U<5%u= zB%&{n5|sllzCV`%k9!pG@yRm2MwN zyH3BNz_l^uP0j?fGm+FxBt7%N#uVLXRZ>&*_r8ZS@uk1DtSzkj)BA@WxrZS0QLB;@ zovSxuwH=o!*h3-f*QMn1RAt>hr7{kEpVb@-J<_u3g5IVpcMe_N^naW z?n>ZdC3PfTvKZ_UxV`{d+B*Hf#GDYo>^}q^?~B|5x-SI70dQGqHdH=7(65Td+36wZ z3LHUi!(-B8(FEPErCAnlq36rKL_yv7&?l+!HxNjhZb}Wgl+}ob$_|u zz}TxOl(Y38fe6?Ed{vfjNbwC>z9q%CBm(QxNxo%+Kh6r8Q+)IN(Z9a%mlweQ@xhm) z8?N!MMl-Id2N#hN@MjI)7?Z>E30)Mjn?hu|!R;MV_e_ob#7C$z~ud z3ciu40t1E_ciefb)Sd$FtCHfejnI62P;Um#omIqqI-&4vRw) zy_UX$U&c)?!tp@PF2TnIsjO%xOJ%RMl%Yo6Qm&Y$G7LlGmS`V5**T&87#qcGL1z%? z5N0d^nbAT}4h6#gMY-e(p7faYYd3~XZ-elN^cKcLC~(>6Q4qS7J%gu>Y7TaYZZMOZ zUcpk4<-qL@J~@hQ@ZixQEE4o0{Vn(c1rG&!d<>sYJ*$!DExtJ_`)>szOJVv;sK*ez z1`#}*m07+%#n)$fSBiHfrq@T4ylaC$ma{ptw#JmLG2W4&>*tfU#;k34!#4b-uvzW= zto6683E|P((nj?k!No0BWW}bG*t7xuB6nx9>v+a}BI_PWxknzm&wO-wr7bI8UGv{&p}gXWnY4{^+g4(K2VMyOk8H!w z*@g4>BZp28SpIpyamKRKEntF~ursBb=r6x;`Z&;$X5F`01`YI6kEYzKJmNoxaz!V` z+r){MKZs;gH+vQ}6Vze>vCKwtlb!AoE2Hq0qtw>-7IRHSyLpBy*p6l|+ySNj#hu?V zqWlZxJG|IYONqr^I4e=1#A*=?yDDle+PE|4jS;xXY{A-{nJMN<=Aso_^o~k=NiJ{n zELlo&ErndcYn6o3^9?87Dp_~rh>#=h$gx3=tu&|43j1o;Y(UD<+bRd{IuH94KlrH% z76@KpcN_3KSb^K?7Xi9lnZHznn*qO(=^om;{54THH2Q@x2AjK!Im59=CkRE4KJ%zD z9~k4u6mLOx+JM-r6l&Trqa6g;F(qFE_F=Xk#h;--YOmk0NDB-$uG4US(H{)cUqSxQ z;io(V0qnb~J<*k}IvkO}=`yXM^9r$u}jN55Y~1 zbhqx_eEh)gfVS4%Q{uzvhW$Cw{dsNjt>fwT6FIRyZcS8V_VghcZb-N@jr~YHug61~ z`flW(OYuL>G#uQj6p>+y7J;1|{Kke;^7*Feb(oBYoAlIacKGlXgO8v1eG;!RrYdO| zVwz(<5(YXTfG$|^>H`p1rlsE{~-|(T&5~W~|(FnFi z*t0?EOD_&Df;AeFPtre!0yy17!4YqnIgb09?EQ|||C2brCLLdsJztahZ2`WWZdZVK zRskP8p4Eeg`I#M@hR?=0WSh_IKn}KBL6>ax625HZ?uNPXKX}{9zK;+5`oJgCzw;(W T)}0&nqmTJP(1&~z8_53wS~wfo From 22fa8400525ad527a49c0b2842838d7bd3a9e88f Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 13:30:50 +0100 Subject: [PATCH 303/332] tweak: slight changes made to method names and processing for last_run attribute which allows the Pipeine to continue running even if no file was found for last attribute - warning is raised in this case --- onsrap/models.py | 2 +- onsrap/pipeline.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index c564688..eee15ed 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -936,7 +936,7 @@ def _pipeline_run_to_dict(self) -> dict[str, Any]: A dictionary representation of the PipelineRun instance. """ return { - "manifest": self.manifest.runmanifest_to_dict(), + "manifest": self.manifest._runmanifest_to_dict(), "status": self.status.value, "started_at": self.started_at.isoformat(), "completed_at": self.completed_at.isoformat(), diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 3b2d6c3..de5e72c 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -10,7 +10,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence import re -from .errors import HistoricalPipelineLoadError, StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError +from .errors import HistoricalPipelineLoadError, StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError, StageLoadError from .warnings import StageConfigurationWarning, PipelineConfigurationWarning from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph @@ -455,8 +455,13 @@ def _load_latest_run(self) -> PipelineRun | None: warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ " will be None.", PipelineConfigurationWarning) return None - - return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + + try: + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + except StageLoadError: + warnings.warn("Historical run file does not exist. Last_run attribute " \ + "will be None.", PipelineConfigurationWarning) + return None def _set_run_output(self) -> Path: """ From e54b771db535fe62fd59e7ecf76f979a2259c333 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 13:43:58 +0100 Subject: [PATCH 304/332] tweak: adjusted tests in test_pipeline_architecture to account for warnings --- tests/test_pipeline_architecture.py | 130 ++++++++++++++++------------ 1 file changed, 74 insertions(+), 56 deletions(-) diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 4515107..59559bf 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -81,16 +81,18 @@ def main(context): ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config=_base_pipeline_config(tmp_path), + ) - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - run = pipeline.run() + run = pipeline.run() assert run.succeeded is True assert [result.name for result in run.stage_results] == [ @@ -144,16 +146,18 @@ def main(context): encoding="utf-8", ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_files( - [writer_stage], - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_files( + [writer_stage], + config=_base_pipeline_config(tmp_path), + ) - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - first_run = pipeline.run() - second_run = pipeline.run() + first_run = pipeline.run() + second_run = pipeline.run() first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) @@ -190,16 +194,18 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( script_stage.write_text("print('script fallback works')\n", encoding="utf-8") with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config=_base_pipeline_config(tmp_path), + ) - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - run = pipeline.run() + run = pipeline.run() assert run.stage_results[0].outputs.strip() == "script fallback works" assert run.stage_results[0].stdout.strip() == "script fallback works" @@ -307,15 +313,17 @@ def run(context): ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - run = pipeline.run() + run = pipeline.run() assert run.stage_outputs["0_data_validation"] == { "stage_name": "0_data_validation", @@ -359,12 +367,15 @@ def run(context): "missing_stage": {"years_to_run": 2017}, } with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_files( - [stage_file], - config=config, - ) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_files( + [stage_file], + config=config, + ) with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() @@ -462,9 +473,12 @@ def run(context): ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" assert pipeline.config.work_dir == tmp_path @@ -582,15 +596,17 @@ def run(context): ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names assert sorted(pipeline.stage_configs) == stage_names - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - run = pipeline.run() + run = pipeline.run() assert run.manifest.stages_run == stage_names assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names @@ -680,26 +696,28 @@ def run(context): ) with pytest.warns( - PipelineConfigurationWarning, match=NO_STAGES_SPECIFIED_WARNING + PipelineConfigurationWarning ): - pipeline = Pipeline.from_files( - [stage_file], - name="config-export-pipeline", - config={ - "pipeline_config": { - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", + with pytest.warns( + StageConfigurationWarning + ): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + }, }, - "stage_configuration": {}, - "global_configuration": { - "dry_run": True, - }, - }, - ) + ) - with pytest.warns(StageConfigurationWarning, match=OUTPUT_DIRECTORY_WARNING): - run = pipeline.run() + run = pipeline.run() run_dir = tmp_path / "runs" / run.manifest.run_id config_file = run_dir / ( From 7b2ecee71b95d1b3a2a021288f3142bb9c9329b5 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 15:30:44 +0100 Subject: [PATCH 305/332] test: adjust tests to account for warnings raised --- tests/test_pipeline.py | 191 +++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 84 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 0bec4d9..b5c785b 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -61,11 +61,12 @@ def test_pipeline_name(self): pipeline_config = PipelineConfig(name="test_pipeline_config") with pytest.warns(PipelineConfigurationWarning): - pipeline_named = Pipeline(name="test_pipeline_name", - stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - pipeline_config = Pipeline(name=None, config=pipeline_config, - stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - pipeline_no_name = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns(StageConfigurationWarning): + pipeline_named = Pipeline(name="test_pipeline_name", + stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + pipeline_config = Pipeline(name=None, config=pipeline_config, + stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + pipeline_no_name = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) assert pipeline_named.name == "test_pipeline_name" assert pipeline_config.name == "test_pipeline_config" @@ -110,28 +111,30 @@ def example_function(): } with pytest.raises(PipelineInitialisationError): - Pipeline(stages=None, dependencies=dependencies_single) + with pytest.warns(PipelineConfigurationWarning): + Pipeline(stages=None, dependencies=dependencies_single) with pytest.warns(PipelineConfigurationWarning): - pipeline_1 = Pipeline( - name="pipeline_1", - stages=[ - Stage("Stage_1", path_1, None, {}), - Stage("Stage_2", example_function, None, {}), - Stage("Stage_0", path_0, None, {}), - ], - dependencies=dependencies_multiple, - ) - - pipeline_2 = Pipeline( - name="pipeline_2", - stages=[ - Stage("Stage_1.py", path_1, None, {}), - Stage("Stage_2", example_function, None, {}), - Stage("Stage_0", path_0, None, {}), - ], - dependencies=dependencies_non_stage_name, - ) + with pytest.warns(StageConfigurationWarning): + pipeline_1 = Pipeline( + name="pipeline_1", + stages=[ + Stage("Stage_1", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_multiple, + ) + + pipeline_2 = Pipeline( + name="pipeline_2", + stages=[ + Stage("Stage_1.py", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_non_stage_name, + ) assert pipeline_1.stages[0].dependencies == ("Stage_0",) assert pipeline_1.stages[1].dependencies == ("Stage_1", "Stage_0") @@ -170,10 +173,11 @@ def test_add_dependencies_single_dict(self, tmp_path): stage_2 = Stage("Stage_2", source=path_2, dependencies={}) stage_0 = Stage("Stage_0", source=path_0, dependencies={}) - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline_dict = Pipeline( - stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline_dict = Pipeline( + stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple + ) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) @@ -214,17 +218,19 @@ def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - stage = stage_factory("Stage_1") - stage_config = StageConfig( - name="Stage_1", _variables={"years_to_run": 2017} - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + stage = stage_factory("Stage_1") + stage_config = StageConfig( + name="Stage_1", _variables={"years_to_run": 2017} + ) + with pytest.warns(PipelineConfigurationWarning): pipeline.add_stage(stage, stage_configs=[stage_config]) - assert pipeline.stages[-1].name == "Stage_1" - assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 def test_add_stage_warns_when_stage_config_count_mismatches( self, stage_factory @@ -245,10 +251,11 @@ def test_add_stage_warns_when_stage_config_count_mismatches( Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline(stages = [Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=())]) - stage_0 = stage_factory("Stage_0") - stage_1 = stage_factory("Stage_1") + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(stages = [Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=())]) + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") with pytest.warns(StageConfigurationWarning) as recorded_warnings: pipeline.add_stage( @@ -281,8 +288,9 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage( Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline(stages=[stage_factory("Stage_0")]) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(stages=[stage_factory("Stage_0")]) pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") @@ -307,10 +315,12 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) - pipeline = Pipeline( - stages=[stage_0, stage_1, stage_2], - config=PipelineConfig(stages_to_run={"Stage_2": True}), - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), + ) assert [stage.name for stage in pipeline.graph.stages] == [ "Stage_0", @@ -370,8 +380,9 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -399,8 +410,9 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - with pytest.warns(PipelineConfigurationWarning, match=NO_STAGES_WARNING): - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -420,10 +432,12 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) pipeline.enable_stage("Stage_1") @@ -444,15 +458,17 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( """ stage_0 = stage_factory("Stage_0") - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - pipeline.add_stage( - stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")] - ) + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")] + ) assert pipeline.config.stages_to_run["Stage_1"] is False @@ -473,16 +489,18 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr """ stage_0 = stage_factory("Stage_0") - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - pipeline.add_stage( - stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")], - enable_stages=True, - ) + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) assert pipeline.config.stages_to_run["Stage_1"] is True assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} @@ -509,10 +527,12 @@ def test_validate_skips_source_check_for_disabled_stages( "Stage_1", source=tmp_path / "missing.py" ) # file intentionally absent - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) pipeline.validate() # must not raise @@ -530,10 +550,13 @@ def test_construct_manifest_inputs_contains_only_effective_stages( """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), + ) runtime_id = pipeline._create_runtime_id() manifest = pipeline._construct_manifest(runtime_id=runtime_id) @@ -547,10 +570,10 @@ def test_generate_context_correctly_assigns_executor(self,) -> None: executor, an error is raised. """ - with pytest.warns(PipelineConfigurationWarning, - match = "No stages specified to run. All stages running by default."): - pipeline = Pipeline(backend="python", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline(backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) assert isinstance(pipeline.executor, PythonStageExecutor) with pytest.raises(PipelineInitialisationError): From 731ffce4bd639862c14379493395b9f477bd4632 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 15:54:27 +0100 Subject: [PATCH 306/332] tests: finish implementation tests for run_history --- tests/test_pipeline.py | 460 +++++++++-------------------------------- 1 file changed, 102 insertions(+), 358 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index b5c785b..4466cca 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -886,351 +886,85 @@ def test_no_errors_raised_success_load_latest_run(self, assert not any(issubclass(warning.category, PipelineConfigurationWarning) for warning in w) -class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): - def test_logger_no_handler_errors(self, - tmp_path: Path) -> None: +class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): + def test_no_previous_runs_pipeline(self, tmp_path: Path) -> None: """ - Tests that if the logger has no handlers, an error is raised when - attempting to extract historical ids as the logger is not writing - to a file that can be checked. + Tests that if a Pipeline instance has no previous runs, the _load_latest_run + method will return None and raise a warning. Assert that it will also store + None in the last_run attribute of the Pipeline instance. Parameters ---------- ``tmp_path`` : Path A temporary directory provided by pytest for creating test files and directories. - - Raises - ------ - ``HistoricalPipelineLoadError`` - Raised when the logger does not have any handlers, indicating that - it is not writing to a file path and cannot extract historical run ids. - """ - logger = Logger(log_dir = tmp_path/"logs") - logger._logger.handlers.clear() # Remove all handlers to simulate no file logging - logger._logger.propagate = False # Prevent checking root logger handlers - with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") - - def test_logger_no_file_handler_errors(self, - tmp_path: Path) -> None: - """ - Tests that if the logger has no file handlers, an error is raised when - attempting to extract historical ids as the logger is not writing - to a file that can be checked. - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - Raises ------ - ``HistoricalPipelineLoadError`` - Raised when the logger does not have any handlers, indicating that - it is not writing to a file path and cannot extract historical run ids. - """ - logger = Logger(log_dir = tmp_path/"logs") - logger._logger.handlers = [logging.StreamHandler()] - with pytest.raises(HistoricalPipelineLoadError, match="does not have a FileHandler"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") - - def test_logger_does_not_exist(self, - tmp_path:Path) -> None: - """ - Checks that the method raises an error if the log doesn't exist at the - location specified. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - file_handler.close() - log_path.unlink(missing_ok=True) # Remove the log file to simulate non-existence - - with pytest.raises(HistoricalPipelineLoadError, - match="does not exist at this location"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") - - def test_return_blank_list_no_matches_in_log(self, - tmp_path) -> None: - """ - Tests that a log file that does not have a record covering "Pipeline started" - will return a blank list from extract_historical_run_ids. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text("2026-08-10 10:00:00,000 Some unrelated log entry\n" \ - "2026-08-10 10:00:01,000 Another unrelated log entry\n") - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == [] - - def test_skips_poor_json_in_log(self, - tmp_path: Path) -> None: - """ - Tests that if a JSON record in the log file is not valid, it will e skipped - and the next valid entry will be extracted. Assert that the returned list - contains only the valid entry. Confirms that only the incorrect record is - skipped. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text("2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" \ - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") - - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" - create_run_dir_1.mkdir(parents=True, exist_ok=True) - - create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" - create_run_dir_2.mkdir(parents=True, exist_ok=True) - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == [ - { - "run_id": "2026-06-23_101719_abc1234", - "timestamp": "2026-08-10 10:00:02,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_abc1234" - }, - { - "run_id": "2026-06-23_101719_878fcb33", - "timestamp": "2026-08-10 10:00:01,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} - ] - - @pytest.mark.parametrize("string, expected", - [('{"some_key":"some_value"}', []), - ('{"run_id":""}',[])]) - - def test_run_id_absent_falsy(self, - tmp_path: Path, - string: str, - expected: list) -> None: - """ - Tests that if the JSON record in the log file does not have a run_id, it will be skipped - and the returned list will be empty. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - ``string`` : str - A dictionary representing a valid JSON record in the log file that excludes - run_id. - ``expected`` : list - The expected output from extract_historical_run_ids when the log file - contains a record without a run_id. - """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text( - f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") - - #creates directory for runs to avoid removal given the directory doesn't exist - (tmp_path/"runs").mkdir(parents=True, exist_ok=True) - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == expected - - def test_records_only_if_directory_exists(self, - tmp_path) -> None: - """ - Checks that a record is only output if the run directory exists. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - """ - - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text( - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") - - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" - create_run_dir_1.mkdir(parents=True, exist_ok=True) - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == [ - { - "run_id": "2026-06-23_101719_878fcb33", - "timestamp": "2026-08-10 10:00:01,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} - ] - - def test_reverse_chronological_order(self, - tmp_path) -> None: - """ - Checks that the run_ids are output in reverse chronological order - based on their positioning in the log file. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text( - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") - - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" - create_run_dir_1.mkdir(parents=True, exist_ok=True) - - create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" - create_run_dir_2.mkdir(parents=True, exist_ok=True) - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result[0]["run_id"] == "2026-06-23_101719_abc1234" - assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + config=PipelineConfig(output_dir=tmp_path / "outputs"), + stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + ) + pipeline.run_output = tmp_path / "runs" + assert pipeline.last_run is None - def test_skip_poor_timestamps(self, - tmp_path) -> None: + def test_last_run_populated_one_run(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: """ - Checks that entries with poor timestamps are skipped. + Tests that if a Pipeline instance has one previous run, this is loaded in + last_run attribute at Pipeline creation. Parameters ---------- ``tmp_path`` : Path A temporary directory provided by pytest for creating test files and directories. - """ - logger = Logger(log_dir = tmp_path/"logs") - - file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - - log_path = Path(file_handler.baseFilename) - - log_path.write_text( - "BADTIMESTAMP Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n") - - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" - create_run_dir_1.mkdir(parents=True, exist_ok=True) - - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == [] - -class TestLoadHistoricalRun(TestLoadLatestRunIntegration): - def test_raises_stageloaderror_no_file(self, - tmp_path: Path) -> None: - """ - Checks that load_historical_run raises a StageLoadError when the specified - directory does not contain any files matching the expected pattern. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - Raises ------ - ``StageLoadError`` - Raised when the specified directory does not contain any files matching - the expected pattern for historical run YAML files. - """ - run_dir = tmp_path/"empty_run" - run_dir.mkdir(parents=True, exist_ok=True) - with pytest.raises(StageLoadError, match="Historical run file does not exist"): - load_historical_run(run_dir=run_dir) - - def test_returns_valid_pipeline_run_from_yaml(self, - tmp_path:Path, - minimal_pipeline_yaml)-> None: - """ - Checks that load_historical_run successfully returns a PipelineRun instance. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. - ``minimal_pipeline_yaml`` : callable - A fixture that returns a minimal YAML configuration for a historical run. + ``PipelineConfigurationWarning`` + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. """ + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + "{\"run_id\": \"2026-08-11_100000_abc12345\", " + " \"run_dir\": \"/path/to/run\"}\n" + ) - run_dir = tmp_path/"valid_run" - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir/"pipeline_attributes_for_test.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id"), encoding="utf-8") + temp_attributes = (tmp_path / "outputs" / "runs" / "2026-08-11_100000_abc12345" + / "pipeline_attributes_for_test.yaml") + temp_attributes.parent.mkdir(parents=True, exist_ok=True) + temp_attributes.write_text(minimal_pipeline_yaml( + run_id = "2026-08-11_100000_abc12345" + ), encoding="utf-8") - result = load_historical_run(run_dir=run_dir) - assert isinstance(result, PipelineRun) - assert result.manifest.run_id == "test_id" + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + config=PipelineConfig(output_dir=tmp_path / "outputs", + log_dir=logs), + stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + ) - def test_correct_yaml_file_chosen(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: - """ - Checks that if there are multiple files within the same run directory, - the method will pass successfully and return a PipelineRun. This does - not assert which file is chosen, only that the method does not raise - an error and returns a PipelineRun instance. + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "2026-08-11_100000_abc12345" - Logically, this should be suitable as we would only ever expect one file - to be present in each run directory. + def test_last_run_most_recent(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: + """ + Tests that if a Pipeline instance has multiple previous runs, the most recent + run is loaded in last_run attribute at Pipeline creation. Parameters ---------- @@ -1239,50 +973,56 @@ def test_correct_yaml_file_chosen(self, and directories. ``minimal_pipeline_yaml`` : callable A fixture that returns a minimal YAML configuration for a historical run. - """ - run_dir = tmp_path/"multiple_runs" - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir/"pipeline_attributes_for_test1.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id_1"), encoding="utf-8") - (run_dir/"pipeline_attributes_for_test2.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id_2"), encoding="utf-8") - - result = load_historical_run(run_dir=run_dir) - assert isinstance(result, PipelineRun) - -class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): - def test_no_previous_runs_pipeline(self, tmp_path: Path) -> None: - """ - Tests that if a Pipeline instance has no previous runs, the _load_latest_run - method will return None and raise a warning. Assert that it will also store - None in the last_run attribute of the Pipeline instance. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files - and directories. Raises ------ ``PipelineConfigurationWarning`` - Raised when no previous runs are found for the Pipeline, indicating - that the last_run attribute will be None. + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. """ + + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + "{\"run_id\": \"run_older\", " + " \"run_dir\": \"/path/to/run\"}\n" + "2026-08-11 10:01:00,000 Pipeline started | " + "{\"run_id\": \"run_newer\", " + " \"run_dir\": \"/path/to/run\"}" + ) + + temp_attributes_1 = (tmp_path / "outputs" / "runs" / "run_older" + / "pipeline_attributes_for_test.yaml") + temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_1.write_text(minimal_pipeline_yaml( + run_id = "run_older" + ), encoding="utf-8") + + temp_attributes_2 = (tmp_path / "outputs" / "runs" / "run_newer" + / "pipeline_attributes_for_test.yaml") + temp_attributes_2.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_2.write_text(minimal_pipeline_yaml( + run_id = "run_newer" + ), encoding="utf-8") + with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( - config=PipelineConfig(output_dir=tmp_path / "outputs"), + config=PipelineConfig(output_dir=tmp_path / "outputs", + log_dir=logs), stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], ) - pipeline.run_output = tmp_path / "runs" - assert pipeline.last_run is None - def test_last_run_populated_one_run(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "run_newer" + + + def test_last_run_most_recent(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: """ - Tests that if a Pipeline instance has one previous run, this is loaded in - last_run attribute at Pipeline creation. + Checks that only the older run is included when the run directory has been + deleted/removed for the most recent run. Parameters ---------- @@ -1298,19 +1038,23 @@ def test_last_run_populated_one_run(self, Raised when there is no stages_to_run parameters to warn the user that all stages will be run by default. """ + logs = tmp_path / "logs" logs.mkdir(parents=True, exist_ok=True) (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " - "{\"run_id\": \"2026-08-11_100000_abc12345\", " - " \"run_dir\": \"/path/to/run\"}\n" + "{\"run_id\": \"run_older\", " + " \"run_dir\": \"/path/to/run\"}\n" + "2026-08-11 10:01:00,000 Pipeline started | " + "{\"run_id\": \"run_newer\", " + " \"run_dir\": \"/path/to/run\"}" ) - temp_attributes = (tmp_path / "outputs" / "runs" / "2026-08-11_100000_abc12345" + temp_attributes_1 = (tmp_path / "outputs" / "runs" / "run_older" / "pipeline_attributes_for_test.yaml") - temp_attributes.parent.mkdir(parents=True, exist_ok=True) - temp_attributes.write_text(minimal_pipeline_yaml( - run_id = "2026-08-11_100000_abc12345" + temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_1.write_text(minimal_pipeline_yaml( + run_id = "run_older" ), encoding="utf-8") with pytest.warns(PipelineConfigurationWarning): @@ -1321,4 +1065,4 @@ def test_last_run_populated_one_run(self, ) assert pipeline.last_run is not None - assert pipeline.last_run.manifest.run_id == "2026-08-11_100000_abc12345" + assert pipeline.last_run.manifest.run_id == "run_older" From 2ba54b3b57b20f7431953a5aab3e1e195863b429 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 15:54:49 +0100 Subject: [PATCH 307/332] tests: separate some of the tests for run_history work into test_loader and test_logger to fit module level testing method --- tests/test_loader.py | 86 +++++++++++++ tests/test_logger.py | 295 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 tests/test_loader.py create mode 100644 tests/test_logger.py diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..1fe34c6 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,86 @@ +import pytest +from pathlib import Path +from onsrap.errors import StageLoadError +from onsrap.loader import load_historical_run +from onsrap.pipeline import PipelineRun + +from tests.test_pipeline import TestLoadLatestRunIntegration + + +class TestLoadHistoricalRun(TestLoadLatestRunIntegration): + def test_raises_stageloaderror_no_file(self, + tmp_path: Path) -> None: + """ + Checks that load_historical_run raises a StageLoadError when the specified + directory does not contain any files matching the expected pattern. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``StageLoadError`` + Raised when the specified directory does not contain any files matching + the expected pattern for historical run YAML files. + """ + run_dir = tmp_path/"empty_run" + run_dir.mkdir(parents=True, exist_ok=True) + with pytest.raises(StageLoadError, match="Historical run file does not exist"): + load_historical_run(run_dir=run_dir) + + def test_returns_valid_pipeline_run_from_yaml(self, + tmp_path:Path, + minimal_pipeline_yaml)-> None: + """ + Checks that load_historical_run successfully returns a PipelineRun instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + + run_dir = tmp_path/"valid_run" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir/"pipeline_attributes_for_test.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id"), encoding="utf-8") + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) + assert result.manifest.run_id == "test_id" + + def test_correct_yaml_file_chosen(self, + tmp_path: Path, + minimal_pipeline_yaml) -> None: + """ + Checks that if there are multiple files within the same run directory, + the method will pass successfully and return a PipelineRun. This does + not assert which file is chosen, only that the method does not raise + an error and returns a PipelineRun instance. + + Logically, this should be suitable as we would only ever expect one file + to be present in each run directory. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + run_dir = tmp_path/"multiple_runs" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir/"pipeline_attributes_for_test1.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id_1"), encoding="utf-8") + (run_dir/"pipeline_attributes_for_test2.yaml").write_text( + minimal_pipeline_yaml(run_id = "test_id_2"), encoding="utf-8") + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) \ No newline at end of file diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..508681e --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,295 @@ +import logging +from pathlib import Path +import pytest + +from onsrap.errors import HistoricalPipelineLoadError +from onsrap.logger import Logger +from tests.test_pipeline import TestLoadLatestRunIntegration + + +class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): + def test_logger_no_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers.clear() # Remove all handlers to simulate no file logging + logger._logger.propagate = False # Prevent checking root logger handlers + with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_no_file_handler_errors(self, + tmp_path: Path) -> None: + """ + Tests that if the logger has no file handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir = tmp_path/"logs") + logger._logger.handlers = [logging.StreamHandler()] + with pytest.raises(HistoricalPipelineLoadError, match="does not have a FileHandler"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_logger_does_not_exist(self, + tmp_path:Path) -> None: + """ + Checks that the method raises an error if the log doesn't exist at the + location specified. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + file_handler.close() + log_path.unlink(missing_ok=True) # Remove the log file to simulate non-existence + + with pytest.raises(HistoricalPipelineLoadError, + match="does not exist at this location"): + logger.extract_historical_run_ids(run_root = tmp_path/"runs") + + def test_return_blank_list_no_matches_in_log(self, + tmp_path) -> None: + """ + Tests that a log file that does not have a record covering "Pipeline started" + will return a blank list from extract_historical_run_ids. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Some unrelated log entry\n" \ + "2026-08-10 10:00:01,000 | Another unrelated log entry\n" \ + "2026-08-10 10:00:02,000 Pipeline Started\n") + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] + + def test_skips_poor_json_in_log(self, + tmp_path: Path) -> None: + """ + Tests that if a JSON record in the log file is not valid, it will e skipped + and the next valid entry will be extracted. Assert that the returned list + contains only the valid entry. Confirms that only the incorrect record is + skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text("2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" \ + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_abc1234", + "timestamp": "2026-08-10 10:00:02,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_abc1234" + }, + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + @pytest.mark.parametrize("string, expected", + [('{"some_key":"some_value"}', []), + ('{"run_id":""}',[])]) + + def test_run_id_absent_falsy(self, + tmp_path: Path, + string: str, + expected: list) -> None: + """ + Tests that if the JSON record in the log file does not have a run_id, it will be skipped + and the returned list will be empty. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``string`` : str + A dictionary representing a valid JSON record in the log file that excludes + run_id. + ``expected`` : list + The expected output from extract_historical_run_ids when the log file + contains a record without a run_id. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") + + #creates directory for runs to avoid removal given the directory doesn't exist + (tmp_path/"runs").mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == expected + + def test_records_only_if_directory_exists(self, + tmp_path) -> None: + """ + Checks that a record is only output if the run directory exists. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [ + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} + ] + + def test_reverse_chronological_order(self, + tmp_path) -> None: + """ + Checks that the run_ids are output in reverse chronological order + based on their positioning in the log file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ + "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result[0]["run_id"] == "2026-06-23_101719_abc1234" + assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" + + def test_skip_poor_timestamps(self, + tmp_path) -> None: + """ + Checks that entries with poor timestamps are skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir = tmp_path/"logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "BADTIMESTAMP Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n") + + create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + assert result == [] \ No newline at end of file From 5800b56450bb31b70cc2ed105aeef4f8f3723ba1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 16:47:50 +0100 Subject: [PATCH 308/332] tweak: provide warning message if a particular run directory does not have a run_id associated with it --- onsrap/pipeline.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index de5e72c..c4348d2 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -113,6 +113,9 @@ def __init__( self.last_run: PipelineRun | None = None self.last_run = self._load_latest_run() + self.all_runs: dict[str, PipelineRun] | None = None + self.all_runs = self._load_all_runs() + self.logger.event( "Pipeline initialized", name=self.name, @@ -463,6 +466,47 @@ def _load_latest_run(self) -> PipelineRun | None: "will be None.", PipelineConfigurationWarning) return None + def _load_all_runs(self) -> dict[str, PipelineRun] | None: + """ + Loads all previous runs of a Pipeline as a dictionary of PipelineRun instances, + keyed by run_id. + + Raises + ------ + ``PipelineConfigurationWarning`` + If there are any errors in loading previous runs, this warning is raised to + indicate a None value will be stored in this attribute. + """ + try: + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output + ) + except HistoricalPipelineLoadError: + warnings.warn("Unable to load previous runs for this Pipeline. All_run" \ + " attribute will be None.", PipelineConfigurationWarning) + return None + + if previous_run_logs == []: + warnings.warn("No previous runs found for this Pipeline. All_run attribute" \ + " will be None.", PipelineConfigurationWarning) + return None + + all_runs = {} + for run_log in previous_run_logs: + run_id = run_log["run_id"] + if run_id is None: + warnings.warn( + f"No run_id found in log for run_dir {run_log.get('run_dir')}. Skipping this run.", + PipelineConfigurationWarning + ) + continue + try: + all_runs[run_id] = load_historical_run(run_dir=Path(self.run_output) / run_id) + except StageLoadError: + warnings.warn(f"Historical run file for run_id {run_id} does not exist. Skipping.", + PipelineConfigurationWarning) + return all_runs if all_runs else None + def _set_run_output(self) -> Path: """ Private method that sets the run output directory for the Pipeline. From 1f4dfbef827093d14c27c030c8fd2e630f42e9f9 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Tue, 11 Aug 2026 17:01:51 +0100 Subject: [PATCH 309/332] tests: started integration tests for all_runs --- onsrap/pipeline.py | 2 +- tests/test_pipeline.py | 282 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 281 insertions(+), 3 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index c4348d2..a61cc98 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -494,7 +494,7 @@ def _load_all_runs(self) -> dict[str, PipelineRun] | None: all_runs = {} for run_log in previous_run_logs: run_id = run_log["run_id"] - if run_id is None: + if run_id is None or run_id == "": warnings.warn( f"No run_id found in log for run_dir {run_log.get('run_dir')}. Skipping this run.", PipelineConfigurationWarning diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 4466cca..c0a7554 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -663,7 +663,6 @@ def _make(run_id: str) -> str: """ return _make -class TestLoadLatestRun(TestLoadLatestRunIntegration): @pytest.fixture def pipeline_no_history(self, tmp_path: Path) -> Pipeline: """ @@ -683,7 +682,9 @@ def pipeline_no_history(self, tmp_path: Path) -> Pipeline: stages = [Stage("Stage_0", source=tmp_path/"Stage_0.py", dependencies=())]) pipeline.run_output = tmp_path/"runs" return pipeline - + + +class TestLoadLatestRun(TestLoadLatestRunIntegration): def test_blank_historical_run_ids(self, pipeline_no_history: Pipeline, monkeypatch) -> None: @@ -1066,3 +1067,280 @@ def test_last_run_most_recent(self, assert pipeline.last_run is not None assert pipeline.last_run.manifest.run_id == "run_older" + +class TestLoadAllRunsIntegration(TestLoadLatestRunIntegration): + def test_returns_none_when_error_in_extract_historical_runs(self, + monkeypatch, + pipeline_no_history + ) -> None: + """ + Checks that all_runs attribute is None when extract_historical_run_ids + raises an error. This is to ensure that the Pipeline instance does not break + when there is an issue with extracting historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue with extracting historical runs, indicating + that the all_runs attribute will be None. + """ + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + mock.Mock(side_effect=HistoricalPipelineLoadError("test"))) + + with pytest.warns(PipelineConfigurationWarning): + assert pipeline_no_history._load_all_runs() is None + assert pipeline_no_history.all_runs is None + + def test_returns_none_when_blank_extract_historical_runs(self, + monkeypatch, + pipeline_no_history + ) -> None: + """ + Checks that all_runs attribute is None when extract_historical_run_ids + returns a blank list. This ensures that the Pipeline instance does not + break when there are no historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there are no historical runs found, indicating that the + all_runs attribute will be None. + """ + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: []) + + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) == [] + with pytest.warns(PipelineConfigurationWarning): + assert pipeline_no_history._load_all_runs() is None + assert pipeline_no_history.all_runs is None + + def test_single_entry_dict_single_run(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + """ + Checks that all_runs attribute is a dictionary with a single entry when + extract_historical_run_ids returns a list with one historical run. This + ensures that the Pipeline instance correctly loads a single historical run. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is one historical run found, indicating that the + all_runs attribute will contain a single entry. + """ + + mock_loader = mock.MagicMock(return_value = mock.sentinel) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [{"run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")}]) + + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 1 + assert "run_A" in result + mock_loader.assert_called_once_with(run_dir = pipeline_no_history.run_output / "run_A") + + def test_multiple_entries_dict_multiple_runs(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + """ + Checks that all_runs attribute is a dictionary with multiple entries when + extract_historical_run_ids returns a list with multiple historical runs. + This ensures that the Pipeline instance correctly loads multiple historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there are multiple historical runs found, indicating that the + all_runs attribute will contain multiple entries. + """ + + mock_loader = mock.MagicMock(side_effect=[mock.sentinel.run_A, mock.sentinel.run_B]) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + {"run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")}, + {"run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B")} + ]) + + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 2 + assert "run_A" in result and "run_B" in result + mock_loader.assert_any_call(run_dir = pipeline_no_history.run_output / "run_A") + mock_loader.assert_any_call(run_dir = pipeline_no_history.run_output / "run_B") + + def test_warning_if_no_run_id(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + """ + Checks that a warning is raised if extract_historical_run_ids returns a + historical run without a run_id. This ensures that the Pipeline instance + correctly handles cases where historical runs are missing identifiers. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + mock_loader = mock.MagicMock(return_value=mock.sentinel) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + {"run_id": "", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")}, + {"run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B")}, + {"run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")} + ]) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 1 + assert "run_A" not in result and "run_B" in result + mock_loader.assert_called_once_with(run_dir = pipeline_no_history.run_output / "run_B") + + def test_None_with_stageloaderror(self, + monkeypatch, + pipeline_no_history: Pipeline) -> None: + """ + Checks that all_runs attribute is None when load_historical_run raises a + StageLoadError. This ensures that the Pipeline instance correctly handles + cases where historical runs cannot be loaded due to errors. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue loading a historical run, indicating that + the all_runs attribute will be None. + """ + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + {"run_id": "good_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")}, + {"run_id": "bad_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B")} + ]) + + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[mock.sentinel.good_run, + StageLoadError("test")])) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + assert isinstance(result,dict) + assert len(result) == 1 + assert "good_run" in result and "bad_run" not in result + + def test_none_if_all_stageloaderrors(self, + monkeypatch, + pipeline_no_history: Pipeline + ) -> None: + """ + Asserts that all_runs attribute is None when load_historical_run raises a + StageLoadError for all historical runs. This ensures that the Pipeline instance + correctly handles cases where all historical runs cannot be loaded due to errors. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue loading all historical runs, indicating that + the all_runs attribute will be None. + """ + + monkeypatch.setattr(pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + {"run_id": "bad_run1", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A")}, + {"run_id": "bad_run2", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B")} + ]) + + monkeypatch.setattr("onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[StageLoadError("test"), + StageLoadError("test")])) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + + assert result is None \ No newline at end of file From 071bea280fd9ba845e232d62f7d71eea4d0399ed Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 09:03:54 +0100 Subject: [PATCH 310/332] tweak: changes to Pipeline docs and code structure. --- onsrap/pipeline.py | 858 +++++++++++++++++++++++++++++---------------- 1 file changed, 562 insertions(+), 296 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7b17244..9f72eb1 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -2,77 +2,99 @@ import getpass import hashlib +import re import subprocess -import warnings import sys +import warnings +from contextlib import suppress from importlib import metadata as importlib_metadata from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence -import re -from .errors import StageConfigurationError, PipelineInitialisationError, PipelineConfigurationError -from .warnings import StageConfigurationWarning, PipelineConfigurationWarning +from .errors import ( + PipelineConfigurationError, + PipelineInitialisationError, + StageConfigurationError, +) from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph from .logger import Logger -from .models import GlobalConfig, PipelineConfig, StageConfig, PipelineRun, RunManifest, RuntimeID, now -from .stage import Stage, _normalize_dependencies - +from .models import ( + GlobalConfig, + PipelineConfig, + PipelineRun, + RunManifest, + RuntimeID, + StageConfig, + now, +) +from .stage import Stage +from .warnings import PipelineConfigurationWarning, StageConfigurationWarning ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") AVAILABLE_EXECUTORS = ("python",) -#captures long and short versions of output, directory, path, location, and file that's not case sensitive. +# captures long and short versions of output, directory, path, location, and file that's not case sensitive. OUTPUT_DIR_KEY_RE = re.compile( - r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", - re.IGNORECASE - ) + r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", + re.IGNORECASE, +) + class Pipeline: """ - Represents an end-to-end code run. This class brings together class instances - from other modules within the package to establish what the Pipeline is. + Represents an end-to-end code run. This class brings together class instances + from other modules within the package to establish what the Pipeline is. - Sets up the metadata, configurations, logging, and executors required to run the - Pipeline. Assigns multiple attributes including those not initialised such as, - ``id``, ``graph``, ``manifest``, and ``last_run``. These take the forms of other - classes defined in other modules within this package. + Sets up the metadata, configurations, logging, and executors required to run the + Pipeline. Assigns multiple attributes including those not initialised such as, + ``id``, ``graph``, ``manifest``, and ``last_run``. These take the forms of other + classes defined in other modules within this package. Parameters ---------- ``name`` : str or None What the pipeline is called. ``backend`` : str, default = "python" - The system used to run the pipeline. - ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None - The instance containing the required information on running the Pipeline. - ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. - The required steps within the Pipeline. + The system used to run the pipeline. + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None + The instance containing the required information on running the Pipeline. + ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. + The required steps within the Pipeline. ``logger`` : Logger or None - The system that is used to track the progress of the Pipeline. + The system that is used to track the progress of the Pipeline. ``executor`` : StageExecutor or None - The way that the Pipeline is actively run. + The way that the Pipeline is actively run. """ + def __init__( self, name: str | None = None, backend: str = "python", - config: PipelineConfig | Mapping[str, Any] | None = None, - stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, - dependencies: tuple[str]| dict[str, Sequence[str]] | None = None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, + stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] + | None = None, + dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, ): - resolved_config, resolved_stage_configs, configured_stages, resolved_global_config = self._resolve_config(config) + ( + resolved_config, + resolved_stage_configs, + configured_stages, + resolved_global_config, + ) = self._resolve_config(config) self.name = name or resolved_config.name or "pipeline" self.backend = backend or resolved_config.backend or "python" if backend == "python" and resolved_config.backend != "python": - raise PipelineInitialisationError(f"Pipeline backend {backend} does not align with PipelineConfig backend {resolved_config.backend}.") + raise PipelineInitialisationError( + f"Pipeline backend {backend} does not align with PipelineConfig backend {resolved_config.backend}." + ) self.config = resolved_config if self.config.name is None: self.config.name = self.name - + self.logger = logger or Logger(log_dir=self.config.log_dir) if executor is not None: self.executor = executor @@ -80,8 +102,10 @@ def __init__( if self.backend == "python": self.executor = PythonStageExecutor() else: - raise PipelineInitialisationError(f"Requested backend does not have a compatible executor. Available executors are: {', '.join(AVAILABLE_EXECUTORS)}.") - + raise PipelineInitialisationError( + f"Requested backend does not have a compatible executor. Available executors are: {', '.join(AVAILABLE_EXECUTORS)}." + ) + if stages is not None and configured_stages: raise PipelineInitialisationError( "Stages parsed through both Pipeline construction and configuration file. Either provide stages through the constructor or the configuration file, not both." @@ -92,19 +116,21 @@ def __init__( else [self._coerce_stage(stage) for stage in stages] ) - self.dependencies = dependencies + self.dependencies = self._normalize_dependency_mapping(dependencies) if dependencies is not None and stages is None: - raise PipelineInitialisationError("Stages need to be defined before you can parse your dependencies " - "for those stages. Try the from_files() method, or create your Stage objects and " \ - "parse them to the Pipeline Constructor.") - if dependencies is not None: - self._assign_dependencies(dependencies, self.stages) + raise PipelineInitialisationError( + "Stages need to be defined before you can parse your dependencies " + "for those stages. Try the from_files() method, or create your Stage objects and " + "parse them to the Pipeline Constructor." + ) + if self.dependencies is not None: + self._assign_dependencies(self.dependencies, self.stages) self.stage_configs = dict(resolved_stage_configs) - self.global_config = resolved_global_config - + self.global_config = resolved_global_config + self._sync_stage_configs() - + self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None @@ -141,11 +167,11 @@ def __str__(self) -> str: def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the ``Pipeline`` class. - This method is structured to be more concise than the ``__str__`` method and is + Representation method that returns a human readable representation of the ``Pipeline`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``Pipeline`` class with its attributes. @@ -160,7 +186,12 @@ def __repr__(self) -> str: def add_stage( self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], - stage_configs: StageConfig | Mapping[str, Any] | str | Path | Iterable[StageConfig | Mapping[str, Any] | str | Path] | None = None, + stage_configs: StageConfig + | Mapping[str, Any] + | str + | Path + | Iterable[StageConfig | Mapping[str, Any] | str | Path] + | None = None, enable_stages: bool = False, ) -> None: """ @@ -193,7 +224,10 @@ def add_stage( parsed_stage_configs: list[StageConfig] = [] if stage_configs is not None: - if isinstance(stage_configs, Mapping) and all(isinstance(value, Mapping) for value in stage_configs.values()): + raw_stage_configs: list[StageConfig | Mapping[str, Any] | str | Path] + if isinstance(stage_configs, Mapping) and all( + isinstance(value, Mapping) for value in stage_configs.values() + ): raw_stage_configs = [ {str(stage_name): stage_payload} for stage_name, stage_payload in stage_configs.items() @@ -214,8 +248,12 @@ def add_stage( known_stage_names.update(stage.name for stage in added_stages) for index, raw_stage_config in enumerate(raw_stage_configs): - stage_name = added_stages[index].name if index < len(added_stages) else None - parsed_stage_config = self._coerce_stage_config(raw_stage_config, name=stage_name) + stage_name = ( + added_stages[index].name if index < len(added_stages) else None + ) + parsed_stage_config = self._coerce_stage_config( + raw_stage_config, name=stage_name + ) if parsed_stage_config.name not in known_stage_names: raise StageConfigurationError( f"Stage configuration was provided for unknown stage: {parsed_stage_config.name}." @@ -227,7 +265,9 @@ def add_stage( for conf in parsed_stage_configs: self.add_stage_config(conf) - self._register_added_stages_in_stage_selection(added_stages, enable_stages=enable_stages) + self._register_added_stages_in_stage_selection( + added_stages, enable_stages=enable_stages + ) self._check_stage_configs(added_stages, self.stage_configs) self._sync_stage_configs() @@ -260,7 +300,49 @@ def add_stage_config( self.stage_configs[parsed_stage_config.name] = parsed_stage_config self._check_output_dir_in_stage_configs(name=parsed_stage_config.name) self.logger.event("Stage configuration added", stage=parsed_stage_config.name) - + + def create_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> StageConfig: + """ + Normalize a stage-configuration payload into a ``StageConfig`` instance. + + This compatibility helper accepts direct stage payloads, stage-name keyed + mappings, and composite configuration payloads or files containing a + ``stage_configuration`` section. + """ + if isinstance(stage_config, StageConfig): + return self._coerce_stage_config(stage_config, name=name) + + if isinstance(stage_config, Mapping): + payload = dict(stage_config) + elif isinstance(stage_config, (str, Path)): + payload = self._load_config_mapping(stage_config) + else: + raise StageConfigurationError( + f"Unsupported stage configuration specification: {type(stage_config)!r}." + ) + + composite_keys = { + "pipeline_variables", + "pipeline_config", + "stage_configuration", + "stage_config", + "global_configuration", + "global_config", + "global_variables", + "global_vars", + } + if composite_keys.intersection(payload): + _, stage_payload, _ = self._split_config_sections(payload) + stage_configs = self._build_stage_configs(stage_payload) + return self._select_stage_config(stage_configs, name=name) + + return self._coerce_stage_config(payload, name=name) + def enable_stage(self, *stage_name: str | list[str]) -> None: """ Mark one or more stages as enabled in the run selection. @@ -283,7 +365,9 @@ def enable_stage(self, *stage_name: str | list[str]) -> None: stage_names.add(name) if not stage_names.issubset({stage.name for stage in self.stages}): - raise PipelineInitialisationError("You're trying to enable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline.") + raise PipelineInitialisationError( + "You're trying to enable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline." + ) if not self.config.stages_to_run: return @@ -315,7 +399,9 @@ def disable_stage(self, *stage_name: str | list[str]) -> None: stage_names.add(name) if not stage_names.issubset({stage.name for stage in self.stages}): - raise PipelineInitialisationError("You're trying to disable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline.") + raise PipelineInitialisationError( + "You're trying to disable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline." + ) if not self.config.stages_to_run: self.config.stages_to_run = {stage.name: True for stage in self.stages} @@ -359,76 +445,92 @@ def validate(self) -> Pipeline: self._output_dir_conflict_check() return self - def run(self) -> PipelineRun: """ - Returns an instance of ``PipelineRunner`` which actually runs the pipeline. + Returns an instance of ``PipelineRunner`` which actually runs the pipeline. """ from .runner import PipelineRunner - + return PipelineRunner(logger=self.logger).run(self) - def add_dependencies(self, - *dependencies: tuple[str]| dict[str, Sequence[str]]) -> None: + def add_dependencies( + self, + *dependencies: Mapping[str, Sequence[str]] | tuple[str, ...], + ) -> None: """ - Adds a set of ``dependencies`` for the Pipeline after the Pipeline initialisation. + Adds a set of ``dependencies`` for the Pipeline after the Pipeline initialisation. This method takes any number of positional arguments and imputes them as ``dependencies``. It looks at each argument parsed, checks the data type against the existing ``Pipeline`` ``dependencies`` and if they are the same data type, it - will take every stage within the ``Pipeline`` instance. It will then run the + will take every stage within the ``Pipeline`` instance. It will then run the ``_dependencies_for_stage()`` class method and normalize any ``dependencies`` before - adding them to the individual ``Stage`` instances. It will then append these + adding them to the individual ``Stage`` instances. It will then append these ``dependencies`` directly to the ``dependencies`` in the ``Pipeline`` instance before - rerunning the ``StageGraph`` creation to ensure the new ``dependencies`` are considered. - A logging entry will be created to track that these ``dependencies`` are added. + rerunning the ``StageGraph`` creation to ensure the new ``dependencies`` are considered. + A logging entry will be created to track that these ``dependencies`` are added. Parameters ---------- ``*dependencies`` : tuple[str]| dict[str, Sequence[str]] - Any number of dependencies that you would like to add to the Pipeline. + Any number of dependencies that you would like to add to the Pipeline. Raises ------ ``PipelineInitializationError`` - If the dependency you are attempting to add to the Pipeline doesn't match + If the dependency you are attempting to add to the Pipeline doesn't match the datatype for dependencies currently in the Pipeline. """ - + for dependency in dependencies: - if self.dependencies is not None and not isinstance(dependency, type(self.dependencies)): - raise PipelineInitialisationError("Existing dependencies are not the same type as new dependencies") - + if not isinstance(dependency, Mapping): + raise PipelineInitialisationError( + "Existing dependencies are not the same type as new dependencies" + ) + + normalized_dependency = self._normalize_dependency_mapping(dependency) + if normalized_dependency is None: + continue + for stage in self.stages: - new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependency) + new_dependencies = self._dependencies_for_stage( + stage.name, stage.source, normalized_dependency + ) existing = stage.dependencies or () - new = existing + tuple(_normalize_dependencies(new_dependencies)) + new = existing + new_dependencies stage.dependencies = tuple(dict.fromkeys(new)) - - if isinstance(dependency, tuple): - existing = self.dependencies or () - self.dependencies = tuple(existing | dependency) - elif isinstance(dependency, dict): - for stage_name, deps in dependency.items(): - existing = self.dependencies.get(stage_name,[]) - combined = existing + tuple(deps) - self.dependencies[stage_name] = tuple(dict.fromkeys(combined)) + + if self.dependencies is None: + self.dependencies = {} + + for stage_name, deps in normalized_dependency.items(): + existing = self.dependencies.get(stage_name, ()) + combined = existing + deps + self.dependencies[stage_name] = tuple(dict.fromkeys(combined)) self.graph = StageGraph.from_stages(self.stages) self.graph.validate() - self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) - + self.logger.event( + "New dependencies added to Pipeline instance and respective Stage instances", + dependencies=dependencies, + ) + def _assign_dependencies( + self, + dependencies: Mapping[str, Sequence[str]] | None = None, + stages: Sequence[Stage] | None = None, + ) -> Sequence[Stage]: + if stages is None: + return () - def _assign_dependencies(self, - dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, - stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: for stage in stages: - new_dependencies = self._dependencies_for_stage(stage.name, stage.source, dependencies) - stage.dependencies = _normalize_dependencies(new_dependencies) + new_dependencies = self._dependencies_for_stage( + stage.name, stage.source, dependencies + ) + stage.dependencies = new_dependencies return stages @@ -437,11 +539,11 @@ def _coerce_stage( stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], ) -> Stage: """ - Extracts the ``Stage`` information from the provided stages in the Pipeline. + Extracts the ``Stage`` information from the provided stages in the Pipeline. - Enables mappings, strings, paths, or callables to be parsed and converted into - a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return - itself. + Enables mappings, strings, paths, or callables to be parsed and converted into + a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return + itself. Parameters ---------- @@ -451,12 +553,12 @@ def _coerce_stage( Raises ------ ``StageConfigurationError`` - If the information parsed is not in a suitable format to be converted into - a ``Stage`` class instance. + If the information parsed is not in a suitable format to be converted into + a ``Stage`` class instance. Returns ------- - ``Stage`` class instance for the stage being run. + ``Stage`` class instance for the stage being run. """ if isinstance(stage, Stage): return stage @@ -470,7 +572,9 @@ def _coerce_stage( if isinstance(stage, (str, Path)): return Stage.from_file(stage) - raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") + raise StageConfigurationError( + f"Unsupported stage specification: {type(stage)!r}." + ) def _coerce_stage_config( self, @@ -489,26 +593,49 @@ def _coerce_stage_config( return stage_config if isinstance(stage_config, Mapping): - if name is not None and all(isinstance(value, Mapping) for value in stage_config.values()) and name not in stage_config: - available_stage_names = ", ".join(str(stage_name) for stage_name in stage_config) + if ( + name is not None + and all(isinstance(value, Mapping) for value in stage_config.values()) + and name not in stage_config + ): + available_stage_names = ", ".join( + str(stage_name) for stage_name in stage_config + ) raise StageConfigurationError( f"Stage configuration '{name}' was not found. Available stage configurations are: {available_stage_names}." ) - return StageConfig.from_mapping(name=name, data=stage_config) + if name is not None and all( + isinstance(value, Mapping) for value in stage_config.values() + ): + selected_config = stage_config.get(name) + if not isinstance(selected_config, Mapping): + raise StageConfigurationError( + f"Stage configuration '{name}' must be defined as a mapping." + ) + return StageConfig.from_mapping(name=name, data=selected_config) + if name is not None: + return StageConfig.from_mapping(name=name, data=stage_config) + if all(isinstance(value, Mapping) for value in stage_config.values()): + return self._select_stage_config( + self._build_stage_configs(stage_config), name=None + ) + raise StageConfigurationError( + "Stage configuration name is required when the configuration payload does not identify a stage." + ) if isinstance(stage_config, (str, Path)): - import yaml + payload = self._load_config_mapping(stage_config) + return self._coerce_stage_config(payload, name=name) - payload = dict(yaml.safe_load(Path(stage_config).read_text())) - return StageConfig.from_mapping(name=name, data=payload) - - raise StageConfigurationError(f"Unsupported stage configuration specification: {type(stage_config)!r}.") + raise StageConfigurationError( + f"Unsupported stage configuration specification: {type(stage_config)!r}." + ) def _rebuild_graph(self) -> None: """ Update and validate the execution graph. - The execution graph is a subset of the full Stage registry - (Pipeline.stages), reflecting the stages that are actually enabled + The execution graph is a subset of the full Stage registry + (Pipeline.stages), reflecting the stages that are actually enabled for execution. The pipeline keeps ``self.stages`` as the complete stage registry, but @@ -520,7 +647,9 @@ def _rebuild_graph(self) -> None: self.graph = StageGraph.from_stages(stages_to_run) self.graph.validate() - def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage], enable_stages: bool = False) -> None: + def _register_added_stages_in_stage_selection( + self, stages: Sequence[Stage], enable_stages: bool = False + ) -> None: """ Default newly added stages to disabled once explicit stage selection is in use. @@ -535,17 +664,16 @@ def _register_added_stages_in_stage_selection(self, stages: Sequence[Stage], ena for stage in stages: self.config.stages_to_run.setdefault(stage.name, enable_stages) - def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: """ - Creates a ``RunManifest`` instance that contains the information about - the run of the Pipeline. + Creates a ``RunManifest`` instance that contains the information about + the run of the Pipeline. Parameters ---------- ``runtime_id`` : RuntimeID - Contains information about the run to be extracted and placed into - the ``RunManifest`` instance. + Contains information about the run to be extracted and placed into + the ``RunManifest`` instance. """ return RunManifest( rap_name=self.name, @@ -553,48 +681,55 @@ def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: git_commit=self._discover_git_commit(), stages_run=[], parameters=self._manifest_parameters(), - inputs={stage.name: list(stage.dependencies) for stage in self.graph.stages}, + inputs={ + stage.name: list(stage.dependencies) for stage in self.graph.stages + }, outputs={}, backend=self.backend, package_versions=self._package_versions(), timestamp=runtime_id.timestamp.isoformat(), reason=self.config.metadata.get("reason"), user=self._current_user(), - config = self._combine_configs(), + config=self._combine_configs(), ) - def _combine_configs(self) -> dict[str,Any]: + def _combine_configs(self) -> dict[str, Any]: """ - Combines all configurations (PipelineConfig, GlobalConfig, StageConfig) within the Pipeline into one dictionary which can - be recorded in the RunManifest for the run. + Combines all configurations (PipelineConfig, GlobalConfig, StageConfig) within the Pipeline into one dictionary which can + be recorded in the RunManifest for the run. - Returns + Returns ------- dict[str,Any] - A dictionary containing the configuration for the Pipeline, the stages, and - the global configuration. + A dictionary containing the configuration for the Pipeline, the stages, and + the global configuration. """ global_variables, exclusions = self.global_config.get_attributes() global_configuration = dict(global_variables or {}) global_configuration["exclusions"] = exclusions - - all_stage_configuration = {name: config.to_dict() for name, config in self.stage_configs.items()} - configuration = {"pipeline_config": self.config.to_dict(), - "stage_configs": all_stage_configuration, - "global_config": global_configuration} - return configuration + all_stage_configuration = { + name: config.to_dict() for name, config in self.stage_configs.items() + } + configuration = { + "pipeline_config": self.config.to_dict(), + "stage_configs": all_stage_configuration, + "global_config": global_configuration, + } + return configuration def _create_runtime_id(self) -> RuntimeID: """ - Creates a RuntimeID instance for the specific run. + Creates a RuntimeID instance for the specific run. - Establishes attributes for this specific run and returns - as a ``RuntimeID`` instance. + Establishes attributes for this specific run and returns + as a ``RuntimeID`` instance. """ current_time = now() - digest = hashlib.sha256(f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8")).hexdigest() + digest = hashlib.sha256( + f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8") + ).hexdigest() short_hash = digest[:8] return RuntimeID( id=f"{current_time.strftime('%Y-%m-%d_%H%M%S')}_{short_hash}", @@ -605,16 +740,16 @@ def _create_runtime_id(self) -> RuntimeID: def _discover_git_commit(self) -> str | None: """ - Finds the specific version of the repository used for this run. + Finds the specific version of the repository used for this run. Attempts to run a Git command to establish the current git commit hash - to be held in the ``RunManifest`` instance for this run. + to be held in the ``RunManifest`` instance for this run. Returns ------- ``OSError`` If Git is unable to be loaded or the Git command cannot be run for - another reason. + another reason. """ try: completed = subprocess.run( @@ -631,24 +766,22 @@ def _discover_git_commit(self) -> str | None: def _package_versions(self) -> list[str]: """ - Creates a list of packages and their versions used in this run. + Creates a list of packages and their versions used in this run. Raises ------ ``importlib_metadata.PackageNotFoundError`` - If the package used cannot be found in the library. + If the package used cannot be found in the library. """ versions = [f"python={sys.version.split()[0]}"] - try: + with suppress(importlib_metadata.PackageNotFoundError): versions.append(f"pyyaml={importlib_metadata.version('PyYAML')}") - except importlib_metadata.PackageNotFoundError: - pass return versions def _current_user(self) -> str | None: """ - Extracts the username for the individual completing the run. Returns a blank - value if the username cannot be extracted. + Extracts the username for the individual completing the run. Returns a blank + value if the username cannot be extracted. """ try: return getpass.getuser() @@ -657,8 +790,8 @@ def _current_user(self) -> str | None: def _resolve_config( self, - config: PipelineConfig | Mapping[str, Any] | None, - ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig]: + config: PipelineConfig | Mapping[str, Any] | str | Path | None, + ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig]: """ Resolve supported configuration inputs into pipeline config, stage config, and stages. @@ -683,7 +816,7 @@ def _resolve_config( raised to indicate that a composite configuration payload is preferred. Output-location warnings are emitted during ``Pipeline.validate()`` when stage configurations are inspected. """ - + if config is None: return PipelineConfig.from_any(config), {}, [], GlobalConfig() @@ -704,10 +837,17 @@ def _resolve_config( "Global configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", StageConfigurationWarning, ) - return config, self._build_stage_configs(stage_configuration), [], GlobalConfig.from_dict(global_configuration) + return ( + config, + self._build_stage_configs(stage_configuration), + [], + GlobalConfig.from_dict(global_configuration), + ) raw_config = self._load_config_mapping(config) - pipeline_payload, stage_config_payload, global_config_payload = self._split_config_sections(raw_config) + pipeline_payload, stage_config_payload, global_config_payload = ( + self._split_config_sections(raw_config) + ) normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) stage_definitions = normalized_pipeline_payload.pop("stages", ()) @@ -722,7 +862,18 @@ def _resolve_config( global_config = GlobalConfig.from_dict(global_config_payload) return pipeline_config, stage_configs, configured_stages, global_config - + + @staticmethod + def _normalize_dependency_mapping( + dependencies: Mapping[str, Sequence[str]] | None, + ) -> dict[str, tuple[str, ...]] | None: + if dependencies is None: + return None + + return { + str(stage_name): tuple(str(dependency) for dependency in stage_dependencies) + for stage_name, stage_dependencies in dependencies.items() + } def _sync_stage_configs(self) -> None: """ @@ -731,13 +882,12 @@ def _sync_stage_configs(self) -> None: """ for stage in self.stages: self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) - def _check_output_dir_in_stage_configs(self, name: str) -> None: """ - Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, - as this will result in overwriting previous run outputs. Users are advised to set their output - location in the stage scripts using the ``resolve_output_path()`` function to ensure unique + Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, + as this will result in overwriting previous run outputs. Users are advised to set their output + location in the stage scripts using the ``resolve_output_path()`` function to ensure unique outputs are saved for each run. Parameters @@ -749,37 +899,47 @@ def _check_output_dir_in_stage_configs(self, name: str) -> None: The stage name to check for output directory keys in the stage configurations. """ - if any(OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[name]._variables): + if any( + OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[name]._variables + ): warnings.warn( f"Stage configuration for {name} contains output directory keys. This will result " f"in overwriting previous run outputs. Please set your output location in the stage scripts " f"using the resolve_output_root() method to ensure unique outputs are saved for each run.", StageConfigurationWarning, ) - self.logger.event(f"Warning: stage configuration for {name} contains output directory keys. Risk of overwriting outputs.", - keys_found = [key for key in self.stage_configs[name]._variables if OUTPUT_DIR_KEY_RE.match(key)] + self.logger.event( + f"Warning: stage configuration for {name} contains output directory keys. Risk of overwriting outputs.", + keys_found=[ + key + for key in self.stage_configs[name]._variables + if OUTPUT_DIR_KEY_RE.match(key) + ], ) def _output_dir_conflict_check(self) -> None: """ - Checks whether the output directory has been assigned in stage configurations and already exists. It raises an error if + Checks whether the output directory has been assigned in stage configurations and already exists. It raises an error if it does, unless the overwrite parameter is set to True. - For each stage in the Pipeline, checks whether an output directory has been defined in the stage configurations. If it has been - defined, it checks whether the Path value for the output directory already exists. If it does already exist, raise either an - error or a warning based on an overwrite configuration. Log either the error or the warning in the Logger. + For each stage in the Pipeline, checks whether an output directory has been defined in the stage configurations. If it has been + defined, it checks whether the Path value for the output directory already exists. If it does already exist, raise either an + error or a warning based on an overwrite configuration. Log either the error or the warning in the Logger. Raises ------ StageConfigurationError If the overwrite parameter in the PipelineConfig is set to False and the output directory already exists. StageConfigurationWarning - If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. + If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. """ - - for stage in self.stages: - available_output_dirs = [key for key in self.stage_configs[stage.name]._variables if OUTPUT_DIR_KEY_RE.match(key)] + for stage in self.stages: + available_output_dirs = [ + key + for key in self.stage_configs[stage.name]._variables + if OUTPUT_DIR_KEY_RE.match(key) + ] self._check_output_dir_in_stage_configs(name=stage.name) for directory in available_output_dirs: @@ -815,7 +975,6 @@ def _output_dir_conflict_check(self) -> None: f"will be overwritten.", overwrite=overwrite, ) - def _validate_stage_configs(self) -> None: """ @@ -823,7 +982,9 @@ def _validate_stage_configs(self) -> None: """ self._sync_stage_configs() stage_names = {stage.name for stage in self.stages} - unknown_stage_configs = sorted(name for name in self.stage_configs if name not in stage_names) + unknown_stage_configs = sorted( + name for name in self.stage_configs if name not in stage_names + ) if unknown_stage_configs and stage_names: missing = ", ".join(unknown_stage_configs) raise StageConfigurationError( @@ -841,34 +1002,40 @@ def _manifest_parameters(self) -> dict[str, Any]: for name, stage_config in self.stage_configs.items() } return parameters - + def _build_stages_from_config( - self, - stage_definitions: Sequence[Any] | None, - *, - backend: str, - work_dir: Path, - ) -> list[Stage]: - """ - Convert configured stage definitions into ``Stage`` instances. + self, + stage_definitions: Sequence[Any] | None, + *, + backend: str, + work_dir: Path, + ) -> list[Stage]: + """ + Convert configured stage definitions into ``Stage`` instances. - Each entry is resolved independently, so the method can process any number of - stage definitions supplied in the pipeline configuration. - """ - if not stage_definitions: - return [] + Each entry is resolved independently, so the method can process any number of + stage definitions supplied in the pipeline configuration. + """ + if not stage_definitions: + return [] - stage_definitions = list(stage_definitions) + stage_definitions = list(stage_definitions) - if not isinstance(stage_definitions, Sequence) or isinstance(stage_definitions, (str, bytes)): - raise StageConfigurationError("Configured stages must be provided as a sequence.") + if not isinstance(stage_definitions, Sequence) or isinstance( + stage_definitions, (str, bytes) + ): + raise StageConfigurationError( + "Configured stages must be provided as a sequence." + ) - configured_stages: list[Stage] = [] - for stage_definition in stage_definitions: - stage = self._stage_from_config_definition(stage_definition, backend=backend, work_dir=work_dir) - if stage is not None: - configured_stages.append(stage) - return configured_stages + configured_stages: list[Stage] = [] + for stage_definition in stage_definitions: + stage = self._stage_from_config_definition( + stage_definition, backend=backend, work_dir=work_dir + ) + if stage is not None: + configured_stages.append(stage) + return configured_stages def _stage_from_config_definition( self, @@ -891,7 +1058,9 @@ def _stage_from_config_definition( return self._coerce_stage(stage_definition) if not isinstance(stage_definition, Mapping): - raise StageConfigurationError("Configured stage entries must be mappings, paths, or callables.") + raise StageConfigurationError( + "Configured stage entries must be mappings, paths, or callables." + ) payload = dict(stage_definition) if any(key in payload for key in ("name", "source", "path", "callable")): @@ -904,13 +1073,17 @@ def _stage_from_config_definition( stage_name, stage_payload = next(iter(payload.items())) if not isinstance(stage_payload, Mapping): - raise StageConfigurationError("Configured stage details must be provided as a mapping.") + raise StageConfigurationError( + "Configured stage details must be provided as a mapping." + ) stage_options = dict(stage_payload) if not bool(stage_options.pop("run", True)): return None - location = stage_options.pop("location", stage_options.pop("source", stage_options.pop("path", None))) + location = stage_options.pop( + "location", stage_options.pop("source", stage_options.pop("path", None)) + ) dependencies = stage_options.pop("dependencies", ()) entrypoint = stage_options.pop("entrypoint", None) metadata = stage_options.pop("metadata", {}) @@ -920,7 +1093,9 @@ def _stage_from_config_definition( metadata = {"metadata": metadata} metadata.update(stage_options) - source = self._resolve_stage_source(stage_name=str(stage_name), location=location, work_dir=work_dir) + source = self._resolve_stage_source( + stage_name=str(stage_name), location=location, work_dir=work_dir + ) return Stage.from_file( source, name=str(stage_name), @@ -929,7 +1104,7 @@ def _stage_from_config_definition( entrypoint=entrypoint, backend=backend, ) - + def _resolve_stages_to_run(self) -> list[Stage]: """ Resolve the effective stage subset that should populate the execution graph. @@ -947,7 +1122,7 @@ def _resolve_stages_to_run(self) -> list[Stage]: if not configured_stages_to_run: warnings.warn( "No stages specified to run. All stages running by default.", - PipelineConfigurationWarning + PipelineConfigurationWarning, ) return self.stages @@ -977,9 +1152,10 @@ def _resolve_stages_to_run(self) -> list[Stage]: } resolved_stage_names: set[str] = set() visiting: set[str] = set() - - def add_stage_with_dependencies(stage_name: str, *, required_by: str | None = None) -> None: + def add_stage_with_dependencies( + stage_name: str, *, required_by: str | None = None + ) -> None: """ Add one selected stage and recursively include everything it depends on. @@ -1026,35 +1202,37 @@ def from_files( ) -> Pipeline: """ Extracts the information from files regarding exactly what is being run in the pipeline and - allows for configuration of how the Pipeline is run. + allows for configuration of how the Pipeline is run. Parameters ---------- ``file_paths`` : Iterable[str or Path] The files that contain the code for each stage in the pipeline. These are what - the Pipeline will run. + the Pipeline will run. ``name`` : str The name of the pipeline. ``backend`` : str, default = "python" - The system that the pipeline is written in. + The system that the pipeline is written in. ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None - The high level information required to run this specific pipeline. + The high level information required to run this specific pipeline. ``dependencies`` : Mapping[str, Sequence[str]] or None - An object containing which stages are required to be run before other stages. + An object containing which stages are required to be run before other stages. ``logger`` : Logger class or None - The logging sysem used for this Pipeline run. + The logging sysem used for this Pipeline run. ``executor`` : StageExecutor class or None - The information on exactly how to run the Pipeline. + The information on exactly how to run the Pipeline. - Returns + Returns ------- - A ``Pipeline`` class instance. + A ``Pipeline`` class instance. """ stages: list[Stage] = [] for file_path in file_paths: path = Path(file_path) stage_name = path.stem - stage_dependencies = cls._dependencies_for_stage(stage_name, path, dependencies) + stage_dependencies = cls._dependencies_for_stage( + stage_name, path, dependencies + ) stages.append( Stage.from_file( path, @@ -1084,39 +1262,41 @@ def from_dict( ) -> Pipeline: """ Extracts information from a dictionary to configure a Pipeline instance as - well as what the Pipeline runs. + well as what the Pipeline runs. - Parameters + Parameters ---------- ``config`` : PipelineConfig | Mapping[str, Any] | str | Path The object containing the information needed to run the Pipeline. - + Returns ------- - A ``Pipeline`` class instance. + A ``Pipeline`` class instance. """ - # REMOVED AS THIS WAS RUNNING TWICE. SHOULD DISCUSS WHAT TO DO ABOUT THIS - #METHOD AND WHETHER IT IS NEEDED - - #pipe_payload, stage_payload = cls._split_config_sections(config) + # REMOVED AS THIS WAS RUNNING TWICE. SHOULD DISCUSS WHAT TO DO ABOUT THIS + # METHOD AND WHETHER IT IS NEEDED + + # pipe_payload, stage_payload = cls._split_config_sections(config) # pipeline_variables contains pipeline information - #name = pipe_payload.get("name", None) - #backend = pipe_payload.get("backend", "python") - #stages = pipe_payload.get("stages", []) - #print(stages) + # name = pipe_payload.get("name", None) + # backend = pipe_payload.get("backend", "python") + # stages = pipe_payload.get("stages", []) + # print(stages) return cls( name=name, backend=backend, config=config, stages=None, + logger=logger, + executor=executor, ) @classmethod def from_config( cls, - config: PipelineConfig | Mapping[str, Any] | str | Path, + config: Mapping[str, Any] | str | Path, name: str | None = None, backend: str = "python", logger: Logger | None = None, @@ -1132,13 +1312,11 @@ def from_config( return cls.from_dict( config=extracted_config, - name=name, - backend=backend, - logger=logger, - executor=executor - ) - - + name=name, + backend=backend, + logger=logger, + executor=executor, + ) @staticmethod def _select_stage_config( @@ -1153,7 +1331,9 @@ def _select_stage_config( """ if name is not None: if name not in stage_configs: - raise StageConfigurationError(f"Stage configuration '{name}' was not found.") + raise StageConfigurationError( + f"Stage configuration '{name}' was not found." + ) return stage_configs[name] if len(stage_configs) != 1: @@ -1185,128 +1365,198 @@ def _load_config_mapping( raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if raw_config is None: - warnings.warn("No configuration has loaded from the configuration file. Please check " \ - "your configurations.") + warnings.warn( + "No configuration has loaded from the configuration file. Please check " + "your configurations." + ) return {} if not isinstance(raw_config, Mapping): - raise TypeError("Configuration file must contain a mapping at the top level. Please ensure" \ - "that your configuration file is structured into key:value pairs in the notation that suits" \ - "the configuration file that you are using. The top level key value pairs should reflect " \ - "the Pipeline and Stage configurations.") + raise TypeError( + "Configuration file must contain a mapping at the top level. Please ensure" + "that your configuration file is structured into key:value pairs in the notation that suits" + "the configuration file that you are using. The top level key value pairs should reflect " + "the Pipeline and Stage configurations." + ) return dict(raw_config) @staticmethod - def _split_config_sections(raw_config: Mapping[str, Any]) -> tuple[Mapping[str, Any] | None, Mapping[str, Any] | None]: + def _split_config_sections( + raw_config: Mapping[str, Any], + ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: """ Split a raw config payload into pipeline-level and stage-level sections. Allows for configurations that have "stage_configuration", "stage_config", "pipeline_variables" - and "pipeline_config" as the key. The key is identified and used to pull the values for the - configuration from the ``raw_config``. It is then Nonetype checked and Type checked to ensure - that appropriate information is extracted and errors are produced if any of these checks fail. + and "pipeline_config" as the key. The key is identified and used to pull the values for the + configuration from the ``raw_config``. It is then Nonetype checked and Type checked to ensure + that appropriate information is extracted and errors are produced if any of these checks fail. Parameters ---------- ``raw_config``: Mapping[str, Any] - Contents of the configuration file previously extracted. + Contents of the configuration file previously extracted. - Returns + Returns ------- ``pipeline_payload``: Mapping[str, Any] | None - Contents of the pipeline configuration settings defined in the configuration file. + Contents of the pipeline configuration settings defined in the configuration file. ``stage_payload``: Mapping[str, Any] | None - Contents of the stage configuration settings defined in the configuration file. + Contents of the stage configuration settings defined in the configuration file. ``global_payload``: Mapping[str, Any] | None Contents of the global configuration settings defined in the configuration file. Raises ------ - ``PipelineConfigurationWarning`` - If blank values for pipeline_payload or global_payload are detected. - If there are remaining keys in the ``raw_config`` that have not been extracted. + ``PipelineConfigurationWarning`` + If blank values for pipeline_payload or global_payload are detected. + If there are remaining keys in the ``raw_config`` that have not been extracted. ``StageConfigurationWarning`` If blank values for stage_payload are detected. - ``PipelineConfigurationError`` - If the pipeline_payload, global_payload or stage_payload are not mapping types. + ``PipelineConfigurationError`` + If the pipeline_payload, global_payload or stage_payload are not mapping types. """ possible_stage_keys = ("stage_configuration", "stage_config") - possible_pipeline_keys = ("pipeline_variables","pipeline_config") - possible_global_keys = ("global_configuration", "global_config", "global_variables", "global_vars") + possible_pipeline_keys = ("pipeline_variables", "pipeline_config") + possible_global_keys = ( + "global_configuration", + "global_config", + "global_variables", + "global_vars", + ) + + stage_payload, stage_configuration = Pipeline._extract_optional_mapping( + possible_stage_keys, raw_config, StageConfigurationWarning + ) + pipeline_payload, pipeline_configuration = Pipeline._extract_mappings( + possible_pipeline_keys, raw_config, PipelineConfigurationWarning + ) + global_payload, global_configuration = Pipeline._extract_optional_mapping( + possible_global_keys, raw_config, PipelineConfigurationWarning + ) - stage_payload, stage_configuration = Pipeline._extract_mappings(possible_stage_keys, raw_config, StageConfigurationWarning) - pipeline_payload, pipeline_configuration = Pipeline._extract_mappings(possible_pipeline_keys, raw_config, PipelineConfigurationWarning) - global_payload, global_configuration = Pipeline._extract_mappings(possible_global_keys, raw_config, PipelineConfigurationWarning) + extracted_keys = {pipeline_configuration} + if stage_configuration is not None: + extracted_keys.add(stage_configuration) + if global_configuration is not None: + extracted_keys.add(global_configuration) - remaining_keys = set(raw_config) - {pipeline_configuration, stage_configuration, global_configuration} + remaining_keys = set(raw_config) - extracted_keys if remaining_keys: - warnings.warn("There are remaining sections in your configuration file that have not been extracted. Please check that all your configurations are in the pipeline or stage configuration keys.", - PipelineConfigurationWarning) + warnings.warn( + "There are remaining sections in your configuration file that have not been extracted. Please check that all your configurations are in the pipeline or stage configuration keys.", + PipelineConfigurationWarning, + ) return pipeline_payload, stage_payload, global_payload @staticmethod - def _extract_mappings(keys: tuple[str,...], - config: Mapping[str, Any], - warning: PipelineConfigurationWarning | StageConfigurationWarning) -> tuple[dict[str, Any], str]: + def _extract_mappings( + keys: tuple[str, ...], + config: Mapping[str, Any], + warning: type[Warning], + ) -> tuple[dict[str, Any], str]: configuration = Pipeline._extract_keys(keys, config) - payload = config.get(configuration,{}) - if payload is None: - warnings.warn(f"Blank {configuration} configuration detected. Please check that this is correct.", warning) + payload = config.get(configuration, {}) + if payload is None: + warnings.warn( + f"Blank {configuration} configuration detected. Please check that this is correct.", + warning, + ) if not isinstance(payload, Mapping): - raise PipelineConfigurationError(f"The {configuration} section must be a mapping.") - return payload, configuration + raise PipelineConfigurationError( + f"The {configuration} section must be a mapping." + ) + return dict(payload), configuration + @staticmethod + def _extract_optional_mapping( + keys: tuple[str, ...], + config: Mapping[str, Any], + warning: type[Warning], + ) -> tuple[dict[str, Any], str | None]: + matches = [key for key in keys if key in config] + if not matches: + return {}, None + + if len(matches) == 1: + configuration = matches[0] + else: + warnings.warn( + f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", + PipelineConfigurationWarning, + ) + configuration = matches[0] + + payload = config.get(configuration, {}) + if payload is None: + warnings.warn( + f"Blank {configuration} configuration detected. Please check that this is correct.", + warning, + ) + return {}, configuration + if not isinstance(payload, Mapping): + raise PipelineConfigurationError( + f"The {configuration} section must be a mapping." + ) + return dict(payload), configuration @staticmethod - def _extract_keys(possible_keys: tuple[str, ...], - dictionary: Mapping[str, Any]) -> str: + def _extract_keys( + possible_keys: tuple[str, ...], dictionary: Mapping[str, Any] + ) -> str: """ - Checks whether a provided dictionary has a key that has been previously defined. + Checks whether a provided dictionary has a key that has been previously defined. Creates a list for all specified keys that are present in the dictionary and checks - the number of keys that match. This should only be 1 so if there are any fewer or - additional then appropriate errors are raised. + the number of keys that match. This should only be 1 so if there are any fewer or + additional then appropriate errors are raised. - Parameters + Parameters ---------- ``possible_keys``: tuple[str, ...] - Set of string keys that are possibly in the dictionary provided. + Set of string keys that are possibly in the dictionary provided. ``dictionary``: Mapping[str, Any] - Dictionary that is being checked for valid keys. + Dictionary that is being checked for valid keys. - Returns + Returns ------- ``key``: str - String value for the key that is present in the ``dictionary`` out of the - ``possible_keys`` values. + String value for the key that is present in the ``dictionary`` out of the + ``possible_keys`` values. - Raises - ------ + Raises + ------ ``PipelineConfigurationError`` - If no keys in the ``dictionary`` are also in the ``possible_keys`` tuple. + If no keys in the ``dictionary`` are also in the ``possible_keys`` tuple. ``PipelineConfigurationWarning`` - If more than one key in the possible_keys is found, alerts user that it will + If more than one key in the possible_keys is found, alerts user that it will default to the first selected option and records the key that is selected. """ - + matches = [key for key in possible_keys if key in dictionary] if len(matches) == 1: key = matches[0] elif len(matches) == 0: - raise PipelineConfigurationError(f"No valid keys were found in the configuration. Please ensure that your top level key is one of: {possible_keys}.") + raise PipelineConfigurationError( + f"No valid keys were found in the configuration. Please ensure that your top level key is one of: {possible_keys}." + ) else: - warnings.warn(f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", PipelineConfigurationWarning) + warnings.warn( + f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", + PipelineConfigurationWarning, + ) key = matches[0] return key - @staticmethod - def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str, Any]: + def _normalize_pipeline_payload( + pipeline_payload: Mapping[str, Any], + ) -> dict[str, Any]: """ Normalize supported aliases in the pipeline section before model construction. @@ -1333,7 +1583,9 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str ) if "stage_to_run" in normalized_payload: if "stages_to_run" not in normalized_payload: - normalized_payload["stages_to_run"] = normalized_payload.pop("stage_to_run") + normalized_payload["stages_to_run"] = normalized_payload.pop( + "stage_to_run" + ) else: warnings.warn( "Both 'stage_to_run' and 'stages_to_run' were found in the pipeline configuration. " @@ -1341,11 +1593,13 @@ def _normalize_pipeline_payload(pipeline_payload: Mapping[str, Any]) -> dict[str UserWarning, stacklevel=2, ) - + return normalized_payload @staticmethod - def _build_stage_configs(stage_configuration: Mapping[str, Any] | None) -> dict[str, StageConfig]: + def _build_stage_configs( + stage_configuration: Mapping[str, Any] | None, + ) -> dict[str, StageConfig]: """ Build a stage-name keyed configuration mapping for any number of configured stages. @@ -1355,14 +1609,21 @@ def _build_stage_configs(stage_configuration: Mapping[str, Any] | None) -> dict[ if stage_configuration is None: return {} if not isinstance(stage_configuration, Mapping): - raise StageConfigurationError("Stage configuration must be a mapping keyed by stage name.") + raise StageConfigurationError( + "Stage configuration must be a mapping keyed by stage name." + ) - return { - str(stage_name): StageConfig.from_mapping(str(stage_name), stage_payload) - for stage_name, stage_payload in stage_configuration.items() - } + stage_configs: dict[str, StageConfig] = {} + for stage_name, stage_payload in stage_configuration.items(): + if stage_payload is not None and not isinstance(stage_payload, Mapping): + raise StageConfigurationError( + f"Stage configuration for {stage_name} must be provided as a mapping." + ) + stage_configs[str(stage_name)] = StageConfig.from_mapping( + str(stage_name), stage_payload + ) + return stage_configs - @staticmethod def _resolve_stage_source(stage_name: str, location: Any, work_dir: Path) -> Path: """ @@ -1391,28 +1652,31 @@ def _dependencies_for_stage( dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: """ - Extracts a tuple of ``dependencies`` for the requested stage. + Extracts a tuple of ``dependencies`` for the requested stage. Will return a blank tuple if there are no ``dependencies`` for the requested - stage. Allows for ``dependencies`` to be found regardless of how the stage is - referenced in the ``dependencies`` mapping. + stage. Allows for ``dependencies`` to be found regardless of how the stage is + referenced in the ``dependencies`` mapping. Parameters ---------- ``stage_name`` : str - The name of the stage that you are extracting the ``dependencies`` for. + The name of the stage that you are extracting the ``dependencies`` for. ``path`` : Path - The filepath for the stage source. + The filepath for the stage source. ``dependencies`` : Mapping[str, Sequence[str]] or None Mapping of the stage source name to their relevant ``dependencies`` (stages - required to run before the ``stage_name`` Stage). + required to run before the ``stage_name`` Stage). """ if not dependencies: return () + candidates: tuple[str, ...] if isinstance(path, Path): candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + elif callable(path): + candidates = (stage_name, str(getattr(path, "__name__", stage_name))) else: - candidates = (stage_name, str(path.__name__)) + candidates = (stage_name,) for candidate in candidates: if candidate in dependencies: return tuple(str(dependency) for dependency in dependencies[candidate]) @@ -1420,7 +1684,9 @@ def _dependencies_for_stage( return () @staticmethod - def _check_stage_configs(stages: list[Stage], stage_configs: Mapping[str, StageConfig]) -> None: + def _check_stage_configs( + stages: list[Stage], stage_configs: Mapping[str, StageConfig] + ) -> None: """ Check that all stages have a corresponding stage configuration. @@ -1437,4 +1703,4 @@ def _check_stage_configs(stages: list[Stage], stage_configs: Mapping[str, StageC warnings.warn( f"Stage(s) {', '.join(stage_no_config)} added to Pipeline without a corresponding StageConfig. ", StageConfigurationWarning, - ) \ No newline at end of file + ) From a34dd0402da67ea950384df6229eaebea9b989ee Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 09:04:22 +0100 Subject: [PATCH 311/332] feat: Added workflows for package build testing and doc deployment. --- .github/workflows/deploy-docs.yml | 56 ++++++++++++++++++++++++++++ .github/workflows/package-build.yml | 28 ++++++++++++++ .github/workflows/python-package.yml | 7 +++- onsrap/run_pipeline.py | 2 +- pyproject.toml | 2 +- 5 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/deploy-docs.yml create mode 100644 .github/workflows/package-build.yml diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000..09f5465 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,56 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Build documentation + run: | + python -m sphinx -b html docs docs/_build/html + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload documentation artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/_build/html + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy documentation + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml new file mode 100644 index 0000000..86b8d55 --- /dev/null +++ b/.github/workflows/package-build.yml @@ -0,0 +1,28 @@ +name: Python Package Builds Successfully + +on: + pull_request: + push: + branches: + - main + - development + +jobs: + build-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install build tools + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build Package + run: | + python -m build --sdist --wheel + - name: Install Built Wheel + run: | + pip install dist/*.whl + diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2ca9e55..abeca7c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,10 +1,15 @@ name: Python CI on: + # Keep push validation narrow to the main integration branches. push: branches: [ "main", "development" ] + + # Run CI for every pull request regardless of target branch so branch-to-branch + # work still gets the same quality and test coverage before merge. pull_request: - branches: [ "main", "development" ] + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: jobs: diff --git a/onsrap/run_pipeline.py b/onsrap/run_pipeline.py index f26acff..c9f2567 100644 --- a/onsrap/run_pipeline.py +++ b/onsrap/run_pipeline.py @@ -8,4 +8,4 @@ the main() function is run, this will then exit the system. """ if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 83f0f80..b20ccda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,4 +57,4 @@ files = ["onsrap"] show_error_codes = true warn_redundant_casts = true warn_unused_configs = true -warn_unused_ignores = true +warn_unused_ignores = true \ No newline at end of file From 72ee4c7c2ea51a836a97cc824e1f102b626a608e Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 09:20:58 +0100 Subject: [PATCH 312/332] tweak: changed Pipeline to utilise _generate_context properly. --- onsrap/pipeline.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 3adb2f7..dc41701 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -96,15 +96,6 @@ def __init__( self.config.name = self.name self.logger = logger or Logger(log_dir=self.config.log_dir) - if executor is not None: - self.executor = executor - else: - if self.backend == "python": - self.executor = PythonStageExecutor() - else: - raise PipelineInitialisationError( - f"Requested backend does not have a compatible executor. Available executors are: {', '.join(AVAILABLE_EXECUTORS)}." - ) if stages is not None and configured_stages: raise PipelineInitialisationError( @@ -116,6 +107,11 @@ def __init__( else [self._coerce_stage(stage) for stage in stages] ) + if executor is not None: + self.executor = executor + else: + self._generate_context() + self.dependencies = self._normalize_dependency_mapping(dependencies) if dependencies is not None and stages is None: raise PipelineInitialisationError( From 08a09af72c768af64b6508ab06aa7e93b305af88 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 09:36:52 +0100 Subject: [PATCH 313/332] fix: improved formatting in workflows yaml files --- .github/workflows/deploy-docs.yml | 91 ++++++++++++++--------------- .github/workflows/package-build.yml | 45 +++++++------- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 09f5465..c7686db 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,56 +1,53 @@ name: Deploy Documentation on: - push: - branches: - - main - - workflow_dispatch: + push: + branches: + - main + workflow_dispatch: permissions: - contents: read - pages: write - id-token: write + contents: read + pages: write + id-token: write concurrency: - group: pages - cancel-in-progress: false + group: pages + cancel-in-progress: false jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: pip - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - name: Build documentation - run: | - python -m sphinx -b html docs docs/_build/html - - name: Configure GitHub Pages - uses: actions/configure-pages@v5 - - - name: Upload documentation artifact - uses: actions/upload-pages-artifact@v4 - with: - path: docs/_build/html - - deploy: - name: Deploy to GitHub Pages - needs: build - runs-on: ubuntu-latest - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Deploy documentation - id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Build documentation + run: | + python -m sphinx -b html docs docs/_build/html + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload documentation artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/_build/html + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy documentation + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml index 86b8d55..3a53cf5 100644 --- a/.github/workflows/package-build.yml +++ b/.github/workflows/package-build.yml @@ -1,28 +1,27 @@ name: Python Package Builds Successfully on: - pull_request: - push: - branches: - - main - - development + pull_request: + push: + branches: + - main + - development jobs: - build-package: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install build tools - run: | - python -m pip install --upgrade pip - python -m pip install build - - name: Build Package - run: | - python -m build --sdist --wheel - - name: Install Built Wheel - run: | - pip install dist/*.whl - + build-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install build tools + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build Package + run: | + python -m build --sdist --wheel + - name: Install Built Wheel + run: | + pip install dist/*.whl From a0a37bf85e10814ab65e877fcf012ed06db5b645 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 12:00:48 +0100 Subject: [PATCH 314/332] tweak: eof whitespace addition --- .github/workflows/deploy-docs.yml | 65 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index c7686db..29b0c82 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -17,37 +17,38 @@ concurrency: jobs: build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: pip - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - name: Build documentation - run: | - python -m sphinx -b html docs docs/_build/html - - name: Configure GitHub Pages - uses: actions/configure-pages@v5 - - - name: Upload documentation artifact - uses: actions/upload-pages-artifact@v4 - with: - path: docs/_build/html + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Build documentation + run: | + python -m sphinx -b html docs docs/_build/html + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload documentation artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/_build/html deploy: - name: Deploy to GitHub Pages - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Deploy documentation - id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy documentation + id: deployment + uses: actions/deploy-pages@v4 + \ No newline at end of file From a56523c98eb5a938fcb773127cc3f275cf20d871 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 13:21:30 +0100 Subject: [PATCH 315/332] fix: added Ruff fix suggestions --- onsrap/logger.py | 2 +- onsrap/models.py | 1 - onsrap/pipeline.py | 4 ++-- onsrap/runner.py | 2 -- onsrap/stage.py | 3 +-- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 7f03801..91ee0c2 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -1,9 +1,9 @@ from __future__ import annotations -from datetime import datetime import json import logging from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Any diff --git a/onsrap/models.py b/onsrap/models.py index 66ff674..b2275ef 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -359,7 +359,6 @@ def to_dict(self) -> dict[str, Any]: "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), - "output_dir": str(self.output_dir) if self.output_dir is not None else None, "allow_subprocess_fallback": self.allow_subprocess_fallback, "python_executable": self.python_executable, } diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index a8de16d..0d1b2be 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -12,15 +12,15 @@ from typing import Any, Callable, Iterable, Mapping, Sequence from .errors import ( + HistoricalPipelineLoadError, PipelineConfigurationError, PipelineInitialisationError, StageConfigurationError, - HistoricalPipelineLoadError, StageLoadError, ) -from .loader import load_historical_run from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph +from .loader import load_historical_run from .logger import Logger from .models import ( GlobalConfig, diff --git a/onsrap/runner.py b/onsrap/runner.py index d3ff4a3..f50500b 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -1,7 +1,6 @@ from __future__ import annotations import argparse -import warnings from pathlib import Path from typing import TYPE_CHECKING @@ -9,7 +8,6 @@ from .execution import ExecutionContext from .logger import Logger from .models import PipelineRun, PipelineStatus, RunManifest, StageResult, now -from .warnings import StageConfigurationWarning if TYPE_CHECKING: from .pipeline import Pipeline diff --git a/onsrap/stage.py b/onsrap/stage.py index f524129..d392229 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -2,8 +2,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING -from datetime import datetime +from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, Optional from .errors import StageConfigurationError, StageDependencyError From 8a6a198bd4298e042b96047e7617b714cab0e845 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 14:19:51 +0100 Subject: [PATCH 316/332] fix: Added Ruff Format changes --- onsrap/errors.py | 3 +- onsrap/execution.py | 10 ++- onsrap/loader.py | 9 ++- onsrap/logger.py | 61 +++++++++++------- onsrap/models.py | 56 +++++++++-------- onsrap/pipeline.py | 150 +++++++++++++++++++++++++------------------- onsrap/runner.py | 39 ++++++------ onsrap/stage.py | 71 ++++++++++++--------- 8 files changed, 228 insertions(+), 171 deletions(-) diff --git a/onsrap/errors.py b/onsrap/errors.py index 9606298..aa37551 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -90,9 +90,10 @@ class PipelineInitialisationError(OnsrapError): class PipelineConfigurationError(OnsrapError): """ Raised when there has been an issue with the PipelineConfig - instance. + instance. """ + class HistoricalPipelineLoadError(OnsrapError): """ Raised when there is an issue loading a previous PipelineRun diff --git a/onsrap/execution.py b/onsrap/execution.py index 504e241..a6d8fe6 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -40,7 +40,7 @@ class ExecutionContext: """ Holds information needed to run the pipeline. - Parameters + Parameters ---------- ``pipeline_name`` : str The name of the pipeline. @@ -596,7 +596,9 @@ def _execute_file(self, stage: Stage, context: ExecutionContext) -> StageResult: return self._execute_subprocess(stage, context) - def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageResult: + def _execute_subprocess( + self, stage: Stage, context: ExecutionContext + ) -> StageResult: """ Run the entire Python file for the ``Stage`` from the top. @@ -686,7 +688,9 @@ def _execute_subprocess(self, stage: Stage, context: ExecutionContext) -> StageR return result -def _invoke_callable(callable_object: Any, stage: Stage, context: ExecutionContext) -> Any: +def _invoke_callable( + callable_object: Any, stage: Stage, context: ExecutionContext +) -> Any: """ Assigns appropriate parameters for a callable and runs it. diff --git a/onsrap/loader.py b/onsrap/loader.py index c077537..3f86dcd 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -162,6 +162,7 @@ def load_python_module(path: Path) -> ModuleType: return module + def load_historical_run(run_dir: Path) -> PipelineRun: """ Load a previously executed pipeline run from a YAML file. @@ -172,13 +173,17 @@ def load_historical_run(run_dir: Path) -> PipelineRun: An instance of ``PipelineRun`` representing the historical run. """ import glob + files = glob.glob(str(run_dir / "pipeline_attributes_for_*.yaml")) if not files: - raise StageLoadError("Historical run file does not exist in: {0}".format(run_dir)) + raise StageLoadError( + "Historical run file does not exist in: {0}".format(run_dir) + ) file_path = Path(files[0]) import yaml + with open(file_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) - return PipelineRun._pipeline_run_from_dict(data) \ No newline at end of file + return PipelineRun._pipeline_run_from_dict(data) diff --git a/onsrap/logger.py b/onsrap/logger.py index 91ee0c2..3d6a02d 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -172,40 +172,51 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: A list of dictionaries containing run_id, timestamp, and run_dir for each historical run. """ - #ensure that logger is writing to a file and extract filepath + # ensure that logger is writing to a file and extract filepath if not self._logger.hasHandlers(): - raise HistoricalPipelineLoadError("The logger does not write to a" \ - "filepath. Please ensure that your logger writes to a file path so that" \ - "we can extract the run_id for historical runs.") + raise HistoricalPipelineLoadError( + "The logger does not write to a" + "filepath. Please ensure that your logger writes to a file path so that" + "we can extract the run_id for historical runs." + ) - logfile_handler = next((h for h in self._logger.handlers if isinstance(h, logging.FileHandler)), None) + logfile_handler = next( + (h for h in self._logger.handlers if isinstance(h, logging.FileHandler)), + None, + ) if logfile_handler is None: - raise HistoricalPipelineLoadError("The logger does not have a FileHandler. " \ - "Please ensure that your logger writes to a file path so that we can extract the run_id " - "for historical runs.") + raise HistoricalPipelineLoadError( + "The logger does not have a FileHandler. " + "Please ensure that your logger writes to a file path so that we can extract the run_id " + "for historical runs." + ) logfile_path = logfile_handler.baseFilename if not Path(logfile_path).exists(): - raise HistoricalPipelineLoadError("The log file does not exist at this location.") + raise HistoricalPipelineLoadError( + "The log file does not exist at this location." + ) matches: list[dict[str, Any]] = [] - #TODO: This method works if the logs are recorded in chronological order. Would there - #ever be a case where a record would appear below another and not be chronological? - #If so, we may need to sort based on the timestamp rather than the ordering. - for raw_line in reversed(Path(logfile_path).read_text(encoding="utf-8").splitlines()): + # TODO: This method works if the logs are recorded in chronological order. Would there + # ever be a case where a record would appear below another and not be chronological? + # If so, we may need to sort based on the timestamp rather than the ordering. + for raw_line in reversed( + Path(logfile_path).read_text(encoding="utf-8").splitlines() + ): if "Pipeline started" not in raw_line or " | " not in raw_line: continue # left: timestamp + message, right: JSON context left, right = raw_line.split(" | ", 1) - #catches where Pipeline started is not recorded in the correct place. - if not left.endswith(" Pipeline started"): + # catches where Pipeline started is not recorded in the correct place. + if not left.endswith(" Pipeline started"): continue - #checks that datetime is valid - timestamp = left[:-len(" Pipeline started")].strip() + # checks that datetime is valid + timestamp = left[: -len(" Pipeline started")].strip() try: datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S,%f") except ValueError: @@ -228,12 +239,14 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: run_dir = run_root / run_id - #only returns run_ids for runs where a run_directory is still present. + # only returns run_ids for runs where a run_directory is still present. if run_dir.exists(): - matches.append({ - "run_id": run_id, - "timestamp": timestamp, - "run_dir": run_dir, - }) - + matches.append( + { + "run_id": run_id, + "timestamp": timestamp, + "run_dir": run_dir, + } + ) + return matches diff --git a/onsrap/models.py b/onsrap/models.py index b2275ef..c92a6ac 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -458,7 +458,9 @@ class StageConfig: metadata: dict[str, Any] = field(default_factory=dict) @classmethod - def from_mapping(cls, name: str, data: Mapping[str, Any] | None = None) -> StageConfig: + def from_mapping( + cls, name: str, data: Mapping[str, Any] | None = None + ) -> StageConfig: """ Build a ``StageConfig`` from a mapping loaded from code or configuration files. @@ -720,9 +722,9 @@ def __repr__(self) -> str: def _runmanifest_to_dict(self) -> dict[str, Any]: """ - Converts the RunManifest instance into a dictionary representation. - This is needed to allow a RunManifest instance to be serialized into a - JSON format for later methods on RunManifest instances not saved in + Converts the RunManifest instance into a dictionary representation. + This is needed to allow a RunManifest instance to be serialized into a + JSON format for later methods on RunManifest instances not saved in memory. Returns @@ -743,25 +745,25 @@ def _runmanifest_to_dict(self) -> dict[str, Any]: "timestamp": self.timestamp, "reason": self.reason, "user": self.user, - "config": self.config + "config": self.config, } @classmethod def _runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: """ - Converts a dictionary representation of a RunManifest instance back into a + Converts a dictionary representation of a RunManifest instance back into a RunManifest instance. Allows for RunManifest instances to be created from a JSON representation of a RunManifest instance. Parameters ---------- ``data`` : dict[str, Any] - A dictionary representation of a RunManifest instance. + A dictionary representation of a RunManifest instance. Returns ------- ``RunManifest`` class instance - A RunManifest instance created from the dictionary representation. + A RunManifest instance created from the dictionary representation. """ return cls( rap_name=data.get("rap_name", ""), @@ -776,7 +778,7 @@ def _runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: timestamp=data.get("timestamp", ""), reason=data.get("reason"), user=data.get("user"), - config=data.get("config") + config=data.get("config"), ) @@ -838,9 +840,9 @@ class StageResult: def _stage_result_to_dict(self) -> dict[str, Any]: """ - Converts the StageResult instance into a dictionary representation. - This is needed to allow a StageResult instance to be serialized into a - JSON format for later methods on StageResult instances not saved in + Converts the StageResult instance into a dictionary representation. + This is needed to allow a StageResult instance to be serialized into a + JSON format for later methods on StageResult instances not saved in memory. Returns @@ -865,19 +867,19 @@ def _stage_result_to_dict(self) -> dict[str, Any]: @classmethod def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: """ - Converts a dictionary representation of a StageResult instance back into a + Converts a dictionary representation of a StageResult instance back into a StageResult instance. Allows for StageResult instances to be created from a JSON representation of a StageResult instance. Parameters ---------- ``data`` : dict[str, Any] - A dictionary representation of a StageResult instance. + A dictionary representation of a StageResult instance. Returns ------- ``StageResult`` class instance - A StageResult instance created from the dictionary representation. + A StageResult instance created from the dictionary representation. """ return cls( name=data["name"], @@ -955,9 +957,9 @@ def result_for(self, stage_name: str) -> Optional[StageResult]: def _pipeline_run_to_dict(self) -> dict[str, Any]: """ - Converts the PipelineRun instance into a dictionary representation. - This is needed to allow a PipelineRun instance to be serialized into a - JSON format for later methods on PipelineRun instances not saved in + Converts the PipelineRun instance into a dictionary representation. + This is needed to allow a PipelineRun instance to be serialized into a + JSON format for later methods on PipelineRun instances not saved in memory. Returns @@ -970,35 +972,39 @@ def _pipeline_run_to_dict(self) -> dict[str, Any]: "status": self.status.value, "started_at": self.started_at.isoformat(), "completed_at": self.completed_at.isoformat(), - "stage_results": {result.name:result._stage_result_to_dict() - for result in self.stage_results}, + "stage_results": { + result.name: result._stage_result_to_dict() + for result in self.stage_results + }, "stage_outputs": self.stage_outputs, } @classmethod def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: """ - Converts a dictionary representation of a PipelineRun instance back into a + Converts a dictionary representation of a PipelineRun instance back into a PipelineRun instance. Allows for PipelineRun instances to be created from a JSON representation of a PipelineRun instance. Parameters ---------- ``data`` : dict[str, Any] - A dictionary representation of a PipelineRun instance. + A dictionary representation of a PipelineRun instance. Returns ------- ``PipelineRun`` class instance - A PipelineRun instance created from the dictionary representation. + A PipelineRun instance created from the dictionary representation. """ return cls( manifest=RunManifest._runmanifest_from_dict(data["manifest"]), status=PipelineStatus(data["status"]), started_at=datetime.fromisoformat(data["started_at"]), completed_at=datetime.fromisoformat(data["completed_at"]), - stage_results=[StageResult._stage_result_from_dict(result) - for result in data.get("stage_results", {}).values()], + stage_results=[ + StageResult._stage_result_from_dict(result) + for result in data.get("stage_results", {}).values() + ], stage_outputs=data.get("stage_outputs", {}), ) diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 0d1b2be..7cb0758 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -150,7 +150,6 @@ def __init__( enabled_stages=[stage.name for stage in self.graph.stages], ) - def __str__(self) -> str: """ String method that returns a human-readable representation of the ``Pipeline`` class. @@ -533,69 +532,85 @@ def _assign_dependencies( if stages is None: return () - self.logger.event("New dependencies added to Pipeline instance and respective Stage instances",dependencies = dependencies) + self.logger.event( + "New dependencies added to Pipeline instance and respective Stage instances", + dependencies=dependencies, + ) - def _load_latest_run(self) -> PipelineRun | None: - """ - Load the most recent run of the Pipeline as a PipelineRun instance. - - Returns - ------- - ``PipelineRun`` or None - An instance of ``PipelineRun`` representing the most recent run of the - Pipeline, or None if no previous runs are found. - """ - try: - previous_run_logs = self.logger.extract_historical_run_ids( - self.run_output - ) - except HistoricalPipelineLoadError: - warnings.warn("Unable to load previous runs for this Pipeline. Last_run" \ - " attribute will be None.", PipelineConfigurationWarning) - return None - - if previous_run_logs == []: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - " will be None.", PipelineConfigurationWarning) - return None - - latest_run_log = previous_run_logs[0] - latest_run_id = latest_run_log["run_id"] - if latest_run_id is None: - warnings.warn("No previous runs found for this Pipeline. Last_run attribute" \ - " will be None.", PipelineConfigurationWarning) - return None + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. - try: - return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) - except StageLoadError: - warnings.warn("Historical run file does not exist. Last_run attribute " \ - "will be None.", PipelineConfigurationWarning) - return None + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + try: + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + except HistoricalPipelineLoadError: + warnings.warn( + "Unable to load previous runs for this Pipeline. Last_run" + " attribute will be None.", + PipelineConfigurationWarning, + ) + return None + + if previous_run_logs == []: + warnings.warn( + "No previous runs found for this Pipeline. Last_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) + return None + + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn( + "No previous runs found for this Pipeline. Last_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) + return None + + try: + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + except StageLoadError: + warnings.warn( + "Historical run file does not exist. Last_run attribute will be None.", + PipelineConfigurationWarning, + ) + return None def _load_all_runs(self) -> dict[str, PipelineRun] | None: """ Loads all previous runs of a Pipeline as a dictionary of PipelineRun instances, - keyed by run_id. + keyed by run_id. Raises ------ ``PipelineConfigurationWarning`` If there are any errors in loading previous runs, this warning is raised to - indicate a None value will be stored in this attribute. + indicate a None value will be stored in this attribute. """ try: - previous_run_logs = self.logger.extract_historical_run_ids( - self.run_output - ) + previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) except HistoricalPipelineLoadError: - warnings.warn("Unable to load previous runs for this Pipeline. All_run" \ - " attribute will be None.", PipelineConfigurationWarning) + warnings.warn( + "Unable to load previous runs for this Pipeline. All_run" + " attribute will be None.", + PipelineConfigurationWarning, + ) return None - + if previous_run_logs == []: - warnings.warn("No previous runs found for this Pipeline. All_run attribute" \ - " will be None.", PipelineConfigurationWarning) + warnings.warn( + "No previous runs found for this Pipeline. All_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) return None all_runs = {} @@ -604,14 +619,18 @@ def _load_all_runs(self) -> dict[str, PipelineRun] | None: if run_id is None or run_id == "": warnings.warn( f"No run_id found in log for run_dir {run_log.get('run_dir')}. Skipping this run.", - PipelineConfigurationWarning + PipelineConfigurationWarning, ) continue try: - all_runs[run_id] = load_historical_run(run_dir=Path(self.run_output) / run_id) + all_runs[run_id] = load_historical_run( + run_dir=Path(self.run_output) / run_id + ) except StageLoadError: - warnings.warn(f"Historical run file for run_id {run_id} does not exist. Skipping.", - PipelineConfigurationWarning) + warnings.warn( + f"Historical run file for run_id {run_id} does not exist. Skipping.", + PipelineConfigurationWarning, + ) return all_runs if all_runs else None def _set_run_output(self) -> Path: @@ -621,28 +640,30 @@ def _set_run_output(self) -> Path: Returns ------- ``Path`` - The path to the run output directory for the Pipeline. + The path to the run output directory for the Pipeline. Raises ------ ``StageConfigurationWarning`` If the output_dir is not specified in the PipelineConfig, a warning is raised to show that the project root or work directory will be used as - the directory for the run outputs. + the directory for the run outputs. """ if self.config.output_dir is not None: run_output = Path(self.config.output_dir) else: warnings.warn( "Output directory is not specified. Using project root or work directory as the run output.", - StageConfigurationWarning + StageConfigurationWarning, ) # TODO: fill with warnings from Pipeline branch run_output = Path(self.config.project_root or self.config.work_dir) return run_output / "runs" - def _assign_dependencies(self, - dependencies:tuple[str]| dict[str, Sequence[str]] | None = None, - stages: Stage | Sequence[Stage] | None = None,) -> Stage | Sequence[Stage]: + def _assign_dependencies( + self, + dependencies: tuple[str] | dict[str, Sequence[str]] | None = None, + stages: Stage | Sequence[Stage] | None = None, + ) -> Stage | Sequence[Stage]: for stage in stages: new_dependencies = self._dependencies_for_stage( stage.name, stage.source, dependencies @@ -1307,12 +1328,12 @@ def add_stage_with_dependencies( def _generate_context(self) -> None: """ - Generates the execution context for the Pipeline based on values parsed. + Generates the execution context for the Pipeline based on values parsed. Validates stage backends and then uses the pipeline backend to generate - the expected StageExecutor class. If this class does not exist within - the orchestration tool, an error is raised. If the class does exist, - it is assigned to the executor attribute of the Pipeline instance. + the expected StageExecutor class. If this class does not exist within + the orchestration tool, an error is raised. If the class does exist, + it is assigned to the executor attribute of the Pipeline instance. Raises ------ @@ -1346,13 +1367,13 @@ def _validate_stage_backends(self) -> None: Checks the backends that have been assigned to each stage. Raise an error if the backends for a stage do not match the Pipeline - backend or if there are multiple backends across the stages. This + backend or if there are multiple backends across the stages. This ensures that the ExecutionContext will run correctly on all stages. Raises ------ ``PipelineInitialisationError`` - If there are multiple backends across the stages or if the stage + If there are multiple backends across the stages or if the stage backend does not match the Pipeline backend. """ backends = [] @@ -1372,7 +1393,6 @@ def _validate_stage_backends(self) -> None: f"Stages have backends '{', '.join(set(backends))}' which do not match pipeline backend '{self.backend}'." ) - @classmethod def from_files( cls, diff --git a/onsrap/runner.py b/onsrap/runner.py index f50500b..f06cb6f 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -157,11 +157,9 @@ def run(self, pipeline: Pipeline) -> PipelineRun: pipeline.manifest = manifest pipeline.last_run = run - #Creates attributes file in the run_directory to log information for later - #analysis of pipeline runs - _log_pipeline_attributes(pipeline_run = run, - run_dir = run_dir, - context = context) + # Creates attributes file in the run_directory to log information for later + # analysis of pipeline runs + _log_pipeline_attributes(pipeline_run=run, run_dir=run_dir, context=context) self.logger.event( "Pipeline failed", @@ -184,11 +182,9 @@ def run(self, pipeline: Pipeline) -> PipelineRun: pipeline.manifest = manifest pipeline.last_run = run - #Creates attributes file in the run_directory to log information for later - #analysis of pipeline runs - _log_pipeline_attributes(pipeline_run = run, - run_dir = run_dir, - context = context) + # Creates attributes file in the run_directory to log information for later + # analysis of pipeline runs + _log_pipeline_attributes(pipeline_run=run, run_dir=run_dir, context=context) self.logger.event( "Pipeline completed", @@ -242,14 +238,15 @@ def main(argv: list[str] | None = None) -> int: pipeline.run() return 0 -def _log_pipeline_attributes(pipeline_run: PipelineRun, - run_dir: Path, - context: ExecutionContext) -> None: + +def _log_pipeline_attributes( + pipeline_run: PipelineRun, run_dir: Path, context: ExecutionContext +) -> None: """ Creates a YAML file within the run directory that contains information - regarding PipelineRun and StageResult instances for the run. This is + regarding PipelineRun and StageResult instances for the run. This is later used to extract information about previous runs which are not - currently stored in memory. + currently stored in memory. Parameters ---------- @@ -260,16 +257,18 @@ def _log_pipeline_attributes(pipeline_run: PipelineRun, ``run_dir`` : Path The directory where the pipeline run is being currently being executed. ``context`` : ExecutionContext - The context of the current pipeline run, containing configuration and + The context of the current pipeline run, containing configuration and state information. """ - attributes_file = run_dir / f"pipeline_attributes_for_{context.pipeline_name}_{context.run_id[-8:]}.yaml" + attributes_file = ( + run_dir + / f"pipeline_attributes_for_{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) import yaml + with open(attributes_file, "w", encoding="utf-8") as f: yaml.safe_dump( - pipeline_run._pipeline_run_to_dict(), - f, - default_flow_style=False + pipeline_run._pipeline_run_to_dict(), f, default_flow_style=False ) diff --git a/onsrap/stage.py b/onsrap/stage.py index d392229..b64cfdb 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -11,11 +11,13 @@ from .models import StageResult -def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: +def _normalize_dependencies( + dependencies: Iterable[str] | str | None, +) -> tuple[str, ...]: """ Standardise the names of any stages dependant on other stages/processes. - Removes trailing or leading white space from the name of any stage/process dependant + Removes trailing or leading white space from the name of any stage/process dependant on another and turns it into a tuple of strings. Parameters @@ -55,16 +57,16 @@ class Stage: """ Represents a single unit of work within a pipeline. - Can be defined by a data source process or a Python script/callable item. - Stages may be dependant on other stages and can hold metadata for themselves. + Can be defined by a data source process or a Python script/callable item. + Stages may be dependant on other stages and can hold metadata for themselves. - Parameters + Parameters ---------- ``name`` : str - The name of the Stage being run. + The name of the Stage being run. ``source`` : Path, Callable, or None Item being implemented in this Stage. E.g. a file path to a Python script - or a function being executed directly. The full file path is gathered if + or a function being executed directly. The full file path is gathered if a path is used. ``dependencies`` : tuple of strings Names of stages that must be completed before this stage is attempted. These @@ -75,12 +77,13 @@ class Stage: Name of the starting script to the pipeline. ``backend`` : str, default = "python" The name of the system that the code runs on. - + Raises ------ ``StageConfigurationError`` If the stage ``name`` is empty or if the source is not a supported type. """ + name: str source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) @@ -98,8 +101,10 @@ def __post_init__(self) -> None: elif isinstance(self.source, Path): self.source = self.source.expanduser() elif self.source is not None and not callable(self.source): - raise StageConfigurationError("Stage source must be a path, callable, or None.") - + raise StageConfigurationError( + "Stage source must be a path, callable, or None." + ) + self.dependencies = _normalize_dependencies(self.dependencies) self.metadata = dict(self.metadata or {}) self.backend = str(self.backend or "python").strip() or "python" @@ -121,11 +126,11 @@ def __str__(self) -> str: def __repr__(self) -> str: """ - Representation method that returns a human readable representation of the ``Stage`` class. - This method is structured to be more concise than the ``__str__`` method and is intended for + Representation method that returns a human readable representation of the ``Stage`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for debugging purposes. - Returns + Returns ------- str A string representation of the ``Stage`` class with its attributes. @@ -135,7 +140,7 @@ def __repr__(self) -> str: f"dependencies={self.dependencies}, metadata={self.metadata}, " f"entrypoint={self.entrypoint}, backend={self.backend})" ) - + @classmethod def from_file( cls, @@ -159,7 +164,7 @@ def from_file( The name or file path for the script that the ``Stage`` will be running. ``name`` : str The name of the ``Stage`` - ``dependencies`` : Iterable[str], str, or None + ``dependencies`` : Iterable[str], str, or None The Stage/s that need to be complete before the ``Stage`` currently attempted. ``metadata`` : Mapping[str, Any], or None Any supporting information for the ``Stage`` being run. @@ -175,14 +180,13 @@ def from_file( Returns ------- - Stage + Stage Stage class instance with cleaned/checked file path, dependencies, and metadata """ path = Path(file_path).expanduser() if not path.exists(): raise StageConfigurationError(f"Stage source file does not exist: {path}") - return cls( name=name or path.stem, source=path.resolve(), @@ -211,7 +215,7 @@ def from_callable( The name or file path for the script that the Stage will be running. ``name`` : str The name of the Stage - ``dependencies`` : Iterable[str], str, or None + ``dependencies`` : Iterable[str], str, or None The Stage/s that need to be complete before the Stage currently attempted. ``metadata`` : Mapping[str, Any], or None Any supporting information for the Stage being run. @@ -222,7 +226,7 @@ def from_callable( Returns ------- - ``Stage`` + ``Stage`` ``Stage class`` instance with collected Stage ``name``, normalised ``dependencies`` and ``metadata``, and defined the source as the callable_object. """ @@ -246,7 +250,7 @@ def from_dict(cls, data: Mapping[str, Any]) -> Stage: ---------- ``data`` : any number of key/value pairs of strings The information to convert into a Stage class. - + Raises ------ ``StageConfigurationError`` @@ -254,7 +258,7 @@ def from_dict(cls, data: Mapping[str, Any]) -> Stage: Returns ------- - ``Stage`` + ``Stage`` ``Stage`` class instance with collected ``Stage`` attributes based on the type of ``source`` provided. """ @@ -302,7 +306,9 @@ def from_dict(cls, data: Mapping[str, Any]) -> Stage: backend=backend, ) - raise StageConfigurationError("Stage dictionary must define a source, path, or callable.") + raise StageConfigurationError( + "Stage dictionary must define a source, path, or callable." + ) def with_dependencies(self, *dependencies: str) -> Stage: """ @@ -315,7 +321,7 @@ def with_dependencies(self, *dependencies: str) -> Stage: Returns ------- - ``Stage`` + ``Stage`` ``Stage`` class instance with normalised ``dependencies`` attribute. """ unpacked_deps: list[str] = [] @@ -327,8 +333,10 @@ def with_dependencies(self, *dependencies: str) -> Stage: for dependency in unpacked_deps: if isinstance(dependency, list): - raise StageDependencyError("Nested lists are not valid arguments for this method! " \ - "Please provided single list or individual string values") + raise StageDependencyError( + "Nested lists are not valid arguments for this method! " + "Please provided single list or individual string values" + ) return replace( self, @@ -341,16 +349,18 @@ def validate(self) -> None: Raises ------- - ``StageConfigurationError`` + ``StageConfigurationError`` If ``source`` attribute does not define a source or does not exist. """ if not (isinstance(self.source, Path) or callable(self.source)): - raise StageConfigurationError(f"Stage '{self.name}' must have a Path or Callable source.") - + raise StageConfigurationError( + f"Stage '{self.name}' must have a Path or Callable source." + ) + if self.source is None or self.source == "": raise StageConfigurationError( f"Stage '{self.name}' does not define a source. Source provided: {self.source}" - ) + ) if isinstance(self.source, Path) and not self.source.is_file(): raise StageConfigurationError(f"Stage source does not exist: {self.source}") @@ -387,7 +397,7 @@ def source_label(self) -> Optional[str]: def run(self, context: ExecutionContext, executor: StageExecutor) -> StageResult: """ - Checks that the ``source`` is valid and then runs the ``source`` + Checks that the ``source`` is valid and then runs the ``source`` Properties ---------- @@ -404,4 +414,3 @@ def run(self, context: ExecutionContext, executor: StageExecutor) -> StageResult """ self.validate() return executor.execute(self, context) - \ No newline at end of file From b82c456d5a1ceb390961e5db354af65f4a18ff2b Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 14:21:59 +0100 Subject: [PATCH 317/332] fix: workflows: added pandas install for dependency requirement prior to pytests --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index abeca7c..631eae5 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -59,6 +59,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e ".[dev]" + python -m pip install pandas - name: Test with pytest run: | python -m pytest From e85ed0f8e1ebf3eb8fb4144f00f1d2834c2d3f11 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 14:35:30 +0100 Subject: [PATCH 318/332] tweak: workflows: remove Sphinx build from python-package.yml due to duplication in documentation workflow. --- .github/workflows/python-package.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 631eae5..fecf263 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -37,9 +37,6 @@ jobs: - name: Type check with mypy run: | python -m mypy onsrap - - name: Build documentation - run: | - python -m sphinx -b html docs docs/_build/html tests: runs-on: ubuntu-latest From 149e22c6a2c7106e929d8a114a893773f5a125dd Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 16:36:08 +0100 Subject: [PATCH 319/332] fix: mypy: resolved type problems, refactored how dependencies are assigned and contract between public and private methods. --- onsrap/models.py | 7 +- onsrap/pipeline.py | 199 ++++++++++++++++++++++++++--------------- tests/test_pipeline.py | 48 ++++++++++ 3 files changed, 180 insertions(+), 74 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index c92a6ac..dc29711 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -608,7 +608,7 @@ def get_attributes( @overload def get_attributes(self, keep_exclusion: Literal[False]) -> dict[str, Any]: ... - def get_attributes(self, keep_exclusion: bool = True) -> dict[str, Any]: + def get_attributes(self, keep_exclusion: bool = True) -> tuple[dict[str, Any], dict[str, Any]] | dict[str, Any]: """ Return a copy of the global variables, optionally excluding any variables specified in the exclusion list. @@ -1052,7 +1052,7 @@ def succeeded(self) -> bool: return self.status == PipelineStatus.SUCCEEDED -def _format_dict(d: dict[str, Any], indent: int = 0) -> str: +def _format_dict(d: dict[str, Any] | dict[str, bool] | None, indent: int = 0) -> str: """ Helper function to format dictionaries for __str__ methods. @@ -1068,6 +1068,9 @@ def _format_dict(d: dict[str, Any], indent: int = 0) -> str: str A formatted string representation of the dictionary. """ + if d is None: + return "" + lines = [] for key, value in d.items(): if isinstance(value, dict): diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 7cb0758..9171b77 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -115,15 +115,9 @@ def __init__( else: self._generate_context() - self.dependencies = self._normalize_dependency_mapping(dependencies) - if dependencies is not None and stages is None: - raise PipelineInitialisationError( - "Stages need to be defined before you can parse your dependencies " - "for those stages. Try the from_files() method, or create your Stage objects and " - "parse them to the Pipeline Constructor." - ) - if self.dependencies is not None: - self._assign_dependencies(self.dependencies, self.stages) + self.dependencies = None + if dependencies is not None: + self.dependencies = self._assign_dependencies(dependencies, self.stages) self.stage_configs = dict(resolved_stage_configs) self.global_config = resolved_global_config @@ -462,80 +456,129 @@ def run(self) -> PipelineRun: def add_dependencies( self, - *dependencies: Mapping[str, Sequence[str]] | tuple[str, ...], + *dependencies: Mapping[str, Sequence[str]], ) -> None: """ - Adds a set of ``dependencies`` for the Pipeline after the Pipeline initialisation. + Add dependency mappings to stages already registered on the Pipeline. - This method takes any number of positional arguments and imputes them as - ``dependencies``. It looks at each argument parsed, checks the data type against - the existing ``Pipeline`` ``dependencies`` and if they are the same data type, it - will take every stage within the ``Pipeline`` instance. It will then run the - ``_dependencies_for_stage()`` class method and normalize any ``dependencies`` before - adding them to the individual ``Stage`` instances. It will then append these - ``dependencies`` directly to the ``dependencies`` in the ``Pipeline`` instance before - rerunning the ``StageGraph`` creation to ensure the new ``dependencies`` are considered. - A logging entry will be created to track that these ``dependencies`` are added. + Each positional argument must be a mapping whose keys identify target + stages by stage name, source-path filename, full source path, or callable + name. Values are normalized, appended to the matching stage's existing + dependencies, de-duplicated in first-seen order, and merged into + ``Pipeline.dependencies`` before the execution graph is rebuilt. Parameters ---------- - ``*dependencies`` : tuple[str]| dict[str, Sequence[str]] - Any number of dependencies that you would like to add to the Pipeline. + ``*dependencies`` : Mapping[str, Sequence[str]] + One or more dependency mappings to merge into the Pipeline. Raises ------ - ``PipelineInitializationError`` - If the dependency you are attempting to add to the Pipeline doesn't match - the datatype for dependencies currently in the Pipeline. + ``PipelineInitialisationError`` + If a dependency payload is not provided as a mapping. """ + if not dependencies: + return + + if self.dependencies is None: + self.dependencies = {} + for dependency in dependencies: if not isinstance(dependency, Mapping): raise PipelineInitialisationError( - "Existing dependencies are not the same type as new dependencies" + "Dependencies added to an existing Pipeline must be provided as a mapping of stage identifiers to dependency names." ) - normalized_dependency = self._normalize_dependency_mapping(dependency) - if normalized_dependency is None: - continue - - for stage in self.stages: - new_dependencies = self._dependencies_for_stage( - stage.name, stage.source, normalized_dependency - ) - existing = stage.dependencies or () - new = existing + new_dependencies - - stage.dependencies = tuple(dict.fromkeys(new)) - - if self.dependencies is None: - self.dependencies = {} - + normalized_dependency = self._assign_dependencies(dependency, self.stages) for stage_name, deps in normalized_dependency.items(): existing = self.dependencies.get(stage_name, ()) combined = existing + deps self.dependencies[stage_name] = tuple(dict.fromkeys(combined)) - self.graph = StageGraph.from_stages(self.stages) - self.graph.validate() - - self.logger.event( - "New dependencies added to Pipeline instance and respective Stage instances", - dependencies=dependencies, - ) + self._rebuild_graph() def _assign_dependencies( self, - dependencies: Mapping[str, Sequence[str]] | None = None, - stages: Sequence[Stage] | None = None, - ) -> Sequence[Stage]: - if stages is None: - return () + dependencies: tuple[str, ...] | Mapping[str, Sequence[str]], + stages: Stage | Sequence[Stage], + ) -> dict[str, tuple[str, ...]]: + """ + Attach dependencies to one stage or a sequence of stages. + + For a single ``Stage``, ``dependencies`` may be either a dependency tuple + for that stage or a mapping keyed by any identifier accepted by + ``_dependencies_for_stage()``. For multiple stages, ``dependencies`` must + be a mapping keyed by stage identifiers. + + The target ``Stage`` objects are mutated in place: new dependency names are + appended to each stage's existing dependencies and de-duplicated while + preserving order. The method returns the normalized dependency mapping that + was applied so callers can keep ``Pipeline.dependencies`` in sync with the + stage objects. + + Parameters + ---------- + ``dependencies`` : tuple[str, ...] | Mapping[str, Sequence[str]] + Dependency names for a single stage or a mapping of stage identifiers + to dependency names. + ``stages`` : Stage | Sequence[Stage] + The stage or stages to mutate. + + Returns + ------- + dict[str, tuple[str, ...]] + The normalized dependency mapping that was applied. + + Raises + ------ + ``PipelineInitialisationError`` + If dependencies are provided before any stages exist, or if a + non-mapping payload is used for multiple stages. + """ + + if isinstance(stages, Stage): + target_stages = [stages] + if isinstance(dependencies, Mapping): + normalized_dependencies = ( + self._normalize_dependency_mapping(dependencies) or {} + ) + else: + normalized_dependencies = { + stages.name: self._normalize_dependency_values(dependencies) + } + else: + target_stages = list(stages) + if not isinstance(dependencies, Mapping): + raise PipelineInitialisationError( + "When assigning dependencies to multiple Stage instances, provide a mapping of stage identifiers to dependency names." + ) + normalized_dependencies = self._normalize_dependency_mapping(dependencies) or {} + + if normalized_dependencies and not target_stages: + raise PipelineInitialisationError( + "Stages need to be defined before you can parse your dependencies " + "for those stages. Try the from_files() method, or create your Stage objects and " + "parse them to the Pipeline Constructor." + ) + + for stage in target_stages: + new_dependencies = self._dependencies_for_stage( + stage.name, stage.source, normalized_dependencies + ) + existing = stage.dependencies or () + combined = existing + new_dependencies + stage.dependencies = tuple(dict.fromkeys(combined)) + + if normalized_dependencies: + self.logger.event( + "Dependencies assigned to Pipeline stages", + dependencies=normalized_dependencies, + stages=[stage.name for stage in target_stages], + ) + + return normalized_dependencies - self.logger.event( - "New dependencies added to Pipeline instance and respective Stage instances", - dependencies=dependencies, - ) def _load_latest_run(self) -> PipelineRun | None: """ @@ -659,18 +702,6 @@ def _set_run_output(self) -> Path: run_output = Path(self.config.project_root or self.config.work_dir) return run_output / "runs" - def _assign_dependencies( - self, - dependencies: tuple[str] | dict[str, Sequence[str]] | None = None, - stages: Stage | Sequence[Stage] | None = None, - ) -> Stage | Sequence[Stage]: - for stage in stages: - new_dependencies = self._dependencies_for_stage( - stage.name, stage.source, dependencies - ) - stage.dependencies = new_dependencies - - return stages def _coerce_stage( self, @@ -1001,6 +1032,30 @@ def _resolve_config( return pipeline_config, stage_configs, configured_stages, global_config + @staticmethod + def _normalize_dependency_values( + dependencies: Sequence[str] | str | None, + ) -> tuple[str, ...]: + """ + Normalize dependency names into a de-duplicated tuple. + """ + if dependencies is None: + return () + + candidate_dependencies: Sequence[str] | tuple[str, ...] + if isinstance(dependencies, str): + candidate_dependencies = (dependencies,) + else: + candidate_dependencies = dependencies + + normalized_dependencies: list[str] = [] + for dependency in candidate_dependencies: + dependency_name = str(dependency).strip() + if dependency_name and dependency_name not in normalized_dependencies: + normalized_dependencies.append(dependency_name) + + return tuple(normalized_dependencies) + @staticmethod def _normalize_dependency_mapping( dependencies: Mapping[str, Sequence[str]] | None, @@ -1009,7 +1064,7 @@ def _normalize_dependency_mapping( return None return { - str(stage_name): tuple(str(dependency) for dependency in stage_dependencies) + str(stage_name): Pipeline._normalize_dependency_values(stage_dependencies) for stage_name, stage_dependencies in dependencies.items() } @@ -1884,7 +1939,7 @@ def _dependencies_for_stage( candidates = (stage_name,) for candidate in candidates: if candidate in dependencies: - return tuple(str(dependency) for dependency in dependencies[candidate]) + return Pipeline._normalize_dependency_values(dependencies[candidate]) return () diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c0a7554..4c3f28d 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -142,6 +142,54 @@ def example_function(): assert pipeline_2.stages[0].dependencies == ("Stage_0",) assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + def test_assign_dependencies_with_config_defined_stages(self, tmp_path): + """ + Test to ensure dependencies can be assigned when stages are loaded from + the pipeline configuration rather than passed directly to the constructor. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + """ + + first_stage = tmp_path / "first_stage.py" + first_stage.write_text("def run(context):\n return 'alpha'\n", encoding="utf-8") + + second_stage = tmp_path / "second_stage.py" + second_stage.write_text("def run(context):\n return 'beta'\n", encoding="utf-8") + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + "\n".join( + [ + "pipeline_variables:", + f" work_dir: \"{tmp_path.as_posix()}\"", + f" project_root: \"{tmp_path.as_posix()}\"", + f" log_dir: \"{(tmp_path / 'logs').as_posix()}\"", + " stages:", + " - first_stage:", + f" location: \"{first_stage.as_posix()}\"", + " - second_stage:", + f" location: \"{second_stage.as_posix()}\"", + "stage_configuration: {}", + ] + ) + + "\n", + encoding="utf-8", + ) + + with pytest.warns(PipelineConfigurationWarning): + with pytest.warns(StageConfigurationWarning): + pipeline = Pipeline( + config=config_file, + dependencies={"second_stage": ("first_stage",)}, + ) + + assert [stage.name for stage in pipeline.stages] == ["first_stage", "second_stage"] + assert pipeline.stages[1].dependencies == ("first_stage",) + assert pipeline.dependencies == {"second_stage": ("first_stage",)} + def test_add_dependencies_single_dict(self, tmp_path): """ Tests that a dictionary correctly assigns dependencies to From 976513caa56587b9864b4b004d1fe8c51b55cd99 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 16:41:26 +0100 Subject: [PATCH 320/332] fix: ruff: format tweaks. --- onsrap/models.py | 6 ++++-- onsrap/pipeline.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index dc29711..8f1cfa1 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -608,7 +608,9 @@ def get_attributes( @overload def get_attributes(self, keep_exclusion: Literal[False]) -> dict[str, Any]: ... - def get_attributes(self, keep_exclusion: bool = True) -> tuple[dict[str, Any], dict[str, Any]] | dict[str, Any]: + def get_attributes( + self, keep_exclusion: bool = True + ) -> tuple[dict[str, Any], dict[str, Any]] | dict[str, Any]: """ Return a copy of the global variables, optionally excluding any variables specified in the exclusion list. @@ -1070,7 +1072,7 @@ def _format_dict(d: dict[str, Any] | dict[str, bool] | None, indent: int = 0) -> """ if d is None: return "" - + lines = [] for key, value in d.items(): if isinstance(value, dict): diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 9171b77..13ed8ea 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -553,7 +553,9 @@ def _assign_dependencies( raise PipelineInitialisationError( "When assigning dependencies to multiple Stage instances, provide a mapping of stage identifiers to dependency names." ) - normalized_dependencies = self._normalize_dependency_mapping(dependencies) or {} + normalized_dependencies = ( + self._normalize_dependency_mapping(dependencies) or {} + ) if normalized_dependencies and not target_stages: raise PipelineInitialisationError( @@ -579,7 +581,6 @@ def _assign_dependencies( return normalized_dependencies - def _load_latest_run(self) -> PipelineRun | None: """ Load the most recent run of the Pipeline as a PipelineRun instance. @@ -702,7 +703,6 @@ def _set_run_output(self) -> Path: run_output = Path(self.config.project_root or self.config.work_dir) return run_output / "runs" - def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], From bc2e58ede8fc1d0a9ca8c41997e325ec9794455a Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Wed, 12 Aug 2026 16:58:54 +0100 Subject: [PATCH 321/332] fix: updated f-string usage to work with older versions of Python. --- examples/pipeline_2/scripts/2_reporting.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index 25ac1d0..2089503 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -60,33 +60,33 @@ def curate_report(report, values): report.append("") report.append("## Summary") report.append("") - report.append(f"Total Orders: **{values["total_count"]}**") + report.append(f"Total Orders: **{values['total_count']}**") report.append("") - report.append(f"Total Profit: **£{values["total_profit"]}**") + report.append(f"Total Profit: **£{values['total_profit']}**") report.append("") report.append("## Region Analysis") - report.append(f"Highest number of orders: **{values["highest_order_region"]}**") + report.append(f"Highest number of orders: **{values['highest_order_region']}**") report.append("") - report.append(f"Highest profit: **{values["highest_prof_region"]}** at **£{values["highest_prof_value"]}**") + report.append(f"Highest profit: **{values['highest_prof_region']}** at **£{values['highest_prof_value']}**") report.append("") - report.append(f"Lowest profit: **{values["lowest_prof_region"]}** at **£{values["lowest_profit_value"]}**") + report.append(f"Lowest profit: **{values['lowest_prof_region']}** at **£{values['lowest_profit_value']}**") report.append("") - report.append(f"Highest quantity of items ordered: **{values["highest_quant_region"]}** at **{values["highest_quant_value"]}**") + report.append(f"Highest quantity of items ordered: **{values['highest_quant_region']}** at **{values['highest_quant_value']}**") report.append("") - report.append(f"Lowest quantity of items ordered: **{values["lowest_quant_region"]}** at **{values["lowest_quant_value"]}**") + report.append(f"Lowest quantity of items ordered: **{values['lowest_quant_region']}** at **{values['lowest_quant_value']}**") report.append("") report.append("## Product Analysis") - report.append(f"Highest profit: **{values["highest_profit_product"]}** at **£{values["highest_profit_value"]}**") + report.append(f"Highest profit: **{values['highest_profit_product']}** at **£{values['highest_profit_value']}**") report.append("") - report.append(f"Lowest profit: **{values["lowest_profit_product"]}** at **£{values["lowest_profit_value"]}**") + report.append(f"Lowest profit: **{values['lowest_profit_product']}** at **£{values['lowest_profit_value']}**") report.append("") report.append("## Order Analysis") report.append("") - report.append(f"The most orders occured on a **{values["highest_delivery_day"]}**.") + report.append(f"The most orders occured on a **{values['highest_delivery_day']}**.") report.append("") - report.append(f"**{values["large_order_num"]}** order/s were Large (greater than 75% of orders for the period).") + report.append(f"**{values['large_order_num']}** order/s were Large (greater than 75% of orders for the period).") report.append("") - report.append(f"**{values["small_order_num"]}** order/s were Small (less than 25% of orders for the period).") + report.append(f"**{values['small_order_num']}** order/s were Small (less than 25% of orders for the period).") def write_report(report): report_file = Path("examples/pipeline_2/outputs/order_analysis.md") From 9e5b5d8bf51430ccffcc44913469a6031b822c3e Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 11:26:40 +0100 Subject: [PATCH 322/332] fix: update to documentation on contributing.md to include more info on MyPy and Bandit and pre-commit hooks. --- .pre-commit-config.yaml | 6 ++++-- docs/contributor_guide/CONTRIBUTING.md | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f185fa3..3877cfb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,8 +28,10 @@ repos: hooks: - id: mypy name: mypy - Type check onsrap package - entry: python -m mypy onsrap - language: system + entry: mypy + language: python + additional_dependencies: [mypy, types-PyYAML] + args: [onsrap] pass_filenames: false - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 diff --git a/docs/contributor_guide/CONTRIBUTING.md b/docs/contributor_guide/CONTRIBUTING.md index 24627ed..0561666 100644 --- a/docs/contributor_guide/CONTRIBUTING.md +++ b/docs/contributor_guide/CONTRIBUTING.md @@ -28,9 +28,13 @@ documentation such as [detect-secrets][detect-secrets-repo] or [nbstripout][nbst ## Code conventions -We mainly follow [PEP8 standards][pep8] in our code conventions, and use flake8 and black +We mainly follow [PEP8 standards][pep8] in our code conventions, and use ruff pre-commit hook for linting and formatting. +We use [MyPy](https://mypy.readthedocs.io/en/stable/) to enforce type-checking to ensure the +rigidity of our framework and [bandit](http://bandit.readthedocs.io/en/latest/) for vulnerability +checking as a point of best practice. These are also pre-commit hooks in the project. + ### Git and GitHub We use Git to version control the source code. Please read From 2d7b52233ae2785e4d25c989afcdce404787605c Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 13:23:21 +0100 Subject: [PATCH 323/332] fix: ruff check: Resolved safe and unsafe ruff errors --- configuration.md | 2 +- examples/pipeline_1/main.py | 9 +- examples/pipeline_1/main2.py | 9 +- .../pipeline_1/scripts/0_data_validation.py | 30 +- .../pipeline_1/scripts/1_preprocessing.py | 12 +- examples/pipeline_1/scripts/2_reporting.py | 34 +- examples/pipeline_2/main.py | 13 +- examples/pipeline_2/scripts/0_clean_data.py | 17 +- examples/pipeline_2/scripts/1_derive_vars.py | 50 +- examples/pipeline_2/scripts/2_reporting.py | 69 +- examples/pipeline_3/main.ipynb | 22 + onsrap/pipeline.py | 1 - tests/test_execution.py | 18 +- tests/test_loader.py | 57 +- tests/test_logger.py | 264 ++--- tests/test_models.py | 177 ++-- tests/test_pipeline.py | 992 ++++++++++-------- tests/test_pipeline_architecture.py | 168 ++- tests/test_runner.py | 75 +- tests/test_stage.py | 208 ++-- 20 files changed, 1195 insertions(+), 1032 deletions(-) diff --git a/configuration.md b/configuration.md index e71cf6f..907b3d1 100644 --- a/configuration.md +++ b/configuration.md @@ -69,7 +69,7 @@ value = context.stage_config.get("years_to_run", default=2020) value = context.stage_config.require("target_variable") # All variables at once -all_vars = context.stage_config.variables # returns a copy +all_vars = context.stage_config.variables # returns a copy # Selected subset (raises if any are missing) subset = context.stage_config.get_variables(["years_to_run", "target_variable"]) diff --git a/examples/pipeline_1/main.py b/examples/pipeline_1/main.py index 08242e3..f9eb273 100644 --- a/examples/pipeline_1/main.py +++ b/examples/pipeline_1/main.py @@ -4,7 +4,6 @@ from onsrap import Pipeline, PipelineConfig - PIPELINE_ROOT = Path(__file__).resolve().parent SCRIPTS_DIR = PIPELINE_ROOT / "scripts" DATA_DIR = PIPELINE_ROOT / "data" @@ -49,11 +48,13 @@ def build_pipeline() -> Pipeline: def main() -> None: run = build_pipeline().run() report = run.manifest.outputs["2_reporting"] - - print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) print(f"Summary report written to: {report['report_path']}") print(f"Cleaned data written to: {report['clean_path']}") if __name__ == "__main__": - main() + main() diff --git a/examples/pipeline_1/main2.py b/examples/pipeline_1/main2.py index f4bf7fb..ad92311 100644 --- a/examples/pipeline_1/main2.py +++ b/examples/pipeline_1/main2.py @@ -4,7 +4,6 @@ from onsrap import Pipeline, PipelineConfig - PIPELINE_ROOT = Path(__file__).resolve().parent SCRIPTS_DIR = PIPELINE_ROOT / "scripts" DATA_DIR = PIPELINE_ROOT / "data" @@ -22,7 +21,7 @@ def build_pipeline() -> Pipeline: - stage_files = [ # Altered Script Order + stage_files = [ # Altered Script Order SCRIPTS_DIR / "1_preprocessing.py", SCRIPTS_DIR / "0_data_validation.py", SCRIPTS_DIR / "2_reporting.py", @@ -59,10 +58,12 @@ def main() -> None: run = build_pipeline().run() report = run.manifest.outputs["2_reporting"] - print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) print(f"Summary report written to: {report['report_path']}") print(f"Cleaned data written to: {report['clean_path']}") if __name__ == "__main__": - main() + main() diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index e54e2bb..9198446 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -3,17 +3,15 @@ import csv import json from pathlib import Path -from typing import Any - REQUIRED_COLUMNS = ( - "order_id", - "customer_name", - "region", - "product", - "quantity", - "unit_price", - "order_date", + "order_id", + "customer_name", + "region", + "product", + "quantity", + "unit_price", + "order_date", ) @@ -52,11 +50,11 @@ def validate_rows(rows: list[dict[str, str]]) -> list[str]: return issues -def build_report(raw_path: Path, rows: list[dict[str, str]], issues: list[str]) -> dict[str, object]: +def build_report( + raw_path: Path, rows: list[dict[str, str]], issues: list[str] +) -> dict[str, object]: order_dates = sorted( - row["order_date"].strip() - for row in rows - if not is_blank(row.get("order_date")) + row["order_date"].strip() for row in rows if not is_blank(row.get("order_date")) ) unique_regions = sorted( { @@ -81,7 +79,9 @@ def build_report(raw_path: Path, rows: list[dict[str, str]], issues: list[str]) def write_report(report_path: Path, report: dict[str, object]) -> None: report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + report_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) def main(context=None) -> dict[str, object]: @@ -106,4 +106,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index 1a2514c..b0d113b 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -4,7 +4,6 @@ import json from datetime import date from pathlib import Path -from typing import Any def load_orders(csv_path: Path) -> list[dict[str, str]]: @@ -73,7 +72,9 @@ def write_clean_rows(clean_path: Path, rows: list[dict[str, object]]) -> None: writer.writerows(rows) -def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, object]]) -> dict[str, object]: +def build_summary( + source_path: Path, clean_path: Path, rows: list[dict[str, object]] +) -> dict[str, object]: total_revenue = round(sum(float(row["order_value"]) for row in rows), 2) return { "source_path": str(source_path), @@ -89,8 +90,9 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: data_root = context.get_data_dir() output_root = context.resolve_output_root() - raw_path = context.resolve_given_path("0_data_validation", "raw_path", - "orders.csv", data_root) + raw_path = context.resolve_given_path( + "0_data_validation", "raw_path", "orders.csv", data_root + ) clean_path = output_root / "interim" / "1_clean_orders.csv" rows = load_orders(raw_path) @@ -102,4 +104,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index bbbd1c2..0e29834 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -4,7 +4,6 @@ import json from collections import defaultdict from pathlib import Path -from typing import Any def load_orders(csv_path: Path) -> list[dict[str, str]]: @@ -31,16 +30,28 @@ def build_summary(rows: list[dict[str, str]]) -> dict[str, object]: revenue_by_product[product] += order_value total_revenue = round(sum(revenue_by_region.values()), 2) - top_region = max(revenue_by_region, key=revenue_by_region.get) if revenue_by_region else None - top_product = max(revenue_by_product, key=revenue_by_product.get) if revenue_by_product else None + top_region = ( + max(revenue_by_region, key=revenue_by_region.get) if revenue_by_region else None + ) + top_product = ( + max(revenue_by_product, key=revenue_by_product.get) + if revenue_by_product + else None + ) return { "total_orders": len(ordered_rows), "total_revenue": total_revenue, - "revenue_by_region": {region: round(amount, 2) for region, amount in sorted(revenue_by_region.items())}, + "revenue_by_region": { + region: round(amount, 2) + for region, amount in sorted(revenue_by_region.items()) + }, "orders_by_region": dict(sorted(orders_by_region.items())), "units_by_product": dict(sorted(units_by_product.items())), - "revenue_by_product": {product: round(amount, 2) for product, amount in sorted(revenue_by_product.items())}, + "revenue_by_product": { + product: round(amount, 2) + for product, amount in sorted(revenue_by_product.items()) + }, "top_region": top_region, "top_product": top_product, "first_order_date": ordered_rows[0]["order_date"] if ordered_rows else None, @@ -50,7 +61,9 @@ def build_summary(rows: list[dict[str, str]]) -> dict[str, object]: def write_summary(summary_path: Path, summary: dict[str, object]) -> None: summary_path.parent.mkdir(parents=True, exist_ok=True) - summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> None: @@ -69,11 +82,10 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = context.get_data_dir() output_root = context.resolve_output_root() - clean_path = context.resolve_given_path("1_preprocessing", "clean_path", - "1_clean_orders.csv", output_root, - "interim") + clean_path = context.resolve_given_path( + "1_preprocessing", "clean_path", "1_clean_orders.csv", output_root, "interim" + ) summary_path = output_root / "processed" / "2_sales_summary.json" region_breakdown_path = output_root / "processed" / "2_revenue_by_region.csv" @@ -94,4 +106,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) \ No newline at end of file + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py index baaf10b..8d40021 100644 --- a/examples/pipeline_2/main.py +++ b/examples/pipeline_2/main.py @@ -1,12 +1,10 @@ -import yaml -from onsrap import Pipeline, PipelineConfig, StageConfig from pathlib import Path - +from onsrap import Pipeline def main() -> None: - config_path = (Path(__file__).resolve().parent)/"conf.yaml" + config_path = (Path(__file__).resolve().parent) / "conf.yaml" print(config_path) pipeline = Pipeline.from_config(config_path) @@ -15,9 +13,12 @@ def main() -> None: report = run.manifest.outputs print(report) - print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) print(f"Summary report written to: {pipeline.config.output_dir}") print(f"Cleaned data written to: {pipeline.config.data_dir}") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py index 885bc09..50ad0a3 100644 --- a/examples/pipeline_2/scripts/0_clean_data.py +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -1,5 +1,4 @@ import pandas as pd -from onsrap import ExecutionContext def check_variables(df, expected_variables): @@ -14,10 +13,10 @@ def check_variables(df, expected_variables): print(f"Missing the following variables: {missing}") -def remove_identifiable(df,identifiable_cols): +def remove_identifiable(df, identifiable_cols): for i in identifiable_cols: if i in df.columns: - df = df.drop(i, axis = 1) + df = df.drop(i, axis=1) else: pass return df @@ -27,12 +26,11 @@ def standardise_columns(df): df.columns = [col.lower() for col in df.columns] df.columns = [col.capitalize() for col in df.columns] for item in df.columns: - df[item] = df[item].apply(lambda x: x.lower() if isinstance(x,str) else x) - df[item] = df[item].apply(lambda x: x.strip() if isinstance(x,str) else x) + df[item] = df[item].apply(lambda x: x.lower() if isinstance(x, str) else x) + df[item] = df[item].apply(lambda x: x.strip() if isinstance(x, str) else x) return df - def main(context=None): config = context.get_stage_config("0_clean_data") print(config) @@ -41,12 +39,13 @@ def main(context=None): expected_variables = config["expected_variables"] identifiable_cols = config["identifiable_cols"] - + check_variables(orders, expected_variables) print(orders.dtypes) orders = remove_identifiable(orders, identifiable_cols) orders = standardise_columns(orders) - orders.to_csv(config["output_location"], index = False) + orders.to_csv(config["output_location"], index=False) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py index 1b06132..94d4d42 100644 --- a/examples/pipeline_2/scripts/1_derive_vars.py +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -1,6 +1,6 @@ -import pandas as pd import numpy as np -from datetime import timedelta +import pandas as pd + def correct_date_time(df): df["Order_date"] = pd.to_datetime(df["Order_date"]) @@ -8,65 +8,58 @@ def correct_date_time(df): def estimate_delivery(df, delivery_times): - df["Estimated_delvery_date"] = df["Order_date"] + pd.to_timedelta(df["Region"].map(delivery_times), unit = "D") + df["Estimated_delvery_date"] = df["Order_date"] + pd.to_timedelta( + df["Region"].map(delivery_times), unit="D" + ) df["Delivery_day"] = df["Estimated_delvery_date"].dt.day_name() - return df + return df + def total_cost(df): df["Total_cost"] = df["Quantity"] * df["Unit_price"] return df + def order_date_values(df): df["Order_day"] = df["Order_date"].dt.day_name() df["Order_month"] = df["Order_date"].dt.month_name() - return(df) + return df + def size_order_alert(df): df["Large_order"] = df["Quantity"] > df["Quantity"].quantile(0.75) df["Small_order"] = df["Quantity"] < df["Quantity"].quantile(0.25) return df + def postage_cost(df): df["Postage"] = np.select( - [ - df["Large_order"], - df["Small_order"] - ], - [ - 5.00, - 1.00 - ], - default = 2.50 + [df["Large_order"], df["Small_order"]], [5.00, 1.00], default=2.50 ) return df + def production_cost(df): df["Total_production_cost"] = np.select( [ df["Product"] == "notebook", df["Product"] == "pen", - df["Product"] == "folder" - ], - [ - (1.00*df["Quantity"]), - (0.3*df["Quantity"]), - (0.75*df["Quantity"]) - + df["Product"] == "folder", ], - default = 0 + [(1.00 * df["Quantity"]), (0.3 * df["Quantity"]), (0.75 * df["Quantity"])], + default=0, ) return df + def profit_per_order(df): - df["Order_profit"] = df["Total_cost"]-df["Total_production_cost"]-df["Postage"] + df["Order_profit"] = df["Total_cost"] - df["Total_production_cost"] - df["Postage"] return df - - def main(context=None): config = context.get_stage_config("1_derive_vars") - + df = pd.read_csv(config["input_location"]) delivery_times = config["delivery_times"] @@ -78,9 +71,8 @@ def main(context=None): df = postage_cost(df) df = production_cost(df) df = profit_per_order(df) - df.to_csv(config["output_location"], index = False) - + df.to_csv(config["output_location"], index=False) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index 2089503..c9023b4 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -1,6 +1,8 @@ -import pandas as pd from pathlib import Path +import pandas as pd + + ##PROFIT PER REGION## def per_region_profits(orders, values, num_format): profit_per_region = orders.groupby("Region")["Order_profit"].sum() @@ -11,6 +13,7 @@ def per_region_profits(orders, values, num_format): values["lowest_prof_region"] = profit_per_region.idxmin().capitalize() values["lowest_profit_value"] = num_format.format((profit_per_region.min())) + ##QUANTITY PER REGION## def per_region_quantity(orders, values): quantity_per_region = orders.groupby("Region")["Quantity"].sum() @@ -21,29 +24,34 @@ def per_region_quantity(orders, values): values["lowest_quant_region"] = quantity_per_region.idxmin().capitalize() values["lowest_quant_value"] = quantity_per_region.min() + ##ORDER DAY POP## def orders_per_day(orders, values): delivery_day_frequency = orders["Order_day"].value_counts() values["highest_delivery_day"] = delivery_day_frequency.idxmin().capitalize() + ##ORDER COUNTS## def order_quantity(orders, values): - values["large_order_num"] = (orders["Large_order"] == True).sum() + values["large_order_num"] = orders["Large_order"].sum() + + values["small_order_num"] = orders["Small_order"].sum() - values["small_order_num"] = (orders["Small_order"] == True).sum() ##ORDERS PER REGION## def per_region_orders(orders, values): orders_per_region = orders["Region"].value_counts() values["highest_order_region"] = orders_per_region.idxmax().capitalize() + ##TOTALS## -def total_summaries(orders,values,num_format): +def total_summaries(orders, values, num_format): values["total_count"] = orders["Order_id"].count() values["total_profit"] = num_format.format(orders["Order_profit"].sum()) + ##PROFIT PER PRODUCT## -def profit_per_product(orders,values, num_format): +def profit_per_product(orders, values, num_format): df = orders.groupby("Product")["Order_profit"].sum().sort_values(ascending=False) values["highest_profit_product"] = df.idxmax().capitalize() @@ -52,11 +60,14 @@ def profit_per_product(orders,values, num_format): values["lowest_profit_product"] = df.idxmin().capitalize() values["lowest_profit_value"] = num_format.format((df.min())) + ##CURATE REPORT## def curate_report(report, values): report.append("# Order Summary") report.append("") - report.append("This is an automatically generated report showing key information on orders of products.") + report.append( + "This is an automatically generated report showing key information on orders of products." + ) report.append("") report.append("## Summary") report.append("") @@ -67,34 +78,49 @@ def curate_report(report, values): report.append("## Region Analysis") report.append(f"Highest number of orders: **{values['highest_order_region']}**") report.append("") - report.append(f"Highest profit: **{values['highest_prof_region']}** at **£{values['highest_prof_value']}**") + report.append( + f"Highest profit: **{values['highest_prof_region']}** at **£{values['highest_prof_value']}**" + ) report.append("") - report.append(f"Lowest profit: **{values['lowest_prof_region']}** at **£{values['lowest_profit_value']}**") + report.append( + f"Lowest profit: **{values['lowest_prof_region']}** at **£{values['lowest_profit_value']}**" + ) report.append("") - report.append(f"Highest quantity of items ordered: **{values['highest_quant_region']}** at **{values['highest_quant_value']}**") + report.append( + f"Highest quantity of items ordered: **{values['highest_quant_region']}** at **{values['highest_quant_value']}**" + ) report.append("") - report.append(f"Lowest quantity of items ordered: **{values['lowest_quant_region']}** at **{values['lowest_quant_value']}**") + report.append( + f"Lowest quantity of items ordered: **{values['lowest_quant_region']}** at **{values['lowest_quant_value']}**" + ) report.append("") report.append("## Product Analysis") - report.append(f"Highest profit: **{values['highest_profit_product']}** at **£{values['highest_profit_value']}**") + report.append( + f"Highest profit: **{values['highest_profit_product']}** at **£{values['highest_profit_value']}**" + ) report.append("") - report.append(f"Lowest profit: **{values['lowest_profit_product']}** at **£{values['lowest_profit_value']}**") + report.append( + f"Lowest profit: **{values['lowest_profit_product']}** at **£{values['lowest_profit_value']}**" + ) report.append("") report.append("## Order Analysis") report.append("") report.append(f"The most orders occured on a **{values['highest_delivery_day']}**.") report.append("") - report.append(f"**{values['large_order_num']}** order/s were Large (greater than 75% of orders for the period).") + report.append( + f"**{values['large_order_num']}** order/s were Large (greater than 75% of orders for the period)." + ) report.append("") - report.append(f"**{values['small_order_num']}** order/s were Small (less than 25% of orders for the period).") + report.append( + f"**{values['small_order_num']}** order/s were Small (less than 25% of orders for the period)." + ) + def write_report(report): report_file = Path("examples/pipeline_2/outputs/order_analysis.md") - report_file.write_text( - "\n".join(report), - encoding="utf-8" - ) + report_file.write_text("\n".join(report), encoding="utf-8") + def main(): orders = pd.read_csv("examples/pipeline_2/data/orders_prepped.csv") @@ -109,10 +135,11 @@ def main(): orders_per_day(orders, values) order_quantity(orders, values) per_region_orders(orders, values) - total_summaries(orders,values,num_format) - profit_per_product(orders,values, num_format) + total_summaries(orders, values, num_format) + profit_per_product(orders, values, num_format) curate_report(report, values) write_report(report) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/pipeline_3/main.ipynb b/examples/pipeline_3/main.ipynb index e69de29..80a457a 100644 --- a/examples/pipeline_3/main.ipynb +++ b/examples/pipeline_3/main.ipynb @@ -0,0 +1,22 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "print(1)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python", + "version": "3.14.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 13ed8ea..1395792 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -1717,7 +1717,6 @@ def _extract_mappings( config: Mapping[str, Any], warning: type[Warning], ) -> tuple[dict[str, Any], str]: - configuration = Pipeline._extract_keys(keys, config) payload = config.get(configuration, {}) if payload is None: diff --git a/tests/test_execution.py b/tests/test_execution.py index 19e191f..fc125a6 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -212,8 +212,8 @@ def test_stage_outputs(self, execution, stageresult) -> None: @pytest.fixture def blank_context_with_config_none(self, stageresult) -> ExecutionContext: """ - Fixture that returns a test ExecutionContext instance with a None config for - testing error handling. + Fixture that returns a test ExecutionContext instance with a None config for + testing error handling. Parameters ---------- @@ -245,7 +245,7 @@ def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: ``execution`` : ExecutionContext An ``ExecutionContext`` object for testing. ``blank_context_with_config_none`` : ExecutionContext - An ``ExecutionContext`` object with a None config for testing error + An ``ExecutionContext`` object with a None config for testing error handling. Raises @@ -309,7 +309,7 @@ def test_stage_config_accessors_return_named_and_active_configs( Raises ------ ``PipelineConfigurationError`` - Requested a StageConfig instance as with_global = True, the output must + Requested a StageConfig instance as with_global = True, the output must be a dictionary however quantifying vars_only as False would demand that the entire StageConfig instance is returned. """ @@ -394,10 +394,10 @@ def test_get_stage_config(self, execution, stage_config) -> None: Raises ------ ``PipelineConfigurationError`` - Requested a StageConfig instance as with_global = True, the output must + Requested a StageConfig instance as with_global = True, the output must be a dictionary however quantifying vars_only as False would demand that the entire StageConfig instance is returned. - + """ assert execution.get_stage_config() == {} with pytest.raises(PipelineConfigurationError): @@ -443,7 +443,7 @@ def test_resolve_given_path_add_folders( ) -> None: """ Tests the add_folder functionality for lists, single strings, or None type in - the resolve_given_path class method as well as when the file_name is a valid + the resolve_given_path class method as well as when the file_name is a valid string or None type. Parameters @@ -548,7 +548,7 @@ def test_combine_vars(self, execution) -> None: def test_combine_vars_errors(self, execution) -> None: """ - Test that confirms that a warning is raised if there is a variable defined in + Test that confirms that a warning is raised if there is a variable defined in both the global and the stage configurations as well as asserting the correct values. @@ -584,7 +584,7 @@ def test_combine_vars_errors(self, execution) -> None: def test_combine_vars_no_exclusion(self, execution) -> None: """ - Test confirming that a dictionary is returned, combining values from a global + Test confirming that a dictionary is returned, combining values from a global configuration and a stage configuration when there are no exclusions defined. Parameters diff --git a/tests/test_loader.py b/tests/test_loader.py index 1fe34c6..768e5f4 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,42 +1,42 @@ -import pytest from pathlib import Path + +import pytest + from onsrap.errors import StageLoadError from onsrap.loader import load_historical_run from onsrap.pipeline import PipelineRun - from tests.test_pipeline import TestLoadLatestRunIntegration class TestLoadHistoricalRun(TestLoadLatestRunIntegration): - def test_raises_stageloaderror_no_file(self, - tmp_path: Path) -> None: + def test_raises_stageloaderror_no_file(self, tmp_path: Path) -> None: """ Checks that load_historical_run raises a StageLoadError when the specified - directory does not contain any files matching the expected pattern. + directory does not contain any files matching the expected pattern. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. - + Raises ------ ``StageLoadError`` Raised when the specified directory does not contain any files matching the expected pattern for historical run YAML files. """ - run_dir = tmp_path/"empty_run" + run_dir = tmp_path / "empty_run" run_dir.mkdir(parents=True, exist_ok=True) with pytest.raises(StageLoadError, match="Historical run file does not exist"): load_historical_run(run_dir=run_dir) - def test_returns_valid_pipeline_run_from_yaml(self, - tmp_path:Path, - minimal_pipeline_yaml)-> None: + def test_returns_valid_pipeline_run_from_yaml( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: """ Checks that load_historical_run successfully returns a PipelineRun instance. - + Parameters ---------- ``tmp_path`` : Path @@ -46,22 +46,23 @@ def test_returns_valid_pipeline_run_from_yaml(self, A fixture that returns a minimal YAML configuration for a historical run. """ - run_dir = tmp_path/"valid_run" + run_dir = tmp_path / "valid_run" run_dir.mkdir(parents=True, exist_ok=True) - (run_dir/"pipeline_attributes_for_test.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id"), encoding="utf-8") + (run_dir / "pipeline_attributes_for_test.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id"), encoding="utf-8" + ) result = load_historical_run(run_dir=run_dir) assert isinstance(result, PipelineRun) assert result.manifest.run_id == "test_id" - def test_correct_yaml_file_chosen(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: + def test_correct_yaml_file_chosen( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: """ - Checks that if there are multiple files within the same run directory, + Checks that if there are multiple files within the same run directory, the method will pass successfully and return a PipelineRun. This does - not assert which file is chosen, only that the method does not raise + not assert which file is chosen, only that the method does not raise an error and returns a PipelineRun instance. Logically, this should be suitable as we would only ever expect one file @@ -70,17 +71,19 @@ def test_correct_yaml_file_chosen(self, Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. ``minimal_pipeline_yaml`` : callable A fixture that returns a minimal YAML configuration for a historical run. """ - run_dir = tmp_path/"multiple_runs" + run_dir = tmp_path / "multiple_runs" run_dir.mkdir(parents=True, exist_ok=True) - (run_dir/"pipeline_attributes_for_test1.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id_1"), encoding="utf-8") - (run_dir/"pipeline_attributes_for_test2.yaml").write_text( - minimal_pipeline_yaml(run_id = "test_id_2"), encoding="utf-8") + (run_dir / "pipeline_attributes_for_test1.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id_1"), encoding="utf-8" + ) + (run_dir / "pipeline_attributes_for_test2.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id_2"), encoding="utf-8" + ) result = load_historical_run(run_dir=run_dir) - assert isinstance(result, PipelineRun) \ No newline at end of file + assert isinstance(result, PipelineRun) diff --git a/tests/test_logger.py b/tests/test_logger.py index 508681e..d85f50d 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,5 +1,6 @@ import logging from pathlib import Path + import pytest from onsrap.errors import HistoricalPipelineLoadError @@ -8,84 +9,85 @@ class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): - def test_logger_no_handler_errors(self, - tmp_path: Path) -> None: + def test_logger_no_handler_errors(self, tmp_path: Path) -> None: """ - Tests that if the logger has no handlers, an error is raised when + Tests that if the logger has no handlers, an error is raised when attempting to extract historical ids as the logger is not writing to a file that can be checked. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. - + Raises ------ ``HistoricalPipelineLoadError`` Raised when the logger does not have any handlers, indicating that it is not writing to a file path and cannot extract historical run ids. """ - logger = Logger(log_dir = tmp_path/"logs") + logger = Logger(log_dir=tmp_path / "logs") logger._logger.handlers.clear() # Remove all handlers to simulate no file logging logger._logger.propagate = False # Prevent checking root logger handlers with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") + logger.extract_historical_run_ids(run_root=tmp_path / "runs") - def test_logger_no_file_handler_errors(self, - tmp_path: Path) -> None: + def test_logger_no_file_handler_errors(self, tmp_path: Path) -> None: """ - Tests that if the logger has no file handlers, an error is raised when + Tests that if the logger has no file handlers, an error is raised when attempting to extract historical ids as the logger is not writing to a file that can be checked. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. - + Raises ------ ``HistoricalPipelineLoadError`` Raised when the logger does not have any handlers, indicating that it is not writing to a file path and cannot extract historical run ids. - """ - logger = Logger(log_dir = tmp_path/"logs") + """ + logger = Logger(log_dir=tmp_path / "logs") logger._logger.handlers = [logging.StreamHandler()] - with pytest.raises(HistoricalPipelineLoadError, match="does not have a FileHandler"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") + with pytest.raises( + HistoricalPipelineLoadError, match="does not have a FileHandler" + ): + logger.extract_historical_run_ids(run_root=tmp_path / "runs") - def test_logger_does_not_exist(self, - tmp_path:Path) -> None: - """ + def test_logger_does_not_exist(self, tmp_path: Path) -> None: + """ Checks that the method raises an error if the log doesn't exist at the - location specified. + location specified. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. - """ - logger = Logger(log_dir = tmp_path/"logs") + """ + logger = Logger(log_dir=tmp_path / "logs") - file_handler = next( + file_handler = next( h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) + ) - log_path = Path(file_handler.baseFilename) + log_path = Path(file_handler.baseFilename) - file_handler.close() - log_path.unlink(missing_ok=True) # Remove the log file to simulate non-existence + file_handler.close() + log_path.unlink( + missing_ok=True + ) # Remove the log file to simulate non-existence - with pytest.raises(HistoricalPipelineLoadError, - match="does not exist at this location"): - logger.extract_historical_run_ids(run_root = tmp_path/"runs") + with pytest.raises( + HistoricalPipelineLoadError, match="does not exist at this location" + ): + logger.extract_historical_run_ids(run_root=tmp_path / "runs") - def test_return_blank_list_no_matches_in_log(self, - tmp_path) -> None: + def test_return_blank_list_no_matches_in_log(self, tmp_path) -> None: """ Tests that a log file that does not have a record covering "Pipeline started" will return a blank list from extract_historical_run_ids. @@ -93,147 +95,148 @@ def test_return_blank_list_no_matches_in_log(self, Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ - logger = Logger(log_dir = tmp_path/"logs") + logger = Logger(log_dir=tmp_path / "logs") file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) - log_path.write_text("2026-08-10 10:00:00,000 Some unrelated log entry\n" \ - "2026-08-10 10:00:01,000 | Another unrelated log entry\n" \ - "2026-08-10 10:00:02,000 Pipeline Started\n") + log_path.write_text( + "2026-08-10 10:00:00,000 Some unrelated log entry\n" + "2026-08-10 10:00:01,000 | Another unrelated log entry\n" + "2026-08-10 10:00:02,000 Pipeline Started\n" + ) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") assert result == [] - def test_skips_poor_json_in_log(self, - tmp_path: Path) -> None: + def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: """ Tests that if a JSON record in the log file is not valid, it will e skipped - and the next valid entry will be extracted. Assert that the returned list - contains only the valid entry. Confirms that only the incorrect record is + and the next valid entry will be extracted. Assert that the returned list + contains only the valid entry. Confirms that only the incorrect record is skipped. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ - logger = Logger(log_dir = tmp_path/"logs") - + logger = Logger(log_dir=tmp_path / "logs") + file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) - log_path.write_text("2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" \ - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + log_path.write_text( + "2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" + '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + ) - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" create_run_dir_2.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") assert result == [ { - "run_id": "2026-06-23_101719_abc1234", - "timestamp": "2026-08-10 10:00:02,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_abc1234" + "run_id": "2026-06-23_101719_abc1234", + "timestamp": "2026-08-10 10:00:02,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_abc1234", }, { - "run_id": "2026-06-23_101719_878fcb33", - "timestamp": "2026-08-10 10:00:01,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} - ] - - @pytest.mark.parametrize("string, expected", - [('{"some_key":"some_value"}', []), - ('{"run_id":""}',[])]) - - def test_run_id_absent_falsy(self, - tmp_path: Path, - string: str, - expected: list) -> None: + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_878fcb33", + }, + ] + + @pytest.mark.parametrize( + "string, expected", [('{"some_key":"some_value"}', []), ('{"run_id":""}', [])] + ) + def test_run_id_absent_falsy( + self, tmp_path: Path, string: str, expected: list + ) -> None: """ Tests that if the JSON record in the log file does not have a run_id, it will be skipped - and the returned list will be empty. + and the returned list will be empty. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. ``string`` : str A dictionary representing a valid JSON record in the log file that excludes run_id. ``expected`` : list - The expected output from extract_historical_run_ids when the log file + The expected output from extract_historical_run_ids when the log file contains a record without a run_id. """ - logger = Logger(log_dir = tmp_path/"logs") - + logger = Logger(log_dir=tmp_path / "logs") + file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) - log_path.write_text( - f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") + log_path.write_text(f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") - #creates directory for runs to avoid removal given the directory doesn't exist - (tmp_path/"runs").mkdir(parents=True, exist_ok=True) + # creates directory for runs to avoid removal given the directory doesn't exist + (tmp_path / "runs").mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") assert result == expected - def test_records_only_if_directory_exists(self, - tmp_path) -> None: + def test_records_only_if_directory_exists(self, tmp_path) -> None: """ Checks that a record is only output if the run directory exists. - + Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ - - logger = Logger(log_dir = tmp_path/"logs") - + + logger = Logger(log_dir=tmp_path / "logs") + file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) log_path.write_text( - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + ) - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") assert result == [ { - "run_id": "2026-06-23_101719_878fcb33", - "timestamp": "2026-08-10 10:00:01,000", - "run_dir": tmp_path/"runs"/"2026-06-23_101719_878fcb33"} - ] + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_878fcb33", + } + ] - def test_reverse_chronological_order(self, - tmp_path) -> None: + def test_reverse_chronological_order(self, tmp_path) -> None: """ Checks that the run_ids are output in reverse chronological order based on their positioning in the log file. @@ -241,55 +244,56 @@ def test_reverse_chronological_order(self, Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ - logger = Logger(log_dir = tmp_path/"logs") - + logger = Logger(log_dir=tmp_path / "logs") + file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) log_path.write_text( - "2026-08-10 10:00:01,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n" \ - "2026-08-10 10:00:02,000 Pipeline started | {\"run_id\": \"2026-06-23_101719_abc1234\"}\n") + '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + ) - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - create_run_dir_2 = tmp_path/"runs"/"2026-06-23_101719_abc1234" + create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" create_run_dir_2.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") assert result[0]["run_id"] == "2026-06-23_101719_abc1234" assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" - def test_skip_poor_timestamps(self, - tmp_path) -> None: + def test_skip_poor_timestamps(self, tmp_path) -> None: """ Checks that entries with poor timestamps are skipped. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ - logger = Logger(log_dir = tmp_path/"logs") - + logger = Logger(log_dir=tmp_path / "logs") + file_handler = next( - h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) - ) - + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + log_path = Path(file_handler.baseFilename) log_path.write_text( - "BADTIMESTAMP Pipeline started | {\"run_id\": \"2026-06-23_101719_878fcb33\"}\n") + 'BADTIMESTAMP Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' + ) - create_run_dir_1 = tmp_path/"runs"/"2026-06-23_101719_878fcb33" + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root = tmp_path/"runs") - assert result == [] \ No newline at end of file + result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + assert result == [] diff --git a/tests/test_models.py b/tests/test_models.py index e7ad5ed..ebeddfc 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,5 +1,3 @@ -from onsrap.models import StageResult, StageStatus, PipelineStatus, RuntimeID, RunManifest, PipelineRun, PipelineConfig -import pytest import datetime from pathlib import Path from textwrap import dedent @@ -12,11 +10,10 @@ PipelineStatus, RunManifest, RuntimeID, + StageResult, StageStatus, ) -from tests.test_execution import stageresult - STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) @@ -115,8 +112,8 @@ def expected_pipeline_config() -> PipelineConfig: def pipelineconfig(expected_pipeline_config) -> PipelineConfig: """ Returns a PipelineConfig instance for testing that is derived - fromthe expected_pipeline_config fixture. Used as a separate - fixture to ensure that the behaviour of the from_any() method is + fromthe expected_pipeline_config fixture. Used as a separate + fixture to ensure that the behaviour of the from_any() method is tested correctly in the TestPipelineConfig class. """ return expected_pipeline_config @@ -146,7 +143,7 @@ def test_from_any( ) -> None: """ Test derivation for a PipelineConfig instance using the from_any() method. This - test checks all methods EXCEPT from_file as this will be covered in another + test checks all methods EXCEPT from_file as this will be covered in another test due to creation of a mock file being required. Parameters @@ -169,12 +166,12 @@ def test_from_any( def test_from_file_errors(self, tmp_path) -> PipelineConfig: """ Checks that a PipelineConfig instance raises the correct exceptions when a - file is not found or the file does not contain a dictionary mapping. + file is not found or the file does not contain a dictionary mapping. Parameters ---------- ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. Raises @@ -204,25 +201,25 @@ def test_from_file_errors(self, tmp_path) -> PipelineConfig: PipelineConfig.from_file(no_map_pipeline_config) def test_from_file_success( - self, tmp_path, expected_pipeline_config - ) -> PipelineConfig: - """ - Checks that a PipelineConfig instance is created successfully from a mock - file. + self, tmp_path, expected_pipeline_config + ) -> PipelineConfig: + """ + Checks that a PipelineConfig instance is created successfully from a mock + file. - Parameters - ---------- - ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and - manipulation. - ``expected_pipeline_config`` : PipelineConfig - A PipelineConfig instance that is expected to be created from the mock - file. - """ - pipeline_config = tmp_path / "configuration.py" - pipeline_config.write_text( - dedent( - """ + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``expected_pipeline_config`` : PipelineConfig + A PipelineConfig instance that is expected to be created from the mock + file. + """ + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ {"name":"test_rap", "backend":"python", "work_dir":"tmp/work", @@ -235,17 +232,16 @@ def test_from_file_success( "num_stages":6} } """ - ).strip() - + "\n", - encoding="utf-8", - ) - configuration = PipelineConfig.from_file(pipeline_config) - assert configuration == expected_pipeline_config - + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config def test_to_dict(self, pipelineconfig) -> None: """ - Test of to_dict() class method for PipelineConfig that it outputs the + Test of to_dict() class method for PipelineConfig that it outputs the PipelineConfig values as a dictionary. Parameters @@ -335,7 +331,7 @@ def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: ``status_stage`` : StageStatus A StageStatus value to set the status of the StageResult instance. ``expected_stage`` : bool - The expected boolean output from the succeeded() method based on the + The expected boolean output from the succeeded() method based on the status of the StageResult instance. """ stageresult.status = status_stage @@ -379,18 +375,19 @@ def pipelinerun(stageresult, runmanifest) -> PipelineRun: {"stage_test": "example output"}, ) + class TestPipelineRun: def test_pipelinerun_configuration( self, pipelinerun, runmanifest, stageresult ) -> None: """ - Checks that the PipelineRun instance is created successfully with the correct + Checks that the PipelineRun instance is created successfully with the correct attributes and values. Parameters ---------- ``pipelinerun`` : PipelineRun - A PipelineRun instance for testing. + A PipelineRun instance for testing. ``runmanifest`` : RunManifest A RunManifest instance for testing. ``stageresult`` : StageResult @@ -433,7 +430,7 @@ def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: ``status`` : PipelineStatus A PipelineStatus value to set the status of the PipelineRun instance. ``expected`` : bool - The expected boolean output from the succeeded() method based on the + The expected boolean output from the succeeded() method based on the status of the PipelineRun instance. """ pipelinerun.status = status @@ -442,51 +439,60 @@ def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: # TODO: Test _extract_stages_run and all methods in StageConfig class + class TestToFromDictMethods: """ Class to store testing methods for to_dict and from_dict, specifically for PipelineRun, RunManifest, and StageResult classes. """ + @pytest.fixture def runmanifest(self) -> RunManifest: - return RunManifest("pipeline", - "1", - None, - ["stage1","stage2"], - {"uniqueID":"example"}, - {"input_path":"input/data/example.csv"}, - {"output_path":"output/data/example.csv"}, - "python", - ["1.3.2"], - "", - None, - None) + return RunManifest( + "pipeline", + "1", + None, + ["stage1", "stage2"], + {"uniqueID": "example"}, + {"input_path": "input/data/example.csv"}, + {"output_path": "output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None, + ) + @pytest.fixture def stageresult(self) -> StageResult: - return StageResult("stage_test", - StageStatus.SUCCEEDED, - datetime.datetime(2024,5,6,15,45,30), - datetime.datetime(2024,5,7,15,45,30), - "example output", - "", - "", - None, - {}, - None, - None) + return StageResult( + "stage_test", + StageStatus.SUCCEEDED, + datetime.datetime(2024, 5, 6, 15, 45, 30), + datetime.datetime(2024, 5, 7, 15, 45, 30), + "example output", + "", + "", + None, + {}, + None, + None, + ) @pytest.fixture def pipelinerun(self, runmanifest, stageresult) -> PipelineRun: - return PipelineRun(runmanifest, - PipelineStatus.SUCCEEDED, - datetime.datetime(2024,5,6,15,45,30), - datetime.datetime(2024,5,7,15,45,30), - [stageresult], - {"stage_test":"example output"}) + return PipelineRun( + runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024, 5, 6, 15, 45, 30), + datetime.datetime(2024, 5, 7, 15, 45, 30), + [stageresult], + {"stage_test": "example output"}, + ) def test_runmanifest_to_dict(self, runmanifest) -> None: """ - Test that the to_dict method for RunManifest outputs the correct dictionary representation. + Test that the to_dict method for RunManifest outputs the correct dictionary representation. Parameters ---------- @@ -506,13 +512,13 @@ def test_runmanifest_to_dict(self, runmanifest) -> None: "timestamp": "", "reason": None, "user": None, - "config":None + "config": None, } assert runmanifest._runmanifest_to_dict() == expected_dict def test_runmanifest_from_dict(self, runmanifest) -> None: """ - Test that the from_dict method for RunManifest correctly creates a RunManifest instance from a dictionary representation. + Test that the from_dict method for RunManifest correctly creates a RunManifest instance from a dictionary representation. Parameters ---------- @@ -532,14 +538,14 @@ def test_runmanifest_from_dict(self, runmanifest) -> None: "timestamp": "", "reason": None, "user": None, - "config":None + "config": None, } new_runmanifest = RunManifest._runmanifest_from_dict(runmanifest_dict) assert new_runmanifest == runmanifest def test_stageresult_to_dict(self, stageresult) -> None: """ - Test that the to_dict method for StageResult outputs the correct dictionary representation. + Test that the to_dict method for StageResult outputs the correct dictionary representation. Parameters ---------- @@ -557,13 +563,13 @@ def test_stageresult_to_dict(self, stageresult) -> None: "return_code": None, "metadata": {}, "error": None, - "source": None + "source": None, } assert stageresult._stage_result_to_dict() == expected_dict def test_stageresult_from_dict(self, stageresult) -> None: """ - Test that the from_dict method for StageResult correctly creates a StageResult instance from a dictionary representation. + Test that the from_dict method for StageResult correctly creates a StageResult instance from a dictionary representation. Parameters ---------- @@ -581,14 +587,14 @@ def test_stageresult_from_dict(self, stageresult) -> None: "return_code": None, "metadata": {}, "error": None, - "source": None + "source": None, } new_stageresult = StageResult._stage_result_from_dict(stageresult_dict) assert new_stageresult == stageresult def test_pipelinerun_to_dict(self, pipelinerun) -> None: """ - Test that the to_dict method for PipelineRun outputs the correct dictionary representation. + Test that the to_dict method for PipelineRun outputs the correct dictionary representation. Parameters ---------- @@ -600,14 +606,17 @@ def test_pipelinerun_to_dict(self, pipelinerun) -> None: "status": "succeeded", "started_at": "2024-05-06T15:45:30", "completed_at": "2024-05-07T15:45:30", - "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, - "stage_outputs": {"stage_test": "example output"} + "stage_results": { + result.name: result._stage_result_to_dict() + for result in pipelinerun.stage_results + }, + "stage_outputs": {"stage_test": "example output"}, } assert pipelinerun._pipeline_run_to_dict() == expected_dict def test_pipelinerun_from_dict(self, pipelinerun) -> None: """ - Test that the from_dict method for PipelineRun correctly creates a PipelineRun instance from a dictionary representation. + Test that the from_dict method for PipelineRun correctly creates a PipelineRun instance from a dictionary representation. Parameters ---------- @@ -619,9 +628,11 @@ def test_pipelinerun_from_dict(self, pipelinerun) -> None: "status": "succeeded", "started_at": "2024-05-06T15:45:30", "completed_at": "2024-05-07T15:45:30", - "stage_results": {result.name: result._stage_result_to_dict() for result in pipelinerun.stage_results}, - "stage_outputs": {"stage_test": "example output"} + "stage_results": { + result.name: result._stage_result_to_dict() + for result in pipelinerun.stage_results + }, + "stage_outputs": {"stage_test": "example output"}, } new_pipelinerun = PipelineRun._pipeline_run_from_dict(pipelinerun_dict) assert new_pipelinerun == pipelinerun - diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 4c3f28d..e0a913e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,19 +1,20 @@ -import logging -from unittest import mock import warnings +from pathlib import Path +from unittest import mock -from onsrap.loader import load_historical_run -from onsrap.pipeline import Pipeline, PipelineConfig +import pytest + +from onsrap.errors import ( + HistoricalPipelineLoadError, + PipelineConfigurationError, + PipelineInitialisationError, + StageLoadError, +) from onsrap.execution import PythonStageExecutor -from onsrap.errors import HistoricalPipelineLoadError, PipelineInitialisationError, PipelineConfigurationError, StageConfigurationError, StageLoadError from onsrap.models import PipelineRun, StageConfig +from onsrap.pipeline import Pipeline, PipelineConfig from onsrap.stage import Stage -from onsrap.warnings import StageConfigurationWarning, PipelineConfigurationWarning -from onsrap.logger import Logger - -from pathlib import Path - -import pytest +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning NO_STAGES_WARNING = "No stages specified to run. All stages running by default." @@ -22,20 +23,20 @@ def stage_factory(): def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: """ - Function that builds a Stage object with a given name, dependencies, and a - source file path that's built out of the name if it is not provided. This - standardises the creation of Stage objects for testing. + Function that builds a Stage object with a given name, dependencies, and a + source file path that's built out of the name if it is not provided. This + standardises the creation of Stage objects for testing. - Parameters - ---------- - ``name`` : str - The name of the stage to be created. - ``dependencies`` : tuple - A tuple of stage names that the created stage depends on. - ``source`` : Path | None - A Path object representing the source file for the stage. If None, a default - source file path is created based on the stage name. - """ + Parameters + ---------- + ``name`` : str + The name of the stage to be created. + ``dependencies`` : tuple + A tuple of stage names that the created stage depends on. + ``source`` : Path | None + A Path object representing the source file for the stage. If None, a default + source file path is created based on the stage name. + """ resolved_source = source if source is not None else Path(f"{name}.py") return Stage(name, source=resolved_source, dependencies=dependencies) @@ -54,19 +55,25 @@ def test_pipeline_name(self): Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ pipeline_config = PipelineConfig(name="test_pipeline_config") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline_named = Pipeline(name="test_pipeline_name", - stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - pipeline_config = Pipeline(name=None, config=pipeline_config, - stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) - pipeline_no_name = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_named = Pipeline( + name="test_pipeline_name", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + pipeline_config = Pipeline( + name=None, + config=pipeline_config, + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + pipeline_no_name = Pipeline( + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())] + ) assert pipeline_named.name == "test_pipeline_name" assert pipeline_config.name == "test_pipeline_config" @@ -87,10 +94,10 @@ def test_assign_dependencies(self, tmp_path): Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'PipelineInitialisationError' - Expected and asserted as there are no stages defined in the Pipeline + Expected and asserted as there are no stages defined in the Pipeline but there are dependencies. """ @@ -110,31 +117,29 @@ def example_function(): "example_function": ("Stage_1.py",), } - with pytest.raises(PipelineInitialisationError): - with pytest.warns(PipelineConfigurationWarning): - Pipeline(stages=None, dependencies=dependencies_single) + with pytest.raises((PipelineInitialisationError, PipelineConfigurationWarning)): + Pipeline(stages=None, dependencies=dependencies_single) - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline_1 = Pipeline( - name="pipeline_1", - stages=[ - Stage("Stage_1", path_1, None, {}), - Stage("Stage_2", example_function, None, {}), - Stage("Stage_0", path_0, None, {}), - ], - dependencies=dependencies_multiple, - ) - - pipeline_2 = Pipeline( - name="pipeline_2", - stages=[ - Stage("Stage_1.py", path_1, None, {}), - Stage("Stage_2", example_function, None, {}), - Stage("Stage_0", path_0, None, {}), - ], - dependencies=dependencies_non_stage_name, - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_1 = Pipeline( + name="pipeline_1", + stages=[ + Stage("Stage_1", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_multiple, + ) + + pipeline_2 = Pipeline( + name="pipeline_2", + stages=[ + Stage("Stage_1.py", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_non_stage_name, + ) assert pipeline_1.stages[0].dependencies == ("Stage_0",) assert pipeline_1.stages[1].dependencies == ("Stage_1", "Stage_0") @@ -154,24 +159,28 @@ def test_assign_dependencies_with_config_defined_stages(self, tmp_path): """ first_stage = tmp_path / "first_stage.py" - first_stage.write_text("def run(context):\n return 'alpha'\n", encoding="utf-8") + first_stage.write_text( + "def run(context):\n return 'alpha'\n", encoding="utf-8" + ) second_stage = tmp_path / "second_stage.py" - second_stage.write_text("def run(context):\n return 'beta'\n", encoding="utf-8") + second_stage.write_text( + "def run(context):\n return 'beta'\n", encoding="utf-8" + ) config_file = tmp_path / "conf.yaml" config_file.write_text( "\n".join( [ "pipeline_variables:", - f" work_dir: \"{tmp_path.as_posix()}\"", - f" project_root: \"{tmp_path.as_posix()}\"", - f" log_dir: \"{(tmp_path / 'logs').as_posix()}\"", + f' work_dir: "{tmp_path.as_posix()}"', + f' project_root: "{tmp_path.as_posix()}"', + f' log_dir: "{(tmp_path / "logs").as_posix()}"', " stages:", " - first_stage:", - f" location: \"{first_stage.as_posix()}\"", + f' location: "{first_stage.as_posix()}"', " - second_stage:", - f" location: \"{second_stage.as_posix()}\"", + f' location: "{second_stage.as_posix()}"', "stage_configuration: {}", ] ) @@ -179,14 +188,16 @@ def test_assign_dependencies_with_config_defined_stages(self, tmp_path): encoding="utf-8", ) - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - config=config_file, - dependencies={"second_stage": ("first_stage",)}, - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + config=config_file, + dependencies={"second_stage": ("first_stage",)}, + ) - assert [stage.name for stage in pipeline.stages] == ["first_stage", "second_stage"] + assert [stage.name for stage in pipeline.stages] == [ + "first_stage", + "second_stage", + ] assert pipeline.stages[1].dependencies == ("first_stage",) assert pipeline.dependencies == {"second_stage": ("first_stage",)} @@ -203,7 +214,7 @@ def test_add_dependencies_single_dict(self, tmp_path): Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'PipelineInitialisationError' Expected and asserted as dependencies are specified for stages that do not @@ -221,11 +232,11 @@ def test_add_dependencies_single_dict(self, tmp_path): stage_2 = Stage("Stage_2", source=path_2, dependencies={}) stage_0 = Stage("Stage_0", source=path_0, dependencies={}) - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline_dict = Pipeline( - stages=[stage_0, stage_1, stage_2], dependencies=dependencies_multiple - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_dict = Pipeline( + stages=[stage_0, stage_1, stage_2], + dependencies=dependencies_multiple, + ) with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) @@ -250,30 +261,29 @@ def test_add_dependencies_single_dict(self, tmp_path): class TestPipelineStageConfigHandling: def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: """ - Tests that when a stage is added after a Pipeline has been initialised, - the stage and the stage_configurations are correctly added to the + Tests that when a stage is added after a Pipeline has been initialised, + the stage and the stage_configurations are correctly added to the Pipeline instance and the stage_configurations are correctly associated - with the stage. + with the stage. Parameter ---------- ``stage_factory`` : Callable A factory function that creates Stage objects for testing. - + Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(stages = [Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())] + ) stage = stage_factory("Stage_1") - stage_config = StageConfig( - name="Stage_1", _variables={"years_to_run": 2017} - ) + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) with pytest.warns(PipelineConfigurationWarning): pipeline.add_stage(stage, stage_configs=[stage_config]) @@ -284,7 +294,7 @@ def test_add_stage_warns_when_stage_config_count_mismatches( self, stage_factory ) -> None: """ - Tests that when a stage is added but there is not the correct number of + Tests that when a stage is added but there is not the correct number of stage_configs provided, a warning is raised and the stage_configuration for that stage is added as a blank StageConfig object. @@ -296,14 +306,17 @@ def test_add_stage_warns_when_stage_config_count_mismatches( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(stages = [Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=())]) - stage_0 = stage_factory("Stage_0") - stage_1 = stage_factory("Stage_1") + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[ + Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=()) + ] + ) + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") with pytest.warns(StageConfigurationWarning) as recorded_warnings: pipeline.add_stage( @@ -321,7 +334,7 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage( self, stage_factory ) -> None: """ - Tests that when a stage_configuration is added to a Pipeline instance, + Tests that when a stage_configuration is added to a Pipeline instance, the configuration is correctly associated with the named stage and that the configuration is coerced into a StageConfig object if it is provided. @@ -333,12 +346,11 @@ def test_add_stage_config_coerces_mapping_payload_for_named_stage( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(stages=[stage_factory("Stage_0")]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_factory("Stage_0")]) pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") @@ -363,12 +375,11 @@ def test_resolve_stages_to_run_includes_transitive_dependencies( stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0, stage_1, stage_2], - config=PipelineConfig(stages_to_run={"Stage_2": True}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), + ) assert [stage.name for stage in pipeline.graph.stages] == [ "Stage_0", @@ -406,8 +417,8 @@ def test_resolve_stages_to_run_rejects_disabled_dependencies( stages=[stage_0, stage_1], config=PipelineConfig( stages_to_run={"Stage_0": False, "Stage_1": True} - ) - ) + ), + ) def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: """ @@ -422,15 +433,14 @@ def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None Raises ------ ``PipelineConfigurationWarning`` - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -443,7 +453,7 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( ) -> None: """ Tests that when a stage is manually disabled in a Pipeline instance, it - is initialised in the stages_to_run configuration. + is initialised in the stages_to_run configuration. Parameters ---------- @@ -453,14 +463,13 @@ def test_disable_stage_in_implicit_mode_creates_explicit_selection( Raises ------ ``PipelineConfigurationWarning`` - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(stages=[stage_0, stage_1]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_0, stage_1]) pipeline.disable_stage("Stage_1") @@ -480,12 +489,13 @@ def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> No """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) pipeline.enable_stage("Stage_1") @@ -503,21 +513,19 @@ def test_add_stage_keeps_new_stage_out_of_explicit_selection( ---------- ``stage_factory`` : Callable A factory function that creates Stage objects for testing. - + """ stage_0 = stage_factory("Stage_0") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - pipeline.add_stage( - stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")] - ) - + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + ) assert pipeline.config.stages_to_run["Stage_1"] is False assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] @@ -537,18 +545,17 @@ def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_tr """ stage_0 = stage_factory("Stage_0") - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0], - config=PipelineConfig(stages_to_run={"Stage_0": True}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) - pipeline.add_stage( - stage_factory("Stage_1"), - stage_configs=[StageConfig(name="Stage_1")], - enable_stages=True, - ) + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) assert pipeline.config.stages_to_run["Stage_1"] is True assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} @@ -575,12 +582,13 @@ def test_validate_skips_source_check_for_disabled_stages( "Stage_1", source=tmp_path / "missing.py" ) # file intentionally absent - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) pipeline.validate() # must not raise @@ -592,50 +600,56 @@ def test_construct_manifest_inputs_contains_only_effective_stages( graph. Parameters - ---------- + ---------- ``stage_factory`` : Callable A factory function that creates Stage objects for testing. """ stage_0 = stage_factory("Stage_0") stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline( - stages=[stage_0, stage_1], - config=PipelineConfig(stages_to_run={"Stage_0": True, "Stage_1": False}), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) runtime_id = pipeline._create_runtime_id() manifest = pipeline._construct_manifest(runtime_id=runtime_id) assert list(manifest.inputs.keys()) == ["Stage_0"] - def test_generate_context_correctly_assigns_executor(self,) -> None: + def test_generate_context_correctly_assigns_executor( + self, + ) -> None: """ Test that the correct executor class is assigned to the Pipeline instance - based on the backend specified. If the backend does not have a compatible - executor, an error is raised. + based on the backend specified. If the backend does not have a compatible + executor, an error is raised. """ - with pytest.warns(PipelineConfigurationWarning): - with pytest.warns(StageConfigurationWarning): - pipeline = Pipeline(backend="python", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) assert isinstance(pipeline.executor, PythonStageExecutor) with pytest.raises(PipelineInitialisationError): - Pipeline(backend="nonexistent_backend", - stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())]) + Pipeline( + backend="nonexistent_backend", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) - def test_validate_stage_backends_errors(self) -> None: + def test_validate_stage_backends_errors(self) -> None: """ - Test that the _validate_stage_backends method correctly raises an error if - the backends for a stage do not match the Pipeline backend or if there are - multiple backends across the stages. + Test that the _validate_stage_backends method correctly raises an error if + the backends for a stage do not match the Pipeline backend or if there are + multiple backends across the stages. Successful runs not tested here as they are covered in test_generate_context_ - correctly_assigns_executor(). + correctly_assigns_executor(). """ with pytest.raises(PipelineInitialisationError): @@ -670,6 +684,7 @@ def test_validate_stage_backends_errors(self) -> None: ], ) + class TestLoadLatestRunIntegration: @pytest.fixture def pipeline_log_line(self): @@ -685,8 +700,11 @@ def _make(run_id: str, timestamp: str) -> str: ``timestamp`` : str The timestamp of when the historical run was initiated. """ - return f"{timestamp} Pipeline started | " \ - f"{{\"run_id\": \"{run_id}\", \"run_dir\": \"/path/to/run\"}}" + return ( + f"{timestamp} Pipeline started | " + f'{{"run_id": "{run_id}", "run_dir": "/path/to/run"}}' + ) + return _make @pytest.fixture @@ -709,6 +727,7 @@ def _make(run_id: str) -> str: stage_results: {{}} stage_outputs: {{}} """ + return _make @pytest.fixture @@ -720,56 +739,64 @@ def pipeline_no_history(self, tmp_path: Path) -> Pipeline: Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. """ with pytest.warns(PipelineConfigurationWarning): - pipeline = Pipeline(config=PipelineConfig( - output_dir = tmp_path/"outputs", - ), - stages = [Stage("Stage_0", source=tmp_path/"Stage_0.py", dependencies=())]) - pipeline.run_output = tmp_path/"runs" + pipeline = Pipeline( + config=PipelineConfig( + output_dir=tmp_path / "outputs", + ), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + pipeline.run_output = tmp_path / "runs" return pipeline - + class TestLoadLatestRun(TestLoadLatestRunIntegration): - def test_blank_historical_run_ids(self, - pipeline_no_history: Pipeline, - monkeypatch) -> None: + def test_blank_historical_run_ids( + self, pipeline_no_history: Pipeline, monkeypatch + ) -> None: """ Tests that if extract_historical_run_ids returns a blank list, the _load_latest_run method will return None and raise a warning. Assert that it will also store None in the last_run attribute of the Pipeline - instance. + instance. Parameters ---------- ``pipeline_no_history`` : Pipeline A Pipeline instance with no historical runs. ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for dynamic modification of attributes, + A pytest fixture that allows for dynamic modification of attributes, methods, or classes during testing. Raises ------ ``PipelineConfigurationWarning`` - Raised when no previous runs are found for the Pipeline, indicating + Raised when no previous runs are found for the Pipeline, indicating that the last_run attribute will be None. """ - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: []) - assert pipeline_no_history.logger.extract_historical_run_ids( - pipeline_no_history.run_output - ) == [] - with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ - "found for this Pipeline. Last_run attribute will be None."): - assert pipeline_no_history._load_latest_run() == None - assert pipeline_no_history.last_run == None + monkeypatch.setattr( + pipeline_no_history.logger, "extract_historical_run_ids", lambda x: [] + ) + assert ( + pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) + == [] + ) + with pytest.warns( + PipelineConfigurationWarning, + match="No previous runs " + "found for this Pipeline. Last_run attribute will be None.", + ): + assert pipeline_no_history._load_latest_run() is None + assert pipeline_no_history.last_run is None - def test_blank_run_ids(self, - pipeline_no_history: Pipeline, - monkeypatch) -> None: + def test_blank_run_ids(self, pipeline_no_history: Pipeline, monkeypatch) -> None: """ Tests that if the found log record does not have a run_id, _load_latest_run will return None and raise a warning. Assert that it will also store None @@ -780,45 +807,53 @@ def test_blank_run_ids(self, ``pipeline_no_history`` : Pipeline A Pipeline instance with no historical runs. ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for dynamic modification of attributes, + A pytest fixture that allows for dynamic modification of attributes, methods, or classes during testing. Raises ------ ``PipelineConfigurationWarning`` - Raised when no previous runs are found for the Pipeline, indicating + Raised when no previous runs are found for the Pipeline, indicating that the last_run attribute will be None. """ - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [{ - "run_id": None, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/")}] - ) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/"), + } + ], + ) assert pipeline_no_history.logger.extract_historical_run_ids( pipeline_no_history.run_output - ) == [{ - "run_id": None, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/") - }] - with pytest.warns(PipelineConfigurationWarning, match="No previous runs " \ - "found for this Pipeline. Last_run attribute will be None."): - assert pipeline_no_history._load_latest_run() == None - assert pipeline_no_history.last_run == None - - def test_load_latest_run_success(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: - - """ - Tests that load_latest_run works successfully with fully mocked data. + ) == [ + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/"), + } + ] + with pytest.warns( + PipelineConfigurationWarning, + match="No previous runs " + "found for this Pipeline. Last_run attribute will be None.", + ): + assert pipeline_no_history._load_latest_run() is None + assert pipeline_no_history.last_run is None + + def test_load_latest_run_success( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Tests that load_latest_run works successfully with fully mocked data. Parameters ---------- ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for dynamic modification of attributes, + A pytest fixture that allows for dynamic modification of attributes, methods, or classes during testing. ``pipeline_no_history`` : Pipeline A Pipeline instance with no historical runs. @@ -828,38 +863,39 @@ def test_load_latest_run_success(self, run_id = "2026-08-10_100000_abc12345" monkeypatch.setattr( pipeline_no_history.logger, - "extract_historical_run_ids", - lambda _: [{ - "run_id": run_id, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run") - }] + "extract_historical_run_ids", + lambda _: [ + { + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run"), + } + ], ) mock_load_historical_run = mock.MagicMock(return_value=expected_run) - monkeypatch.setattr("onsrap.pipeline.load_historical_run", - mock_load_historical_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) result = pipeline_no_history._load_latest_run() assert result is expected_run expected_path = pipeline_no_history.run_output / run_id - mock_load_historical_run.assert_called_once_with(run_dir = expected_path) - - def test_which_run_is_selected_load_latest_run(self, - monkeypatch, - pipeline_no_history: Pipeline - ) -> None: + mock_load_historical_run.assert_called_once_with(run_dir=expected_path) + def test_which_run_is_selected_load_latest_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ - Checks that the first item is selected from the list of historical runs + Checks that the first item is selected from the list of historical runs returned by extract_historical_run_ids. Parameters ---------- ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for dynamic modification of attributes, + A pytest fixture that allows for dynamic modification of attributes, methods, or classes during testing. ``pipeline_no_history`` : Pipeline A Pipeline instance with no historical runs. @@ -869,45 +905,45 @@ def test_which_run_is_selected_load_latest_run(self, run_id_2 = "2026-08-10_100000_def67890" monkeypatch.setattr( pipeline_no_history.logger, - "extract_historical_run_ids", + "extract_historical_run_ids", lambda _: [ { - "run_id": run_id_1, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": "run_A" - }, - { - "run_id": run_id_2, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": "run_B" - } - ] + "run_id": run_id_1, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_A", + }, + { + "run_id": run_id_2, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_B", + }, + ], ) mock_load_historical_run = mock.MagicMock(return_value=expected_run) - monkeypatch.setattr("onsrap.pipeline.load_historical_run", - mock_load_historical_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) pipeline_no_history._load_latest_run() expected_path = pipeline_no_history.run_output / run_id_1 - mock_load_historical_run.assert_called_once_with(run_dir = expected_path) + mock_load_historical_run.assert_called_once_with(run_dir=expected_path) - #does not refer to run_dir in the extract_historical_run_ids list but the - #parameter required in load_historical_run. + # does not refer to run_dir in the extract_historical_run_ids list but the + # parameter required in load_historical_run. assert mock_load_historical_run.call_args.kwargs["run_dir"].name == run_id_1 - def test_no_errors_raised_success_load_latest_run(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: - + def test_no_errors_raised_success_load_latest_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ - Tests that no errors are raised when load_latest_run is successful. + Tests that no errors are raised when load_latest_run is successful. Parameters ---------- ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for dynamic modification of attributes, + A pytest fixture that allows for dynamic modification of attributes, methods, or classes during testing. ``pipeline_no_history`` : Pipeline A Pipeline instance with no historical runs. @@ -917,62 +953,69 @@ def test_no_errors_raised_success_load_latest_run(self, run_id = "2026-08-10_100000_abc12345" monkeypatch.setattr( pipeline_no_history.logger, - "extract_historical_run_ids", - lambda _: [{ - "run_id": run_id, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run") - }] + "extract_historical_run_ids", + lambda _: [ + { + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run"), + } + ], ) mock_load_historical_run = mock.MagicMock(return_value=expected_run) - monkeypatch.setattr("onsrap.pipeline.load_historical_run", - mock_load_historical_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) with warnings.catch_warnings(record=True) as w: pipeline_no_history._load_latest_run() - assert not any(issubclass(warning.category, PipelineConfigurationWarning) - for warning in w) + assert not any( + issubclass(warning.category, PipelineConfigurationWarning) for warning in w + ) + class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): def test_no_previous_runs_pipeline(self, tmp_path: Path) -> None: """ Tests that if a Pipeline instance has no previous runs, the _load_latest_run - method will return None and raise a warning. Assert that it will also store + method will return None and raise a warning. Assert that it will also store None in the last_run attribute of the Pipeline instance. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. Raises ------ ``PipelineConfigurationWarning`` - Raised when no previous runs are found for the Pipeline, indicating + Raised when no previous runs are found for the Pipeline, indicating that the last_run attribute will be None. """ with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( config=PipelineConfig(output_dir=tmp_path / "outputs"), - stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], ) pipeline.run_output = tmp_path / "runs" assert pipeline.last_run is None - def test_last_run_populated_one_run(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: + def test_last_run_populated_one_run( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: """ - Tests that if a Pipeline instance has one previous run, this is loaded in - last_run attribute at Pipeline creation. + Tests that if a Pipeline instance has one previous run, this is loaded in + last_run attribute at Pipeline creation. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. ``minimal_pipeline_yaml`` : callable A fixture that returns a minimal YAML configuration for a historical run. @@ -980,45 +1023,49 @@ def test_last_run_populated_one_run(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is no stages_to_run parameters to warn the user that + Raised when there is no stages_to_run parameters to warn the user that all stages will be run by default. """ logs = tmp_path / "logs" logs.mkdir(parents=True, exist_ok=True) (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " - "{\"run_id\": \"2026-08-11_100000_abc12345\", " - " \"run_dir\": \"/path/to/run\"}\n" + '{"run_id": "2026-08-11_100000_abc12345", ' + ' "run_dir": "/path/to/run"}\n' ) - temp_attributes = (tmp_path / "outputs" / "runs" / "2026-08-11_100000_abc12345" - / "pipeline_attributes_for_test.yaml") + temp_attributes = ( + tmp_path + / "outputs" + / "runs" + / "2026-08-11_100000_abc12345" + / "pipeline_attributes_for_test.yaml" + ) temp_attributes.parent.mkdir(parents=True, exist_ok=True) - temp_attributes.write_text(minimal_pipeline_yaml( - run_id = "2026-08-11_100000_abc12345" - ), encoding="utf-8") + temp_attributes.write_text( + minimal_pipeline_yaml(run_id="2026-08-11_100000_abc12345"), encoding="utf-8" + ) with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( - config=PipelineConfig(output_dir=tmp_path / "outputs", - log_dir=logs), - stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], ) assert pipeline.last_run is not None assert pipeline.last_run.manifest.run_id == "2026-08-11_100000_abc12345" - def test_last_run_most_recent(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: + def test_last_run_most_recent(self, tmp_path: Path, minimal_pipeline_yaml) -> None: """ Tests that if a Pipeline instance has multiple previous runs, the most recent - run is loaded in last_run attribute at Pipeline creation. + run is loaded in last_run attribute at Pipeline creation. Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. ``minimal_pipeline_yaml`` : callable A fixture that returns a minimal YAML configuration for a historical run. @@ -1026,7 +1073,7 @@ def test_last_run_most_recent(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is no stages_to_run parameters to warn the user that + Raised when there is no stages_to_run parameters to warn the user that all stages will be run by default. """ @@ -1034,41 +1081,51 @@ def test_last_run_most_recent(self, logs.mkdir(parents=True, exist_ok=True) (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " - "{\"run_id\": \"run_older\", " - " \"run_dir\": \"/path/to/run\"}\n" + '{"run_id": "run_older", ' + ' "run_dir": "/path/to/run"}\n' "2026-08-11 10:01:00,000 Pipeline started | " - "{\"run_id\": \"run_newer\", " - " \"run_dir\": \"/path/to/run\"}" + '{"run_id": "run_newer", ' + ' "run_dir": "/path/to/run"}' ) - temp_attributes_1 = (tmp_path / "outputs" / "runs" / "run_older" - / "pipeline_attributes_for_test.yaml") + temp_attributes_1 = ( + tmp_path + / "outputs" + / "runs" + / "run_older" + / "pipeline_attributes_for_test.yaml" + ) temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) - temp_attributes_1.write_text(minimal_pipeline_yaml( - run_id = "run_older" - ), encoding="utf-8") + temp_attributes_1.write_text( + minimal_pipeline_yaml(run_id="run_older"), encoding="utf-8" + ) - temp_attributes_2 = (tmp_path / "outputs" / "runs" / "run_newer" - / "pipeline_attributes_for_test.yaml") + temp_attributes_2 = ( + tmp_path + / "outputs" + / "runs" + / "run_newer" + / "pipeline_attributes_for_test.yaml" + ) temp_attributes_2.parent.mkdir(parents=True, exist_ok=True) - temp_attributes_2.write_text(minimal_pipeline_yaml( - run_id = "run_newer" - ), encoding="utf-8") + temp_attributes_2.write_text( + minimal_pipeline_yaml(run_id="run_newer"), encoding="utf-8" + ) with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( - config=PipelineConfig(output_dir=tmp_path / "outputs", - log_dir=logs), - stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], ) assert pipeline.last_run is not None assert pipeline.last_run.manifest.run_id == "run_newer" - - def test_last_run_most_recent(self, - tmp_path: Path, - minimal_pipeline_yaml) -> None: + def test_last_run_most_recent_no_directory( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: """ Checks that only the older run is included when the run directory has been deleted/removed for the most recent run. @@ -1076,7 +1133,7 @@ def test_last_run_most_recent(self, Parameters ---------- ``tmp_path`` : Path - A temporary directory provided by pytest for creating test files + A temporary directory provided by pytest for creating test files and directories. ``minimal_pipeline_yaml`` : callable A fixture that returns a minimal YAML configuration for a historical run. @@ -1084,7 +1141,7 @@ def test_last_run_most_recent(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is no stages_to_run parameters to warn the user that + Raised when there is no stages_to_run parameters to warn the user that all stages will be run by default. """ @@ -1092,35 +1149,41 @@ def test_last_run_most_recent(self, logs.mkdir(parents=True, exist_ok=True) (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " - "{\"run_id\": \"run_older\", " - " \"run_dir\": \"/path/to/run\"}\n" + '{"run_id": "run_older", ' + ' "run_dir": "/path/to/run"}\n' "2026-08-11 10:01:00,000 Pipeline started | " - "{\"run_id\": \"run_newer\", " - " \"run_dir\": \"/path/to/run\"}" + '{"run_id": "run_newer", ' + ' "run_dir": "/path/to/run"}' ) - temp_attributes_1 = (tmp_path / "outputs" / "runs" / "run_older" - / "pipeline_attributes_for_test.yaml") + temp_attributes_1 = ( + tmp_path + / "outputs" + / "runs" + / "run_older" + / "pipeline_attributes_for_test.yaml" + ) temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) - temp_attributes_1.write_text(minimal_pipeline_yaml( - run_id = "run_older" - ), encoding="utf-8") + temp_attributes_1.write_text( + minimal_pipeline_yaml(run_id="run_older"), encoding="utf-8" + ) with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( - config=PipelineConfig(output_dir=tmp_path / "outputs", - log_dir=logs), - stages=[Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=())], + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], ) assert pipeline.last_run is not None assert pipeline.last_run.manifest.run_id == "run_older" + class TestLoadAllRunsIntegration(TestLoadLatestRunIntegration): - def test_returns_none_when_error_in_extract_historical_runs(self, - monkeypatch, - pipeline_no_history - ) -> None: + def test_returns_none_when_error_in_extract_historical_runs( + self, monkeypatch, pipeline_no_history + ) -> None: """ Checks that all_runs attribute is None when extract_historical_run_ids raises an error. This is to ensure that the Pipeline instance does not break @@ -1137,22 +1200,23 @@ def test_returns_none_when_error_in_extract_historical_runs(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is an issue with extracting historical runs, indicating + Raised when there is an issue with extracting historical runs, indicating that the all_runs attribute will be None. """ - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - mock.Mock(side_effect=HistoricalPipelineLoadError("test"))) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + mock.Mock(side_effect=HistoricalPipelineLoadError("test")), + ) with pytest.warns(PipelineConfigurationWarning): assert pipeline_no_history._load_all_runs() is None assert pipeline_no_history.all_runs is None - def test_returns_none_when_blank_extract_historical_runs(self, - monkeypatch, - pipeline_no_history - ) -> None: + def test_returns_none_when_blank_extract_historical_runs( + self, monkeypatch, pipeline_no_history + ) -> None: """ Checks that all_runs attribute is None when extract_historical_run_ids returns a blank list. This ensures that the Pipeline instance does not @@ -1169,23 +1233,26 @@ def test_returns_none_when_blank_extract_historical_runs(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there are no historical runs found, indicating that the + Raised when there are no historical runs found, indicating that the all_runs attribute will be None. """ - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: []) - - assert pipeline_no_history.logger.extract_historical_run_ids( - pipeline_no_history.run_output - ) == [] + monkeypatch.setattr( + pipeline_no_history.logger, "extract_historical_run_ids", lambda x: [] + ) + + assert ( + pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output + ) + == [] + ) with pytest.warns(PipelineConfigurationWarning): assert pipeline_no_history._load_all_runs() is None assert pipeline_no_history.all_runs is None - def test_single_entry_dict_single_run(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: + def test_single_entry_dict_single_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ Checks that all_runs attribute is a dictionary with a single entry when extract_historical_run_ids returns a list with one historical run. This @@ -1202,31 +1269,39 @@ def test_single_entry_dict_single_run(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is one historical run found, indicating that the + Raised when there is one historical run found, indicating that the all_runs attribute will contain a single entry. """ - mock_loader = mock.MagicMock(return_value = mock.sentinel) + mock_loader = mock.MagicMock(return_value=mock.sentinel) monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [{"run_id": "run_A", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")}]) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + } + ], + ) result = pipeline_no_history._load_all_runs() assert isinstance(result, dict) assert len(result) == 1 assert "run_A" in result - mock_loader.assert_called_once_with(run_dir = pipeline_no_history.run_output / "run_A") - - def test_multiple_entries_dict_multiple_runs(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: + mock_loader.assert_called_once_with( + run_dir=pipeline_no_history.run_output / "run_A" + ) + + def test_multiple_entries_dict_multiple_runs( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ Checks that all_runs attribute is a dictionary with multiple entries when - extract_historical_run_ids returns a list with multiple historical runs. + extract_historical_run_ids returns a list with multiple historical runs. This ensures that the Pipeline instance correctly loads multiple historical runs. Parameters @@ -1240,34 +1315,42 @@ def test_multiple_entries_dict_multiple_runs(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there are multiple historical runs found, indicating that the + Raised when there are multiple historical runs found, indicating that the all_runs attribute will contain multiple entries. """ - mock_loader = mock.MagicMock(side_effect=[mock.sentinel.run_A, mock.sentinel.run_B]) + mock_loader = mock.MagicMock( + side_effect=[mock.sentinel.run_A, mock.sentinel.run_B] + ) monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [ - {"run_id": "run_A", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")}, - {"run_id": "run_B", - "timestamp": "2026-08-10 10:01:00,000", - "run_dir": Path("/path/to/run_B")} - ]) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) result = pipeline_no_history._load_all_runs() assert isinstance(result, dict) assert len(result) == 2 assert "run_A" in result and "run_B" in result - mock_loader.assert_any_call(run_dir = pipeline_no_history.run_output / "run_A") - mock_loader.assert_any_call(run_dir = pipeline_no_history.run_output / "run_B") + mock_loader.assert_any_call(run_dir=pipeline_no_history.run_output / "run_A") + mock_loader.assert_any_call(run_dir=pipeline_no_history.run_output / "run_B") - def test_warning_if_no_run_id(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: + def test_warning_if_no_run_id( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ Checks that a warning is raised if extract_historical_run_ids returns a historical run without a run_id. This ensures that the Pipeline instance @@ -1284,30 +1367,40 @@ def test_warning_if_no_run_id(self, mock_loader = mock.MagicMock(return_value=mock.sentinel) monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [ - {"run_id": "", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")}, - {"run_id": "run_B", - "timestamp": "2026-08-10 10:01:00,000", - "run_dir": Path("/path/to/run_B")}, - {"run_id": None, - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")} - ]) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": "", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B"), + }, + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + ], + ) with pytest.warns(PipelineConfigurationWarning): result = pipeline_no_history._load_all_runs() assert isinstance(result, dict) assert len(result) == 1 assert "run_A" not in result and "run_B" in result - mock_loader.assert_called_once_with(run_dir = pipeline_no_history.run_output / "run_B") + mock_loader.assert_called_once_with( + run_dir=pipeline_no_history.run_output / "run_B" + ) - def test_None_with_stageloaderror(self, - monkeypatch, - pipeline_no_history: Pipeline) -> None: + def test_None_with_stageloaderror( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ Checks that all_runs attribute is None when load_historical_run raises a StageLoadError. This ensures that the Pipeline instance correctly handles @@ -1324,35 +1417,41 @@ def test_None_with_stageloaderror(self, Raises ------ ``PipelineConfigurationWarning`` - Raised when there is an issue loading a historical run, indicating that + Raised when there is an issue loading a historical run, indicating that the all_runs attribute will be None. """ - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [ - {"run_id": "good_run", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")}, - {"run_id": "bad_run", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_B")} - ]) + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": "good_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "bad_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) - monkeypatch.setattr("onsrap.pipeline.load_historical_run", - mock.Mock(side_effect=[mock.sentinel.good_run, - StageLoadError("test")])) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[mock.sentinel.good_run, StageLoadError("test")]), + ) with pytest.warns(PipelineConfigurationWarning): result = pipeline_no_history._load_all_runs() - assert isinstance(result,dict) + assert isinstance(result, dict) assert len(result) == 1 assert "good_run" in result and "bad_run" not in result - def test_none_if_all_stageloaderrors(self, - monkeypatch, - pipeline_no_history: Pipeline - ) -> None: + def test_none_if_all_stageloaderrors( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: """ Asserts that all_runs attribute is None when load_historical_run raises a StageLoadError for all historical runs. This ensures that the Pipeline instance @@ -1372,23 +1471,30 @@ def test_none_if_all_stageloaderrors(self, Raised when there is an issue loading all historical runs, indicating that the all_runs attribute will be None. """ - - monkeypatch.setattr(pipeline_no_history.logger, - "extract_historical_run_ids", - lambda x: [ - {"run_id": "bad_run1", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_A")}, - {"run_id": "bad_run2", - "timestamp": "2026-08-10 10:00:00,000", - "run_dir": Path("/path/to/run_B")} - ]) - - monkeypatch.setattr("onsrap.pipeline.load_historical_run", - mock.Mock(side_effect=[StageLoadError("test"), - StageLoadError("test")])) + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x: [ + { + "run_id": "bad_run1", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "bad_run2", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) + + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[StageLoadError("test"), StageLoadError("test")]), + ) with pytest.warns(PipelineConfigurationWarning): result = pipeline_no_history._load_all_runs() - assert result is None \ No newline at end of file + assert result is None diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 59559bf..d4f8014 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -17,8 +17,10 @@ NO_STAGES_SPECIFIED_WARNING = ( "No stages specified to run. All stages running by default." ) -OUTPUT_DIRECTORY_WARNING = "Output directory is not specified. Using project root or " \ - "work directory as the run output." +OUTPUT_DIRECTORY_WARNING = ( + "Output directory is not specified. Using project root or " + "work directory as the run output." +) def _base_pipeline_config(tmp_path: Path) -> dict: @@ -49,12 +51,12 @@ def test_pipeline_from_files_executes_python_entrypoints( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified in the configuration. This means that the Pipeline defaults to using - the project root or working directory as the run output. + the project root or working directory as the run output. """ first_stage = tmp_path / "first_stage.py" first_stage.write_text( @@ -80,17 +82,12 @@ def main(context): encoding="utf-8", ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config=_base_pipeline_config(tmp_path), + ) run = pipeline.run() @@ -119,7 +116,7 @@ def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> Non Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -145,16 +142,11 @@ def main(context): + "\n", encoding="utf-8", ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_files( - [writer_stage], - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [writer_stage], + config=_base_pipeline_config(tmp_path), + ) first_run = pipeline.run() second_run = pipeline.run() @@ -173,7 +165,7 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( self, tmp_path: Path ) -> None: """ - Checks that a pipeline will run with a non-module based Python script by + Checks that a pipeline will run with a non-module based Python script by running the entire script. Parameters @@ -184,7 +176,7 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -193,17 +185,12 @@ def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( script_stage = tmp_path / "script_stage.py" script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_files( - [script_stage], - name="script-pipeline", - config=_base_pipeline_config(tmp_path), - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config=_base_pipeline_config(tmp_path), + ) run = pipeline.run() @@ -250,11 +237,11 @@ def test_pipeline_from_config_builds_stages_and_injects_stage_config( ---------- ``tmp_path`` : Path A temporary directory provided by pytest for testing. - + Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -294,8 +281,8 @@ def run(context): stages: - 0_data_validation: location: "{ - (tmp_path / "scripts" / "0_data_validation.py").as_posix() - }" + (tmp_path / "scripts" / "0_data_validation.py").as_posix() + }" run: true dependencies: [] @@ -312,13 +299,8 @@ def run(context): encoding="utf-8", ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 @@ -340,11 +322,11 @@ def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> N ---------- ``tmp_path`` : Path A temporary directory provided by pytest for testing. - + Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -366,16 +348,11 @@ def run(context): config["stage_configuration"] = { "missing_stage": {"years_to_run": 2017}, } - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_files( - [stage_file], - config=config, - ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [stage_file], + config=config, + ) with pytest.raises(StageConfigurationError, match="unknown stages"): pipeline.validate() @@ -390,14 +367,14 @@ def test_pipeline_from_config_parses_stage_configuration_payloads( the stage. Parameters - ---------- + ---------- ``tmp_path`` : Path A temporary directory provided by pytest for testing. Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. """ @@ -472,13 +449,8 @@ def run(context): yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8" ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) assert pipeline.name == "parse-test" assert pipeline.config.work_dir == tmp_path @@ -515,7 +487,7 @@ def test_pipeline_from_config_scales_stage_configuration_to_many_stages( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -595,13 +567,8 @@ def run(context): encoding="utf-8", ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_config(config_file) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) assert [stage.name for stage in pipeline.stages] == stage_names assert sorted(pipeline.stage_configs) == stage_names @@ -661,11 +628,11 @@ def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( self, tmp_path: Path ) -> None: """ - Integration test that checks that the _log_config method is correctly called - within PipelineRunner.run() and that the information is parsed in a suitable + Integration test that checks that the _log_config method is correctly called + within PipelineRunner.run() and that the information is parsed in a suitable format to a YAML file in the run directory. - This test also captures that _combine_configs() correctly converts all + This test also captures that _combine_configs() correctly converts all configuration information into a single dictionary that can be serialized to YAML. @@ -677,7 +644,7 @@ def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( Raises ------ 'PipelineConfigurationWarning' - Expected and asserted as there is no stage run specification in the + Expected and asserted as there is no stage run specification in the Pipeline configuration. This does not affect the test capability. 'StageConfigurationWarning' Expected and asserted as there is no output directory specified @@ -695,27 +662,22 @@ def run(context): encoding="utf-8", ) - with pytest.warns( - PipelineConfigurationWarning - ): - with pytest.warns( - StageConfigurationWarning - ): - pipeline = Pipeline.from_files( - [stage_file], - name="config-export-pipeline", - config={ - "pipeline_config": { - "work_dir": tmp_path, - "project_root": tmp_path, - "log_dir": tmp_path / "logs", - }, - "stage_configuration": {}, - "global_configuration": { - "dry_run": True, - }, + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, }, - ) + }, + ) run = pipeline.run() diff --git a/tests/test_runner.py b/tests/test_runner.py index 8801828..dfe3f69 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -6,8 +6,16 @@ from onsrap.execution import ExecutionContext from onsrap.logger import Logger -from onsrap.models import PipelineConfig, PipelineStatus, RunManifest, PipelineRun, StageResult, StageStatus, now -from onsrap.runner import _log_config, print_config_diffs, _log_pipeline_attributes +from onsrap.models import ( + PipelineConfig, + PipelineRun, + PipelineStatus, + RunManifest, + StageResult, + StageStatus, + now, +) +from onsrap.runner import _log_config, _log_pipeline_attributes, print_config_diffs class TestLogConfig: @@ -106,7 +114,7 @@ def _write_yaml(path: Path, content: str) -> None: Parameters ---------- ``path`` : Path - The path where the YAML file will be written. + The path where the YAML file will be written. ``content`` : str The YAML content to write to the file. """ @@ -186,14 +194,19 @@ def test_returns_and_prints_differences( global_config: dry_run: True """).strip() - + "\n", - encoding="utf-8", - ) + + "\n", + encoding="utf-8", + ) assert print_config_diffs(test_file_a, test_file_b) == { - "changed": {"stage_configs.stage_a.target_variable": ("classification", "identification")}, + "changed": { + "stage_configs.stage_a.target_variable": ( + "classification", + "identification", + ) + }, "added": {"pipeline_config.backend": "python"}, - "removed": {"pipeline_config.output_dir": "outputs"} + "removed": {"pipeline_config.output_dir": "outputs"}, } captured = capsys.readouterr() @@ -202,6 +215,7 @@ def test_returns_and_prints_differences( assert "REMOVED in second configuration (1)" in captured.out assert "stage_configs.stage_a.target_variable" in captured.out + class TestRunInfoWriteOut: def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: """ @@ -226,31 +240,31 @@ def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: ) context = ExecutionContext( - pipeline_name="synthetic_pipeline", - run_id="run_1234", - config=pipeline_config, - logger=Logger(), - run_dir=run_dir, - working_directory=tmp_path, - stage_configs={}, - global_config=None, - ) + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=pipeline_config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) stage_results = [ - StageResult( - name="stage_a", - status=StageStatus.SUCCEEDED, - started_at=now(), - finished_at= now(), - error=None, - source=None, - ) - ] - + StageResult( + name="stage_a", + status=StageStatus.SUCCEEDED, + started_at=now(), + finished_at=now(), + error=None, + source=None, + ) + ] + run_manifest = RunManifest( rap_name="synthetic_pipeline", run_id="run_1234", - ) + ) pipeline_run = PipelineRun( manifest=run_manifest, @@ -262,9 +276,7 @@ def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: ) _log_pipeline_attributes( - pipeline_run=pipeline_run, - run_dir=run_dir, - context=context + pipeline_run=pipeline_run, run_dir=run_dir, context=context ) expected_file = run_dir / ( @@ -281,4 +293,3 @@ def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: assert parsed_yaml == expected_contents assert PipelineRun._pipeline_run_from_dict(parsed_yaml) == pipeline_run - diff --git a/tests/test_stage.py b/tests/test_stage.py index c4dd016..b300082 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -30,22 +30,21 @@ def test_normalize_dependencies_dedupe(self) -> None: Tests that duplicate values are removed from the normalized dependencies whilst preserving first seen order. """ - assert _normalize_dependencies( - ["Stage_1.py", "Stage_2.py", "Stage_1.py"] - ) == ("Stage_1.py", "Stage_2.py") + assert _normalize_dependencies(["Stage_1.py", "Stage_2.py", "Stage_1.py"]) == ( + "Stage_1.py", + "Stage_2.py", + ) assert _normalize_dependencies( - ["Stage_2.py","Stage_1.py", "Stage_2.py", "Stage_1.py"] - ) == ("Stage_2.py","Stage_1.py") + ["Stage_2.py", "Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_2.py", "Stage_1.py") def test_normalize_dependencies_whitespace_handling(self) -> None: """ - Tests that whitespace only or blank dependency values are removed from + Tests that whitespace only or blank dependency values are removed from the normalised dependencies. """ - assert _normalize_dependencies( - [" ", ""] - ) == () + assert _normalize_dependencies([" ", ""]) == () def test_normalize_dependencies_type_check(self) -> None: """ @@ -69,9 +68,10 @@ def test_normalize_dependencies_mixed_types(self) -> None: dependency iterable. """ assert _normalize_dependencies(["Stage_1.py", 11, "Stage_2.py"]) == ( - "Stage_1.py", "11", "Stage_2.py" - ) - + "Stage_1.py", + "11", + "Stage_2.py", + ) @pytest.fixture @@ -120,14 +120,14 @@ def test_stage_name_error(self, example_function) -> None: Raises ------ ``StageConfigurationError`` - If the name is left blank or entirely whitespace in a ``Stage`` class - instance. + If the name is left blank or entirely whitespace in a ``Stage`` class + instance. """ with pytest.raises(StageConfigurationError): Stage("", example_function, ["stage_1"], {"info": "example"}) with pytest.raises(StageConfigurationError): - Stage(" ", example_function, ["stage_1"], {"info": "example"}) + Stage(" ", example_function, ["stage_1"], {"info": "example"}) def test_stage_source_type(self) -> None: """ @@ -136,7 +136,7 @@ def test_stage_source_type(self) -> None: Raises ------ ``StageConfigurationError`` - If the source is not a valid callable or file path in a ``Stage`` class + If the source is not a valid callable or file path in a ``Stage`` class instance. """ with pytest.raises(StageConfigurationError): @@ -195,41 +195,39 @@ def test_stage_backend_irregular_values(self, example_function) -> None: ) stage_blank = Stage( - "callable_stage", - example_function, - ["stage_1"], - {"info": "example"}, - backend=" ", - ) + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=" ", + ) stage_non_string = Stage( - "callable_stage", - example_function, - ["stage_1"], - {"info": "example"}, - backend=11, - ) - + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=11, + ) + assert stage_none.backend == "python" assert stage_blank.backend == "python" assert stage_non_string.backend == "11" - def test_stage_constructor_expands_string_source_with_home( - self, - monkeypatch, - tmp_path): + self, monkeypatch, tmp_path + ): """ - Check that a string source is converted to a Path and expanded with + Check that a string source is converted to a Path and expanded with expanduser(). This uses fake environmental variables to make sure that the tests are not dependent on the actual user's home directory. Parameters ---------- ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for temporary modification of environment + A pytest fixture that allows for temporary modification of environment variables and other attributes during testing. ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. """ fake_home = tmp_path / "fake_home" @@ -249,23 +247,21 @@ def test_stage_constructor_expands_string_source_with_home( assert isinstance(stage.source, Path) assert stage.source == expected - def test_stage_constructor_expands_path_source_with_home( - self, - monkeypatch, - tmp_path): + self, monkeypatch, tmp_path + ): """ - Check that a Path source is expanded with expanduser(). This uses fake - environmental variables to make sure that the tests are not dependent on the + Check that a Path source is expanded with expanduser(). This uses fake + environmental variables to make sure that the tests are not dependent on the actual user's home directory. Parameters ---------- ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for temporary modification of environment + A pytest fixture that allows for temporary modification of environment variables and other attributes during testing. ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. """ fake_home = tmp_path / "fake_home" @@ -285,10 +281,10 @@ def test_stage_constructor_expands_path_source_with_home( assert isinstance(stage.source, Path) assert stage.source == expected - def test_normalise_dependencies_within_stage_init(self, example_function) -> None: + def test_normalise_dependencies_within_stage_init(self, example_function) -> None: """ - Thin smoke test to check that _normalize_dependencies is called within the Stage - post_init method. + Thin smoke test to check that _normalize_dependencies is called within the Stage + post_init method. Parameters ---------- @@ -305,7 +301,7 @@ def test_normalise_dependencies_within_stage_init(self, example_function) -> Non def test_source_path(self, stage_test, tmp_path, example_function) -> None: """ - Tests whether source_path detects a path vs other valid and invalid source + Tests whether source_path detects a path vs other valid and invalid source types. Parameters @@ -313,7 +309,7 @@ def test_source_path(self, stage_test, tmp_path, example_function) -> None: ``stage_test`` : Stage A ``Stage`` object created with a callable source for testing. ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. ``example_function`` : callable A callable function to pass as a source for a ``Stage`` class instance. @@ -338,7 +334,7 @@ def test_source_label(self, stage_test, tmp_path, example_function) -> None: ``stage_test`` : Stage A ``Stage`` object created with a callable source for testing. ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. ``example_function`` : callable A callable function to pass as a source for a ``Stage`` class instance. @@ -356,7 +352,7 @@ def test_source_label(self, stage_test, tmp_path, example_function) -> None: class NoName: def __call__(self): pass - + stage_test.source = NoName() assert stage_test.source_label == f"tests.test_stage.{stage_test.name}" @@ -365,8 +361,8 @@ def test_metadata_copy_safely(self) -> None: Checks that if the original metadata dictionary is modified after the Stage instance is created, the Stage instance's metadata remains unchanged. """ - #TODO: do we want this to be how it works? Or would the user assume that if - #they modify the original dict, it modifies the Stage instance. + # TODO: do we want this to be how it works? Or would the user assume that if + # they modify the original dict, it modifies the Stage instance. original_metadata = {"info": "example"} stage = Stage( name="callable_stage", @@ -377,7 +373,7 @@ def test_metadata_copy_safely(self) -> None: original_metadata["info"] = "modified" assert stage.metadata["info"] == "example" - def test_repr_function(self) -> None: + def test_repr_function(self) -> None: """ Tests that the __repr__ function returns a string representation of the Stage instance with the correct attributes. @@ -423,6 +419,7 @@ def test_str_function(self) -> None: ) assert str(stage) == expected_str + class TestValidateStage: def test_validate(self, stage_test, tmp_path) -> None: """ @@ -433,13 +430,13 @@ def test_validate(self, stage_test, tmp_path) -> None: ``stage_test`` : Stage A ``Stage`` object created with a callable source for testing. ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. Raises ------ ``StageConfigurationError`` - If the source is not a valid callable or file path in a ``Stage`` class + If the source is not a valid callable or file path in a ``Stage`` class instance. In this instance, it raises if the source is None, an empty string, or a Path object that is not a file. """ @@ -453,11 +450,13 @@ def test_validate(self, stage_test, tmp_path) -> None: stage_test.source = "" with pytest.raises(StageConfigurationError): stage_test.validate() - - def test_validate_successes(self, stage_test, example_function, temp_script) -> None: + + def test_validate_successes( + self, stage_test, example_function, temp_script + ) -> None: """ - Tests that validate successfully approves of callables and file paths as - sources for a stage instance. + Tests that validate successfully approves of callables and file paths as + sources for a stage instance. Parameters ---------- @@ -469,17 +468,18 @@ def test_validate_successes(self, stage_test, example_function, temp_script) -> A fixture factory that creates temporary Python scripts. """ stage_test.source = example_function - assert stage_test.validate() == None + assert stage_test.validate() is None stage_test.source = temp_script(filename="valid_script.py") - assert stage_test.validate() == None + assert stage_test.validate() is None + class TestWithDependencies: def test_with_dependencies_list(self, stage_test) -> None: """ Tests adding different types of dependencies when the original dependency is a list. Also checks that the original stage_test instance is not modified when - with_dependencies is called. + with_dependencies is called. Parameters ---------- @@ -500,7 +500,7 @@ def test_with_dependencies_list(self, stage_test) -> None: stage_test.with_dependencies("stage2", "stage3") assert stage_test.dependencies == original_deps - def test_with_dependencies_errors(self, stage_test) -> None: + def test_with_dependencies_errors(self, stage_test) -> None: """ Tests that a StageDependencyError is raised if a nested list is provided in dependencies. @@ -513,7 +513,7 @@ def test_with_dependencies_errors(self, stage_test) -> None: with pytest.raises(StageDependencyError): stage_test.with_dependencies(["stage2", ["nested_stage"]]) - def test_with_dependencies_list_positional_args(self, stage_test) -> None: + def test_with_dependencies_list_positional_args(self, stage_test) -> None: """ Tests that with_dependencies can accept a list and positional arguments in the same call and combine them into a single normalized dependencies tuple. @@ -529,7 +529,7 @@ def test_with_dependencies_list_positional_args(self, stage_test) -> None: "stage_1", "stage2", "stage3", - "stage4" + "stage4", ) def test_with_dependencies_duplicates(self, stage_test) -> None: @@ -537,45 +537,51 @@ def test_with_dependencies_duplicates(self, stage_test) -> None: Tests that when the same dependency is added through with_dependencies, it is not duplicated in the dependencies tuple of the new Stage instance. - Caution that this only deduplicates due to Stage post_init calling - _normalize_dependencies however if that moves, + Caution that this only deduplicates due to Stage post_init calling + _normalize_dependencies however if that moves, test_normalise_dependencies_within_stage_init will capture the issue. Parameters ---------- ``stage_test`` : Stage A ``Stage`` object created with a callable source for testing. - """ + """ new_stage = stage_test.with_dependencies(["stage_1"]) assert new_stage.dependencies == ("stage_1",) # TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST + @pytest.fixture def temp_script(tmp_path): """ Fixture factory that creates temporary Python scripts. - + Usage: script = temp_script("def main(): pass") script = temp_script("def process(): return 42", "processor.py") """ + def _create_script(content="def main(): pass\n", filename="temp_script.py"): script = tmp_path / filename script.write_text(content, encoding="utf-8") return script + return _create_script + class TestStageFactories: """ Parent class for tests which create Stage class instances from different methods. """ + class TestStageFromFile(TestStageFactories): """ Class which tests the creation of Stage class instances from a file path. """ + def test_stage_instance_from_file(self, temp_script) -> None: """ Tests that a Stage instance is created from a filepath where name is either @@ -584,7 +590,7 @@ def test_stage_instance_from_file(self, temp_script) -> None: Parameters ---------- ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. """ test_stage = temp_script( @@ -608,24 +614,24 @@ def main(): ) def test_stage_from_files_error(self, tmp_path: Path) -> None: - """ - Tests that if the file doesn't exist, a StageConfigurationError is raised. - - Parameters - ---------- - ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and - manipulation. - - Raises - ------ - ``StageConfigurationError`` - If the source file doesn't exist when attempting to create a - ``Stage`` instance - """ - source_file = tmp_path / "not_an_actual_file.py" - with pytest.raises(StageConfigurationError): - Stage.from_file(source_file) + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + Stage.from_file(source_file) def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch): """ @@ -637,7 +643,7 @@ def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch) ``temp_script`` : callable A fixture factory that creates temporary Python scripts. ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for temporary modification of environment + A pytest fixture that allows for temporary modification of environment variables and other attributes during testing. """ script = temp_script() @@ -652,16 +658,16 @@ def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch) def test_from_file_expands_source_path(self, tmp_path, monkeypatch): """ Tests that a path with a tilde (~) is expanded to the user's home directory - when passed to from_file. Uses monkeypatch to set a fake home directory for + when passed to from_file. Uses monkeypatch to set a fake home directory for testing purposes. Parameters ---------- ``tmp_path`` : Path - A temporary path provided by pytest for testing file creation and + A temporary path provided by pytest for testing file creation and manipulation. ``monkeypatch`` : pytest.MonkeyPatch - A pytest fixture that allows for temporary modification of environment + A pytest fixture that allows for temporary modification of environment variables and other attributes during testing. """ fake_home = tmp_path / "fake_home" @@ -684,6 +690,7 @@ class TestStageFromCallable(TestStageFactories): """ Class which tests the creation of Stage class instances from a callable object. """ + def test_stage_from_callable_name(self, example_function) -> None: """ Tests that a stage name is extracted from a callable object stage. @@ -701,9 +708,11 @@ def test_from_callable_fallback_name(self): Tests that a fallback name is assigned to a stage instance if the callable object does not have a name attribute. """ + class NoName: - def __call__(self): pass - + def __call__(self): + pass + stage = Stage.from_callable(NoName()) assert stage.name == "stage" @@ -719,6 +728,7 @@ def test_from_callable_explicit_name(self, example_function) -> None: test = Stage.from_callable(example_function, name="explicit_name") assert test.name == "explicit_name" + class TestStageFromDict(TestStageFactories): """ Class which tests the creation of Stage class instances from a dictionary. @@ -729,7 +739,7 @@ def test_from_dict_callable_sources(self, example_function) -> None: Tests that callable sources are correctly used to create a stage instance from a dictionary regardless of whether the key is source or callable. Also validates that the name is correctly assigned from the dictionary or - derived from the callable. + derived from the callable. Parameters ---------- @@ -750,7 +760,7 @@ def test_from_dict_callable_sources(self, example_function) -> None: def test_from_dict_aliases(self, temp_script) -> None: """ Tests that a stage instance is created from a dictionary item with aliases - source and path as source options. + source and path as source options. Parameters ---------- @@ -801,11 +811,11 @@ def test_from_dict_errors(self) -> None: with pytest.raises(StageConfigurationError): Stage.from_dict(data) - def test_all_keys_from_dict_in_stage(self, example_function) -> None: + def test_all_keys_from_dict_in_stage(self, example_function) -> None: """ Checks that from_dict does not change the originally parsed dictionary so that if the dictionary is needed later, it is not permanently changed when - creating a stage instance from it. + creating a stage instance from it. Parameters ---------- From 6e36d63242380c89013bf5bfb005a8af328fb984 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 13:35:15 +0100 Subject: [PATCH 324/332] fix: Added fixture to allow StageResult testing to work properly. --- tests/test_models.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_models.py b/tests/test_models.py index ebeddfc..5ab8849 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -286,8 +286,23 @@ def runmanifest() -> RunManifest: ) +@pytest.fixture +def stageresult() -> StageResult: + """ + Example StageResult instance for tests that use the module-level fixture. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs="example output", + ) + + class TestStageResult: - def test_stage_result(self, stageresult) -> None: + def test_stage_result(self, stageresult: StageResult) -> None: """ Uses a StageResult instance created in test_execution to ensure that the class instance is created suitably with required defaults. From cd441a1a04b2d41dde0b6e8c82957193d2234605 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 13:49:00 +0100 Subject: [PATCH 325/332] feat: added py.typed for best practice downstream type-checking information. added whitespace to toml --- onsrap/py.typed | 0 pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 onsrap/py.typed diff --git a/onsrap/py.typed b/onsrap/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index b20ccda..83f0f80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,4 +57,4 @@ files = ["onsrap"] show_error_codes = true warn_redundant_casts = true warn_unused_configs = true -warn_unused_ignores = true \ No newline at end of file +warn_unused_ignores = true From 5dc068811fa1e8886d990fb283d1f74c6735f00b Mon Sep 17 00:00:00 2001 From: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:46:38 +0100 Subject: [PATCH 326/332] fix: change configuration.md to better reflect Configuration structure Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration.md b/configuration.md index 907b3d1..d66d58b 100644 --- a/configuration.md +++ b/configuration.md @@ -8,11 +8,12 @@ point where individual stage scripts read their own variables at execution time. ## Overview -onsrap uses two distinct levels of configuration. +onsrap uses three distinct levels of configuration. | Level | Object | Scope | |---|---|---| | Pipeline | `PipelineConfig` | Execution environment, directories, backend, run metadata | +| Global | `GlobalConfig` | Variables shared across stages, with per-stage exclusions | | Stage | `StageConfig` | Per-stage variables injected at execution time | Both objects are constructed during `Pipeline` initialisation and are immutable From be1adc739465233bebe168749f5a49e16830a1ca Mon Sep 17 00:00:00 2001 From: Alex Sweet <148556854+BelowBayesline@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:47:30 +0100 Subject: [PATCH 327/332] fix: readme.md language around Stage tweaked to better reflect use case. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b063a8..a278756 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ requirements. It's suggested that you install this package and its requirements within a virtual environment. -Stages must be written in functional programming. A ``stage`` can be a file or a callable item, such as a function. If the ``stage`` is a file, it must have an entrypoint function (a function that, when called, runs the entirety of the stage). The ``stage`` file can run without an entrypoint, however the package has less control over the implementation and therefore best practice is inclusion of an entrypoint. +Stages should use a functional style. A ``stage`` can be a file or a callable item, such as a function. File stages should define an entrypoint function that runs the stage; files without an entrypoint can use subprocess fallback, but the package has less control over that execution mode. There should be a parent file that sets out configuration, required directories and file paths, and builds the ``Pipeline`` instance. It is recommended that this is named something similar to ``main.py`` so that it is easy for users to see where the ``Pipeline`` starts. This file will be what is run through the terminal to run the entire pipeline. From 4b88644dced6639a63bf19675d6f5dceede6ce53 Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 15:42:47 +0100 Subject: [PATCH 328/332] fix: dedented dunder methods for PipelineRunner. Changed Logger._logger.setLevel to outside if-statement scope so Logging level is consistent. --- onsrap/logger.py | 7 +++---- onsrap/runner.py | 52 ++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 3d6a02d..ae8d9f0 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -54,13 +54,12 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.config = LogConfig(log_dir=str(log_dir), log_level=log_level) self.log_dir = Path(self.config.log_dir) self.log_dir.mkdir(parents=True, exist_ok=True) - logger_name = f"{self.config.logger_name}:{self.log_dir.resolve()}" self._logger = logging.getLogger(logger_name) + self._logger.setLevel( + getattr(logging, self.config.log_level.upper(), logging.INFO) + ) if logger_name not in self._configured_loggers: - self._logger.setLevel( - getattr(logging, self.config.log_level.upper(), logging.INFO) - ) self._logger.propagate = False stream_handler = logging.StreamHandler() diff --git a/onsrap/runner.py b/onsrap/runner.py index f06cb6f..fdfee28 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -26,33 +26,33 @@ class PipelineRunner: def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() - def __str__(self) -> str: - """ - String method that returns a human-readable representation of the ``PipelineRunner`` class. - - Returns - ------- - str - A string representation of the ``PipelineRunner`` class with its attributes. - """ - return ( - f"PipelineRunner Instance Attributes\n" - f"--------------------------\n" - f"Logger: {self.logger} \n" - ) + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``PipelineRunner`` class. - def __repr__(self) -> str: - """ - Representation method that returns a human readable representation of the ``PipelineRunner`` class. - This method is structured to be more concise than the ``__str__`` method and is - intended for debugging purposes. - - Returns - ------- - str - A string representation of the ``PipelineRunner`` class with its attributes. - """ - return f"PipelineRunner(logger={self.logger})" + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return ( + f"PipelineRunner Instance Attributes\n" + f"--------------------------\n" + f"Logger: {self.logger} \n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``PipelineRunner`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return f"PipelineRunner(logger={self.logger})" def run(self, pipeline: Pipeline) -> PipelineRun: """ From e6135a4ca7973d43d9097ea29e7e05a312d7d1fb Mon Sep 17 00:00:00 2001 From: BelowBayesline Date: Thu, 13 Aug 2026 15:45:09 +0100 Subject: [PATCH 329/332] fix: set orders_per_day to use idxmax not idxmin for greatest value. --- examples/pipeline_2/scripts/2_reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py index c9023b4..ba71526 100644 --- a/examples/pipeline_2/scripts/2_reporting.py +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -28,7 +28,7 @@ def per_region_quantity(orders, values): ##ORDER DAY POP## def orders_per_day(orders, values): delivery_day_frequency = orders["Order_day"].value_counts() - values["highest_delivery_day"] = delivery_day_frequency.idxmin().capitalize() + values["highest_delivery_day"] = delivery_day_frequency.idxmax().capitalize() ##ORDER COUNTS## From 905229450990a1b47ee4a8d826f22d361da161b9 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 17 Aug 2026 13:36:21 +0100 Subject: [PATCH 330/332] fix: Fix as part of copilot review for PR #66 - implement a safe encode/decode methodology that reviews values typed as Any and codes them into a YAML safe input. The decoder then puts it back into the appropriate typing for use back in the pipeline --- onsrap/models.py | 184 +++++++++++++++++++++++++++++++++++++++++-- onsrap/runner.py | 13 ++- tests/test_runner.py | 76 ++++++++++++++++++ 3 files changed, 264 insertions(+), 9 deletions(-) diff --git a/onsrap/models.py b/onsrap/models.py index 8f1cfa1..ca35572 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,8 +1,9 @@ from __future__ import annotations import warnings -from dataclasses import dataclass, field -from datetime import datetime +from base64 import b64decode, b64encode +from dataclasses import asdict, dataclass, field, is_dataclass +from datetime import date, datetime, time from enum import Enum from pathlib import Path from typing import Any, Iterable, Literal, Mapping, Optional, overload @@ -975,10 +976,10 @@ def _pipeline_run_to_dict(self) -> dict[str, Any]: "started_at": self.started_at.isoformat(), "completed_at": self.completed_at.isoformat(), "stage_results": { - result.name: result._stage_result_to_dict() + result.name: _yaml_safe_encode(result._stage_result_to_dict()) for result in self.stage_results }, - "stage_outputs": self.stage_outputs, + "stage_outputs": _yaml_safe_encode(self.stage_outputs), } @classmethod @@ -998,16 +999,20 @@ def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: ``PipelineRun`` class instance A PipelineRun instance created from the dictionary representation. """ + manifest_data = _yaml_safe_decode(data["manifest"]) + stage_results_data = _yaml_safe_decode(data.get("stage_results", {})) + stage_outputs_data = _yaml_safe_decode(data.get("stage_outputs", {})) + return cls( - manifest=RunManifest._runmanifest_from_dict(data["manifest"]), + manifest=RunManifest._runmanifest_from_dict(manifest_data), status=PipelineStatus(data["status"]), started_at=datetime.fromisoformat(data["started_at"]), completed_at=datetime.fromisoformat(data["completed_at"]), stage_results=[ StageResult._stage_result_from_dict(result) - for result in data.get("stage_results", {}).values() + for result in stage_results_data.values() ], - stage_outputs=data.get("stage_outputs", {}), + stage_outputs=stage_outputs_data, ) @classmethod @@ -1081,3 +1086,168 @@ def _format_dict(d: dict[str, Any] | dict[str, bool] | None, indent: int = 0) -> else: lines.append(f"{' ' * indent}{key}: {value}") return "\n".join(lines) + + +_YAML_TYPE_KEY = "__onsrap_yaml_type__" +_YAML_VALUE_KEY = "value" + + +def _yaml_safe_mapping_key(value: Any) -> str | int | float | bool | None: + """ + Convert mapping keys to YAML-safe scalar values. + + Complex key types are coerced to strings because YAML mappings require + hashable scalar-like keys to round-trip predictably with ``yaml.safe_load``. + """ + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, Enum): + return str(value.value) + if isinstance(value, Path): + return str(value) + return repr(value) + + +def _yaml_safe_encode(value: Any) -> Any: + """ + Convert arbitrary Python values into structures accepted by ``yaml.safe_dump``. + """ + if isinstance(value, (str, int, float, bool)) or value is None: + return value + + if isinstance(value, datetime): + return {_YAML_TYPE_KEY: "datetime", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, date): + return {_YAML_TYPE_KEY: "date", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, time): + return {_YAML_TYPE_KEY: "time", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, Path): + return {_YAML_TYPE_KEY: "path", _YAML_VALUE_KEY: str(value)} + + if isinstance(value, Enum): + return {_YAML_TYPE_KEY: "enum", _YAML_VALUE_KEY: _yaml_safe_encode(value.value)} + + if isinstance(value, bytes): + return { + _YAML_TYPE_KEY: "bytes", + _YAML_VALUE_KEY: b64encode(value).decode("ascii"), + } + + if isinstance(value, bytearray): + return { + _YAML_TYPE_KEY: "bytearray", + _YAML_VALUE_KEY: b64encode(bytes(value)).decode("ascii"), + } + + if isinstance(value, tuple): + return { + _YAML_TYPE_KEY: "tuple", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, set): + return { + _YAML_TYPE_KEY: "set", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, frozenset): + return { + _YAML_TYPE_KEY: "frozenset", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, list): + return [_yaml_safe_encode(item) for item in value] + + if isinstance(value, Mapping): + return { + _yaml_safe_mapping_key(key): _yaml_safe_encode(item) + for key, item in value.items() + } + + if is_dataclass(value) and not isinstance(value, type): + return { + _YAML_TYPE_KEY: "dataclass", + "python_type": f"{value.__class__.__module__}.{value.__class__.__qualname__}", + _YAML_VALUE_KEY: _yaml_safe_encode(asdict(value)), + } + + return { + _YAML_TYPE_KEY: "repr", + "python_type": f"{value.__class__.__module__}.{value.__class__.__qualname__}", + _YAML_VALUE_KEY: repr(value), + } + + +def _yaml_safe_decode(value: Any) -> Any: + """ + Decode values previously produced by ``_yaml_safe_encode``. + """ + if isinstance(value, list): + return [_yaml_safe_decode(item) for item in value] + + if not isinstance(value, Mapping): + return value + + marker = value.get(_YAML_TYPE_KEY) + if marker is None: + return {key: _yaml_safe_decode(item) for key, item in value.items()} + + encoded_value = value.get(_YAML_VALUE_KEY) + + if marker == "datetime": + try: + return datetime.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "date": + try: + return date.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "time": + try: + return time.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "path": + return Path(str(encoded_value)) + + if marker == "enum": + return _yaml_safe_decode(encoded_value) + + if marker == "bytes": + try: + return b64decode(str(encoded_value).encode("ascii")) + except Exception: + return encoded_value + + if marker == "bytearray": + try: + return bytearray(b64decode(str(encoded_value).encode("ascii"))) + except Exception: + return encoded_value + + if marker == "tuple": + return tuple(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "set": + return set(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "frozenset": + return frozenset(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "dataclass": + return _yaml_safe_decode(encoded_value) + + if marker == "repr": + return encoded_value + + return {key: _yaml_safe_decode(item) for key, item in value.items()} diff --git a/onsrap/runner.py b/onsrap/runner.py index f06cb6f..22c55d3 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -7,7 +7,14 @@ from .errors import StageExecutionError from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, RunManifest, StageResult, now +from .models import ( + PipelineRun, + PipelineStatus, + RunManifest, + StageResult, + _yaml_safe_encode, + now, +) if TYPE_CHECKING: from .pipeline import Pipeline @@ -297,8 +304,10 @@ def _log_config( ) import yaml + config_to_dump = _yaml_safe_encode(manifest.config) + with open(config_file, "w", encoding="utf-8") as f: - yaml.safe_dump(manifest.config or {}, f, default_flow_style=False) + yaml.safe_dump(config_to_dump or {}, f, default_flow_style=False) def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: diff --git a/tests/test_runner.py b/tests/test_runner.py index dfe3f69..531f159 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path from textwrap import dedent @@ -293,3 +294,78 @@ def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: assert parsed_yaml == expected_contents assert PipelineRun._pipeline_run_from_dict(parsed_yaml) == pipeline_run + + def test_log_pipeline_attributes_serializes_arbitrary_stage_outputs( + self, tmp_path: Path + ) -> None: + """ + Tests that stage outputs containing non-YAML-native Python objects are + serialized safely and can be reconstructed for supported types. + """ + + class CustomOutput: + def __repr__(self) -> str: + return "CustomOutput(example)" + + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True, exist_ok=True) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=PipelineConfig(name="synthetic_pipeline"), + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + run_manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + ) + + pipeline_run = PipelineRun( + manifest=run_manifest, + status=PipelineStatus.SUCCEEDED, + started_at=context.started_at, + completed_at=now(), + stage_results=[], + stage_outputs={ + "path_value": Path("data/interim/output.csv"), + "datetime_value": datetime(2026, 8, 17, 12, 30, 45), + "tuple_value": (1, "a"), + "set_value": {1, 2}, + "bytes_value": b"abc", + "bytearray_value": bytearray(b"xyz"), + "custom_value": CustomOutput(), + }, + ) + + _log_pipeline_attributes( + pipeline_run=pipeline_run, run_dir=run_dir, context=context + ) + + expected_file = run_dir / ( + "pipeline_attributes_for_" + f"{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + parsed_yaml = yaml.safe_load(expected_file.read_text(encoding="utf-8")) + + assert isinstance(parsed_yaml["stage_outputs"]["path_value"], dict) + loaded_pipeline_run = PipelineRun._pipeline_run_from_dict(parsed_yaml) + + assert loaded_pipeline_run.stage_outputs["path_value"] == Path( + "data/interim/output.csv" + ) + assert loaded_pipeline_run.stage_outputs["datetime_value"] == datetime( + 2026, 8, 17, 12, 30, 45 + ) + assert loaded_pipeline_run.stage_outputs["tuple_value"] == (1, "a") + assert loaded_pipeline_run.stage_outputs["set_value"] == {1, 2} + assert loaded_pipeline_run.stage_outputs["bytes_value"] == b"abc" + assert loaded_pipeline_run.stage_outputs["bytearray_value"] == bytearray(b"xyz") + assert ( + loaded_pipeline_run.stage_outputs["custom_value"] == "CustomOutput(example)" + ) From 58ec369b9eaab7f545e2f817bbaf3b904839f1ba Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 17 Aug 2026 13:37:18 +0100 Subject: [PATCH 331/332] test: assert warnings in test_pipeline TestPipelineNamingAndInit --- tests/test_pipeline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index e0a913e..eefd4a4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -117,7 +117,10 @@ def example_function(): "example_function": ("Stage_1.py",), } - with pytest.raises((PipelineInitialisationError, PipelineConfigurationWarning)): + with ( + pytest.raises(PipelineInitialisationError), + pytest.warns(PipelineConfigurationWarning), + ): Pipeline(stages=None, dependencies=dependencies_single) with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): @@ -131,6 +134,7 @@ def example_function(): dependencies=dependencies_multiple, ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): pipeline_2 = Pipeline( name="pipeline_2", stages=[ @@ -241,7 +245,8 @@ def test_add_dependencies_single_dict(self, tmp_path): with pytest.raises(PipelineInitialisationError): pipeline_dict.add_dependencies(dep_tuple) - pipeline_dict.add_dependencies(dep_dict) + with pytest.warns(PipelineConfigurationWarning): + pipeline_dict.add_dependencies(dep_dict) assert stage_1.dependencies == ("Stage_0",) assert stage_2.dependencies == ( "Stage_1", From 324f1274c2e591382f68e4a4425e8f4a0194f6a1 Mon Sep 17 00:00:00 2001 From: Sophie Pike Date: Mon, 17 Aug 2026 15:00:52 +0100 Subject: [PATCH 332/332] fix: action comment on PR #66 to check that run_ids compare names against the Pipeline as well to add assurance that only these pipeline runs are returned --- onsrap/logger.py | 11 +++++--- onsrap/pipeline.py | 8 ++++-- tests/test_logger.py | 59 ++++++++++++++++++++++++++++++------------ tests/test_pipeline.py | 39 ++++++++++++++++------------ 4 files changed, 79 insertions(+), 38 deletions(-) diff --git a/onsrap/logger.py b/onsrap/logger.py index 3d6a02d..e7b5bab 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -157,7 +157,9 @@ def warning(self, message: str, **kwargs: Any) -> None: else: self._logger.warning(message) - def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: + def extract_historical_run_ids( + self, run_root: Path, name: str + ) -> list[dict[str, Any]]: """ Extracts historical run IDs from the log files. @@ -165,6 +167,8 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: ---------- ``run_root`` : Path The root directory where the historical runs are stored. + ``name`` : str + The name of the pipeline for which to extract historical run IDs. Returns ------- @@ -238,9 +242,10 @@ def extract_historical_run_ids(self, run_root: Path) -> list[dict[str, Any]]: timestamp = f"{parts[0]} {parts[1]}" run_dir = run_root / run_id - # only returns run_ids for runs where a run_directory is still present. - if run_dir.exists(): + + log_name = payload.get("name") + if run_dir.exists() and log_name == name: matches.append( { "run_id": run_id, diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index 1395792..cebb27e 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -592,7 +592,9 @@ def _load_latest_run(self) -> PipelineRun | None: Pipeline, or None if no previous runs are found. """ try: - previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output, self.name + ) except HistoricalPipelineLoadError: warnings.warn( "Unable to load previous runs for this Pipeline. Last_run" @@ -640,7 +642,9 @@ def _load_all_runs(self) -> dict[str, PipelineRun] | None: indicate a None value will be stored in this attribute. """ try: - previous_run_logs = self.logger.extract_historical_run_ids(self.run_output) + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output, self.name + ) except HistoricalPipelineLoadError: warnings.warn( "Unable to load previous runs for this Pipeline. All_run" diff --git a/tests/test_logger.py b/tests/test_logger.py index d85f50d..40bcf88 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -31,7 +31,9 @@ def test_logger_no_handler_errors(self, tmp_path: Path) -> None: logger._logger.handlers.clear() # Remove all handlers to simulate no file logging logger._logger.propagate = False # Prevent checking root logger handlers with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): - logger.extract_historical_run_ids(run_root=tmp_path / "runs") + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) def test_logger_no_file_handler_errors(self, tmp_path: Path) -> None: """ @@ -56,7 +58,9 @@ def test_logger_no_file_handler_errors(self, tmp_path: Path) -> None: with pytest.raises( HistoricalPipelineLoadError, match="does not have a FileHandler" ): - logger.extract_historical_run_ids(run_root=tmp_path / "runs") + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) def test_logger_does_not_exist(self, tmp_path: Path) -> None: """ @@ -85,7 +89,9 @@ def test_logger_does_not_exist(self, tmp_path: Path) -> None: with pytest.raises( HistoricalPipelineLoadError, match="does not exist at this location" ): - logger.extract_historical_run_ids(run_root=tmp_path / "runs") + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) def test_return_blank_list_no_matches_in_log(self, tmp_path) -> None: """ @@ -112,7 +118,9 @@ def test_return_blank_list_no_matches_in_log(self, tmp_path) -> None: "2026-08-10 10:00:02,000 Pipeline Started\n" ) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result == [] def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: @@ -138,8 +146,10 @@ def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: log_path.write_text( "2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" - '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' - '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' ) create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" @@ -148,7 +158,9 @@ def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" create_run_dir_2.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result == [ { "run_id": "2026-06-23_101719_abc1234", @@ -163,7 +175,7 @@ def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: ] @pytest.mark.parametrize( - "string, expected", [('{"some_key":"some_value"}', []), ('{"run_id":""}', [])] + "string, expected", [('{"name":"test_pipeline"}', []), ('{"run_id":""}', [])] ) def test_run_id_absent_falsy( self, tmp_path: Path, string: str, expected: list @@ -197,7 +209,9 @@ def test_run_id_absent_falsy( # creates directory for runs to avoid removal given the directory doesn't exist (tmp_path / "runs").mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result == expected def test_records_only_if_directory_exists(self, tmp_path) -> None: @@ -220,14 +234,18 @@ def test_records_only_if_directory_exists(self, tmp_path) -> None: log_path = Path(file_handler.baseFilename) log_path.write_text( - '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' - '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' ) create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result == [ { "run_id": "2026-06-23_101719_878fcb33", @@ -256,8 +274,10 @@ def test_reverse_chronological_order(self, tmp_path) -> None: log_path = Path(file_handler.baseFilename) log_path.write_text( - '2026-08-10 10:00:01,000 Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' - '2026-08-10 10:00:02,000 Pipeline started | {"run_id": "2026-06-23_101719_abc1234"}\n' + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline",' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' ) create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" @@ -266,7 +286,9 @@ def test_reverse_chronological_order(self, tmp_path) -> None: create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" create_run_dir_2.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result[0]["run_id"] == "2026-06-23_101719_abc1234" assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" @@ -289,11 +311,14 @@ def test_skip_poor_timestamps(self, tmp_path) -> None: log_path = Path(file_handler.baseFilename) log_path.write_text( - 'BADTIMESTAMP Pipeline started | {"run_id": "2026-06-23_101719_878fcb33"}\n' + 'BADTIMESTAMP Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' ) create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" create_run_dir_1.mkdir(parents=True, exist_ok=True) - result = logger.extract_historical_run_ids(run_root=tmp_path / "runs") + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) assert result == [] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index eefd4a4..980bb97 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -749,6 +749,7 @@ def pipeline_no_history(self, tmp_path: Path) -> Pipeline: """ with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( + name="test_pipeline", config=PipelineConfig( output_dir=tmp_path / "outputs", ), @@ -785,11 +786,11 @@ def test_blank_historical_run_ids( that the last_run attribute will be None. """ monkeypatch.setattr( - pipeline_no_history.logger, "extract_historical_run_ids", lambda x: [] + pipeline_no_history.logger, "extract_historical_run_ids", lambda x, y: [] ) assert ( pipeline_no_history.logger.extract_historical_run_ids( - pipeline_no_history.run_output + pipeline_no_history.run_output, pipeline_no_history.name ) == [] ) @@ -824,7 +825,7 @@ def test_blank_run_ids(self, pipeline_no_history: Pipeline, monkeypatch) -> None monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": None, "timestamp": "2026-08-10 10:00:00,000", @@ -833,7 +834,7 @@ def test_blank_run_ids(self, pipeline_no_history: Pipeline, monkeypatch) -> None ], ) assert pipeline_no_history.logger.extract_historical_run_ids( - pipeline_no_history.run_output + pipeline_no_history.run_output, pipeline_no_history.name ) == [ { "run_id": None, @@ -869,7 +870,7 @@ def test_load_latest_run_success( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda _: [ + lambda x, y: [ { "run_id": run_id, "timestamp": "2026-08-10 10:00:00,000", @@ -911,7 +912,7 @@ def test_which_run_is_selected_load_latest_run( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda _: [ + lambda x, y: [ { "run_id": run_id_1, "timestamp": "2026-08-10 10:00:00,000", @@ -959,7 +960,7 @@ def test_no_errors_raised_success_load_latest_run( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda _: [ + lambda x, y: [ { "run_id": run_id, "timestamp": "2026-08-10 10:00:00,000", @@ -1036,6 +1037,7 @@ def test_last_run_populated_one_run( (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " '{"run_id": "2026-08-11_100000_abc12345", ' + '"name": "test_pipeline", ' ' "run_dir": "/path/to/run"}\n' ) @@ -1053,6 +1055,7 @@ def test_last_run_populated_one_run( with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( + name="test_pipeline", config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), stages=[ Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) @@ -1087,9 +1090,11 @@ def test_last_run_most_recent(self, tmp_path: Path, minimal_pipeline_yaml) -> No (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " '{"run_id": "run_older", ' + '"name": "test_pipeline", ' ' "run_dir": "/path/to/run"}\n' "2026-08-11 10:01:00,000 Pipeline started | " '{"run_id": "run_newer", ' + '"name": "test_pipeline", ' ' "run_dir": "/path/to/run"}' ) @@ -1119,6 +1124,7 @@ def test_last_run_most_recent(self, tmp_path: Path, minimal_pipeline_yaml) -> No with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( + name="test_pipeline", config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), stages=[ Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) @@ -1154,10 +1160,10 @@ def test_last_run_most_recent_no_directory( logs.mkdir(parents=True, exist_ok=True) (logs / "onsrap.log").write_text( "2026-08-11 10:00:00,000 Pipeline started | " - '{"run_id": "run_older", ' + '{"name": "test_pipeline", "run_id": "run_older", ' ' "run_dir": "/path/to/run"}\n' "2026-08-11 10:01:00,000 Pipeline started | " - '{"run_id": "run_newer", ' + '{"name": "test_pipeline", "run_id": "run_newer", ' ' "run_dir": "/path/to/run"}' ) @@ -1175,6 +1181,7 @@ def test_last_run_most_recent_no_directory( with pytest.warns(PipelineConfigurationWarning): pipeline = Pipeline( + name="test_pipeline", config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), stages=[ Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) @@ -1242,12 +1249,12 @@ def test_returns_none_when_blank_extract_historical_runs( all_runs attribute will be None. """ monkeypatch.setattr( - pipeline_no_history.logger, "extract_historical_run_ids", lambda x: [] + pipeline_no_history.logger, "extract_historical_run_ids", lambda x, y: [] ) assert ( pipeline_no_history.logger.extract_historical_run_ids( - pipeline_no_history.run_output + pipeline_no_history.run_output, pipeline_no_history.name ) == [] ) @@ -1284,7 +1291,7 @@ def test_single_entry_dict_single_run( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": "run_A", "timestamp": "2026-08-10 10:00:00,000", @@ -1332,7 +1339,7 @@ def test_multiple_entries_dict_multiple_runs( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": "run_A", "timestamp": "2026-08-10 10:00:00,000", @@ -1375,7 +1382,7 @@ def test_warning_if_no_run_id( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": "", "timestamp": "2026-08-10 10:00:00,000", @@ -1429,7 +1436,7 @@ def test_None_with_stageloaderror( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": "good_run", "timestamp": "2026-08-10 10:00:00,000", @@ -1480,7 +1487,7 @@ def test_none_if_all_stageloaderrors( monkeypatch.setattr( pipeline_no_history.logger, "extract_historical_run_ids", - lambda x: [ + lambda x, y: [ { "run_id": "bad_run1", "timestamp": "2026-08-10 10:00:00,000",