diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000..29b0c82 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,54 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Build documentation + run: | + python -m sphinx -b html docs docs/_build/html + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload documentation artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/_build/html + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy documentation + id: deployment + uses: actions/deploy-pages@v4 + \ No newline at end of file diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml new file mode 100644 index 0000000..3a53cf5 --- /dev/null +++ b/.github/workflows/package-build.yml @@ -0,0 +1,27 @@ +name: Python Package Builds Successfully + +on: + pull_request: + push: + branches: + - main + - development + +jobs: + build-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install build tools + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build Package + run: | + python -m build --sdist --wheel + - name: Install Built Wheel + run: | + pip install dist/*.whl diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f321dd8..fecf263 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,40 +1,62 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python package +name: Python CI on: + # Keep push validation narrow to the main integration branches. push: branches: [ "main", "development" ] + + # Run CI for every pull request regardless of target branch so branch-to-branch + # work still gets the same quality and test coverage before merge. pull_request: - branches: [ "main", "development" ] + types: [opened, synchronize, reopened, ready_for_review] + + workflow_dispatch: jobs: - build: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + - name: Install development dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Lint with Ruff + run: | + python -m ruff check onsrap + python -m ruff format --check onsrap + - name: Run security checks + run: | + python -m bandit -ll -c pyproject.toml -r onsrap + - name: Type check with mypy + run: | + python -m mypy onsrap + + tests: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + cache: pip + - name: Install development dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + python -m pip install -e ".[dev]" + python -m pip install pandas - name: Test with pytest run: | - pytest + python -m pytest diff --git a/.gitignore b/.gitignore index 87a11f2..59f0e36 100644 --- a/.gitignore +++ b/.gitignore @@ -895,18 +895,24 @@ docs/_linkcheck/ !examples/ !examples/** -# Allow logs folders and their contents inside examples -!examples/**/logs/ -!examples/**/logs/** -!examples/**/*.log # Allow scripts folders and their contents inside examples !examples/**/scripts/ !examples/**/scripts/** +examples/**/scripts/__pycache__/ # Allow example data files !examples/**/*.csv !examples/**/*.xls !examples/**/*.xlsx -examples/**/runs/**/ \ No newline at end of file +examples/**/runs/**/ + +# Ignore coding playground +tests/playground/ +tests/sandbox/ + +# Allow logs folders and their contents inside examples +examples/**/logs/ +examples/**/logs/** +examples/**/*.log \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03ad356..3877cfb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,15 @@ repos: args: [ --fix ] # Run ruff formatter. - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy - Type check onsrap package + entry: mypy + language: python + additional_dependencies: [mypy, types-PyYAML] + args: [onsrap] + pass_filenames: false - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 hooks: @@ -36,5 +45,5 @@ repos: hooks: - id: bandit name: bandit - Checks for vulnerabilities - args: ["-c", "pyproject.toml"] + args: ["-ll", "-c", "pyproject.toml"] additional_dependencies: ["bandit[toml]"] diff --git a/CHANGELOG.md b/CHANGELOG.md index f519e85..855862e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be tracked here. +## [0.1.1] - 2026-06-24 + +This release adds run-scoped output handling for the example pipeline and aligns runtime timestamps with local time. + +### Added + +- Introduced per-run output directories so each pipeline execution writes into its own `runs//` tree. +- Anchored the example pipeline to its own `main.py` directory so runs stay inside `examples/pipeline_1/` instead of the repository working directory. +- Routed stage output paths through `ExecutionContext.run_dir` so stages can record artifacts in the active run directory. +- Switched runtime timestamp generation to local time and kept a compatibility alias for the previous helper. +- Added regression coverage for repeated runs and run-specific output paths. + + ## [0.1.0] - 2026-06-22 First tracked version of `onsrap`. diff --git a/README.md b/README.md index db12f9c..a278756 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,20 @@ located. ``` ## What is `onsrap`? -Add a summary of your project here. +Reproducible Analytical Pipelines (RAPs) are a cornerstone of high quality statistics. Reproducible refers to the concept that if code is run multiple times with the same inputs, it will produce the same outputs. A pipeline is a series of stages (small chunks of work) which are run in a specified order to produce desired outputs. Pipelines are crucial to reproducible work as they ensure that the code is run consistently. This increases the quality of the outputs by ensuring as little manual input as possible. + +ONSRap is a Python package that automatically orchestrates and runs these RAPs. The goal is to standardise how pipelines are run to reduce developer time required to convert existing code into RAP standards. As well as reducing developer time, this package also supports achieving RAP standards through items 4, 6, and 10. These are accomplished through this package by: + + - Item 4: Document everything that is needed to write and run the code + - This package includes inbuilt logging that records when the pipeline was run as well as the Pipeline configuration used. + +- Item 6: Code modules should run end-to-end without manual intervention + - This package is designed whereby once the configuration has been provided by the user and the main.py file is run, no further human input is required. + +- Item 10: Don't reinvent the wheel + - Multiple pipelines exist within the ONS and each have the potential to be orchestrated in different ways. This package aims to standardise the orchestration, ensuring consistency across pipelines. This consistency means that developers are more easily able to move between pipelines as they will all be structured in a similar way. + +For more information on the ONS Rap Minimum Standards, please see the full [standards documentation][standards]. ## Getting started @@ -18,11 +31,13 @@ requirements. It's suggested that you install this package and its requirements within a virtual environment. +Stages should use a functional style. A ``stage`` can be a file or a callable item, such as a function. File stages should define an entrypoint function that runs the stage; files without an entrypoint can use subprocess fallback, but the package has less control over that execution mode. + +There should be a parent file that sets out configuration, required directories and file paths, and builds the ``Pipeline`` instance. It is recommended that this is named something similar to ``main.py`` so that it is easy for users to see where the ``Pipeline`` starts. This file will be what is run through the terminal to run the entire pipeline. + ## Requirements -- Python 3.9+ installed -- a `.secrets` file with the required [secrets and credentials](#required-secrets-and-credentials) -- to have [loaded environment variables][docs-loading-environment-variables] from `.env` +- Python 3.14.6 installed Contributors have some additional requirements - please see our [contributing guidance][contributing]. @@ -53,7 +68,18 @@ Remember to update the setup and requirement files inline with any changes to yo package. ## Running the pipeline (Python only) +### Running Your Own Pipeline +To run your own Pipeline, you will need to build a ``Pipeline`` instance. This can be built using the from_files() method which requires a list of strings or Paths for your individual ``stages``. These are then compiled into a ``Pipeline`` instance. A ``Pipeline`` instance can also be created using the from_dict() method which takes a dictionary containing each attribute of the intented ``Pipeline`` instance and converts it. + +You will also need to define your ``PipelineConfig`` instance. This contains information regarding the working directory, project root, data directory, and log directory required to run the ``Pipeline`` as well as any metadata that you feel needs to be logged. + +Lastly, you need to define any ``dependencies`` required for the ``stages``. These are whether any stage needs to be run before another stage. These should be structured as a dictionary with the name of the stage as the key and the value is the stage/s that need to run before it as a tuple. +Both ``PipelineConfig`` and ``dependencies`` should be parsed into the ``Pipeline`` instance. + +Once you have your ``Pipeline`` instance, you can run the ``Pipeline.run()`` method which will run the entire ``Pipeline`` instance that has been created. + +### Example Pipeline The main runnable example now lives in `examples/pipeline_1/main.py`. It builds a three-stage pipeline from numbered scripts under `examples/pipeline_1/scripts/`. To run the example, use: @@ -66,21 +92,12 @@ Alternatively, most Python IDEs allow you to run the code directly using a `run` ## Required secrets and credentials -To run this project, you need a `.secrets` file with [secrets/credentials as -environmental variables][docs-loading-environment-variables-secrets]. The -secrets/credentials should have the following environment variable name(s): - -| Secret/credential | Environment variable name | Description | -|-------------------|---------------------------|--------------------------------------------| -| Secret 1 | `SECRET_VARIABLE_1` | Plain English description of Secret 1. | -| Credential 1 | `CREDENTIAL_VARIABLE_1` | Plain English description of Credential 1. | +No secrets or credentials are required for running this package. -Once you've added them, [load these environment variables][docs-loading-environment-variables] using -`.env`. ## Project structure layout -The cookiecutter template generated for each project will follow this folder structure: +The ONSRap repository has the following structure: ```shell . @@ -89,14 +106,46 @@ The cookiecutter template generated for each project will follow this folder str │ │ ├── raw/ │ │ ├── interim/ │ │ └── processed/ -│ └── onsrap/ -│ ├── example_modules/ -│ │ ├── __init__.py -│ │ └── example_module.py -│ ├── __init__.py -│ ├── example_config.yml -│ └── run_pipeline.py -└── ... +│ ├── onsrap/ +│ │ ├── example_modules/ +│ │ │ ├── __init__.py +│ │ │ └── example_module.py +│ │ ├── __init__.py +│ │ ├── errors.py +│ │ ├── execution.py +│ │ ├── graph.py +│ │ ├── loader.py +│ │ ├── models.py +│ │ ├── pipeline.py +│ │ ├── run_pipeline.py +│ │ ├── runner.py +│ │ └── stage.py +│ ├── examples/ +│ │ ├── pipeline_1/ +│ │ │ ├── data/ +│ │ │ │ └── orders.csv +│ │ │ ├── logs/ +│ │ │ │ └── onsrap.log +│ │ │ ├── runs/ +│ │ │ │ └── README.md +│ │ │ └── scripts/ +│ │ │ │ ├── 0_data_validation.py +│ │ │ │ ├── 1_preprocessing.py +│ │ │ │ └── 2_reporting.py +│ │ │ ├── Example.md +│ │ │ └── main.py +│ │ ├── pipeline_2/ +│ │ └── pipeline_3/ +│ ├── tests/ +│ │ ├── __init__.py +│ │ ├── repo_tests_README.md +│ │ ├── test_execution.py +│ │ ├── test_models.py +│ │ ├── test_pipeline_architecture.py +│ │ ├── test_pipeline.py +│ │ └── test_stage.py +│ └── +└── ``` ## Licence @@ -118,3 +167,4 @@ This project structure is based on the [`govcookiecutter` template project][govc [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter [docs-loading-environment-variables]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md [docs-loading-environment-variables-secrets]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md#storing-secrets-and-credentials +[standards]: https://best-practice-and-impact.github.io/ONS_minimum_RAP/ \ No newline at end of file diff --git a/configuration.md b/configuration.md new file mode 100644 index 0000000..d66d58b --- /dev/null +++ b/configuration.md @@ -0,0 +1,491 @@ +# Configuration + +This document describes how configuration flows through the onsrap pipeline +architecture, from the initial input accepted at construction time through to the +point where individual stage scripts read their own variables at execution time. + +--- + +## Overview + +onsrap uses three distinct levels of configuration. + +| Level | Object | Scope | +|---|---|---| +| Pipeline | `PipelineConfig` | Execution environment, directories, backend, run metadata | +| Global | `GlobalConfig` | Variables shared across stages, with per-stage exclusions | +| Stage | `StageConfig` | Per-stage variables injected at execution time | + +Both objects are constructed during `Pipeline` initialisation and are immutable +for the duration of a run. They are kept separate so that pipeline orchestration +concerns (where to write logs, which Python interpreter to use) never bleed into +the domain logic a stage script contains. + +--- + +## Configuration Objects + +### `PipelineConfig` + +Defined in `onsrap/models.py`. Holds every setting that controls how the runner +behaves. + +| Field | Type | Default | Description | +|---|---|---|---| +| `name` | `str \| None` | `None` | Pipeline name. Back-filled from the `Pipeline.name` argument when absent. | +| `backend` | `str` | `"python"` | Execution backend. Currently only `"python"` is implemented. | +| `work_dir` | `Path` | `Path.cwd()` | Working directory used for stage file discovery and subprocess execution. | +| `project_root` | `Path \| None` | `None` → falls back to `work_dir` | Root used to construct `runs/` output directories. | +| `log_dir` | `Path` | `Path("logs")` | Directory where `onsrap.log` is written. | +| `data_dir` | `Path` | `Path("data")` | Conventional location for input data. Not enforced by the runner; available to stages via `context.config.data_dir`. | +| `output_dir` | `Path \| None` | `None` | Conventional location for pipeline outputs. Not enforced by the runner; available to stages via `context.config.output_dir`. | +| `allow_subprocess_fallback` | `bool` | `True` | When `True`, stage files without a recognised entrypoint function (`run`, `main`, `execute`) are executed as plain scripts via subprocess. Set to `False` to require entrypoints everywhere. | +| `python_executable` | `str \| None` | `None` → `sys.executable` | Python interpreter used in subprocess fallback mode. | +| `metadata` | `dict[str, Any]` | `{}` | Arbitrary additional values. Any unrecognised key from a raw config mapping is absorbed here rather than raising an error. | + +`PipelineConfig` can be constructed directly in code, loaded with +`PipelineConfig.from_any()`, or produced automatically by `Pipeline._resolve_config()` +from a raw mapping or YAML file. + +### `StageConfig` + +Defined in `onsrap/models.py`. Holds every setting that should be visible to one +specific stage script. + +| Attribute | Access | Description | +|---|---|---| +| `name` | `stage_config.name` | The stage name this config belongs to. | +| `_variables` | `.get(key)`, `.require(key)`, `.variables`, `.get_variables(...)` | Arbitrary key/value pairs — the main carrier for stage parameters. All YAML keys not named `datasets` or `metadata` end up here. | +| `datasets` | `stage_config.datasets` | Mapping of dataset identifiers to their properties (e.g. file path, format). | +| `metadata` | `stage_config.metadata` | Supporting metadata about the stage configuration itself (e.g. purpose, owner). | + +Accessing variables from stage code: + +```python +# Optional: returns default if the key is absent +value = context.stage_config.get("years_to_run") +value = context.stage_config.get("years_to_run", default=2020) + +# Mandatory: raises StageConfigurationError if the key is absent +value = context.stage_config.require("target_variable") + +# All variables at once +all_vars = context.stage_config.variables # returns a copy + +# Selected subset (raises if any are missing) +subset = context.stage_config.get_variables(["years_to_run", "target_variable"]) +``` + +--- + +## Entry Points + +`Pipeline` is the single public entry point for all configuration. Four class +methods accept configuration in different ways. + +``` +Pipeline(config=...) — direct constructor; most flexible +Pipeline.from_config(path) — preferred when a composite config file defines everything +Pipeline.from_files([...], config=...) — explicit stage file list with optional config +Pipeline.from_dict({...}) — construct from an in-memory mapping +``` + +All four funnel into `Pipeline.__init__`, which calls `_resolve_config()` as its +first action. That single call produces the three objects the pipeline needs before +execution can start: `PipelineConfig`, `dict[str, StageConfig]`, and +`list[Stage]`. + +--- + +## Supported Config Input Types + +`Pipeline._resolve_config()` and `Pipeline._load_config_mapping()` together +handle the following types for the `config` parameter. + +| Type | Behaviour | +|---|---| +| `None` | Constructs a fully-defaulted `PipelineConfig`. No stage configs. | +| `PipelineConfig` instance | Used directly. Stage configuration may optionally be embedded in `config.metadata["stage_configuration"]` (backwards-compatibility path; emits a `StageConfigurationWarning`). | +| `Mapping[str, Any]` | Parsed as a composite or flat config payload (see below). | +| `str` / `Path` | Read and YAML-parsed; the resulting mapping is treated as above. Only `.yaml` / `.yml` files are accepted. | + +--- + +## Config Payload Formats + +A raw mapping (or YAML file loaded into a mapping) is classified by +`_split_config_sections()` into one of two shapes. + +### Composite format (recommended) + +Used when any of the keys `pipeline_variables`, `stage_configuration`, or +`stage_config` are present at the top level. This is the format used by +`examples/pipeline_2/conf.yaml`. + +```yaml +pipeline_variables: + name: "My Pipeline" + backend: python + working_dir: "path/to/pipeline" # alias for work_dir + project_root: "path/to/pipeline" + log_dir: "path/to/pipeline/logs" + data_dir: "path/to/pipeline/data" + output_dir: "path/to/pipeline/output" + stages: + - 0_data_validation: + location: "" # empty → scripts/0_data_validation.py + run: true + dependencies: [] + - 1_preprocessing: + location: "" + run: true + dependencies: + - 0_data_validation + metadata: + description: "Example pipeline" + +stage_configuration: + 0_data_validation: + years_to_run: 2017 + time_col: "order_date" + target_variable: "classification" + datasets: + orders: + path: "data/orders.csv" + metadata: + purpose: "validate raw inputs" + 1_preprocessing: + drop_columns: ["id", "notes"] +``` + +In this format: +- Everything under `pipeline_variables` becomes `PipelineConfig`. +- Everything under `stage_configuration` becomes the `stage_configs` mapping. +- Any key at the top level outside these two sections is **ignored**. + +### Flat format + +Used when none of the composite section markers are present. The entire mapping +is treated as pipeline config; an optional nested `stage_configuration` or +`stage_config` key within it carries the stage-level config. + +```python +Pipeline.from_files( + ["scripts/0_data_validation.py", "scripts/1_preprocessing.py"], + config={ + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + "stage_configuration": { + "0_data_validation": { + "years_to_run": 2017, + "target_variable": "classification", + } + }, + }, +) +``` + +--- + +## Pipeline-Level Parsing Flow + +``` +config input (any supported type) + │ + ▼ +Pipeline._resolve_config() + │ + ├─ None ─────────────────────────────► PipelineConfig() (defaults) + │ + ├─ PipelineConfig instance ──────────► used as-is + │ (stage config read from .metadata if present) + │ + └─ everything else + │ + ▼ + Pipeline._load_config_mapping() + Normalises input to dict[str, Any]: + Mapping → dict(mapping) + str / Path → yaml.safe_load(file) + │ + ▼ + Pipeline._split_config_sections() + Detects composite vs flat shape. + Returns (pipeline_payload, stage_config_payload). + │ + ▼ + Pipeline._normalize_pipeline_payload() + Maps recognised field aliases: + working_dir → work_dir (only when work_dir absent) + Pops the stages list before handing payload on. + │ + ▼ + PipelineConfig.from_mapping(pipeline_payload) + Extracts recognised fields; remaining keys + are absorbed into PipelineConfig.metadata. +``` + +### Recognised `pipeline_variables` keys + +`name`, `backend`, `work_dir` / `working_dir`, `project_root`, `log_dir`, +`data_dir`, `output_dir`, `allow_subprocess_fallback`, `python_executable`, +`metadata`, `stages`. + +Any other key is silently absorbed into `PipelineConfig.metadata`. This is +intentional — it allows pipelines to carry arbitrary project metadata — but it +also means a typo in a recognised key name will not raise an error. + +--- + +## Stage Configuration Parsing Flow + +``` +stage_configuration section (Mapping[str, Any]) + │ + ▼ +Pipeline._build_stage_configs() +Iterates every stage name in the mapping. +For each stage: + │ + ▼ +StageConfig.from_mapping(stage_name, stage_payload) +Splits the stage payload into three buckets: + datasets → StageConfig.datasets + metadata → StageConfig.metadata + everything else → StageConfig._variables + │ + ▼ +Pipeline.stage_configs (dict[str, StageConfig]) +One entry per configured stage. Stages without an +explicit entry get a default empty StageConfig via +Pipeline._sync_stage_configs(). +``` + +--- + +## Stage Definition Parsing Flow + +When stages are listed under `pipeline_variables.stages` in a composite YAML, +`Pipeline._build_stages_from_config()` converts each entry into a `Stage` object. + +``` +stages: list (each entry one of several forms) + │ + ▼ +Pipeline._stage_from_config_definition() +Dispatches on the type of the entry: + + Stage instance ─────────────────────────────► returned as-is + str / Path / callable ──────────────────────► Pipeline._coerce_stage() + Mapping with name/source/path/callable keys ► Stage.from_dict() + {stage_name: {...}} single-key mapping ─────► stage_name-keyed form (most common in YAML) + │ + ▼ (for single-key YAML form) +Extracts from the inner mapping: + run → if false, stage is skipped entirely + location → source file path (alias: source, path) + dependencies → list of prerequisite stage names + entrypoint → explicit function name (optional) + metadata → stage metadata dict + all others → merged into metadata + │ + ▼ +Pipeline._resolve_stage_source(stage_name, location, work_dir) + location is None or "" ─► work_dir / "scripts" / ".py" + absolute path ──────────► used as-is + relative, exists ───────► used as-is + relative, absent ───────► work_dir / location tried first + neither exists ─────────► raw candidate returned (Stage.validate() will fail later) + │ + ▼ +Stage.from_file(source, name, dependencies, metadata, entrypoint, backend) +Expands the path, verifies it exists, stores resolved absolute path. +``` + +--- + +## Config Consumption at Execution Time + +`PipelineRunner.run()` is the bridge between the `Pipeline` object (which holds +all parsed configuration) and the actual execution of stage scripts. + +``` +Pipeline.stage_configs (dict[str, StageConfig]) + │ + │ copied at run start + ▼ +PipelineRunner.run() + Creates ExecutionContext with: + config = pipeline.config (PipelineConfig) + stage_configs = dict(pipeline.stage_configs) + │ + │ for each stage in topological order + ▼ +context.set_active_stage(stage.name) + Sets ExecutionContext.active_stage_name + │ + ▼ +stage.run(context, executor) + → PythonStageExecutor.execute(stage, context) + → loads stage module; calls entrypoint(context) + +Within the stage function: + context.stage_config # StageConfig for THIS stage + context.stage_config.get(key) # optional variable lookup + context.stage_config.require(key) # mandatory variable lookup + context.stage_config_for(name) # any other stage's StageConfig + context.config # PipelineConfig (directories etc.) + context.run_dir # project_root/runs/ + context.result_for(name) # StageResult from an earlier stage + │ + ▼ +context.set_active_stage(None) # reset after stage finishes +``` + +### What stages can read + +| `context` attribute | Type | Contains | +|---|---|---| +| `context.config` | `PipelineConfig` | Directories, backend, subprocess settings | +| `context.stage_config` | `StageConfig \| None` | This stage's variables, datasets, metadata | +| `context.stage_config_for(name)` | `StageConfig \| None` | Any stage's config by name | +| `context.stage_configs` | `dict[str, StageConfig]` | All stage configs | +| `context.run_id` | `str` | Unique run identifier | +| `context.run_dir` | `Path` | `project_root / "runs" / run_id` | +| `context.result_for(name)` | `StageResult \| None` | Output of a previously run stage | +| `context.stage_outputs` | `dict[str, Any]` | All outputs from stages run so far | + +--- + +## Validation Model + +Configuration parsing and execution validation are intentionally separate phases. + +| Phase | When | What is checked | +|---|---|---| +| **Parse** | `Pipeline.__init__` | Type validity; YAML syntax; that stage source files exist (`Stage.from_file` calls `path.exists()`); `StageConfig` is built for each configured stage name | +| **Validate** | `Pipeline.validate()`, called automatically at the start of every `Pipeline.run()` | Stage source files still exist; dependency graph is acyclic and complete; every name in `stage_configuration` matches a real stage | + +Calling `pipeline.validate()` explicitly before `pipeline.run()` is safe and +useful in test suites or CI pipelines. + +--- + +## Manifest Serialisation + +When a pipeline run completes, `Pipeline._manifest_parameters()` serialises the +full configuration state into `RunManifest.parameters`. This includes the +`PipelineConfig` fields and a nested `stage_configuration` block containing every +`StageConfig.to_dict()`. This means the exact parameters used for any given run +are reproducible from the manifest alone. + +--- + +## Known Pitfalls + +### 1 — Relative paths are resolved against the process working directory + +`work_dir`, `log_dir`, `data_dir`, and `output_dir` are stored as `Path` objects +constructed directly from whatever string is in the config. A value like +`"examples/pipeline_2"` is therefore resolved relative to wherever Python is +running when the pipeline is constructed, not relative to the YAML file's +location. + +**Mitigation**: Use absolute paths in YAML configs, or construct `PipelineConfig` +in Python code where you can use `Path(__file__).parent` to anchor paths relative +to the config module. + +### 2 — Unrecognised `pipeline_variables` keys are silently absorbed into metadata + +`PipelineConfig.from_mapping()` pops every recognised field and then calls +`metadata.update(remaining_payload)`. A typo such as `log_dirs:` instead of +`log_dir:` will not raise; instead the default `Path("logs")` will be used and +the misspelled key will appear in `pipeline.config.metadata`. + +**Mitigation**: When debugging unexpected defaults, inspect +`pipeline.config.metadata` to see which keys were not recognised. + +### 3 — Stage configs are not cross-checked until `validate()` runs + +`Pipeline.__init__` does not validate that every stage named in +`stage_configuration` has a matching stage in the `stages` list. That check runs +in `_validate_stage_configs()`, which is called inside `validate()`, which is +called at the start of `run()`. A name mismatch (e.g., a renamed stage script) +will therefore only surface when the pipeline is actually executed. + +**Mitigation**: Call `pipeline.validate()` explicitly after construction in +environments where you want early failure. + +### 4 — Keys outside `pipeline_variables` in a composite YAML are ignored + +When the composite format is detected, `_split_config_sections()` reads only +`pipeline_variables` and `stage_configuration` / `stage_config`. Any other top-level +key in the YAML file is silently discarded. + +```yaml +pipeline_variables: + name: "my-pipeline" +work_dir: "/path/that/will/be/ignored" # ← this key is outside pipeline_variables +stage_configuration: ... +``` + +### 5 — `allow_subprocess_fallback` as a quoted YAML string + +YAML `false` (unquoted) parses to Python `False`. The quoted string `"false"` +parses to Python `"false"`. Because `PipelineConfig` now explicitly checks for +string values and maps common representations to booleans, a quoted `"false"` +will emit a `UserWarning` and be interpreted as `False`. However, to avoid any +ambiguity, use unquoted YAML booleans: + +```yaml +allow_subprocess_fallback: false # correct — unquoted YAML boolean +allow_subprocess_fallback: "false" # warns — will be treated as False +``` + +### 6 — Passing a composite YAML to `PipelineConfig.from_file()` directly + +`PipelineConfig.from_file()` (and `PipelineConfig.from_any()` when given a path) +does not understand the `pipeline_variables` / `stage_configuration` structure. +If a composite YAML is loaded this way, `pipeline_variables` and +`stage_configuration` are treated as unknown keys and absorbed into +`PipelineConfig.metadata`. The resulting `PipelineConfig` will have default +values for all fields. + +**Mitigation**: Always pass composite YAML files to `Pipeline.from_config()` or +as the `config=` argument to `Pipeline(...)`. Reserve `PipelineConfig.from_file()` +for flat, pipeline-only YAML files. + +### 7 — Stage source resolution for relative non-existent paths + +`_resolve_stage_source()` tries the literal path, then `work_dir / path`. If +neither exists it returns the raw candidate. `Stage.from_file()` then calls +`path.exists()` and raises `StageConfigurationError`. This means a typo in a +`location` field is caught at construction time, not silently deferred, but the +error message will point to the stage file rather than the config key. + +### 8 — Unknown stage definition keys are absorbed into `Stage.metadata` + +In `_stage_from_config_definition()`, any key under a stage entry that is not +`run`, `location` / `source` / `path`, `dependencies`, `entrypoint`, or +`metadata` is merged into `Stage.metadata`. This is intentional — it lets you +attach arbitrary properties to a stage definition (e.g. `owner: analytics`) — +but it also means a misspelled reserved key such as `dependancies` will silently +appear in metadata rather than being recognised as a dependency list. + +--- + +## Extensibility + +Stage configuration scales linearly: `_build_stage_configs()` iterates every +key in the `stage_configuration` mapping and constructs one `StageConfig` per +entry. Adding a new stage to the pipeline requires: + +1. Adding the stage entry to `pipeline_variables.stages` in the YAML (or passing + it to `from_files`). +2. Adding the corresponding entry to `stage_configuration` in the YAML (or + passing it in the flat config mapping). + +No other changes are needed. `Pipeline._sync_stage_configs()` ensures that every +stage that does not have an explicit entry still receives an empty `StageConfig` +so that `context.stage_config` is never `None` during execution. diff --git a/docs/contributor_guide/CONTRIBUTING.md b/docs/contributor_guide/CONTRIBUTING.md index 24627ed..0561666 100644 --- a/docs/contributor_guide/CONTRIBUTING.md +++ b/docs/contributor_guide/CONTRIBUTING.md @@ -28,9 +28,13 @@ documentation such as [detect-secrets][detect-secrets-repo] or [nbstripout][nbst ## Code conventions -We mainly follow [PEP8 standards][pep8] in our code conventions, and use flake8 and black +We mainly follow [PEP8 standards][pep8] in our code conventions, and use ruff pre-commit hook for linting and formatting. +We use [MyPy](https://mypy.readthedocs.io/en/stable/) to enforce type-checking to ensure the +rigidity of our framework and [bandit](http://bandit.readthedocs.io/en/latest/) for vulnerability +checking as a point of best practice. These are also pre-commit hooks in the project. + ### Git and GitHub We use Git to version control the source code. Please read diff --git a/examples/pipeline_1/Example.md b/examples/pipeline_1/Example.md index 32c05ae..5105843 100644 --- a/examples/pipeline_1/Example.md +++ b/examples/pipeline_1/Example.md @@ -1,3 +1,5 @@ # Purpose -This example is designed to show the typical use case of using Callable entry points for \ No newline at end of file +This example is designed to show the typical use case of using Callable entry points for the package. + +The contents of this directory is trying to show how a Functional Programming approach can be orchestrated using the package. \ No newline at end of file diff --git a/examples/pipeline_1/logs/onsrap.log b/examples/pipeline_1/logs/onsrap.log deleted file mode 100644 index d9ab511..0000000 --- a/examples/pipeline_1/logs/onsrap.log +++ /dev/null @@ -1,45 +0,0 @@ -2026-06-19 17:45:43,313 Pipeline initialized | {"backend": "python", "id": "2026-06-19_164543_e9f91a60", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-19 17:45:43,313 Validating pipeline | {"name": "pipeline_1"} -2026-06-19 17:45:43,348 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-19_164543_a1816f20", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-19 17:45:43,348 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-19 17:45:43,352 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-19 17:45:43,353 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-19 17:45:43,354 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-19 17:45:43,357 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-19 17:45:43,358 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-19 17:45:43,358 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-19 17:45:43,362 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-19 17:45:43,365 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-19 17:45:43,366 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-19_164543_a1816f20", "stages": 3} -2026-06-22 13:17:50,023 Pipeline initialized | {"backend": "python", "id": "2026-06-22_121749_ae8ff387", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-22 13:17:50,024 Validating pipeline | {"name": "pipeline_1"} -2026-06-22 13:17:50,068 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-22_121749_ae8ff387", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-22 13:17:50,069 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-22 13:17:50,101 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-22 13:17:50,128 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-22 13:17:50,128 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-22 13:17:50,161 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-22 13:17:50,163 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-22 13:17:50,163 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-22 13:17:50,195 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-22 13:17:50,222 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-22 13:17:50,223 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-22_121749_ae8ff387", "stages": 3} -2026-06-23 10:17:19,051 Pipeline initialized | {"backend": "python", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:17:19,052 Validating pipeline | {"name": "pipeline_1"} -2026-06-23 10:17:19,109 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-23_101719_878fcb33", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:17:19,109 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-23 10:17:19,114 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-23 10:17:19,114 Pipeline failed | {"error": "Callable stage failed.", "name": "pipeline_1", "run_id": "2026-06-23_101719_878fcb33"} -2026-06-23 10:19:03,621 Pipeline initialized | {"backend": "python", "name": "pipeline_1", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:19:03,621 Validating pipeline | {"name": "pipeline_1"} -2026-06-23 10:19:03,676 Pipeline started | {"name": "pipeline_1", "run_id": "2026-06-23_101903_e95b29dc", "stages": ["0_data_validation", "1_preprocessing", "2_reporting"]} -2026-06-23 10:19:03,677 Executing stage | {"name": "0_data_validation", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py"} -2026-06-23 10:19:03,681 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\0_data_validation.py", "stage": "0_data_validation"} -2026-06-23 10:19:03,683 Stage finished | {"mode": "callable", "stage": "0_data_validation", "status": "succeeded"} -2026-06-23 10:19:03,683 Executing stage | {"name": "1_preprocessing", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py"} -2026-06-23 10:19:03,687 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\1_preprocessing.py", "stage": "1_preprocessing"} -2026-06-23 10:19:03,688 Stage finished | {"mode": "callable", "stage": "1_preprocessing", "status": "succeeded"} -2026-06-23 10:19:03,688 Executing stage | {"name": "2_reporting", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py"} -2026-06-23 10:19:03,692 Stage started | {"mode": "callable", "source": "D:\\hub\\onsrap\\examples\\pipeline_1\\scripts\\2_reporting.py", "stage": "2_reporting"} -2026-06-23 10:19:03,695 Stage finished | {"mode": "callable", "stage": "2_reporting", "status": "succeeded"} -2026-06-23 10:19:03,695 Pipeline completed | {"name": "pipeline_1", "run_id": "2026-06-23_101903_e95b29dc", "stages": 3} diff --git a/examples/pipeline_1/main.py b/examples/pipeline_1/main.py index 543fff9..f9eb273 100644 --- a/examples/pipeline_1/main.py +++ b/examples/pipeline_1/main.py @@ -4,7 +4,6 @@ from onsrap import Pipeline, PipelineConfig - PIPELINE_ROOT = Path(__file__).resolve().parent SCRIPTS_DIR = PIPELINE_ROOT / "scripts" DATA_DIR = PIPELINE_ROOT / "data" @@ -28,6 +27,7 @@ def build_pipeline() -> Pipeline: backend="python", work_dir=PIPELINE_ROOT, project_root=PIPELINE_ROOT, + output_dir=PIPELINE_ROOT, data_dir=DATA_DIR, log_dir=LOG_DIR, metadata={ @@ -49,10 +49,12 @@ def main() -> None: run = build_pipeline().run() report = run.manifest.outputs["2_reporting"] - print(f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages.") + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) print(f"Summary report written to: {report['report_path']}") print(f"Cleaned data written to: {report['clean_path']}") if __name__ == "__main__": - main() + main() diff --git a/examples/pipeline_1/main2.py b/examples/pipeline_1/main2.py new file mode 100644 index 0000000..ad92311 --- /dev/null +++ b/examples/pipeline_1/main2.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from pathlib import Path + +from onsrap import Pipeline, PipelineConfig + +PIPELINE_ROOT = Path(__file__).resolve().parent +SCRIPTS_DIR = PIPELINE_ROOT / "scripts" +DATA_DIR = PIPELINE_ROOT / "data" +LOG_DIR = PIPELINE_ROOT / "logs" + +""" +In this version of the main pipeline script, the order of the stage files has been altered. +The preprocessing stage is now listed before the data validation stage in the `stage_files` list. +However, the dependencies remain unchanged, meaning that the pipeline will still enforce that data +validation must be completed before preprocessing can run. + +This means that changing the order of the stage files in pipeline does not affect the execution order +of the stages, which is defined by the dependencies. +""" + + +def build_pipeline() -> Pipeline: + stage_files = [ # Altered Script Order + SCRIPTS_DIR / "1_preprocessing.py", + SCRIPTS_DIR / "0_data_validation.py", + SCRIPTS_DIR / "2_reporting.py", + ] + + dependencies = { + "1_preprocessing": ("0_data_validation",), + "2_reporting": ("1_preprocessing",), + } + + config = PipelineConfig( + name="pipeline_1", + backend="python", + work_dir=PIPELINE_ROOT, + project_root=PIPELINE_ROOT, + data_dir=DATA_DIR, + log_dir=LOG_DIR, + metadata={ + "example": "retail-orders", + "description": "Validate, clean, and summarize a small orders dataset.", + }, + ) + + return Pipeline.from_files( + stage_files, + name="pipeline_1", + backend="python", + config=config, + dependencies=dependencies, + ) + + +def main() -> None: + run = build_pipeline().run() + report = run.manifest.outputs["2_reporting"] + + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) + print(f"Summary report written to: {report['report_path']}") + print(f"Cleaned data written to: {report['clean_path']}") + + +if __name__ == "__main__": + main() diff --git a/examples/pipeline_1/runs/README.md b/examples/pipeline_1/runs/README.md index 862c9bd..dc632ef 100644 --- a/examples/pipeline_1/runs/README.md +++ b/examples/pipeline_1/runs/README.md @@ -1,6 +1,6 @@ # Run Output Directory -In this directory is the outputs of the example example pipeline. This directory's contents are untracked if the contents are further directories. This means there are no example pipeline outputs! +In this directory is the outputs of the example pipeline. This directory's contents are untracked if the contents are further directories. This means there are no example pipeline outputs! However, you can run pipeline_1 to generate its outputs yourself. Simply type the following command into a powershell or cmd terminal: ```powershell diff --git a/examples/pipeline_1/scripts/0_data_validation.py b/examples/pipeline_1/scripts/0_data_validation.py index eb331cc..9198446 100644 --- a/examples/pipeline_1/scripts/0_data_validation.py +++ b/examples/pipeline_1/scripts/0_data_validation.py @@ -3,34 +3,18 @@ import csv import json from pathlib import Path -from typing import Any - REQUIRED_COLUMNS = ( - "order_id", - "customer_name", - "region", - "product", - "quantity", - "unit_price", - "order_date", + "order_id", + "customer_name", + "region", + "product", + "quantity", + "unit_price", + "order_date", ) -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - def load_orders(csv_path: Path) -> list[dict[str, str]]: with csv_path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle)) @@ -66,11 +50,11 @@ def validate_rows(rows: list[dict[str, str]]) -> list[str]: return issues -def build_report(raw_path: Path, rows: list[dict[str, str]], issues: list[str]) -> dict[str, object]: +def build_report( + raw_path: Path, rows: list[dict[str, str]], issues: list[str] +) -> dict[str, object]: order_dates = sorted( - row["order_date"].strip() - for row in rows - if not is_blank(row.get("order_date")) + row["order_date"].strip() for row in rows if not is_blank(row.get("order_date")) ) unique_regions = sorted( { @@ -95,12 +79,14 @@ def build_report(raw_path: Path, rows: list[dict[str, str]], issues: list[str]) def write_report(report_path: Path, report: dict[str, object]) -> None: report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + report_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() raw_path = data_root / "orders.csv" report_path = output_root / "interim" / "0_validation_report.json" @@ -120,4 +106,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_1/scripts/1_preprocessing.py b/examples/pipeline_1/scripts/1_preprocessing.py index 0d8ad1c..b0d113b 100644 --- a/examples/pipeline_1/scripts/1_preprocessing.py +++ b/examples/pipeline_1/scripts/1_preprocessing.py @@ -4,32 +4,6 @@ import json from datetime import date from pathlib import Path -from typing import Any - - -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_raw_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - validation_result = context.result_for("0_data_validation") - if validation_result is not None: - raw_path = validation_result.outputs.get("raw_path") - if raw_path: - return Path(raw_path) - - return data_root / "orders.csv" def load_orders(csv_path: Path) -> list[dict[str, str]]: @@ -98,7 +72,9 @@ def write_clean_rows(clean_path: Path, rows: list[dict[str, object]]) -> None: writer.writerows(rows) -def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, object]]) -> dict[str, object]: +def build_summary( + source_path: Path, clean_path: Path, rows: list[dict[str, object]] +) -> dict[str, object]: total_revenue = round(sum(float(row["order_value"]) for row in rows), 2) return { "source_path": str(source_path), @@ -112,9 +88,11 @@ def build_summary(source_path: Path, clean_path: Path, rows: list[dict[str, obje def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - raw_path = resolve_raw_path(context, data_root) + data_root = context.get_data_dir() + output_root = context.resolve_output_root() + raw_path = context.resolve_given_path( + "0_data_validation", "raw_path", "orders.csv", data_root + ) clean_path = output_root / "interim" / "1_clean_orders.csv" rows = load_orders(raw_path) @@ -126,4 +104,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_1/scripts/2_reporting.py b/examples/pipeline_1/scripts/2_reporting.py index 337c2e5..0e29834 100644 --- a/examples/pipeline_1/scripts/2_reporting.py +++ b/examples/pipeline_1/scripts/2_reporting.py @@ -4,32 +4,6 @@ import json from collections import defaultdict from pathlib import Path -from typing import Any - - -def resolve_data_root(context: Any | None = None) -> Path: - if context is not None: - return Path(context.config.data_dir) - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_output_root(context: Any | None = None) -> Path: - if context is not None and getattr(context, "run_dir", None) is not None: - return Path(context.run_dir) / "data" - - return Path(__file__).resolve().parents[1] / "data" - - -def resolve_clean_path(context: Any | None, data_root: Path) -> Path: - if context is not None: - preprocessing_result = context.result_for("1_preprocessing") - if preprocessing_result is not None: - clean_path = preprocessing_result.outputs.get("clean_path") - if clean_path: - return Path(clean_path) - - return data_root / "interim" / "1_clean_orders.csv" def load_orders(csv_path: Path) -> list[dict[str, str]]: @@ -56,16 +30,28 @@ def build_summary(rows: list[dict[str, str]]) -> dict[str, object]: revenue_by_product[product] += order_value total_revenue = round(sum(revenue_by_region.values()), 2) - top_region = max(revenue_by_region, key=revenue_by_region.get) if revenue_by_region else None - top_product = max(revenue_by_product, key=revenue_by_product.get) if revenue_by_product else None + top_region = ( + max(revenue_by_region, key=revenue_by_region.get) if revenue_by_region else None + ) + top_product = ( + max(revenue_by_product, key=revenue_by_product.get) + if revenue_by_product + else None + ) return { "total_orders": len(ordered_rows), "total_revenue": total_revenue, - "revenue_by_region": {region: round(amount, 2) for region, amount in sorted(revenue_by_region.items())}, + "revenue_by_region": { + region: round(amount, 2) + for region, amount in sorted(revenue_by_region.items()) + }, "orders_by_region": dict(sorted(orders_by_region.items())), "units_by_product": dict(sorted(units_by_product.items())), - "revenue_by_product": {product: round(amount, 2) for product, amount in sorted(revenue_by_product.items())}, + "revenue_by_product": { + product: round(amount, 2) + for product, amount in sorted(revenue_by_product.items()) + }, "top_region": top_region, "top_product": top_product, "first_order_date": ordered_rows[0]["order_date"] if ordered_rows else None, @@ -75,7 +61,9 @@ def build_summary(rows: list[dict[str, str]]) -> dict[str, object]: def write_summary(summary_path: Path, summary: dict[str, object]) -> None: summary_path.parent.mkdir(parents=True, exist_ok=True) - summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + summary_path.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> None: @@ -94,9 +82,10 @@ def write_region_breakdown(region_path: Path, summary: dict[str, object]) -> Non def main(context=None) -> dict[str, object]: - data_root = resolve_data_root(context) - output_root = resolve_output_root(context) - clean_path = resolve_clean_path(context, data_root) + output_root = context.resolve_output_root() + clean_path = context.resolve_given_path( + "1_preprocessing", "clean_path", "1_clean_orders.csv", output_root, "interim" + ) summary_path = output_root / "processed" / "2_sales_summary.json" region_breakdown_path = output_root / "processed" / "2_revenue_by_region.csv" @@ -117,4 +106,4 @@ def main(context=None) -> dict[str, object]: if __name__ == "__main__": - print(json.dumps(main(), indent=2, sort_keys=True)) \ No newline at end of file + print(json.dumps(main(), indent=2, sort_keys=True)) diff --git a/examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc b/examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc deleted file mode 100644 index 5ef1c31..0000000 Binary files a/examples/pipeline_1/scripts/__pycache__/0_data_validation.cpython-314.pyc and /dev/null differ diff --git a/examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc b/examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc deleted file mode 100644 index 91c1a90..0000000 Binary files a/examples/pipeline_1/scripts/__pycache__/1_preprocessing.cpython-314.pyc and /dev/null differ diff --git a/examples/pipeline_1/scripts/__pycache__/2_reporting.cpython-314.pyc b/examples/pipeline_1/scripts/__pycache__/2_reporting.cpython-314.pyc deleted file mode 100644 index 3af5dea..0000000 Binary files a/examples/pipeline_1/scripts/__pycache__/2_reporting.cpython-314.pyc and /dev/null differ diff --git a/examples/pipeline_2/conf.yaml b/examples/pipeline_2/conf.yaml new file mode 100644 index 0000000..58d076b --- /dev/null +++ b/examples/pipeline_2/conf.yaml @@ -0,0 +1,95 @@ +pipeline_variables: + name: "Config Example Pipeline" + backend: python + stages: + - 0_clean_data: + name: "0_clean_data" + location: examples/pipeline_2/scripts/0_clean_data.py + dependencies: [] + - 1_derive_vars: + location: examples/pipeline_2/scripts/1_derive_vars.py + dependencies: + - "0_clean_data" + - 2_reporting: + location: examples/pipeline_2/scripts/2_reporting.py + dependencies: + - "1_derive_vars" + working_dir: examples/pipeline_2 + project_root: examples/pipeline_2 + data_dir: examples/pipeline_2/data + output_dir: examples/pipeline_2/outputs + log_dir: examples/pipeline_2/logs + overwrite: True + metadata: + example: retail-orders-using-configuration + description: "This is an example of a pipeline that uses configuration files to run a retail orders pipeline." + stages_to_run: + 0_clean_data: True + 1_derive_vars: True + 2_reporting: True + + +global_variables: + year_of_run: 2020 + +stage_configuration: + 0_clean_data: + expected_variables: + - "order_id" + - "customer_name" + - "region" + - "product" + - "quantity" + - "unit_price" + - "order_date" + - "order_method" + identifiable_cols: + - "customer_name" + - "age" + - "dob" + - "address" + output_location: examples/pipeline_2/data/orders_cleaned.csv + input_location: examples/pipeline_2/data/orders.csv + 1_derive_vars: + input_location: examples/pipeline_2/data/orders_cleaned.csv + output_location: examples/pipeline_2/data/orders_prepped.csv + delivery_times: + north: 14 + south: 4 + east: 7 + west: 7 + order_date: "Order_date" + estimated_delivery_date: "Estimated_delivery_date" + delivery_day: "Delivery_day" + region: "Region" + total_cost: "Total_cost" + quantity: "Quantity" + unit_price: "Unit_price" + order_day: "Order_day" + order_month: "Order_month" + large_order: "Large_order" + small_order: "Small_order" + postage: "Postage" + postage_values: + large_order: 5.00 + small_order: 1.00 + default: 2.50 + product: "Product" + product_costs: + notebook: 1.00 + pen: 0.3 + folder: 0.75 + total_production_cost: "Total_production_cost" + order_profit: "Order_profit" + 2_reporting: + input_location: examples/pipeline_2/data/orders_prepped.csv + report_location: examples/pipeline_2/outputs/order_analysis.md + region: "Region" + order_profit: "Order_profit" + quantity: "Quantity" + order_day: "Order_day" + large_order: "Large_order" + small_order: "Small_order" + order_id: "Order_id" + product: "Product" + num_format: "{:.2f}" diff --git a/examples/pipeline_2/data/orders.csv b/examples/pipeline_2/data/orders.csv new file mode 100644 index 0000000..3ffb04b --- /dev/null +++ b/examples/pipeline_2/data/orders.csv @@ -0,0 +1,7 @@ +order_id,customer_name,region,product,quantity,unit_price,order_date +1001,Alice Johnson,North,Notebook,2,3.50,2026-06-01 +1002,Ben Carter,South,Pen,5,1.20,2026-06-01 +1003,Chloe Nguyen,North,Notebook,1,3.50,2026-06-02 +1004,Daniel Patel,West,Folder,3,2.75,2026-06-03 +1005,Emma Stone,East,Pen,4,1.20,2026-06-03 +1006,Frank White, north , notebook ,2,3.50,2026-06-04 \ No newline at end of file diff --git a/examples/pipeline_2/data/orders_cleaned.csv b/examples/pipeline_2/data/orders_cleaned.csv new file mode 100644 index 0000000..c3e721b --- /dev/null +++ b/examples/pipeline_2/data/orders_cleaned.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date +1001,north,notebook,2,3.5,2026-06-01 +1002,south,pen,5,1.2,2026-06-01 +1003,north,notebook,1,3.5,2026-06-02 +1004,west,folder,3,2.75,2026-06-03 +1005,east,pen,4,1.2,2026-06-03 +1006,north,notebook,2,3.5,2026-06-04 diff --git a/examples/pipeline_2/data/orders_prepped.csv b/examples/pipeline_2/data/orders_prepped.csv new file mode 100644 index 0000000..b6c5d70 --- /dev/null +++ b/examples/pipeline_2/data/orders_prepped.csv @@ -0,0 +1,7 @@ +Order_id,Region,Product,Quantity,Unit_price,Order_date,Estimated_delvery_date,Delivery_day,Total_cost,Order_day,Order_month,Large_order,Small_order,Postage,Total_production_cost,Order_profit +1001,north,notebook,2,3.5,2026-06-01,2026-06-15,Monday,7.0,Monday,June,False,False,2.5,2.0,2.5 +1002,south,pen,5,1.2,2026-06-01,2026-06-05,Friday,6.0,Monday,June,True,False,5.0,1.5,-0.5 +1003,north,notebook,1,3.5,2026-06-02,2026-06-16,Tuesday,3.5,Tuesday,June,False,True,1.0,1.0,1.5 +1004,west,folder,3,2.75,2026-06-03,2026-06-10,Wednesday,8.25,Wednesday,June,False,False,2.5,2.25,3.5 +1005,east,pen,4,1.2,2026-06-03,2026-06-10,Wednesday,4.8,Wednesday,June,True,False,5.0,1.2,-1.4000000000000004 +1006,north,notebook,2,3.5,2026-06-04,2026-06-18,Thursday,7.0,Thursday,June,False,False,2.5,2.0,2.5 diff --git a/examples/pipeline_2/main.py b/examples/pipeline_2/main.py new file mode 100644 index 0000000..8d40021 --- /dev/null +++ b/examples/pipeline_2/main.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from onsrap import Pipeline + + +def main() -> None: + config_path = (Path(__file__).resolve().parent) / "conf.yaml" + print(config_path) + + pipeline = Pipeline.from_config(config_path) + + run = pipeline.run() + report = run.manifest.outputs + print(report) + + print( + f"Pipeline '{run.manifest.rap_name}' completed with {len(run.stage_results)} stages." + ) + print(f"Summary report written to: {pipeline.config.output_dir}") + print(f"Cleaned data written to: {pipeline.config.data_dir}") + + +if __name__ == "__main__": + main() diff --git a/examples/pipeline_2/outputs/order_analysis.md b/examples/pipeline_2/outputs/order_analysis.md new file mode 100644 index 0000000..d1ea234 --- /dev/null +++ b/examples/pipeline_2/outputs/order_analysis.md @@ -0,0 +1,33 @@ +# Order Summary + +This is an automatically generated report showing key information on orders of products. + +## Summary + +Total Orders: **6** + +Total Profit: **£8.10** + +## Region Analysis +Highest number of orders: **North** + +Highest profit: **North** at **£6.50** + +Lowest profit: **East** at **£-1.90** + +Highest quantity of items ordered: **North** at **5** + +Lowest quantity of items ordered: **West** at **3** + +## Product Analysis +Highest profit: **Notebook** at **£6.50** + +Lowest profit: **Pen** at **£-1.90** + +## Order Analysis + +The most orders occured on a **Tuesday**. + +**2** order/s were Large (greater than 75% of orders for the period). + +**1** order/s were Small (less than 25% of orders for the period). \ No newline at end of file diff --git a/examples/pipeline_2/scripts/0_clean_data.py b/examples/pipeline_2/scripts/0_clean_data.py new file mode 100644 index 0000000..50ad0a3 --- /dev/null +++ b/examples/pipeline_2/scripts/0_clean_data.py @@ -0,0 +1,51 @@ +import pandas as pd + + +def check_variables(df, expected_variables): + missing = [] + for i in expected_variables: + if i in df.columns: + pass + else: + missing.append(i) + if missing == []: + print("All variables present") + print(f"Missing the following variables: {missing}") + + +def remove_identifiable(df, identifiable_cols): + for i in identifiable_cols: + if i in df.columns: + df = df.drop(i, axis=1) + else: + pass + return df + + +def standardise_columns(df): + df.columns = [col.lower() for col in df.columns] + df.columns = [col.capitalize() for col in df.columns] + for item in df.columns: + df[item] = df[item].apply(lambda x: x.lower() if isinstance(x, str) else x) + df[item] = df[item].apply(lambda x: x.strip() if isinstance(x, str) else x) + return df + + +def main(context=None): + config = context.get_stage_config("0_clean_data") + print(config) + + orders = pd.read_csv(config["input_location"]) + + expected_variables = config["expected_variables"] + identifiable_cols = config["identifiable_cols"] + + check_variables(orders, expected_variables) + print(orders.dtypes) + orders = remove_identifiable(orders, identifiable_cols) + orders = standardise_columns(orders) + orders.to_csv(config["output_location"], index=False) + + +if __name__ == "__main__": + main() diff --git a/examples/pipeline_2/scripts/1_derive_vars.py b/examples/pipeline_2/scripts/1_derive_vars.py new file mode 100644 index 0000000..94d4d42 --- /dev/null +++ b/examples/pipeline_2/scripts/1_derive_vars.py @@ -0,0 +1,78 @@ +import numpy as np +import pandas as pd + + +def correct_date_time(df): + df["Order_date"] = pd.to_datetime(df["Order_date"]) + return df + + +def estimate_delivery(df, delivery_times): + df["Estimated_delvery_date"] = df["Order_date"] + pd.to_timedelta( + df["Region"].map(delivery_times), unit="D" + ) + df["Delivery_day"] = df["Estimated_delvery_date"].dt.day_name() + return df + + +def total_cost(df): + df["Total_cost"] = df["Quantity"] * df["Unit_price"] + return df + + +def order_date_values(df): + df["Order_day"] = df["Order_date"].dt.day_name() + df["Order_month"] = df["Order_date"].dt.month_name() + return df + + +def size_order_alert(df): + df["Large_order"] = df["Quantity"] > df["Quantity"].quantile(0.75) + df["Small_order"] = df["Quantity"] < df["Quantity"].quantile(0.25) + return df + + +def postage_cost(df): + df["Postage"] = np.select( + [df["Large_order"], df["Small_order"]], [5.00, 1.00], default=2.50 + ) + return df + + +def production_cost(df): + df["Total_production_cost"] = np.select( + [ + df["Product"] == "notebook", + df["Product"] == "pen", + df["Product"] == "folder", + ], + [(1.00 * df["Quantity"]), (0.3 * df["Quantity"]), (0.75 * df["Quantity"])], + default=0, + ) + return df + + +def profit_per_order(df): + df["Order_profit"] = df["Total_cost"] - df["Total_production_cost"] - df["Postage"] + return df + + +def main(context=None): + config = context.get_stage_config("1_derive_vars") + + df = pd.read_csv(config["input_location"]) + delivery_times = config["delivery_times"] + + df = correct_date_time(df) + df = estimate_delivery(df, delivery_times) + df = total_cost(df) + df = order_date_values(df) + df = size_order_alert(df) + df = postage_cost(df) + df = production_cost(df) + df = profit_per_order(df) + df.to_csv(config["output_location"], index=False) + + +if __name__ == "__main__": + main() diff --git a/examples/pipeline_2/scripts/2_reporting.py b/examples/pipeline_2/scripts/2_reporting.py new file mode 100644 index 0000000..ba71526 --- /dev/null +++ b/examples/pipeline_2/scripts/2_reporting.py @@ -0,0 +1,145 @@ +from pathlib import Path + +import pandas as pd + + +##PROFIT PER REGION## +def per_region_profits(orders, values, num_format): + profit_per_region = orders.groupby("Region")["Order_profit"].sum() + + values["highest_prof_region"] = profit_per_region.idxmax().capitalize() + values["highest_prof_value"] = num_format.format((profit_per_region.max())) + + values["lowest_prof_region"] = profit_per_region.idxmin().capitalize() + values["lowest_profit_value"] = num_format.format((profit_per_region.min())) + + +##QUANTITY PER REGION## +def per_region_quantity(orders, values): + quantity_per_region = orders.groupby("Region")["Quantity"].sum() + + values["highest_quant_region"] = quantity_per_region.idxmax().capitalize() + values["highest_quant_value"] = quantity_per_region.max() + + values["lowest_quant_region"] = quantity_per_region.idxmin().capitalize() + values["lowest_quant_value"] = quantity_per_region.min() + + +##ORDER DAY POP## +def orders_per_day(orders, values): + delivery_day_frequency = orders["Order_day"].value_counts() + values["highest_delivery_day"] = delivery_day_frequency.idxmax().capitalize() + + +##ORDER COUNTS## +def order_quantity(orders, values): + values["large_order_num"] = orders["Large_order"].sum() + + values["small_order_num"] = orders["Small_order"].sum() + + +##ORDERS PER REGION## +def per_region_orders(orders, values): + orders_per_region = orders["Region"].value_counts() + values["highest_order_region"] = orders_per_region.idxmax().capitalize() + + +##TOTALS## +def total_summaries(orders, values, num_format): + values["total_count"] = orders["Order_id"].count() + values["total_profit"] = num_format.format(orders["Order_profit"].sum()) + + +##PROFIT PER PRODUCT## +def profit_per_product(orders, values, num_format): + df = orders.groupby("Product")["Order_profit"].sum().sort_values(ascending=False) + + values["highest_profit_product"] = df.idxmax().capitalize() + values["highest_profit_value"] = num_format.format((df.max())) + + values["lowest_profit_product"] = df.idxmin().capitalize() + values["lowest_profit_value"] = num_format.format((df.min())) + + +##CURATE REPORT## +def curate_report(report, values): + report.append("# Order Summary") + report.append("") + report.append( + "This is an automatically generated report showing key information on orders of products." + ) + report.append("") + report.append("## Summary") + report.append("") + report.append(f"Total Orders: **{values['total_count']}**") + report.append("") + report.append(f"Total Profit: **£{values['total_profit']}**") + report.append("") + report.append("## Region Analysis") + report.append(f"Highest number of orders: **{values['highest_order_region']}**") + report.append("") + report.append( + f"Highest profit: **{values['highest_prof_region']}** at **£{values['highest_prof_value']}**" + ) + report.append("") + report.append( + f"Lowest profit: **{values['lowest_prof_region']}** at **£{values['lowest_profit_value']}**" + ) + report.append("") + report.append( + f"Highest quantity of items ordered: **{values['highest_quant_region']}** at **{values['highest_quant_value']}**" + ) + report.append("") + report.append( + f"Lowest quantity of items ordered: **{values['lowest_quant_region']}** at **{values['lowest_quant_value']}**" + ) + report.append("") + report.append("## Product Analysis") + report.append( + f"Highest profit: **{values['highest_profit_product']}** at **£{values['highest_profit_value']}**" + ) + report.append("") + report.append( + f"Lowest profit: **{values['lowest_profit_product']}** at **£{values['lowest_profit_value']}**" + ) + report.append("") + report.append("## Order Analysis") + report.append("") + report.append(f"The most orders occured on a **{values['highest_delivery_day']}**.") + report.append("") + report.append( + f"**{values['large_order_num']}** order/s were Large (greater than 75% of orders for the period)." + ) + report.append("") + report.append( + f"**{values['small_order_num']}** order/s were Small (less than 25% of orders for the period)." + ) + + +def write_report(report): + report_file = Path("examples/pipeline_2/outputs/order_analysis.md") + + report_file.write_text("\n".join(report), encoding="utf-8") + + +def main(): + orders = pd.read_csv("examples/pipeline_2/data/orders_prepped.csv") + + report = [] + values = {} + + num_format = "{:.2f}" + + per_region_profits(orders, values, num_format) + per_region_quantity(orders, values) + orders_per_day(orders, values) + order_quantity(orders, values) + per_region_orders(orders, values) + total_summaries(orders, values, num_format) + profit_per_product(orders, values, num_format) + curate_report(report, values) + write_report(report) + + +if __name__ == "__main__": + main() diff --git a/examples/pipeline_3/main.ipynb b/examples/pipeline_3/main.ipynb index e69de29..80a457a 100644 --- a/examples/pipeline_3/main.ipynb +++ b/examples/pipeline_3/main.ipynb @@ -0,0 +1,22 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "print(1)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python", + "version": "3.14.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/onsrap/__init__.py b/onsrap/__init__.py index 908f017..b23aab9 100644 --- a/onsrap/__init__.py +++ b/onsrap/__init__.py @@ -13,13 +13,14 @@ from .logger import LogConfig, Logger from .models import ( Catalog, + GlobalConfig, PipelineConfig, PipelineRun, PipelineStatus, - RAPConfig, RAPDataset, RunManifest, RuntimeID, + StageConfig, StageResult, StageStatus, ) @@ -32,6 +33,7 @@ "DependencyCycleError", "DuplicateStageError", "ExecutionContext", + "GlobalConfig", "LogConfig", "Logger", "MissingDependencyError", @@ -43,10 +45,10 @@ "PipelineStatus", "PipelineValidationError", "PythonStageExecutor", - "RAPConfig", "RAPDataset", "RunManifest", "RuntimeID", + "StageConfig", "Stage", "StageConfigurationError", "StageExecutionError", diff --git a/onsrap/errors.py b/onsrap/errors.py index 372d6df..aa37551 100644 --- a/onsrap/errors.py +++ b/onsrap/errors.py @@ -1,32 +1,55 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .models import StageResult + class OnsrapError(Exception): """Base exception for onsrap.""" class PipelineValidationError(OnsrapError): - """Raised when the pipeline definition is invalid.""" + """ + Raised when the pipeline definition is invalid. + Child class with ``OnsrapError`` as the parent class. + """ class StageConfigurationError(PipelineValidationError): - """Raised when a stage definition is malformed.""" + """ + Raised when a stage definition is malformed. + Child class with ``PipelineValidationError`` as the parent class. + """ class DuplicateStageError(PipelineValidationError): - """Raised when two stages share the same name.""" + """ + Raised when two stages share the same name. + Child class with ``PipelineValidationError`` as the parent class. + """ class MissingDependencyError(PipelineValidationError): - """Raised when a stage depends on an unknown stage.""" + """ + Raised when a stage depends on an unknown stage. + Child class with ``PipelineValidationError`` as the parent class. + """ class DependencyCycleError(PipelineValidationError): - """Raised when the stage graph contains a cycle.""" + """ + Raised when the stage graph contains a cycle. + Child class with ``PipelineValidationError`` as the parent class. + """ class StageExecutionError(OnsrapError): - """Raised when a stage fails during execution.""" + """ + Raised when a stage fails during execution. + Child class with ``OnsrapError`` as the parent class. + """ def __init__( self, @@ -34,7 +57,7 @@ def __init__( stage_name: str | None = None, source: str | None = None, original_exception: Exception | None = None, - result: object | None = None, + result: StageResult | None = None, ): super().__init__(message) self.stage_name = stage_name @@ -44,4 +67,35 @@ def __init__( class StageLoadError(StageExecutionError): - """Raised when a file-backed stage cannot be loaded.""" + """ + Raised when a file-backed stage cannot be loaded. + Child class with ``StageExecutionError`` as the parent class. + """ + + +class StageDependencyError(OnsrapError): + """ + Raised when incorrect inputs are provided to the dependency + attribute of a Stage. + """ + + +class PipelineInitialisationError(OnsrapError): + """ + Raised when there is an error in definition of the Pipeline + instance + """ + + +class PipelineConfigurationError(OnsrapError): + """ + Raised when there has been an issue with the PipelineConfig + instance. + """ + + +class HistoricalPipelineLoadError(OnsrapError): + """ + Raised when there is an issue loading a previous PipelineRun + instance. + """ diff --git a/onsrap/execution.py b/onsrap/execution.py index 7c52c6e..a6d8fe6 100644 --- a/onsrap/execution.py +++ b/onsrap/execution.py @@ -3,15 +3,33 @@ import inspect import subprocess import sys +import warnings from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import Any, Protocol, TYPE_CHECKING - -from .errors import StageExecutionError, StageLoadError -from .loader import PREFERRED_ENTRYPOINTS, discover_python_entrypoint, load_python_callable +from typing import TYPE_CHECKING, Any, Protocol + +from onsrap.warnings import StageConfigurationWarning + +from .errors import ( + PipelineConfigurationError, + StageExecutionError, + StageLoadError, +) +from .loader import ( + PREFERRED_ENTRYPOINTS, + discover_python_entrypoint, + load_python_callable, +) from .logger import Logger -from .models import PipelineConfig, StageResult, StageStatus, now +from .models import ( + GlobalConfig, + PipelineConfig, + StageConfig, + StageResult, + StageStatus, + now, +) if TYPE_CHECKING: from .stage import Stage @@ -19,6 +37,37 @@ @dataclass class ExecutionContext: + """ + Holds information needed to run the pipeline. + + Parameters + ---------- + ``pipeline_name`` : str + The name of the pipeline. + ``run_id`` : str + The unique identifier for the current run of the pipeline. + ``config`` : ``PipelineConfig`` class instance + The configuration required for the pipeline. + ``logger`` : ``Logger`` class instance + The logger used for this pipeline run. + ``run_dir``: Path + The directory that the run saved to. + ``started_at`` : datetime, default = current time + The time that the pipeline run started. + ``working_directory`` : Path, default = current working directory + The directory that the work is taking place in. + ``stage_results`` : dict[str, StageResult], default = dict + Stores the logs for the stage run. + ``stage_configs`` : dict[str, StageConfig], default = dict + Stage-name keyed configuration mapping resolved by the ``Pipeline``. + ``variables`` : dict[str, Any], default = dict + Stores relevant variables regarding the stage run and their results. + ``active_stage_name`` : str or None, default = None + Name of the stage currently being executed. Used to expose ``stage_config``. + ``global_config`` : ``GlobalConfig`` or None, default = None + Variables which are parsed to all stages throughout the pipeline. + """ + pipeline_name: str run_id: str config: PipelineConfig @@ -27,33 +76,350 @@ class ExecutionContext: started_at: datetime = field(default_factory=now) working_directory: Path = field(default_factory=Path.cwd) stage_results: dict[str, StageResult] = field(default_factory=dict) + stage_configs: dict[str, StageConfig] = field(default_factory=dict) variables: dict[str, Any] = field(default_factory=dict) + active_stage_name: str | None = None + global_config: GlobalConfig | None = None def record(self, result: StageResult) -> StageResult: + """ + Extracts key information from ``StageResult``. + + Saves all information on the results of the Stage to the ``stage_results`` attribute + and exclusively metadata outputs regarding the run to the ``variables`` attribute. + + Parameters + ---------- + ``result`` : ``StageResult`` + An instance of a ``StageResult`` class which is created from the Executor classes (StageExecutor, PythonStageExecutor). + + Returns + ------ + ``result`` + An unchanged ``StageResult`` instance. + """ self.stage_results[result.name] = result self.variables[result.name] = result.outputs return result def result_for(self, stage_name: str) -> StageResult | None: + """ + Getter function that returns the stage_results for a specific ``Stage``. + + Parameters + ---------- + ``stage_name`` : str + The name of the ``Stage`` that you are calling the results for. + + Returns + ------- + ``stage_results`` + Attribute for the specific `Stage` named. + """ return self.stage_results.get(stage_name) + def set_active_stage(self, stage_name: str | None) -> None: + """ + Mark the stage currently being executed so ``stage_config`` resolves correctly. + """ + self.active_stage_name = stage_name + + @property + def stage_config(self) -> StageConfig | None: + """ + Return the configuration for the stage currently being executed. + + The preferred access method for this is ``get_stage_config()`` which allows + for optional arguments to return the full ``StageConfig`` instance or just + the variables dictionary. + + This property is ``None`` outside an active stage run. + """ + return self.stage_config_for(self.active_stage_name) + + def stage_config_for(self, stage_name: str | None) -> StageConfig | None: + """ + Return the configuration registered for ``stage_name``. + + Unlike ``stage_config``, this helper does not depend on the currently + active stage and can be used to inspect any known stage configuration. + + Parameters + ---------- + ``stage_name`` : str or None + Name of the stage whose configuration should be returned. + """ + if stage_name is None: + return None + return self.stage_configs.get(stage_name) + @property def stage_outputs(self) -> dict[str, Any]: + """ + Creates a ``stage_outputs`` attribute for the ``ExecutionContext`` class. + + Extracts the ```outputs`` attribute from the ``stage_results`` class for each + ``Stage`` name. + + Returns + ------- + ``stage_outputs`` + Dictionary containing the name of the stage and the associated outputs of + the run. + """ return {name: result.outputs for name, result in self.stage_results.items()} + def get_data_dir(self) -> Path: + """ + Establishes the filepath that the data is held in. + + Returns + ------- + Path + The file path for the location of the data being used in the pipeline. + """ + if self.config is not None: + return Path(self.config.data_dir) + + raise PipelineConfigurationError( + "Please parse a PipelineConfig instance to the ExecutionContext." + ) + + def resolve_output_root(self) -> Path: + """ + Establishes the filepath that the outputs are going to be saved to. + + Returns + ------- + Path + The file path for the outputs of the run to be saved to. + """ + if self.run_dir is not None: + return Path(self.run_dir) + + raise PipelineConfigurationError( + "Please parse a run directory to the ExecutionContext." + ) + + def get_stage_config( + self, stage: str | None = None, with_global: bool = True, vars_only: bool = True + ) -> dict[str, Any] | StageConfig | None: + """ + Returns the configuration for the stage currently being executed, with optional arguments. + + Optional argument ``vars_only`` can be set to ``False`` to return the full ``StageConfig`` instance, + rather than just the variables dictionary. + + If you want to access ``metadata`` or ``dataframes`` from the ``StageConfig``, you must set ``vars_only`` to False. + + Parameters + ---------- + ``stage`` : str + The name of the stage to get the configuration for. + ``vars_only`` : bool, default = True + If True, returns only the variables dictionary from the ``StageConfig``. If False, returns the full ``StageConfig`` instance. + + Returns + ------- + dict[str, Any] or StageConfig or None + The parameters contained within the configuration for the currently active stage. + If ``vars_only`` is set to False, returns the StageConfig object itself, containing all attributes including variables, metadata, and dataframes. + """ + stage_config: StageConfig | None = ( + self.stage_config_for(stage) if stage is not None else self.stage_config + ) + + if with_global and not vars_only: + raise PipelineConfigurationError( + "get_stage_config() cannot return a StageConfig when with_global=True. " + "Global and stage variables are combined into a dictionary; use vars_only=True " + "or set with_global=False to retrieve the raw StageConfig object." + ) + + if stage_config is None: + if with_global: + return self._combine_vars() + return {} if vars_only else None + if with_global: + return self._combine_vars(stage_config) + if vars_only: + return stage_config.variables + return stage_config + + def resolve_given_path( + self, + stage_name: str | None, + path_name: str | None, + file_name: str | None, + root: Path, + add_folder: list[str] | str | None = None, + ) -> Path: + """ + Returns a file path for a requested item. + + This investigates the result of a previous stage to extract a selected path. + If the path is not available, it creates a path using a root previously derived + in main.py, the chosen directory within the root (optional), and the file path. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage where the path was outputted. + ``path_name`` : str + The name for the path within the stage results. This will be the key from the + key/value pair within the output of the previous stage. + ``file_name`` : str + The name of the file that you are trying to access the Path for. + ``root`` : Path + The file path for the root of the directory. This should be denoted through + other methods. + ``add_folder`` : list[str] | str | None, default = None + Additional folder name/s to add into the returned file path. + + Returns + ------- + Path + The file path where data has previously been saved to to allow for extraction of + that data throughout the pipeline. + """ + result = self.result_for(stage_name) if stage_name is not None else None + if result is not None and path_name is not None: + selected_path = result.outputs.get(path_name) + if selected_path: + return Path(selected_path) + if isinstance(add_folder, list): + if file_name is not None: + new_path = root.joinpath(*add_folder, file_name) + return new_path + new_path = root.joinpath(*add_folder) + return new_path + if isinstance(add_folder, str): + if file_name is not None: + return root / add_folder / file_name + return root / add_folder + if file_name is not None: + return root / file_name + return root + + def _combine_vars(self, stage: StageConfig | None = None) -> dict[str, Any]: + """ + Private method that combines the global variables and the stage + variables for the current stage. + + Global variables are extracted from the ``global_config`` attribute + and any variables which are marked as to be excluded from the exclusion + attribute are removed. The global variables are then combined with the stage + specific variables and returned as a dictionary. Conflicts raise a warning + to alert the user that the stage configuration definition will be used as a + priority. + + Returns + ------- + ``combined``: dict[str, Any] + A dictionary of all variables required for the stage that are sourced + through the configuration. + + Raises + ------ + ``StageConfigurationWarning`` + If there are conflicting variables between the global and stage configuration, + a warning is raised to alert the user that the stage configuration will take + precedence. + """ + resolved_stage = stage or self.stage_config + global_vars: dict[str, Any] + exclusions: dict[str, Any] + if self.global_config is not None: + global_vars, exclusions = self.global_config.get_attributes() + else: + global_vars, exclusions = {}, {} + exclusions = exclusions or {} + + if resolved_stage is None: + return dict(global_vars) + + stage_exclusions_extract = exclusions.get(resolved_stage.name, []) + stage_exclusions = [exclusion for exclusion in stage_exclusions_extract] + + combined = { + key: value + for key, value in global_vars.items() + if key not in stage_exclusions + } + stage_vars = resolved_stage.variables + + conflicts = stage_vars.keys() & combined.keys() + if conflicts: + conflicting = ", ".join(sorted(conflicts)) + warnings.warn( + f"Stage defines variable(s) that are also defined in global " + f"variables: {conflicting}. Stage variables will take precedence.", + StageConfigurationWarning, + ) + combined.update(stage_vars) + return combined + class StageExecutor(Protocol): - def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + """ + Child class of ``Protocol`` + Implementation required + """ + + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: + """ + Method to run ``Stage`` however implementation required + """ ... class PythonStageExecutor: + """ + Class to run Python `Stage`. + + Contains methods that allow automatic running of individual `Stage` processes for + a pipeline. + """ + def __init__(self, preferred_entrypoints: tuple[str, ...] = PREFERRED_ENTRYPOINTS): self.preferred_entrypoints = preferred_entrypoints - def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def __str__(self) -> str: + return f"PythonStageExecutor: \n Preferred Entrypoints: {self.preferred_entrypoints})" + + def __repr__(self) -> str: + return ( + f"PythonStageExecutor(preferred_entrypoints={self.preferred_entrypoints})" + ) + + def execute(self, stage: Stage, context: ExecutionContext) -> StageResult: + """ + Main function to select how ``Stage`` is run. + + Identifies the type of ``source`` within the ``Stage`` and runs the relevant + function for that type. + + Parameters + ---------- + ``stage`` : ``Stage`` class + The ``Stage`` that is attempting to be run. + + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``StageResult`` instance. + + Raise + ----- + ``StageExecutionError`` + If the ``source`` is not a Path or a callable object. + """ if callable(stage.source): - return self._execute_callable(stage, context, stage.source, stage.source_label) + return self._execute_callable( + stage, context, stage.source, stage.source_label + ) if isinstance(stage.source, Path): return self._execute_file(stage, context) @@ -66,11 +432,41 @@ def execute(self, stage: "Stage", context: ExecutionContext) -> StageResult: def _execute_callable( self, - stage: "Stage", + stage: Stage, context: ExecutionContext, callable_object: Any, source_label: str | None, ) -> StageResult: + """ + Attempt to run a callable object. + + Calls the logger.event() method to record an event and attempts to + run the callable parsed. If the callable cannot be run, an error is flagged + and the ``StageResult`` instance created shows a failure. If it can be run, + the callable is run and the ``StageResult`` instance shows a success. + Metadata is kept for the attempt including ``duration``, ``name``, ``outputs``, + ``source``, ``mode`` attempted, and ``errors``. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + ``callable_object`` : Any + The callable attempting to be run. + ``source_label`` : str or None + The type of ``source`` for the ``Stage``. + + Return + ------ + ``StageResult`` class instance + + Raise + ----- + ``StageExecutionError`` + If the callable object cannot be run + """ started_at = now() context.logger.event( "Stage started", @@ -117,7 +513,40 @@ def _execute_callable( ) return result - def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def _execute_file(self, stage: Stage, context: ExecutionContext) -> StageResult: + """ + Attempt to run a file. + Attempt to run a callable object. + + Calls the logger.event() method to record an event and attempts to run + the callable parsed. If the callable cannot be run, an error is flagged and + the ``StageResult`` instance created shows a failure. If it can be run, the + callable is run and the ``StageResult`` instance shows a success. Metadata + is kept for the attempt including ``duration``, ``name``, ``outputs``, ``source``, + ``mode`` attempted, and ``errors``. + If there is no entrypoint or the entrypoint is not a callable object, an error will be + raised. ``_execute_subprocess()`` method called if no entrypoint is found. A + ``StageResult`` instance will be created to log the results of the ``Stage``run regardless + of success or failure. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``StageResult`` class instance + + Raise + ----- + ``StageLoadError`` + If the entrypoint in the stage is unable to be run. + ``StageExecutionError`` + If the entrypoint is not found. + """ path = stage.source assert isinstance(path, Path) @@ -167,7 +596,36 @@ def _execute_file(self, stage: "Stage", context: ExecutionContext) -> StageResul return self._execute_subprocess(stage, context) - def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> StageResult: + def _execute_subprocess( + self, stage: Stage, context: ExecutionContext + ) -> StageResult: + """ + Run the entire Python file for the ``Stage`` from the top. + + Not desired method. Uses black-box design and obfuscates Pipeline running. Please refer + to Wiki documentation on how to implement callable solutions instead. + + If the ``Stage`` source is a file but does not have a callable entrypoint, this method + will run the entire script top to bottom. The results of the ``Stage`` are recorded + as a ``StageResult`` instance and logging processes are complete. + + Parameters + ---------- + ``stage`` : ``Stage`` class + A ``Stage`` class instance for the stage being run. + ``context`` : ``ExecutionContext`` class + The metadata required to run the ``Stage``. + + Return + ------ + ``result`` + A ``StageResult`` instance holding information on the ``Stage``. + + Raise + ----- + ``StageExecutionError`` + If the ``Stage`` script was unable to be run successfully. + """ path = stage.source assert isinstance(path, Path) @@ -194,7 +652,9 @@ def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> Stag finished_at = now() result = StageResult( name=stage.name, - status=StageStatus.SUCCEEDED if completed.returncode == 0 else StageStatus.FAILED, + status=StageStatus.SUCCEEDED + if completed.returncode == 0 + else StageStatus.FAILED, started_at=started_at, finished_at=finished_at, outputs=completed.stdout, @@ -204,7 +664,8 @@ def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> Stag metadata=dict(stage.metadata), error=None if completed.returncode == 0 - else completed.stderr.strip() or "Subprocess returned a non-zero exit code.", + else completed.stderr.strip() + or "Subprocess returned a non-zero exit code.", source=str(path), ) @@ -227,7 +688,29 @@ def _execute_subprocess(self, stage: "Stage", context: ExecutionContext) -> Stag return result -def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionContext) -> Any: +def _invoke_callable( + callable_object: Any, stage: Stage, context: ExecutionContext +) -> Any: + """ + Assigns appropriate parameters for a callable and runs it. + + Searches for parameter terms that likely refer to context or stage. If none of these are found, + assigns ``context`` as the first parameter and ``stage`` as the second. + + Parameters + ---------- + ``callable_object`` : Any + The callable item that is going to be run. + ``stage`` : ``Stage`` class + The ``Stage`` class instance to be a parameter for the ``callable_object``. + ``context`` : ``ExecutionContext`` class + The ``ExecutionContext`` class instance to be a parameter for the ``callable_object``. + + Returns + ------- + ``callable_object`` + An invocation of the ``callable_object`` with appropriately assigned parameters. + """ signature = inspect.signature(callable_object) parameters = list(signature.parameters.values()) @@ -251,7 +734,9 @@ def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionCon if parameter.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) ] - has_varargs = any(parameter.kind == inspect.Parameter.VAR_POSITIONAL for parameter in parameters) + has_varargs = any( + parameter.kind == inspect.Parameter.VAR_POSITIONAL for parameter in parameters + ) if not positional_parameters and not has_varargs: return callable_object() @@ -263,8 +748,14 @@ def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionCon return callable_object(context) if len(positional_parameters) >= 2 or has_varargs: - first_name = positional_parameters[0].name.lower() if positional_parameters else "" - second_name = positional_parameters[1].name.lower() if len(positional_parameters) > 1 else "" + first_name = ( + positional_parameters[0].name.lower() if positional_parameters else "" + ) + second_name = ( + positional_parameters[1].name.lower() + if len(positional_parameters) > 1 + else "" + ) if first_name in ("stage", "task") and second_name in ("context", "ctx"): return callable_object(stage, context) if first_name in ("context", "ctx") and second_name in ("stage", "task"): @@ -275,13 +766,37 @@ def _invoke_callable(callable_object: Any, stage: "Stage", context: ExecutionCon def _build_success_result( - stage: "Stage", + stage: Stage, started_at: datetime, finished_at: datetime, output: Any, *, source: str | None = None, ) -> StageResult: + """ + Create a ``StageResult`` instance showing a successful stage run. + + If the output of a ``Stage`` run is a ``StageResult`` class, set missing attributes to relevant + information from the ``Stage``. + + Parameters + ---------- + ``stage`` : ``Stage`` class + The ``Stage`` class instance being run. + ``started_at`` : datetime + The time and date that the run started. + ``finished_at`` : datetime + The time and date that the run ended. + ``output`` : Any + The output produced from the stage run. + ``source`` : str or None + The file/callable being run in the stage. + + Return + ------ + ``StageResult`` instance + Containing metadata for the stage run and showing that the run was a success. + """ if isinstance(output, StageResult): if output.name != stage.name: output.name = stage.name @@ -299,4 +814,4 @@ def _build_success_result( outputs=output, metadata=dict(stage.metadata), source=source, - ) \ No newline at end of file + ) diff --git a/onsrap/graph.py b/onsrap/graph.py index 18d2eab..efacc13 100644 --- a/onsrap/graph.py +++ b/onsrap/graph.py @@ -9,18 +9,43 @@ @dataclass class StageGraph: + """ + Represents an order to run stages. + + Holds an order that stages need to run in based on dependencies and logic. + + Parameters + ---------- + ``stages`` : list of ``Stage`` class items + """ + stages: list[Stage] = field(default_factory=list) @classmethod - def from_stages(cls, stages: Iterable[Stage]) -> "StageGraph": + def from_stages(cls, stages: Iterable[Stage]) -> StageGraph: """ This is the primary constructor for StageGraph, which performs validation and normalization of the stage list. + + Parameters + ---------- + ``stages`` : Iterable of ``Stage`` class instances + + Returns + ------- + The ``stages`` parameter as a list. """ return cls(list(stages)) def validate(self) -> None: """ Validate the stage graph for issues such as duplicate stage names, missing dependencies, and cycles. + + Raises + ------ + ``DuplicateStageError`` + If the stage name appears multiple times in the stage list. + ``MissingDependencyError`` + If there are unknown dependencies. """ # Check for duplicate stages @@ -70,21 +95,34 @@ def topological_order(self) -> list[Stage]: stages that depend on it. That may free up more stages, which are then added to the ready list. - If the algorithm cannot place every stage, the graph contains either a - cycle or a dependency that could not be resolved. In that case a - ``DependencyCycleError`` is raised. + Returns + ------- + A list of stages ordered in the way that they need to be run through the + pipeline. + + Raises + ------ + ``DependencyCycleError`` + If the algorithm cannot place every stage, the graph contains either a + cycle or a dependency that could not be resolved. """ stage_by_name = {stage.name: stage for stage in self.stages} - incoming = {stage.name: set(stage.dependencies) for stage in self.stages} - dependents = {stage.name: set() for stage in self.stages} + incoming: dict[str, set[str]] = { + stage.name: set(stage.dependencies) for stage in self.stages + } + dependents: dict[str, set[str]] = {stage.name: set() for stage in self.stages} for stage in self.stages: for dependency in stage.dependencies: + if dependency not in dependents: + raise MissingDependencyError( + f"Unknown stage dependency: {stage.name} -> {dependency}" + ) dependents[dependency].add(stage.name) original_order = [stage.name for stage in self.stages] ready = [name for name in original_order if not incoming[name]] - ordered = [] + ordered: list[str] = [] while ready: current = ready.pop(0) diff --git a/onsrap/loader.py b/onsrap/loader.py index 21d8277..3f86dcd 100644 --- a/onsrap/loader.py +++ b/onsrap/loader.py @@ -6,8 +6,10 @@ import sys from pathlib import Path from types import ModuleType +from typing import Any from .errors import StageConfigurationError, StageLoadError +from .models import PipelineRun PREFERRED_ENTRYPOINTS = ("run", "main", "execute") @@ -23,14 +25,30 @@ def discover_python_entrypoint(path: Path) -> str | None: the module. That keeps discovery fast and avoids running stage code just to learn how it should be invoked. - Returns ``None`` when the file exists but does not define a preferred + Parameters + ---------- + ``path`` : Path + File path for the stage being run. + + Returns + ------- + String item containing the name of the ``PREFERRED_ENTRYPOINTS`` item relevant + for the stages. + ``None`` when the file exists but does not define a preferred callable, which signals to the executor that it should treat the file as a script-style stage instead. + + Raises + ------ + ``StageConfigurationError`` + If the file path requested for the ``Stage`` does not exist. """ - + file_path = Path(path) if not file_path.exists(): - raise StageConfigurationError("Stage source file does not exist: {0}".format(file_path)) + raise StageConfigurationError( + "Stage source file does not exist: {0}".format(file_path) + ) try: tree = ast.parse(file_path.read_text(encoding="utf-8"), filename=str(file_path)) @@ -51,7 +69,7 @@ def discover_python_entrypoint(path: Path) -> str | None: return None -def load_python_callable(path: Path, entrypoint: str): +def load_python_callable(path: Path, entrypoint: str) -> Any: """ Import a stage module and return the named callable from it. @@ -61,9 +79,23 @@ def load_python_callable(path: Path, entrypoint: str): receive the execution context. It is kept separate from module loading so the executor can reuse the same import path for multiple runtime strategies. - A ``StageConfigurationError`` is raised if the chosen entrypoint does not - exist or is not callable, because that means the stage definition and the - executable surface no longer agree. + Parameters + ---------- + ``path`` : Path + The file path for the stage being run. + ``entrypoint`` : str + The name of the entrypoint function defined in the stage script. + + Raises + ------ + ``StageConfigurationError`` + If the chosen entrypoint does not exist or is not callable, because that means + the stage definition and the executable surface no longer agree. + + Returns + ------- + ``target`` + The ``entrypoint`` attribute of the module called to run the stage. """ module = load_python_module(path) target = getattr(module, entrypoint, None) @@ -87,9 +119,23 @@ def load_python_module(path: Path) -> ModuleType: The generated name is derived from the file path so repeated loads of the same stage remain stable during a run, while still avoiding collisions with - other Python modules. Import failures are converted into ``StageLoadError`` - so callers can report a stage-specific problem rather than a raw import - exception. + other Python modules. + + Parameters + ---------- + ``path`` : Path + The path for the stage. + + Returns + ------- + ``module`` + The set of code being run for the stage. + + Raises + ------ + ``StageLoadError`` + If the file is unable to be imported so callers can report a stage-specific + problem rather than a raw import exception. """ file_path = Path(path) if not file_path.exists(): @@ -97,7 +143,7 @@ def load_python_module(path: Path) -> ModuleType: module_name = "onsrap_stage_{0}_{1}".format( file_path.stem, - hashlib.sha1(str(file_path.resolve()).encode("utf-8")).hexdigest()[:12], + hashlib.sha256(str(file_path.resolve()).encode("utf-8")).hexdigest()[:12], ) spec = importlib.util.spec_from_file_location(module_name, str(file_path)) if spec is None or spec.loader is None: @@ -115,3 +161,29 @@ def load_python_module(path: Path) -> ModuleType: ) from exc return module + + +def load_historical_run(run_dir: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + Returns + ------- + ``PipelineRun`` + An instance of ``PipelineRun`` representing the historical run. + """ + import glob + + files = glob.glob(str(run_dir / "pipeline_attributes_for_*.yaml")) + if not files: + raise StageLoadError( + "Historical run file does not exist in: {0}".format(run_dir) + ) + file_path = Path(files[0]) + + import yaml + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return PipelineRun._pipeline_run_from_dict(data) diff --git a/onsrap/logger.py b/onsrap/logger.py index 0eafe7f..875e19f 100644 --- a/onsrap/logger.py +++ b/onsrap/logger.py @@ -3,26 +3,63 @@ import json import logging from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Any +from .errors import HistoricalPipelineLoadError + @dataclass class LogConfig: + """ + Data class which holds information regarding how the logs are set up. + + Parameters + ---------- + ``log_dir`` : str, default = "logs/" + The directory where all logs are stored for the Pipeline. + ``log_level`` : str, default = "INFO" + Denotes how severe the log message is. + ``logger_name`` : str, default = "onsrap" + The name of the logging system. + """ + log_dir: str = "logs/" log_level: str = "INFO" logger_name: str = "onsrap" class Logger: + """ + Creates a logging system. + + This system creates a logging directory and enables writing the log messages + to both console and the logging files. It allows configurable logging levels + to adjust for severity and avoids duplicating logging messages or handlers. + If the logger is unable to write to a file, the logging continues using only + the console handler. + + Parameters + ---------- + ``log_dir`` : str or Path, default = "logs/" + The directory where you'd like your logs stored. + ``log_level`` : str, default = "INFO" + The severity of the log. + """ + + _configured_loggers: set[str] = set() + def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self.config = LogConfig(log_dir=str(log_dir), log_level=log_level) self.log_dir = Path(self.config.log_dir) self.log_dir.mkdir(parents=True, exist_ok=True) - - self._logger = logging.getLogger(self.config.logger_name) - if not getattr(self._logger, "_onsrap_configured", False): - self._logger.setLevel(getattr(logging, self.config.log_level.upper(), logging.INFO)) + logger_name = f"{self.config.logger_name}:{self.log_dir.resolve()}" + self._logger = logging.getLogger(logger_name) + self._logger.setLevel( + getattr(logging, self.config.log_level.upper(), logging.INFO) + ) + if logger_name not in self._configured_loggers: self._logger.propagate = False stream_handler = logging.StreamHandler() @@ -30,24 +67,190 @@ def __init__(self, log_dir: str | Path = "logs/", log_level: str = "INFO"): self._logger.addHandler(stream_handler) try: - file_handler = logging.FileHandler(self.log_dir / "onsrap.log", encoding="utf-8") + file_handler = logging.FileHandler( + self.log_dir / "onsrap.log", encoding="utf-8" + ) file_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self._logger.addHandler(file_handler) except OSError: pass - setattr(self._logger, "_onsrap_configured", True) + self._configured_loggers.add(logger_name) def __call__(self, *args: Any, **kwargs: Any) -> None: + """ + Converts Logger instances to be callable, enabling easier implementation + of logging. + + Positional arguemnts are converted to strings and joined with spaces. + Keyword arguments are serialised as JSON and appended as structured + context. + """ message = " ".join(str(arg) for arg in args) if kwargs: context = json.dumps(kwargs, default=str, sort_keys=True) message = f"{message} | {context}" if message else context self._logger.info(message) + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Logger`` + class. + + Returns + ------- + str + A string representation of the ``Logger`` class with its attributes. + """ + return ( + f"Log Directory: {self.log_dir.resolve()}\n" + f" Log Level: {self.config.log_level}" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the + ``Logger`` class. This method is structured to be more concise than + the ``__str__`` method and is intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``Logger`` class with its attributes. + """ + return f"Logger(log_dir={self.log_dir.resolve()}, log_level={self.config.log_level})" + def event(self, message: str, **kwargs: Any) -> None: + """ + Logs a named event with optional structured context. + + Parameters + ---------- + ``message`` : str + The main description of the event to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. + """ if kwargs: - self._logger.info("%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True)) + self._logger.info( + "%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True) + ) else: self._logger.info(message) + def warning(self, message: str, **kwargs: Any) -> None: + """ + Logs a warning message with optional structured context. + + Parameters + ---------- + ``message`` : str + The main description of the warning to be logged. + ``**kwargs`` : Any + Additional information to be recorded in the log record. + """ + if kwargs: + self._logger.warning( + "%s | %s", message, json.dumps(kwargs, default=str, sort_keys=True) + ) + else: + self._logger.warning(message) + + def extract_historical_run_ids( + self, run_root: Path, name: str + ) -> list[dict[str, Any]]: + """ + Extracts historical run IDs from the log files. + + Parameters + ---------- + ``run_root`` : Path + The root directory where the historical runs are stored. + ``name`` : str + The name of the pipeline for which to extract historical run IDs. + + Returns + ------- + list[dict[str, Any]] + A list of dictionaries containing run_id, timestamp, and run_dir for each historical run. + """ + + # ensure that logger is writing to a file and extract filepath + if not self._logger.hasHandlers(): + raise HistoricalPipelineLoadError( + "The logger does not write to a" + "filepath. Please ensure that your logger writes to a file path so that" + "we can extract the run_id for historical runs." + ) + + logfile_handler = next( + (h for h in self._logger.handlers if isinstance(h, logging.FileHandler)), + None, + ) + if logfile_handler is None: + raise HistoricalPipelineLoadError( + "The logger does not have a FileHandler. " + "Please ensure that your logger writes to a file path so that we can extract the run_id " + "for historical runs." + ) + + logfile_path = logfile_handler.baseFilename + if not Path(logfile_path).exists(): + raise HistoricalPipelineLoadError( + "The log file does not exist at this location." + ) + + matches: list[dict[str, Any]] = [] + + # TODO: This method works if the logs are recorded in chronological order. Would there + # ever be a case where a record would appear below another and not be chronological? + # If so, we may need to sort based on the timestamp rather than the ordering. + for raw_line in reversed( + Path(logfile_path).read_text(encoding="utf-8").splitlines() + ): + if "Pipeline started" not in raw_line or " | " not in raw_line: + continue + + # left: timestamp + message, right: JSON context + left, right = raw_line.split(" | ", 1) + + # catches where Pipeline started is not recorded in the correct place. + if not left.endswith(" Pipeline started"): + continue + + # checks that datetime is valid + timestamp = left[: -len(" Pipeline started")].strip() + try: + datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S,%f") + except ValueError: + continue + + try: + payload = json.loads(right) + except json.JSONDecodeError: + continue + + run_id = payload.get("run_id") + if not run_id: + continue + + # timestamp is the first two space-separated tokens: YYYY-MM-DD HH:MM:SS,mmm + parts = left.split(" ", 2) + if len(parts) < 2: + continue + timestamp = f"{parts[0]} {parts[1]}" + + run_dir = run_root / run_id + # only returns run_ids for runs where a run_directory is still present. + + log_name = payload.get("name") + if run_dir.exists() and log_name == name: + matches.append( + { + "run_id": run_id, + "timestamp": timestamp, + "run_dir": run_dir, + } + ) + + return matches diff --git a/onsrap/models.py b/onsrap/models.py index c7977b7..ca35572 100644 --- a/onsrap/models.py +++ b/onsrap/models.py @@ -1,13 +1,21 @@ from __future__ import annotations -from dataclasses import dataclass, field -from datetime import datetime, timezone +import warnings +from base64 import b64decode, b64encode +from dataclasses import asdict, dataclass, field, is_dataclass +from datetime import date, datetime, time from enum import Enum from pathlib import Path -from typing import Any, Mapping, Optional, Union +from typing import Any, Iterable, Literal, Mapping, Optional, overload + +from .errors import PipelineConfigurationError, StageConfigurationError class StageStatus(str, Enum): + """ + Class to hold information on how the Stage has run. + """ + PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -16,6 +24,10 @@ class StageStatus(str, Enum): class PipelineStatus(str, Enum): + """ + Class to hold information on how the Pipeline has run. + """ + PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" @@ -23,64 +35,192 @@ class PipelineStatus(str, Enum): def now() -> datetime: + """ + Function to extract the current time in a datetime format. + """ return datetime.now() def utcnow() -> datetime: + """ + Function to extract the current time in UTC in a datetime format. + """ return now() @dataclass class RuntimeID: + """ + Holds information regarding individual runs. + + Parameters + ---------- + ``id`` : str + The id number for the run. + ``timestamp`` : datetime + The time that the run started. + ``hash`` : str + A hashed identifier created with the combined ID and + timestamp to create a unique identifier for the run. + ``short_hash`` : str + A shortened version of the ``hash`` attribute to be used + in file names for the runs. + """ + id: str timestamp: datetime hash: str short_hash: str def get_id(self) -> str: + """ + Getter function to extract the ``id`` attribute. + """ return self.id def get_timestamp(self) -> datetime: + """ + Getter function to extract the ``timestamp`` attribute. + """ return self.timestamp def get_hash(self) -> str: + """ + Getter function to extract the ``hash`` attribute. + """ return self.hash def get_short_hash(self) -> str: + """ + Getter function to extract the ``short_hash`` attribute. + """ return self.short_hash @dataclass -class RAPConfig: - contents: dict[str, Any] = field(default_factory=dict) +class PipelineConfig: + """ + Holds information required to run the whole pipeline. + Parameters + ---------- + ``name`` : str, optional + The name of the pipeline. + ``stages_to_run`` : dict[str, bool], optional + A dictionary of all stage names alongside a boolean value that indicates + whether the stage should be run or not. + ``backend`` : str, default = "python" + The system that the pipeline is run on. + ``work_dir`` : Path + The directory to run the Pipeline in. + ``project_root`` : Path + The top level directory for the whole project. + ``log_dir`` : Path + The directory to store the logs in. + ``data_dir`` : Path + The directory where the data is stored. + ``output_dir`` : Path, optional + The directory where pipeline outputs should be written. Not used internally + by the runner; exposed for stage code to read via ``context.config.output_dir``. + ``allow_subprocess_fallback`` : bool + Indicates whether the subprocess system (running the whole file + rather than an entrypoint function) should be allowed. + ``python_executable`` : str, optional + The name of the executable function for the entrypoint of the + pipeline. + ``metadata`` : dict[str, Any] + Any additional information on the pipeline. + ``overwrite`` : bool, default = False + Indicates whether the pipeline should overwrite previous outputs. + """ -@dataclass -class PipelineConfig: name: Optional[str] = None + stages_to_run: Optional[dict[str, bool]] = None backend: str = "python" work_dir: Path = field(default_factory=Path.cwd) project_root: Optional[Path] = None + output_dir: Optional[Path] = None log_dir: Path = field(default_factory=lambda: Path("logs")) data_dir: Path = field(default_factory=lambda: Path("data")) allow_subprocess_fallback: bool = True python_executable: Optional[str] = None metadata: dict[str, Any] = field(default_factory=dict) + overwrite: bool = False + + def __post_init__(self) -> None: + """ + Post-initialization method to ensure that the ``work_dir`` and ``project_root`` + attributes are set correctly. + """ + if self.stages_to_run is None: + self.stages_to_run = {} + + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``PipelineConfig`` class. + + Returns + ------- + str + A string representation of the ``PipelineConfig`` class with its attributes. + """ + return ( + f" Name: {self.name}\n Stages To Run: {_format_dict(self.stages_to_run, indent=4)}\n" + f" Backend: {self.backend} \n" + f" Work Directory: {self.work_dir}\n Project Root: {self.project_root}\n" + f" Output Directory: {self.output_dir}\n Log Directory: {self.log_dir}\n" + f" Data Directory: {self.data_dir}\n Allow Subprocess Fallback: {self.allow_subprocess_fallback}\n" + f" Python Executable: {self.python_executable}\n Overwrite: {self.overwrite}\n" + f" Metadata: \n{_format_dict(self.metadata, indent=8)}" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``PipelineConfig`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``PipelineConfig`` class with its attributes. + """ + return ( + f"PipelineConfig(name={self.name}, stages_to_run={self.stages_to_run}, " + f"backend={self.backend}, " + f"work_dir={self.work_dir}, project_root={self.project_root}, " + f"output_dir={self.output_dir}, log_dir={self.log_dir}, data_dir={self.data_dir}, " + f"allow_subprocess_fallback={self.allow_subprocess_fallback}, " + f"python_executable={self.python_executable}, overwrite={self.overwrite}, " + f"metadata={self.metadata})" + ) @classmethod def from_any( cls, - value: Union["PipelineConfig", RAPConfig, Mapping[str, Any], str, Path, None], - ) -> "PipelineConfig": + value: PipelineConfig | Mapping[str, Any] | str | Path | None, + ) -> PipelineConfig: + """ + Converts one of several datatypes into a PipelineConfig class instance. + + Parameters + ---------- + ``value`` : PipelineConfig, Mapping[str, Any], str, Path, or None + The object holding metadata on how the Pipeline should run to be converted + into a PipelineConfig class instance. + + Raises + ------ + ``TypeError`` + If the datatype for the object holding information on how the pipeline is run + is not a datatype that can be converted to a PipelineConfig. + """ if value is None: return cls() if isinstance(value, cls): return value - if isinstance(value, RAPConfig): - return cls.from_mapping(value.contents) - if isinstance(value, Mapping): return cls.from_mapping(dict(value)) @@ -90,7 +230,20 @@ def from_any( raise TypeError("Unsupported pipeline config type: {0!r}".format(type(value))) @classmethod - def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": + def from_mapping(cls, data: Mapping[str, Any]) -> PipelineConfig: + """ + Extracts information from a mapping datatype and returns a PipelineConfig + instance. + + Parameters + ---------- + ``data`` : Mapping[str, Any] + The information to be converted into a ``PipelineConfig`` instance. + + Returns + ------- + ``PipelineConfig`` class instance + """ payload = dict(data) metadata = payload.pop("metadata", {}) @@ -100,34 +253,84 @@ def from_mapping(cls, data: Mapping[str, Any]) -> "PipelineConfig": metadata = {"metadata": metadata} name = payload.pop("name", None) + backend = payload.pop("backend", "python") + stages_to_run = PipelineConfig._extract_stages_run(payload) work_dir = Path(payload.pop("work_dir", Path.cwd())) project_root_value = payload.pop("project_root", None) - project_root = Path(project_root_value) if project_root_value is not None else work_dir + output_dir_value = payload.pop("output_dir", None) + project_root = ( + Path(project_root_value) if project_root_value is not None else work_dir + ) log_dir = Path(payload.pop("log_dir", "logs")) data_dir = Path(payload.pop("data_dir", "data")) - allow_subprocess_fallback = bool(payload.pop("allow_subprocess_fallback", True)) + raw_subprocess_fallback = payload.pop("allow_subprocess_fallback", True) + overwrite = PipelineConfig._to_bool(payload.pop("overwrite", False)) + if isinstance(raw_subprocess_fallback, str): + warnings.warn( + "allow_subprocess_fallback should be a boolean, not a string. " + f"Received {raw_subprocess_fallback!r}. Use an unquoted YAML boolean.", + UserWarning, + stacklevel=2, + ) + allow_subprocess_fallback = raw_subprocess_fallback.strip().lower() not in ( + "false", + "0", + "no", + "off", + ) + else: + allow_subprocess_fallback = bool(raw_subprocess_fallback) python_executable = payload.pop("python_executable", None) metadata.update(payload) return cls( name=name, + stages_to_run=stages_to_run, backend=backend, work_dir=work_dir, project_root=project_root, + output_dir=output_dir_value, log_dir=log_dir, data_dir=data_dir, allow_subprocess_fallback=allow_subprocess_fallback, + overwrite=overwrite, python_executable=python_executable, metadata=metadata, ) @classmethod - def from_file(cls, path: Path) -> "PipelineConfig": + def from_file(cls, path: Path) -> PipelineConfig: + """ + Extracts a mapping item from a file containing information about how the + pipeline should run. + + Then calls the from_mapping() method to extract the information. + + Parameters + ---------- + ``path`` : Path + The file path containing information to be converted into a PipelineConfig + instance. + + Returns + ------- + ``PipelineConfig`` class instance. + + Raises + ------ + ``FileNotFoundError`` + If the file path does not exist. + ``TypeError`` + If the file containing information about how the Pipeline runs does not + contain a mapping type. + """ config_path = Path(path).expanduser() if not config_path.exists(): - raise FileNotFoundError("Config file does not exist: {0}".format(config_path)) + raise FileNotFoundError( + "Config file does not exist: {0}".format(config_path) + ) import yaml @@ -136,16 +339,25 @@ def from_file(cls, path: Path) -> "PipelineConfig": return cls() if not isinstance(raw_config, Mapping): - raise TypeError("Pipeline config file must contain a mapping at the top level.") + raise TypeError( + "Pipeline config file must contain a mapping at the top level." + ) return cls.from_mapping(raw_config) def to_dict(self) -> dict[str, Any]: + """ + Returns a prescriptive expression of the attributes within the PipelineConfig instance + that allows for easier processing by the user. + """ data = { "name": self.name, "backend": self.backend, "work_dir": str(self.work_dir), - "project_root": str(self.project_root) if self.project_root is not None else None, + "project_root": str(self.project_root) + if self.project_root is not None + else None, + "output_dir": str(self.output_dir) if self.output_dir is not None else None, "log_dir": str(self.log_dir), "data_dir": str(self.data_dir), "allow_subprocess_fallback": self.allow_subprocess_fallback, @@ -154,9 +366,308 @@ def to_dict(self) -> dict[str, Any]: data.update(self.metadata) return data + @staticmethod + def _extract_stages_run(payload: dict[str, Any]) -> dict[str, bool] | None: + """ + Method to extract stages_to_run configuration and convert all values to boolean values. + + Parameters + ---------- + ``payload`` : Mapping[str, Any] + The dictionary where the stages_to_run configuration is being extracted from. + + Returns + ------- + ``boolean_dict`` + A dictionary of stage_name:bool to indicate whether a stage is being run. + + None + If stages_to_run does not exist within the configuration. + """ + stages_to_run = payload.pop("stages_to_run", None) + if stages_to_run is None: + return None + + boolean_dict = { + stage_name: PipelineConfig._to_bool(value) + for stage_name, value in stages_to_run.items() + } + + return boolean_dict + + @staticmethod + def _to_bool(value: bool | int | str) -> bool: + """ + Method to convert values to boolean True/False values. + + Integers convert to boolean where 0 = False and 1 = True. A certain subset of strings are + accepted for conversion. Any other strings will. raise an error. + + Parameters + ---------- + ``value`` : bool | int | str + + Returns + ------- + ``value`` + The value input but converted to a boolean value. + + Raises + ------ + ``ValueError`` + When the value has not been able to be converted to a boolean. + """ + + if isinstance(value, bool): + return value + + if isinstance(value, int): + return bool(value) + + if isinstance(value, str): + value = value.strip().lower() + + if value in {"true", "yes", "y", "1"}: + return True + + if value in {"false", "no", "n", "0"}: + return False + + raise ValueError(f"Cannot convert {value!r} to bool") + + raise ValueError(f"Cannot convert {value!r} to bool") + + +@dataclass +class StageConfig: + """ + Holds configuration that should be exposed to an individual stage at runtime. + + Parameters + ---------- + ``name`` : str + The name of the stage that this configuration applies to. + ``_variables`` : dict[str, Any] + Arbitrary stage-scoped variables. + ``metadata`` : dict[str, Any] + Additional supporting metadata for the stage configuration. + """ + + # TODO: output location for stages potentially problematic for output overwrites! + name: str + _variables: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_mapping( + cls, name: str, data: Mapping[str, Any] | None = None + ) -> StageConfig: + """ + Build a ``StageConfig`` from a mapping loaded from code or configuration files. + + The ``datasets`` and ``metadata`` keys are extracted into their dedicated + attributes. All remaining keys are treated as stage variables that should be + exposed to the stage at runtime. + + Parameters + ---------- + ``name`` : str + Stage name that this configuration applies to. + ``data`` : Mapping[str, Any] or None + Raw configuration payload for that stage. + # Removed global_vars parameter + + Returns + ------- + ``StageConfig`` + A normalized stage configuration object. + """ + payload = dict(data or {}) + + metadata = payload.pop("metadata", {}) + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} + + return cls( + name=str(name).strip(), + _variables=payload, + metadata=metadata, + ) + + @property + def variables(self) -> dict[str, Any]: + """ + Return a copy of the stage variables without datasets or metadata. + """ + return dict(self._variables) + + def get(self, variable: str, default: Any = None) -> Any: + """ + Return a configured variable if present, otherwise return ``default``. + """ + return self._variables.get(variable, default) + + def require(self, variable: str) -> Any: + """ + Return a configured variable and raise if the stage does not define it. + """ + if variable not in self._variables: + raise StageConfigurationError( + f"Stage configuration '{self.name}' does not define '{variable}'." + ) + return self._variables[variable] + + def get_variables(self, variable: Iterable[str] | str | None = None) -> Any: + """ + Return all configured variables, one configured variable, or a selected subset. + """ + if variable is None: + return dict(self._variables) + + if isinstance(variable, str): + return self.require(variable) + + requested_variables: dict[str, Any] = {} + missing_variables: list[str] = [] + for requested_name in variable: + if requested_name in self._variables: + requested_variables[requested_name] = self._variables[requested_name] + else: + missing_variables.append(requested_name) + + if missing_variables: + missing = ", ".join(sorted(missing_variables)) + raise StageConfigurationError( + f"Stage configuration '{self.name}' does not define: {missing}." + ) + + return requested_variables + + def to_dict(self) -> dict[str, Any]: + """ + Serialize the stage configuration back to a mapping suitable for manifests. + """ + data = dict(self._variables) + if self.metadata: + data["metadata"] = dict(self.metadata) + return data + + +@dataclass +class GlobalConfig: + """ + Holds configuration that should be exposed to all stages at runtime. + + Parameters + ---------- + ``_variables`` : dict[str, Any] + Variables that should be parsed to all stages throughout the pipeline. + """ + + _variables: dict[str, Any] = field(default_factory=dict) + exclusion: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_dict(cls, data: Mapping[str, Any] | None) -> GlobalConfig: + """ + Build a ``GlobalConfig`` from a mapping loaded from code or configuration files. + + Parameters + ---------- + ``data`` : Mapping[str, Any] | None + Raw configuration payload for the global configuration. + ``exclusions`` : dict[str, Any] or None + A lookup of which global variables should be excluded from each stage. + + Returns + ------- + ``GlobalConfig`` + A global configuration object. + """ + if data is None: + return cls() + + payload = dict(data or {}) + + exclusions = payload.pop("exclusions", {}) + if exclusions is None: + exclusions = {} + elif not isinstance(exclusions, Mapping): + raise PipelineConfigurationError("Global exclusions must be a mapping.") + else: + exclusions = dict(exclusions) + + return cls(_variables=payload, exclusion=exclusions) + + @overload + def get_attributes( + self, keep_exclusion: Literal[True] = True + ) -> tuple[dict[str, Any], dict[str, Any]]: ... + + @overload + def get_attributes(self, keep_exclusion: Literal[False]) -> dict[str, Any]: ... + + def get_attributes( + self, keep_exclusion: bool = True + ) -> tuple[dict[str, Any], dict[str, Any]] | dict[str, Any]: + """ + Return a copy of the global variables, optionally excluding any variables + specified in the exclusion list. + + Parameters + ---------- + ``keep_exclusion`` : bool, default = True + If True, return both _variables and exclusion. + If False, return only the variables and not the exclusion list. + + Returns + ------- + ``self._variables`` : dict[str, Any] + All global variables for the pipeline. + ``self.exclusion`` : dict[str, Any] + The exclusion list of global variables for each stage. Only returned + if ``keep_exclusion`` is True. + """ + if keep_exclusion: + return dict(self._variables), dict(self.exclusion or {}) + return dict(self._variables) + @dataclass class RunManifest: + """ + Holds metadata information about the run. + + Parameters + ---------- + ``rap_name`` : str, default = "" + The name of the Pipeline. + ``run_id`` : str, default = "" + The unique ID of the run. + ``git_commit`` : str, default = None + The git commit number for the run, indicating the exact state of the code. + ``stages_run`` : list[str] + List of the names of stages that were included in this run. + ``parameters`` : dict[str, Any] + + ``inputs`` : dict[str, Any] + + ``outputs`` : dict[str, Any] + + ``backend`` : str, default = "python" + The system that the Pipeline will run in. + ``package_versions``: list[str] or str + The package versions that are used in this run. + ``timestamp`` : str, default = "" + The time that this run started. + ``reason`` : str, optional, default = None + The reason that this run took place. + ``user`` : str, optional, default = None + The person running this specific run. + """ + rap_name: str = "" run_id: str = "" git_commit: Optional[str] = None @@ -165,10 +676,113 @@ class RunManifest: inputs: dict[str, Any] = field(default_factory=dict) outputs: dict[str, Any] = field(default_factory=dict) backend: str = "python" - package_versions: Union[list[str], str] = field(default_factory=list) + package_versions: list[str] | str = field(default_factory=list) timestamp: str = "" reason: Optional[str] = None user: Optional[str] = None + config: Optional[dict[str, Any]] = None + + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``RunManifest`` + class. + + Returns + ------- + str + A string representation of the ``RunManifest`` class with its attributes. + """ + + return ( + f"\nRAP Name: {self.rap_name}\nRun ID: {self.run_id} \n" + f"Git Commit: {self.git_commit}\nStages Run: {self.stages_run} \n" + f"Parameters: \n{_format_dict(self.parameters, indent=4)} \n" + f"Inputs: \n{_format_dict(self.inputs, indent=4)} \n" + f"Outputs: \n{_format_dict(self.outputs, indent=4)} \nBackend: {self.backend} \n" + f"Package Versions: {self.package_versions} \nTimestamp: {self.timestamp}\n" + f"Reason: {self.reason} \nUser: {self.user}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the + ``RunManifest`` class. This method is structured to be more concise than + the ``__str__`` method and is intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``RunManifest`` class with its attributes. + """ + return ( + f"RunManifest(rap_name={self.rap_name}, run_id={self.run_id}, " + f"git_commit={self.git_commit}, stages_run={self.stages_run}, " + f"parameters={self.parameters}, inputs={self.inputs}, " + f"outputs={self.outputs}, backend = {self.backend}, " + f"package_versions={self.package_versions}, " + f"timestamp={self.timestamp}, reason={self.reason}, user={self.user})" + ) + + def _runmanifest_to_dict(self) -> dict[str, Any]: + """ + Converts the RunManifest instance into a dictionary representation. + This is needed to allow a RunManifest instance to be serialized into a + JSON format for later methods on RunManifest instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the RunManifest instance. + """ + return { + "rap_name": self.rap_name, + "run_id": self.run_id, + "git_commit": self.git_commit, + "stages_run": self.stages_run, + "parameters": self.parameters, + "inputs": self.inputs, + "outputs": self.outputs, + "backend": self.backend, + "package_versions": self.package_versions, + "timestamp": self.timestamp, + "reason": self.reason, + "user": self.user, + "config": self.config, + } + + @classmethod + def _runmanifest_from_dict(cls, data: dict[str, Any]) -> RunManifest: + """ + Converts a dictionary representation of a RunManifest instance back into a + RunManifest instance. Allows for RunManifest instances to be created from + a JSON representation of a RunManifest instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a RunManifest instance. + + Returns + ------- + ``RunManifest`` class instance + A RunManifest instance created from the dictionary representation. + """ + return cls( + rap_name=data.get("rap_name", ""), + run_id=data.get("run_id", ""), + git_commit=data.get("git_commit"), + stages_run=data.get("stages_run", []), + parameters=data.get("parameters", {}), + inputs=data.get("inputs", {}), + outputs=data.get("outputs", {}), + backend=data.get("backend", "python"), + package_versions=data.get("package_versions", []), + timestamp=data.get("timestamp", ""), + reason=data.get("reason"), + user=data.get("user"), + config=data.get("config"), + ) class RAPDataset: @@ -185,6 +799,36 @@ class Catalog: @dataclass class StageResult: + """ + Holds information about how the stage ran. + + Parameters + ---------- + ``name`` : str + The name of the Stage run. + ``status`` : StageStatus + The status of the run at completion. + ``started_at`` : datetime + The date and time that the Stage started. + ``finished_at`` : datetime + The date and time that the Stage finished. + ``outputs`` : Any, default = None + Captures outputs of the stage being run. + ``stdout`` : str, default = "" + Captures outputs of the stage being run. + ``stderr`` : str, default = "" + Captures any errors produced during the run. + ``return_code`` : int, optional, default = None + Indicates whether the stage has run successfully or if there + was an error. + ``metadata``: dict[str, Any] + Holds information about the Stage such as file directories. + ``error`` : str, optional, default = None + Any errors produced during the run. + ``source`` : str, optional, default = None + The name/location of the code for that Stage run. + """ + name: str status: StageStatus started_at: datetime @@ -197,17 +841,102 @@ class StageResult: error: Optional[str] = None source: Optional[str] = None + def _stage_result_to_dict(self) -> dict[str, Any]: + """ + Converts the StageResult instance into a dictionary representation. + This is needed to allow a StageResult instance to be serialized into a + JSON format for later methods on StageResult instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the StageResult instance. + """ + return { + "name": self.name, + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "finished_at": self.finished_at.isoformat(), + "outputs": self.outputs, + "stdout": self.stdout, + "stderr": self.stderr, + "return_code": self.return_code, + "metadata": self.metadata, + "error": self.error, + "source": self.source, + } + + @classmethod + def _stage_result_from_dict(cls, data: dict[str, Any]) -> StageResult: + """ + Converts a dictionary representation of a StageResult instance back into a + StageResult instance. Allows for StageResult instances to be created from + a JSON representation of a StageResult instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a StageResult instance. + + Returns + ------- + ``StageResult`` class instance + A StageResult instance created from the dictionary representation. + """ + return cls( + name=data["name"], + status=StageStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + finished_at=datetime.fromisoformat(data["finished_at"]), + outputs=data.get("outputs"), + stdout=data.get("stdout", ""), + stderr=data.get("stderr", ""), + return_code=data.get("return_code"), + metadata=data.get("metadata", {}), + error=data.get("error"), + source=data.get("source"), + ) + @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``StageResult`` class called ``succeeded`` that + contains a boolean value indicating if the run was a success or not. + Updates the ``status`` attribute to record that the Stage ran successfully. + """ return self.status == StageStatus.SUCCEEDED @property def duration_seconds(self) -> float: + """ + Creates a new attribute in the ``StageResult`` class called ``duration_seconds`` + that holds the exact duration of the stage in seconds. + """ return max((self.finished_at - self.started_at).total_seconds(), 0.0) @dataclass class PipelineRun: + """ + Holds information about how the whole Pipeline ran. + + Parameters + ---------- + ``manifest`` : RunManifest class instance + Metadata on how the specific run has gone. + ``status`` : PipelineStatus class instance + Whether the Pipeline ran successfully or if there were errors. + ``started_at`` : datetime + The date and time the Pipeline started. + ``completed_at`` : datetime + The date and time the Pipeline ended. + ``stage_results`` : list[StageResult] + Holds the results for every stage run as part of the Pipeline. + ``stage_outputs`` : dict[str, Any] + Holds the outputs from all stages run as part of the Pipeline. + """ + manifest: RunManifest status: PipelineStatus started_at: datetime @@ -216,11 +945,309 @@ class PipelineRun: stage_outputs: dict[str, Any] = field(default_factory=dict) def result_for(self, stage_name: str) -> Optional[StageResult]: + """ + Extracts the results for a specific stage. + + Parameters + ---------- + ``stage_name`` : str + The name of the Stage that you are requesting the results for. + """ for result in self.stage_results: if result.name == stage_name: return result return None + def _pipeline_run_to_dict(self) -> dict[str, Any]: + """ + Converts the PipelineRun instance into a dictionary representation. + This is needed to allow a PipelineRun instance to be serialized into a + JSON format for later methods on PipelineRun instances not saved in + memory. + + Returns + ------- + dict[str, Any] + A dictionary representation of the PipelineRun instance. + """ + return { + "manifest": self.manifest._runmanifest_to_dict(), + "status": self.status.value, + "started_at": self.started_at.isoformat(), + "completed_at": self.completed_at.isoformat(), + "stage_results": { + result.name: _yaml_safe_encode(result._stage_result_to_dict()) + for result in self.stage_results + }, + "stage_outputs": _yaml_safe_encode(self.stage_outputs), + } + + @classmethod + def _pipeline_run_from_dict(cls, data: dict[str, Any]) -> PipelineRun: + """ + Converts a dictionary representation of a PipelineRun instance back into a + PipelineRun instance. Allows for PipelineRun instances to be created from + a JSON representation of a PipelineRun instance. + + Parameters + ---------- + ``data`` : dict[str, Any] + A dictionary representation of a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` class instance + A PipelineRun instance created from the dictionary representation. + """ + manifest_data = _yaml_safe_decode(data["manifest"]) + stage_results_data = _yaml_safe_decode(data.get("stage_results", {})) + stage_outputs_data = _yaml_safe_decode(data.get("stage_outputs", {})) + + return cls( + manifest=RunManifest._runmanifest_from_dict(manifest_data), + status=PipelineStatus(data["status"]), + started_at=datetime.fromisoformat(data["started_at"]), + completed_at=datetime.fromisoformat(data["completed_at"]), + stage_results=[ + StageResult._stage_result_from_dict(result) + for result in stage_results_data.values() + ], + stage_outputs=stage_outputs_data, + ) + + @classmethod + def load_pipeline_run_for_historical_run(cls, file_path: Path) -> PipelineRun: + """ + Load a previously executed pipeline run from a YAML file. + + This function is used to load the state of a pipeline run that has been + saved to a YAML file. It reads the file, parses the YAML content, and + reconstructs the PipelineRun object. + + Parameters + ---------- + ``file_path`` : Path + The path to the YAML file containing the saved pipeline run. + + Returns + ------- + ``PipelineRun`` + The reconstructed PipelineRun object. + + Raises + ------ + ``FileNotFoundError`` + If the specified file does not exist. + """ + import yaml + + if not file_path.exists(): + raise FileNotFoundError(f"Pipeline run file does not exist: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + return cls._pipeline_run_from_dict(data) + @property def succeeded(self) -> bool: + """ + Creates a new attribute in the ``PipelineRun`` class called ``succeeded`` that + contains a boolean value indicating if the Pipeline was a success or not. + Updates the ``status`` attribute to record that the Pipeline ran successfully. + """ return self.status == PipelineStatus.SUCCEEDED + + +def _format_dict(d: dict[str, Any] | dict[str, bool] | None, indent: int = 0) -> str: + """ + Helper function to format dictionaries for __str__ methods. + + Parameters + ---------- + ``d`` : dict + The dictionary to format. + ``indent`` : int, default = 0 + The number of spaces to indent the dictionary representation. + + Returns + ------- + str + A formatted string representation of the dictionary. + """ + if d is None: + return "" + + lines = [] + for key, value in d.items(): + if isinstance(value, dict): + lines.append(f"{' ' * indent}{key}:") + lines.append(_format_dict(value, indent + 4)) + else: + lines.append(f"{' ' * indent}{key}: {value}") + return "\n".join(lines) + + +_YAML_TYPE_KEY = "__onsrap_yaml_type__" +_YAML_VALUE_KEY = "value" + + +def _yaml_safe_mapping_key(value: Any) -> str | int | float | bool | None: + """ + Convert mapping keys to YAML-safe scalar values. + + Complex key types are coerced to strings because YAML mappings require + hashable scalar-like keys to round-trip predictably with ``yaml.safe_load``. + """ + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, Enum): + return str(value.value) + if isinstance(value, Path): + return str(value) + return repr(value) + + +def _yaml_safe_encode(value: Any) -> Any: + """ + Convert arbitrary Python values into structures accepted by ``yaml.safe_dump``. + """ + if isinstance(value, (str, int, float, bool)) or value is None: + return value + + if isinstance(value, datetime): + return {_YAML_TYPE_KEY: "datetime", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, date): + return {_YAML_TYPE_KEY: "date", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, time): + return {_YAML_TYPE_KEY: "time", _YAML_VALUE_KEY: value.isoformat()} + + if isinstance(value, Path): + return {_YAML_TYPE_KEY: "path", _YAML_VALUE_KEY: str(value)} + + if isinstance(value, Enum): + return {_YAML_TYPE_KEY: "enum", _YAML_VALUE_KEY: _yaml_safe_encode(value.value)} + + if isinstance(value, bytes): + return { + _YAML_TYPE_KEY: "bytes", + _YAML_VALUE_KEY: b64encode(value).decode("ascii"), + } + + if isinstance(value, bytearray): + return { + _YAML_TYPE_KEY: "bytearray", + _YAML_VALUE_KEY: b64encode(bytes(value)).decode("ascii"), + } + + if isinstance(value, tuple): + return { + _YAML_TYPE_KEY: "tuple", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, set): + return { + _YAML_TYPE_KEY: "set", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, frozenset): + return { + _YAML_TYPE_KEY: "frozenset", + _YAML_VALUE_KEY: [_yaml_safe_encode(item) for item in value], + } + + if isinstance(value, list): + return [_yaml_safe_encode(item) for item in value] + + if isinstance(value, Mapping): + return { + _yaml_safe_mapping_key(key): _yaml_safe_encode(item) + for key, item in value.items() + } + + if is_dataclass(value) and not isinstance(value, type): + return { + _YAML_TYPE_KEY: "dataclass", + "python_type": f"{value.__class__.__module__}.{value.__class__.__qualname__}", + _YAML_VALUE_KEY: _yaml_safe_encode(asdict(value)), + } + + return { + _YAML_TYPE_KEY: "repr", + "python_type": f"{value.__class__.__module__}.{value.__class__.__qualname__}", + _YAML_VALUE_KEY: repr(value), + } + + +def _yaml_safe_decode(value: Any) -> Any: + """ + Decode values previously produced by ``_yaml_safe_encode``. + """ + if isinstance(value, list): + return [_yaml_safe_decode(item) for item in value] + + if not isinstance(value, Mapping): + return value + + marker = value.get(_YAML_TYPE_KEY) + if marker is None: + return {key: _yaml_safe_decode(item) for key, item in value.items()} + + encoded_value = value.get(_YAML_VALUE_KEY) + + if marker == "datetime": + try: + return datetime.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "date": + try: + return date.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "time": + try: + return time.fromisoformat(str(encoded_value)) + except ValueError: + return encoded_value + + if marker == "path": + return Path(str(encoded_value)) + + if marker == "enum": + return _yaml_safe_decode(encoded_value) + + if marker == "bytes": + try: + return b64decode(str(encoded_value).encode("ascii")) + except Exception: + return encoded_value + + if marker == "bytearray": + try: + return bytearray(b64decode(str(encoded_value).encode("ascii"))) + except Exception: + return encoded_value + + if marker == "tuple": + return tuple(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "set": + return set(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "frozenset": + return frozenset(_yaml_safe_decode(item) for item in encoded_value or []) + + if marker == "dataclass": + return _yaml_safe_decode(encoded_value) + + if marker == "repr": + return encoded_value + + return {key: _yaml_safe_decode(item) for key, item in value.items()} diff --git a/onsrap/pipeline.py b/onsrap/pipeline.py index ffa75ad..cebb27e 100644 --- a/onsrap/pipeline.py +++ b/onsrap/pipeline.py @@ -2,56 +2,737 @@ import getpass import hashlib +import re import subprocess import sys +import warnings +from contextlib import suppress from importlib import metadata as importlib_metadata from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence -from .errors import StageConfigurationError +from .errors import ( + HistoricalPipelineLoadError, + PipelineConfigurationError, + PipelineInitialisationError, + StageConfigurationError, + StageLoadError, +) from .execution import PythonStageExecutor, StageExecutor from .graph import StageGraph +from .loader import load_historical_run from .logger import Logger -from .models import PipelineConfig, PipelineRun, PipelineStatus, RAPConfig, RunManifest, RuntimeID, StageResult, now +from .models import ( + GlobalConfig, + PipelineConfig, + PipelineRun, + RunManifest, + RuntimeID, + StageConfig, + now, +) from .stage import Stage +from .warnings import PipelineConfigurationWarning, StageConfigurationWarning + +ACCEPTED_CONFIG_TYPES = (".yaml", ".yml") +AVAILABLE_EXECUTORS = ("python",) +# captures long and short versions of output, directory, path, location, and file that's not case sensitive. +OUTPUT_DIR_KEY_RE = re.compile( + r"^(?:out(?:put)?)(?:$|[_\-\s]?(?:dir(?:ectory)?|path|loc(?:ation)?|file))$", + re.IGNORECASE, +) class Pipeline: + """ + Represents an end-to-end code run. This class brings together class instances + from other modules within the package to establish what the Pipeline is. + + Sets up the metadata, configurations, logging, and executors required to run the + Pipeline. Assigns multiple attributes including those not initialised such as, + ``id``, ``graph``, ``manifest``, and ``last_run``. These take the forms of other + classes defined in other modules within this package. + + Parameters + ---------- + ``name`` : str or None + What the pipeline is called. + ``backend`` : str, default = "python" + The system used to run the pipeline. + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None + The instance containing the required information on running the Pipeline. + ``stages`` : sequence of Stage, Mapping[str, Any], str, Path, Callable, or None. + The required steps within the Pipeline. + ``logger`` : Logger or None + The system that is used to track the progress of the Pipeline. + ``executor`` : StageExecutor or None + The way that the Pipeline is actively run. + """ + def __init__( self, name: str | None = None, backend: str = "python", - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, - stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] | None = None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, + stages: Sequence[Stage | Mapping[str, Any] | str | Path | Callable[..., Any]] + | None = None, + dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, - executor: StageExecutor | None = None, + executor: StageExecutor | PythonStageExecutor | None = None, ): - self.name = name or "pipeline" - self.backend = backend or "python" - self.config = PipelineConfig.from_any(config) + ( + resolved_config, + resolved_stage_configs, + configured_stages, + resolved_global_config, + ) = self._resolve_config(config) + + self.name = name or resolved_config.name or "pipeline" + self.backend = backend or resolved_config.backend or "python" + if backend == "python" and resolved_config.backend != "python": + raise PipelineInitialisationError( + f"Pipeline backend {backend} does not align with PipelineConfig backend {resolved_config.backend}." + ) + + self.config = resolved_config if self.config.name is None: self.config.name = self.name - self.config.backend = self.backend self.logger = logger or Logger(log_dir=self.config.log_dir) - self.executor = executor or PythonStageExecutor() - self.stages = [self._coerce_stage(stage) for stage in (stages or [])] - self.graph = StageGraph.from_stages(self.stages) + + if stages is not None and configured_stages: + raise PipelineInitialisationError( + "Stages parsed through both Pipeline construction and configuration file. Either provide stages through the constructor or the configuration file, not both." + ) + self.stages = ( + configured_stages + if stages is None + else [self._coerce_stage(stage) for stage in stages] + ) + + if executor is not None: + self.executor = executor + else: + self._generate_context() + + self.dependencies = None + if dependencies is not None: + self.dependencies = self._assign_dependencies(dependencies, self.stages) + + self.stage_configs = dict(resolved_stage_configs) + self.global_config = resolved_global_config + + self._sync_stage_configs() + + self._rebuild_graph() self.id: RuntimeID | None = None self.manifest: RunManifest | None = None + + self.run_output = self._set_run_output() + self.last_run: PipelineRun | None = None + self.last_run = self._load_latest_run() + + self.all_runs: dict[str, PipelineRun] | None = None + self.all_runs = self._load_all_runs() self.logger.event( "Pipeline initialized", name=self.name, backend=self.backend, stages=[stage.name for stage in self.stages], + enabled_stages=[stage.name for stage in self.graph.stages], ) + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Pipeline`` class. + + Returns + ------- + str + A string representation of the ``Pipeline`` class with its attributes. + """ + stages = "\n".join(f"{str(stage)}\n" for stage in self.stages) + graph = [stage.name for stage in self.graph.stages] + return ( + f"\nPipeline Instance Attributes\n" + f"--------------------------\n" + f"Name:\n {self.name}\n\nBackend:\n {self.backend} \n\n" + f"Configuration:\n{self.config}\n\nStages:\n{stages} \n" + f"Dependencies:\n {self.dependencies} \n\nLogger:\n {self.logger} \n\n" + f"Executor:\n {self.executor} \n\nGraph:\n {graph} \n\n" + f"ID:\n {self.id} \n\nManifest:\n {self.manifest} \n\nLast Run:\n {self.last_run}\n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``Pipeline`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``Pipeline`` class with its attributes. + """ + return ( + f"Pipeline(name={self.name}, backend={self.backend}, " + f"stages={self.stages}, dependencies={self.dependencies}, " + f"logger={self.logger}, executor={self.executor}, graph={self.graph}, " + f"id={self.id}, manifest={self.manifest}, last_run={self.last_run})" + ) + + def add_stage( + self, + *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], + stage_configs: StageConfig + | Mapping[str, Any] + | str + | Path + | Iterable[StageConfig | Mapping[str, Any] | str | Path] + | None = None, + enable_stages: bool = False, + ) -> None: + """ + Adds one or more steps to the Pipeline. + + ``enable_stages`` : bool, default False + Whether to enable the added stages immediately. Default is False and is recommended. + + Creates a list called ``added_stages`` that runs the _coerce_stage() method + to extract the information from the given ``stages`` parameter. It then appends + this list to the ``stages`` attribute of the ``Pipeline`` class, adds any stage + configuration that was provided alongside those stages, and updates the StageGraph + using the _rebuild_graph() method. + + Parameters + ---------- + ``stages`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The new steps being added to the Pipeline. + ``stage_configs`` : StageConfig | Mapping[str, Any] | str | Path | Iterable[StageConfig | Mapping[str, Any] | str | Path] | None + Optional stage configuration payloads to add alongside the stages. + """ + if not stages: + warnings.warn( + "No stages provided to add_stage(). No changes made to the Pipeline.", + PipelineConfigurationWarning, + ) + return + + added_stages = [self._coerce_stage(stage) for stage in stages] + parsed_stage_configs: list[StageConfig] = [] + + if stage_configs is not None: + raw_stage_configs: list[StageConfig | Mapping[str, Any] | str | Path] + if isinstance(stage_configs, Mapping) and all( + isinstance(value, Mapping) for value in stage_configs.values() + ): + raw_stage_configs = [ + {str(stage_name): stage_payload} + for stage_name, stage_payload in stage_configs.items() + ] + elif isinstance(stage_configs, (StageConfig, Mapping, str, Path)): + raw_stage_configs = [stage_configs] + else: + raw_stage_configs = list(stage_configs) + + if len(raw_stage_configs) != len(added_stages): + warnings.warn( + "The number of stage configurations passed to add_stage() does not match the number of stages. " + f"Received {len(raw_stage_configs)} stage configuration(s) for {len(added_stages)} stage(s).", + StageConfigurationWarning, + ) + + known_stage_names = {stage.name for stage in self.stages} + known_stage_names.update(stage.name for stage in added_stages) + + for index, raw_stage_config in enumerate(raw_stage_configs): + stage_name = ( + added_stages[index].name if index < len(added_stages) else None + ) + parsed_stage_config = self._coerce_stage_config( + raw_stage_config, name=stage_name + ) + if parsed_stage_config.name not in known_stage_names: + raise StageConfigurationError( + f"Stage configuration was provided for unknown stage: {parsed_stage_config.name}." + ) + parsed_stage_configs.append(parsed_stage_config) + + self.stages.extend(added_stages) + + for conf in parsed_stage_configs: + self.add_stage_config(conf) + + self._register_added_stages_in_stage_selection( + added_stages, enable_stages=enable_stages + ) + self._check_stage_configs(added_stages, self.stage_configs) + + self._sync_stage_configs() + self._rebuild_graph() + + self.logger.event( + "Stage added", + stages=[stage.name for stage in added_stages], + stage_configs=[stage_config.name for stage_config in parsed_stage_configs], + ) + + def add_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> None: + """ + Add or replace a ``StageConfig`` attached to the Pipeline. + + Parameters + ---------- + ``stage_config`` : StageConfig | Mapping[str, Any] | str | Path + The stage configuration information to add to the Pipeline. + ``name`` : str or None, keyword-only + Optional stage name used when the parsed configuration payload does not + identify the stage on its own. + """ + parsed_stage_config = self._coerce_stage_config(stage_config, name=name) + self.stage_configs[parsed_stage_config.name] = parsed_stage_config + self._check_output_dir_in_stage_configs(name=parsed_stage_config.name) + self.logger.event("Stage configuration added", stage=parsed_stage_config.name) + + def create_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> StageConfig: + """ + Normalize a stage-configuration payload into a ``StageConfig`` instance. + + This compatibility helper accepts direct stage payloads, stage-name keyed + mappings, and composite configuration payloads or files containing a + ``stage_configuration`` section. + """ + if isinstance(stage_config, StageConfig): + return self._coerce_stage_config(stage_config, name=name) + + if isinstance(stage_config, Mapping): + payload = dict(stage_config) + elif isinstance(stage_config, (str, Path)): + payload = self._load_config_mapping(stage_config) + else: + raise StageConfigurationError( + f"Unsupported stage configuration specification: {type(stage_config)!r}." + ) + + composite_keys = { + "pipeline_variables", + "pipeline_config", + "stage_configuration", + "stage_config", + "global_configuration", + "global_config", + "global_variables", + "global_vars", + } + if composite_keys.intersection(payload): + _, stage_payload, _ = self._split_config_sections(payload) + stage_configs = self._build_stage_configs(stage_payload) + return self._select_stage_config(stage_configs, name=name) + + return self._coerce_stage_config(payload, name=name) + + def enable_stage(self, *stage_name: str | list[str]) -> None: + """ + Mark one or more stages as enabled in the run selection. + + In implicit "run all" mode (``stages_to_run`` is empty), this is a no-op + because every registered stage already participates in the execution graph. + In explicit mode the requested stages are marked ``True`` in + ``stages_to_run`` and the execution graph is rebuilt to reflect the change. + + Parameters + ---------- + ``stage_name`` : str or list[str] + One or more stage names to enable. + """ + stage_names = set() + for name in stage_name: + if isinstance(name, list): + stage_names.update(name) + else: + stage_names.add(name) + + if not stage_names.issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError( + "You're trying to enable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline." + ) + + if not self.config.stages_to_run: + return + + for name in stage_names: + self.config.stages_to_run[name] = True + self._rebuild_graph() + self.logger.event("Stages enabled", stages=sorted(stage_names)) + + def disable_stage(self, *stage_name: str | list[str]) -> None: + """ + Mark one or more stages as disabled in the run selection. + + When in implicit "run all" mode (``stages_to_run`` is empty), calling + ``disable_stage`` switches the pipeline into explicit stage-selection mode: + every currently registered stage is first marked enabled, then the requested + stages are set to ``False``. The execution graph is rebuilt after the change. + + Parameters + ---------- + ``stage_name`` : str or list[str] + One or more stage names to disable. + """ + stage_names = set() + for name in stage_name: + if isinstance(name, list): + stage_names.update(name) + else: + stage_names.add(name) + + if not stage_names.issubset({stage.name for stage in self.stages}): + raise PipelineInitialisationError( + "You're trying to disable a stage that does not exist in the Pipeline. Please add the stage to the Pipeline." + ) + + if not self.config.stages_to_run: + self.config.stages_to_run = {stage.name: True for stage in self.stages} + + for name in stage_names: + self.config.stages_to_run[name] = False + self._rebuild_graph() + self.logger.event("Stages disabled", stages=sorted(stage_names)) + + def ordered_stages(self) -> list[Stage]: + """ + Return the effective stages in dependency-respecting execution order. + + This is the primary method used by ``PipelineRunner`` to determine what to + execute. Only stages that are part of the current execution graph appear + here; stages disabled via ``PipelineConfig.stages_to_run`` are absent even + if they are registered in ``Pipeline.stages``. + """ + return self.graph.topological_order() + + def validate(self) -> Pipeline: + """ + Confirm that the pipeline is ready to run. + + Validates source files for every stage in the current execution graph, + checks that all stage-configuration names correspond to a known stage, and + validates the execution graph for structural consistency. Disabled stages + are excluded from source-file validation because they will not be executed. + """ + self.logger.event( + "Validating pipeline", + name=self.name, + stages=len(self.stages), + enabled_stages=len(self.graph.stages), + ) + self._validate_stage_configs() + for stage in self.graph.stages: + stage.validate() + self.graph.validate() + + self._output_dir_conflict_check() + + return self + + def run(self) -> PipelineRun: + """ + Returns an instance of ``PipelineRunner`` which actually runs the pipeline. + """ + from .runner import PipelineRunner + + return PipelineRunner(logger=self.logger).run(self) + + def add_dependencies( + self, + *dependencies: Mapping[str, Sequence[str]], + ) -> None: + """ + Add dependency mappings to stages already registered on the Pipeline. + + Each positional argument must be a mapping whose keys identify target + stages by stage name, source-path filename, full source path, or callable + name. Values are normalized, appended to the matching stage's existing + dependencies, de-duplicated in first-seen order, and merged into + ``Pipeline.dependencies`` before the execution graph is rebuilt. + + Parameters + ---------- + ``*dependencies`` : Mapping[str, Sequence[str]] + One or more dependency mappings to merge into the Pipeline. + + Raises + ------ + ``PipelineInitialisationError`` + If a dependency payload is not provided as a mapping. + """ + + if not dependencies: + return + + if self.dependencies is None: + self.dependencies = {} + + for dependency in dependencies: + if not isinstance(dependency, Mapping): + raise PipelineInitialisationError( + "Dependencies added to an existing Pipeline must be provided as a mapping of stage identifiers to dependency names." + ) + + normalized_dependency = self._assign_dependencies(dependency, self.stages) + for stage_name, deps in normalized_dependency.items(): + existing = self.dependencies.get(stage_name, ()) + combined = existing + deps + self.dependencies[stage_name] = tuple(dict.fromkeys(combined)) + + self._rebuild_graph() + + def _assign_dependencies( + self, + dependencies: tuple[str, ...] | Mapping[str, Sequence[str]], + stages: Stage | Sequence[Stage], + ) -> dict[str, tuple[str, ...]]: + """ + Attach dependencies to one stage or a sequence of stages. + + For a single ``Stage``, ``dependencies`` may be either a dependency tuple + for that stage or a mapping keyed by any identifier accepted by + ``_dependencies_for_stage()``. For multiple stages, ``dependencies`` must + be a mapping keyed by stage identifiers. + + The target ``Stage`` objects are mutated in place: new dependency names are + appended to each stage's existing dependencies and de-duplicated while + preserving order. The method returns the normalized dependency mapping that + was applied so callers can keep ``Pipeline.dependencies`` in sync with the + stage objects. + + Parameters + ---------- + ``dependencies`` : tuple[str, ...] | Mapping[str, Sequence[str]] + Dependency names for a single stage or a mapping of stage identifiers + to dependency names. + ``stages`` : Stage | Sequence[Stage] + The stage or stages to mutate. + + Returns + ------- + dict[str, tuple[str, ...]] + The normalized dependency mapping that was applied. + + Raises + ------ + ``PipelineInitialisationError`` + If dependencies are provided before any stages exist, or if a + non-mapping payload is used for multiple stages. + """ + + if isinstance(stages, Stage): + target_stages = [stages] + if isinstance(dependencies, Mapping): + normalized_dependencies = ( + self._normalize_dependency_mapping(dependencies) or {} + ) + else: + normalized_dependencies = { + stages.name: self._normalize_dependency_values(dependencies) + } + else: + target_stages = list(stages) + if not isinstance(dependencies, Mapping): + raise PipelineInitialisationError( + "When assigning dependencies to multiple Stage instances, provide a mapping of stage identifiers to dependency names." + ) + normalized_dependencies = ( + self._normalize_dependency_mapping(dependencies) or {} + ) + + if normalized_dependencies and not target_stages: + raise PipelineInitialisationError( + "Stages need to be defined before you can parse your dependencies " + "for those stages. Try the from_files() method, or create your Stage objects and " + "parse them to the Pipeline Constructor." + ) + + for stage in target_stages: + new_dependencies = self._dependencies_for_stage( + stage.name, stage.source, normalized_dependencies + ) + existing = stage.dependencies or () + combined = existing + new_dependencies + stage.dependencies = tuple(dict.fromkeys(combined)) + + if normalized_dependencies: + self.logger.event( + "Dependencies assigned to Pipeline stages", + dependencies=normalized_dependencies, + stages=[stage.name for stage in target_stages], + ) + + return normalized_dependencies + + def _load_latest_run(self) -> PipelineRun | None: + """ + Load the most recent run of the Pipeline as a PipelineRun instance. + + Returns + ------- + ``PipelineRun`` or None + An instance of ``PipelineRun`` representing the most recent run of the + Pipeline, or None if no previous runs are found. + """ + try: + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output, self.name + ) + except HistoricalPipelineLoadError: + warnings.warn( + "Unable to load previous runs for this Pipeline. Last_run" + " attribute will be None.", + PipelineConfigurationWarning, + ) + return None + + if previous_run_logs == []: + warnings.warn( + "No previous runs found for this Pipeline. Last_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) + return None + + latest_run_log = previous_run_logs[0] + latest_run_id = latest_run_log["run_id"] + if latest_run_id is None: + warnings.warn( + "No previous runs found for this Pipeline. Last_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) + return None + + try: + return load_historical_run(run_dir=Path(self.run_output) / latest_run_id) + except StageLoadError: + warnings.warn( + "Historical run file does not exist. Last_run attribute will be None.", + PipelineConfigurationWarning, + ) + return None + + def _load_all_runs(self) -> dict[str, PipelineRun] | None: + """ + Loads all previous runs of a Pipeline as a dictionary of PipelineRun instances, + keyed by run_id. + + Raises + ------ + ``PipelineConfigurationWarning`` + If there are any errors in loading previous runs, this warning is raised to + indicate a None value will be stored in this attribute. + """ + try: + previous_run_logs = self.logger.extract_historical_run_ids( + self.run_output, self.name + ) + except HistoricalPipelineLoadError: + warnings.warn( + "Unable to load previous runs for this Pipeline. All_run" + " attribute will be None.", + PipelineConfigurationWarning, + ) + return None + + if previous_run_logs == []: + warnings.warn( + "No previous runs found for this Pipeline. All_run attribute" + " will be None.", + PipelineConfigurationWarning, + ) + return None + + all_runs = {} + for run_log in previous_run_logs: + run_id = run_log["run_id"] + if run_id is None or run_id == "": + warnings.warn( + f"No run_id found in log for run_dir {run_log.get('run_dir')}. Skipping this run.", + PipelineConfigurationWarning, + ) + continue + try: + all_runs[run_id] = load_historical_run( + run_dir=Path(self.run_output) / run_id + ) + except StageLoadError: + warnings.warn( + f"Historical run file for run_id {run_id} does not exist. Skipping.", + PipelineConfigurationWarning, + ) + return all_runs if all_runs else None + + def _set_run_output(self) -> Path: + """ + Private method that sets the run output directory for the Pipeline. + + Returns + ------- + ``Path`` + The path to the run output directory for the Pipeline. + + Raises + ------ + ``StageConfigurationWarning`` + If the output_dir is not specified in the PipelineConfig, a warning is + raised to show that the project root or work directory will be used as + the directory for the run outputs. + """ + if self.config.output_dir is not None: + run_output = Path(self.config.output_dir) + else: + warnings.warn( + "Output directory is not specified. Using project root or work directory as the run output.", + StageConfigurationWarning, + ) # TODO: fill with warnings from Pipeline branch + run_output = Path(self.config.project_root or self.config.work_dir) + return run_output / "runs" + def _coerce_stage( self, stage: Stage | Mapping[str, Any] | str | Path | Callable[..., Any], ) -> Stage: + """ + Extracts the ``Stage`` information from the provided stages in the Pipeline. + + Enables mappings, strings, paths, or callables to be parsed and converted into + a useable ``Stage`` class instance. If a ``Stage`` class instance is parsed, return + itself. + + Parameters + ---------- + ``stage`` : Stage | Mapping[str, Any] | str | Path | Callable[..., Any] + The information attempting to be converted into a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the information parsed is not in a suitable format to be converted into + a ``Stage`` class instance. + + Returns + ------- + ``Stage`` class instance for the stage being run. + """ if isinstance(stage, Stage): return stage @@ -64,51 +745,164 @@ def _coerce_stage( if isinstance(stage, (str, Path)): return Stage.from_file(stage) - raise StageConfigurationError(f"Unsupported stage specification: {type(stage)!r}.") + raise StageConfigurationError( + f"Unsupported stage specification: {type(stage)!r}." + ) - def _rebuild_graph(self) -> None: - self.graph = StageGraph.from_stages(self.stages) + def _coerce_stage_config( + self, + stage_config: StageConfig | Mapping[str, Any] | str | Path, + *, + name: str | None = None, + ) -> StageConfig: + """ + Normalize supported stage-configuration inputs into a ``StageConfig``. + """ + if isinstance(stage_config, StageConfig): + if name is not None and stage_config.name != name: + raise StageConfigurationError( + f"Stage configuration '{stage_config.name}' does not match stage '{name}'." + ) + return stage_config - def add_stage(self, *stages: Stage | Mapping[str, Any] | str | Path | Callable[..., Any]) -> None: - added_stages = [self._coerce_stage(stage) for stage in stages] - self.stages.extend(added_stages) - self._rebuild_graph() - self.logger.event("Stage added", stages=[stage.name for stage in added_stages]) + if isinstance(stage_config, Mapping): + if ( + name is not None + and all(isinstance(value, Mapping) for value in stage_config.values()) + and name not in stage_config + ): + available_stage_names = ", ".join( + str(stage_name) for stage_name in stage_config + ) + raise StageConfigurationError( + f"Stage configuration '{name}' was not found. Available stage configurations are: {available_stage_names}." + ) + if name is not None and all( + isinstance(value, Mapping) for value in stage_config.values() + ): + selected_config = stage_config.get(name) + if not isinstance(selected_config, Mapping): + raise StageConfigurationError( + f"Stage configuration '{name}' must be defined as a mapping." + ) + return StageConfig.from_mapping(name=name, data=selected_config) + if name is not None: + return StageConfig.from_mapping(name=name, data=stage_config) + if all(isinstance(value, Mapping) for value in stage_config.values()): + return self._select_stage_config( + self._build_stage_configs(stage_config), name=None + ) + raise StageConfigurationError( + "Stage configuration name is required when the configuration payload does not identify a stage." + ) - def ordered_stages(self) -> list[Stage]: - return self.graph.topological_order() + if isinstance(stage_config, (str, Path)): + payload = self._load_config_mapping(stage_config) + return self._coerce_stage_config(payload, name=name) - def validate(self) -> "Pipeline": - self.logger.event("Validating pipeline", name=self.name) - for stage in self.stages: - stage.validate() + raise StageConfigurationError( + f"Unsupported stage configuration specification: {type(stage_config)!r}." + ) + + def _rebuild_graph(self) -> None: + """ + Update and validate the execution graph. + The execution graph is a subset of the full Stage registry + (Pipeline.stages), reflecting the stages that are actually enabled + for execution. + + The pipeline keeps ``self.stages`` as the complete stage registry, but + ``self.graph`` represents the effective run set after applying + ``PipelineConfig.stages_to_run`` and expanding any selected stage's + dependencies. + """ + stages_to_run = self._resolve_stages_to_run() + self.graph = StageGraph.from_stages(stages_to_run) self.graph.validate() - return self - def run(self) -> PipelineRun: - from .runner import PipelineRunner + def _register_added_stages_in_stage_selection( + self, stages: Sequence[Stage], enable_stages: bool = False + ) -> None: + """ + Default newly added stages to disabled once explicit stage selection is in use. - return PipelineRunner(logger=self.logger).run(self) + When ``stages_to_run`` is empty, the pipeline is in implicit "run all" + mode and new stages should immediately participate in the execution graph. + Once the configuration has switched to an explicit stage-selection mapping, + newly added stages stay out of the execution graph until they are enabled. + """ + if not self.config.stages_to_run: + return + + for stage in stages: + self.config.stages_to_run.setdefault(stage.name, enable_stages) def _construct_manifest(self, *, runtime_id: RuntimeID) -> RunManifest: + """ + Creates a ``RunManifest`` instance that contains the information about + the run of the Pipeline. + + Parameters + ---------- + ``runtime_id`` : RuntimeID + Contains information about the run to be extracted and placed into + the ``RunManifest`` instance. + """ return RunManifest( rap_name=self.name, run_id=runtime_id.get_id(), git_commit=self._discover_git_commit(), stages_run=[], - parameters=self.config.to_dict(), - inputs={stage.name: list(stage.dependencies) for stage in self.stages}, + parameters=self._manifest_parameters(), + inputs={ + stage.name: list(stage.dependencies) for stage in self.graph.stages + }, outputs={}, backend=self.backend, package_versions=self._package_versions(), timestamp=runtime_id.timestamp.isoformat(), reason=self.config.metadata.get("reason"), user=self._current_user(), + config=self._combine_configs(), ) + def _combine_configs(self) -> dict[str, Any]: + """ + Combines all configurations (PipelineConfig, GlobalConfig, StageConfig) within the Pipeline into one dictionary which can + be recorded in the RunManifest for the run. + + Returns + ------- + dict[str,Any] + A dictionary containing the configuration for the Pipeline, the stages, and + the global configuration. + """ + global_variables, exclusions = self.global_config.get_attributes() + global_configuration = dict(global_variables or {}) + global_configuration["exclusions"] = exclusions + + all_stage_configuration = { + name: config.to_dict() for name, config in self.stage_configs.items() + } + configuration = { + "pipeline_config": self.config.to_dict(), + "stage_configs": all_stage_configuration, + "global_config": global_configuration, + } + + return configuration + def _create_runtime_id(self) -> RuntimeID: + """ + Creates a RuntimeID instance for the specific run. + + Establishes attributes for this specific run and returns + as a ``RuntimeID`` instance. + """ current_time = now() - digest = hashlib.sha256(f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8")).hexdigest() + digest = hashlib.sha256( + f"{self.name}:{self.backend}:{current_time.isoformat()}".encode("utf-8") + ).hexdigest() short_hash = digest[:8] return RuntimeID( id=f"{current_time.strftime('%Y-%m-%d_%H%M%S')}_{short_hash}", @@ -118,6 +912,18 @@ def _create_runtime_id(self) -> RuntimeID: ) def _discover_git_commit(self) -> str | None: + """ + Finds the specific version of the repository used for this run. + + Attempts to run a Git command to establish the current git commit hash + to be held in the ``RunManifest`` instance for this run. + + Returns + ------- + ``OSError`` + If Git is unable to be loaded or the Git command cannot be run for + another reason. + """ try: completed = subprocess.run( ["git", "rev-parse", "HEAD"], @@ -132,19 +938,520 @@ def _discover_git_commit(self) -> str | None: return commit or None def _package_versions(self) -> list[str]: + """ + Creates a list of packages and their versions used in this run. + + Raises + ------ + ``importlib_metadata.PackageNotFoundError`` + If the package used cannot be found in the library. + """ versions = [f"python={sys.version.split()[0]}"] - try: + with suppress(importlib_metadata.PackageNotFoundError): versions.append(f"pyyaml={importlib_metadata.version('PyYAML')}") - except importlib_metadata.PackageNotFoundError: - pass return versions def _current_user(self) -> str | None: + """ + Extracts the username for the individual completing the run. Returns a blank + value if the username cannot be extracted. + """ try: return getpass.getuser() except Exception: return None + def _resolve_config( + self, + config: PipelineConfig | Mapping[str, Any] | str | Path | None, + ) -> tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig]: + """ + Resolve supported configuration inputs into pipeline config, stage config, and stages. + + This method is the main normalization step for configuration injection. It accepts + already-constructed config objects, raw mappings, and YAML files, and converts them + into the three objects the pipeline needs before execution starts. + + Parameters + ---------- + ``config`` : PipelineConfig | Mapping[str, Any] | None + The configuration input to resolve into the three objects required for execution. + + Returns + ------- + tuple[PipelineConfig, dict[str, StageConfig], list[Stage], GlobalConfig] + The resolved pipeline configuration, stage configurations, configured stages, and global configuration. + + Raises + ------ + ``StageConfigurationWarning`` + If a stage configuration is found within the metadata section of a PipelineConfig instance, a warning is + raised to indicate that a composite configuration payload is preferred. + Output-location warnings are emitted during ``Pipeline.validate()`` when stage configurations are inspected. + """ + + if config is None: + return PipelineConfig.from_any(config), {}, [], GlobalConfig() + + if isinstance(config, PipelineConfig): + stage_configuration = config.metadata.get("stage_configuration", None) + if stage_configuration is None: + stage_configuration = config.metadata.get("stage_config", None) + + if stage_configuration is not None: + warnings.warn( + "Stage configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", + StageConfigurationWarning, + ) + + global_configuration = config.metadata.get("global_config", None) + if global_configuration is not None: + warnings.warn( + "Global configuration found in PipelineConfig metadata. This is supported for backwards compatibility but a composite config payload is preferred.", + StageConfigurationWarning, + ) + return ( + config, + self._build_stage_configs(stage_configuration), + [], + GlobalConfig.from_dict(global_configuration), + ) + + raw_config = self._load_config_mapping(config) + pipeline_payload, stage_config_payload, global_config_payload = ( + self._split_config_sections(raw_config) + ) + normalized_pipeline_payload = self._normalize_pipeline_payload(pipeline_payload) + + stage_definitions = normalized_pipeline_payload.pop("stages", ()) + + pipeline_config = PipelineConfig.from_mapping(normalized_pipeline_payload) + stage_configs = self._build_stage_configs(stage_config_payload) + configured_stages = self._build_stages_from_config( + stage_definitions, + backend=pipeline_config.backend, + work_dir=pipeline_config.work_dir, + ) + global_config = GlobalConfig.from_dict(global_config_payload) + + return pipeline_config, stage_configs, configured_stages, global_config + + @staticmethod + def _normalize_dependency_values( + dependencies: Sequence[str] | str | None, + ) -> tuple[str, ...]: + """ + Normalize dependency names into a de-duplicated tuple. + """ + if dependencies is None: + return () + + candidate_dependencies: Sequence[str] | tuple[str, ...] + if isinstance(dependencies, str): + candidate_dependencies = (dependencies,) + else: + candidate_dependencies = dependencies + + normalized_dependencies: list[str] = [] + for dependency in candidate_dependencies: + dependency_name = str(dependency).strip() + if dependency_name and dependency_name not in normalized_dependencies: + normalized_dependencies.append(dependency_name) + + return tuple(normalized_dependencies) + + @staticmethod + def _normalize_dependency_mapping( + dependencies: Mapping[str, Sequence[str]] | None, + ) -> dict[str, tuple[str, ...]] | None: + if dependencies is None: + return None + + return { + str(stage_name): Pipeline._normalize_dependency_values(stage_dependencies) + for stage_name, stage_dependencies in dependencies.items() + } + + def _sync_stage_configs(self) -> None: + """ + Ensure every known stage has a ``StageConfig`` entry, even if it is empty. + + """ + for stage in self.stages: + self.stage_configs.setdefault(stage.name, StageConfig(name=stage.name)) + + def _check_output_dir_in_stage_configs(self, name: str) -> None: + """ + Checks ``StageConfig`` instances for output directory keys and raises a warning if any are found, + as this will result in overwriting previous run outputs. Users are advised to set their output + location in the stage scripts using the ``resolve_output_path()`` function to ensure unique + outputs are saved for each run. + + Parameters + ---------- + ``regex_pattern`` + The regular expression pattern used to identify output directory keys in the stage configurations. + + ``name`` + The stage name to check for output directory keys in the stage configurations. + """ + + if any( + OUTPUT_DIR_KEY_RE.match(key) for key in self.stage_configs[name]._variables + ): + warnings.warn( + f"Stage configuration for {name} contains output directory keys. This will result " + f"in overwriting previous run outputs. Please set your output location in the stage scripts " + f"using the resolve_output_root() method to ensure unique outputs are saved for each run.", + StageConfigurationWarning, + ) + self.logger.event( + f"Warning: stage configuration for {name} contains output directory keys. Risk of overwriting outputs.", + keys_found=[ + key + for key in self.stage_configs[name]._variables + if OUTPUT_DIR_KEY_RE.match(key) + ], + ) + + def _output_dir_conflict_check(self) -> None: + """ + Checks whether the output directory has been assigned in stage configurations and already exists. It raises an error if + it does, unless the overwrite parameter is set to True. + + For each stage in the Pipeline, checks whether an output directory has been defined in the stage configurations. If it has been + defined, it checks whether the Path value for the output directory already exists. If it does already exist, raise either an + error or a warning based on an overwrite configuration. Log either the error or the warning in the Logger. + + Raises + ------ + StageConfigurationError + If the overwrite parameter in the PipelineConfig is set to False and the output directory already exists. + StageConfigurationWarning + If the overwrite parameter in the PipelineConfig is set to True and the output directory already exists. + """ + + for stage in self.stages: + available_output_dirs = [ + key + for key in self.stage_configs[stage.name]._variables + if OUTPUT_DIR_KEY_RE.match(key) + ] + self._check_output_dir_in_stage_configs(name=stage.name) + + for directory in available_output_dirs: + output_dir = self.stage_configs[stage.name].get(directory) + if not output_dir: + continue + + output_path = Path(output_dir) + if not output_path.is_absolute(): + output_path = self.config.work_dir / output_path + + exists = output_path.exists() + overwrite = bool(self.config.overwrite) + + if exists and not overwrite: + self.logger.event( + f"Error: Output directory {output_path} already exists. Pipeline will crash to prevent overwrite.", + overwrite=overwrite, + ) + raise StageConfigurationError( + f"Stage configuration for {stage.name} contains an output directory path that already exists. Please set a unique " + f"output directory for this stage to prevent overwriting." + ) + + if exists and overwrite: + warnings.warn( + f"Stage configuration for {stage.name} contains an output directory path that already exists. As the overwrite " + f"parameter is True, the pipeline will proceed and will overwrite the previous run file.", + StageConfigurationWarning, + ) + self.logger.event( + f"Warning: Output directory {output_path} already exists however permissions allow overwriting. The previous file " + f"will be overwritten.", + overwrite=overwrite, + ) + + def _validate_stage_configs(self) -> None: + """ + Confirm that every configured stage name matches a stage present in the pipeline. + """ + self._sync_stage_configs() + stage_names = {stage.name for stage in self.stages} + unknown_stage_configs = sorted( + name for name in self.stage_configs if name not in stage_names + ) + if unknown_stage_configs and stage_names: + missing = ", ".join(unknown_stage_configs) + raise StageConfigurationError( + f"Stage configuration was provided for unknown stages: {missing}." + ) + + def _manifest_parameters(self) -> dict[str, Any]: + """ + Build the manifest parameter payload, including per-stage configuration. + """ + parameters = self.config.to_dict() + if self.stage_configs: + parameters["stage_configuration"] = { + name: stage_config.to_dict() + for name, stage_config in self.stage_configs.items() + } + return parameters + + def _build_stages_from_config( + self, + stage_definitions: Sequence[Any] | None, + *, + backend: str, + work_dir: Path, + ) -> list[Stage]: + """ + Convert configured stage definitions into ``Stage`` instances. + + Each entry is resolved independently, so the method can process any number of + stage definitions supplied in the pipeline configuration. + """ + if not stage_definitions: + return [] + + stage_definitions = list(stage_definitions) + + if not isinstance(stage_definitions, Sequence) or isinstance( + stage_definitions, (str, bytes) + ): + raise StageConfigurationError( + "Configured stages must be provided as a sequence." + ) + + configured_stages: list[Stage] = [] + for stage_definition in stage_definitions: + stage = self._stage_from_config_definition( + stage_definition, backend=backend, work_dir=work_dir + ) + if stage is not None: + configured_stages.append(stage) + return configured_stages + + def _stage_from_config_definition( + self, + stage_definition: Any, + *, + backend: str, + work_dir: Path, + ) -> Stage | None: + """ + Resolve one configured stage entry into a ``Stage`` instance. + + Supported forms include ready-made ``Stage`` objects, paths, callables, full stage + dictionaries, and compact ``{stage_name: {...}}`` definitions from YAML config files. + Entries with ``run: false`` are skipped. + """ + if isinstance(stage_definition, Stage): + return stage_definition + + if isinstance(stage_definition, (str, Path)) or callable(stage_definition): + return self._coerce_stage(stage_definition) + + if not isinstance(stage_definition, Mapping): + raise StageConfigurationError( + "Configured stage entries must be mappings, paths, or callables." + ) + + payload = dict(stage_definition) + if any(key in payload for key in ("name", "source", "path", "callable")): + return Stage.from_dict(payload) + + if len(payload) != 1: + raise StageConfigurationError( + "Configured stage mappings must define exactly one stage name." + ) + + stage_name, stage_payload = next(iter(payload.items())) + if not isinstance(stage_payload, Mapping): + raise StageConfigurationError( + "Configured stage details must be provided as a mapping." + ) + + stage_options = dict(stage_payload) + if not bool(stage_options.pop("run", True)): + return None + + location = stage_options.pop( + "location", stage_options.pop("source", stage_options.pop("path", None)) + ) + dependencies = stage_options.pop("dependencies", ()) + entrypoint = stage_options.pop("entrypoint", None) + metadata = stage_options.pop("metadata", {}) + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} + metadata.update(stage_options) + + source = self._resolve_stage_source( + stage_name=str(stage_name), location=location, work_dir=work_dir + ) + return Stage.from_file( + source, + name=str(stage_name), + dependencies=dependencies, + metadata=metadata, + entrypoint=entrypoint, + backend=backend, + ) + + def _resolve_stages_to_run(self) -> list[Stage]: + """ + Resolve the effective stage subset that should populate the execution graph. + + An empty ``stages_to_run`` mapping means the pipeline runs all known stages. + Otherwise, stages explicitly marked ``True`` are selected and their transitive + dependencies are pulled in automatically. A dependency that is explicitly + disabled in ``stages_to_run`` while another enabled stage requires it raises + a ``PipelineConfigurationError``. + """ + stage_lookup = {stage.name: stage for stage in self.stages} + configured_stages_to_run = dict(self.config.stages_to_run or {}) + + # When PipelineConfig.stages_to_run is empty, the pipeline is in implicit "run all" mode. + if not configured_stages_to_run: + warnings.warn( + "No stages specified to run. All stages running by default.", + PipelineConfigurationWarning, + ) + return self.stages + + unknown_stage_names = sorted( + stage_name + for stage_name in configured_stages_to_run + if stage_name not in stage_lookup + ) + if unknown_stage_names: + missing = ", ".join(unknown_stage_names) + raise PipelineInitialisationError( + f"Pipeline configuration references unknown stages in stages_to_run: {missing}." + ) + + explicitly_enabled = [ + stage_name + for stage_name, value in configured_stages_to_run.items() + if value + ] + if not explicitly_enabled: + return [] + + explicitly_disabled = { + stage_name + for stage_name, value in configured_stages_to_run.items() + if not value + } + resolved_stage_names: set[str] = set() + visiting: set[str] = set() + + def add_stage_with_dependencies( + stage_name: str, *, required_by: str | None = None + ) -> None: + """ + Add one selected stage and recursively include everything it depends on. + + ``_resolve_stages_to_run()`` uses this helper to turn the user-facing + ``stages_to_run`` selection into a runnable execution set for the + ``StageGraph``. It also guards against invalid configurations where an + enabled stage depends on a stage that has been explicitly disabled. + """ + if stage_name in resolved_stage_names: + return + if required_by is not None and stage_name in explicitly_disabled: + raise PipelineConfigurationError( + f"Stage '{required_by}' is enabled but depends on disabled stage '{stage_name}'." + ) + if stage_name in visiting: + return + if stage_name not in stage_lookup: + raise PipelineInitialisationError( + f"You're trying to run a stage that does not exist: '{stage_name}'. Please add the stage to the Pipeline." + ) + + visiting.add(stage_name) + resolved_stage_names.add(stage_name) + for dependency_name in stage_lookup[stage_name].dependencies: + add_stage_with_dependencies(dependency_name, required_by=stage_name) + visiting.remove(stage_name) + + for stage_name in explicitly_enabled: + add_stage_with_dependencies(stage_name) + + return [stage for stage in self.stages if stage.name in resolved_stage_names] + + def _generate_context(self) -> None: + """ + Generates the execution context for the Pipeline based on values parsed. + + Validates stage backends and then uses the pipeline backend to generate + the expected StageExecutor class. If this class does not exist within + the orchestration tool, an error is raised. If the class does exist, + it is assigned to the executor attribute of the Pipeline instance. + + Raises + ------ + ``PipelineInitialisationError`` + If the backend for the Pipeline does not have a compatible executor class + """ + + if len(self.stages) == 0: + warnings.warn( + "No stages have been defined in the Pipeline. The Pipeline will not run any stages.", + PipelineConfigurationWarning, + ) + + else: + # Check all stages have the same backend as Pipeline + self._validate_stage_backends() + + backend_key = str(self.backend).strip().lower() + execution_class_name = f"{backend_key.capitalize()}StageExecutor" + + if (executor_class := globals().get(execution_class_name)) is not None: + self.executor = executor_class() + else: + raise PipelineInitialisationError( + f"Requested backend {backend_key} does not have a compatible executor. " + f"Available executors are: {', '.join(AVAILABLE_EXECUTORS)}." + ) + + def _validate_stage_backends(self) -> None: + """ + Checks the backends that have been assigned to each stage. + + Raise an error if the backends for a stage do not match the Pipeline + backend or if there are multiple backends across the stages. This + ensures that the ExecutionContext will run correctly on all stages. + + Raises + ------ + ``PipelineInitialisationError`` + If there are multiple backends across the stages or if the stage + backend does not match the Pipeline backend. + """ + backends = [] + for stage in self.stages: + backends.append(stage.backend) + + backends = [backend.lower() for backend in backends] + + if len(set(backends)) > 1: + raise PipelineInitialisationError( + f"Not all stages have the same backend. Found backends: {', '.join(set(backends))}. This means " + "that the execution context will not work on all stages." + ) + + if set(backends) != {self.backend}: + raise PipelineInitialisationError( + f"Stages have backends '{', '.join(set(backends))}' which do not match pipeline backend '{self.backend}'." + ) + @classmethod def from_files( cls, @@ -152,16 +1459,44 @@ def from_files( *, name: str | None = None, backend: str = "python", - config: PipelineConfig | RAPConfig | Mapping[str, Any] | str | Path | None = None, + config: PipelineConfig | Mapping[str, Any] | str | Path | None = None, dependencies: Mapping[str, Sequence[str]] | None = None, logger: Logger | None = None, executor: StageExecutor | None = None, - ) -> "Pipeline": + ) -> Pipeline: + """ + Extracts the information from files regarding exactly what is being run in the pipeline and + allows for configuration of how the Pipeline is run. + + Parameters + ---------- + ``file_paths`` : Iterable[str or Path] + The files that contain the code for each stage in the pipeline. These are what + the Pipeline will run. + ``name`` : str + The name of the pipeline. + ``backend`` : str, default = "python" + The system that the pipeline is written in. + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path | None + The high level information required to run this specific pipeline. + ``dependencies`` : Mapping[str, Sequence[str]] or None + An object containing which stages are required to be run before other stages. + ``logger`` : Logger class or None + The logging sysem used for this Pipeline run. + ``executor`` : StageExecutor class or None + The information on exactly how to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ stages: list[Stage] = [] - for position, file_path in enumerate(file_paths): + for file_path in file_paths: path = Path(file_path) stage_name = path.stem - stage_dependencies = cls._dependencies_for_stage(stage_name, path, dependencies) + stage_dependencies = cls._dependencies_for_stage( + stage_name, path, dependencies + ) stages.append( Stage.from_file( path, @@ -181,40 +1516,454 @@ def from_files( ) @classmethod - def from_dict(cls, cfg: Mapping[str, Any]) -> "Pipeline": - payload = dict(cfg) + def from_dict( + cls, + config: PipelineConfig | Mapping[str, Any] | str | Path, + name: str | None = None, + backend: str = "python", + logger: Logger | None = None, + executor: StageExecutor | None = None, + ) -> Pipeline: + """ + Extracts information from a dictionary to configure a Pipeline instance as + well as what the Pipeline runs. + + Parameters + ---------- + ``config`` : PipelineConfig | Mapping[str, Any] | str | Path + The object containing the information needed to run the Pipeline. + + Returns + ------- + A ``Pipeline`` class instance. + """ + # REMOVED AS THIS WAS RUNNING TWICE. SHOULD DISCUSS WHAT TO DO ABOUT THIS + # METHOD AND WHETHER IT IS NEEDED - name = payload.pop("name", None) - backend = payload.pop("backend", "python") - config = payload.pop("config", None) - stages = payload.pop("stages", []) + # pipe_payload, stage_payload = cls._split_config_sections(config) - if config is None and payload: - config = payload - elif isinstance(config, Mapping) and payload: - combined_config = dict(config) - combined_config.update(payload) - config = combined_config + # pipeline_variables contains pipeline information + # name = pipe_payload.get("name", None) + # backend = pipe_payload.get("backend", "python") + # stages = pipe_payload.get("stages", []) + # print(stages) return cls( name=name, backend=backend, config=config, - stages=stages, + stages=None, + logger=logger, + executor=executor, + ) + + @classmethod + def from_config( + cls, + config: Mapping[str, Any] | str | Path, + name: str | None = None, + backend: str = "python", + logger: Logger | None = None, + executor: StageExecutor | None = None, + ) -> Pipeline: + """ + Construct a pipeline directly from a composite configuration payload or file. + + This is the preferred entrypoint when configuration defines both pipeline-level + settings and the stage-level configuration that should be injected at runtime. + """ + extracted_config = Pipeline._load_config_mapping(config) + + return cls.from_dict( + config=extracted_config, + name=name, + backend=backend, + logger=logger, + executor=executor, + ) + + @staticmethod + def _select_stage_config( + stage_configs: Mapping[str, StageConfig], + *, + name: str | None, + ) -> StageConfig: + """ + Select one stage configuration from a stage-name keyed mapping. + + When ``name`` is omitted, exactly one stage configuration must be present. + """ + if name is not None: + if name not in stage_configs: + raise StageConfigurationError( + f"Stage configuration '{name}' was not found." + ) + return stage_configs[name] + + if len(stage_configs) != 1: + raise StageConfigurationError( + "The provided input resolves to multiple stage configurations; specify a stage name." + ) + + return next(iter(stage_configs.values())) + + @staticmethod + def _load_config_mapping( + config: Mapping[str, Any] | str | Path, + ) -> dict[str, Any]: + """ + Load raw configuration data from a mapping or YAML file. + """ + if isinstance(config, Mapping): + return dict(config) + + config_path = Path(config).expanduser() + if config_path.suffix.lower() not in ACCEPTED_CONFIG_TYPES: + raise StageConfigurationError( + f"Unsupported config file format parsed as Stage Configuration: {config!r}." + ) + if not config_path.exists(): + raise FileNotFoundError(f"Config file does not exist: {config_path}") + + import yaml + + raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if raw_config is None: + warnings.warn( + "No configuration has loaded from the configuration file. Please check " + "your configurations." + ) + return {} + if not isinstance(raw_config, Mapping): + raise TypeError( + "Configuration file must contain a mapping at the top level. Please ensure" + "that your configuration file is structured into key:value pairs in the notation that suits" + "the configuration file that you are using. The top level key value pairs should reflect " + "the Pipeline and Stage configurations." + ) + return dict(raw_config) + + @staticmethod + def _split_config_sections( + raw_config: Mapping[str, Any], + ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + """ + Split a raw config payload into pipeline-level and stage-level sections. + + Allows for configurations that have "stage_configuration", "stage_config", "pipeline_variables" + and "pipeline_config" as the key. The key is identified and used to pull the values for the + configuration from the ``raw_config``. It is then Nonetype checked and Type checked to ensure + that appropriate information is extracted and errors are produced if any of these checks fail. + + Parameters + ---------- + ``raw_config``: Mapping[str, Any] + Contents of the configuration file previously extracted. + + Returns + ------- + ``pipeline_payload``: Mapping[str, Any] | None + Contents of the pipeline configuration settings defined in the configuration file. + ``stage_payload``: Mapping[str, Any] | None + Contents of the stage configuration settings defined in the configuration file. + ``global_payload``: Mapping[str, Any] | None + Contents of the global configuration settings defined in the configuration file. + + Raises + ------ + ``PipelineConfigurationWarning`` + If blank values for pipeline_payload or global_payload are detected. + If there are remaining keys in the ``raw_config`` that have not been extracted. + + ``StageConfigurationWarning`` + If blank values for stage_payload are detected. + + ``PipelineConfigurationError`` + If the pipeline_payload, global_payload or stage_payload are not mapping types. + """ + possible_stage_keys = ("stage_configuration", "stage_config") + possible_pipeline_keys = ("pipeline_variables", "pipeline_config") + possible_global_keys = ( + "global_configuration", + "global_config", + "global_variables", + "global_vars", + ) + + stage_payload, stage_configuration = Pipeline._extract_optional_mapping( + possible_stage_keys, raw_config, StageConfigurationWarning ) + pipeline_payload, pipeline_configuration = Pipeline._extract_mappings( + possible_pipeline_keys, raw_config, PipelineConfigurationWarning + ) + global_payload, global_configuration = Pipeline._extract_optional_mapping( + possible_global_keys, raw_config, PipelineConfigurationWarning + ) + + extracted_keys = {pipeline_configuration} + if stage_configuration is not None: + extracted_keys.add(stage_configuration) + if global_configuration is not None: + extracted_keys.add(global_configuration) + + remaining_keys = set(raw_config) - extracted_keys + if remaining_keys: + warnings.warn( + "There are remaining sections in your configuration file that have not been extracted. Please check that all your configurations are in the pipeline or stage configuration keys.", + PipelineConfigurationWarning, + ) + + return pipeline_payload, stage_payload, global_payload + + @staticmethod + def _extract_mappings( + keys: tuple[str, ...], + config: Mapping[str, Any], + warning: type[Warning], + ) -> tuple[dict[str, Any], str]: + configuration = Pipeline._extract_keys(keys, config) + payload = config.get(configuration, {}) + if payload is None: + warnings.warn( + f"Blank {configuration} configuration detected. Please check that this is correct.", + warning, + ) + if not isinstance(payload, Mapping): + raise PipelineConfigurationError( + f"The {configuration} section must be a mapping." + ) + return dict(payload), configuration + + @staticmethod + def _extract_optional_mapping( + keys: tuple[str, ...], + config: Mapping[str, Any], + warning: type[Warning], + ) -> tuple[dict[str, Any], str | None]: + matches = [key for key in keys if key in config] + if not matches: + return {}, None + + if len(matches) == 1: + configuration = matches[0] + else: + warnings.warn( + f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", + PipelineConfigurationWarning, + ) + configuration = matches[0] + + payload = config.get(configuration, {}) + if payload is None: + warnings.warn( + f"Blank {configuration} configuration detected. Please check that this is correct.", + warning, + ) + return {}, configuration + if not isinstance(payload, Mapping): + raise PipelineConfigurationError( + f"The {configuration} section must be a mapping." + ) + return dict(payload), configuration + + @staticmethod + def _extract_keys( + possible_keys: tuple[str, ...], dictionary: Mapping[str, Any] + ) -> str: + """ + Checks whether a provided dictionary has a key that has been previously defined. + + Creates a list for all specified keys that are present in the dictionary and checks + the number of keys that match. This should only be 1 so if there are any fewer or + additional then appropriate errors are raised. + + Parameters + ---------- + ``possible_keys``: tuple[str, ...] + Set of string keys that are possibly in the dictionary provided. + ``dictionary``: Mapping[str, Any] + Dictionary that is being checked for valid keys. + + Returns + ------- + ``key``: str + String value for the key that is present in the ``dictionary`` out of the + ``possible_keys`` values. + + Raises + ------ + ``PipelineConfigurationError`` + If no keys in the ``dictionary`` are also in the ``possible_keys`` tuple. + + ``PipelineConfigurationWarning`` + If more than one key in the possible_keys is found, alerts user that it will + default to the first selected option and records the key that is selected. + """ + + matches = [key for key in possible_keys if key in dictionary] + if len(matches) == 1: + key = matches[0] + elif len(matches) == 0: + raise PipelineConfigurationError( + f"No valid keys were found in the configuration. Please ensure that your top level key is one of: {possible_keys}." + ) + else: + warnings.warn( + f"Multiple configuration keys were found, defaulting to the first option: {matches[0]}", + PipelineConfigurationWarning, + ) + key = matches[0] + + return key + + @staticmethod + def _normalize_pipeline_payload( + pipeline_payload: Mapping[str, Any], + ) -> dict[str, Any]: + """ + Normalize supported aliases in the pipeline section before model construction. + + Recognized aliases: + + - ``working_dir`` → ``work_dir`` (only when ``work_dir`` is absent). + - ``stage_to_run`` → ``stages_to_run`` (only when ``stages_to_run`` is absent). + + If both ``working_dir`` and ``work_dir`` are present at the same time, a + ``UserWarning`` is emitted and ``working_dir`` is left in the payload where + it will be silently absorbed into ``PipelineConfig.metadata``. + """ + + normalized_payload = dict(pipeline_payload) + if "working_dir" in normalized_payload: + if "work_dir" not in normalized_payload: + normalized_payload["work_dir"] = normalized_payload.pop("working_dir") + else: + warnings.warn( + "Both 'working_dir' and 'work_dir' were found in the pipeline configuration. " + "'work_dir' will be used and 'working_dir' will be ignored.", + UserWarning, + stacklevel=2, + ) + if "stage_to_run" in normalized_payload: + if "stages_to_run" not in normalized_payload: + normalized_payload["stages_to_run"] = normalized_payload.pop( + "stage_to_run" + ) + else: + warnings.warn( + "Both 'stage_to_run' and 'stages_to_run' were found in the pipeline configuration. " + "'stages_to_run' will be used and 'stage_to_run' will be ignored.", + UserWarning, + stacklevel=2, + ) + + return normalized_payload + + @staticmethod + def _build_stage_configs( + stage_configuration: Mapping[str, Any] | None, + ) -> dict[str, StageConfig]: + """ + Build a stage-name keyed configuration mapping for any number of configured stages. + + The returned mapping scales linearly with the provided stage entries and is used as + the canonical runtime lookup structure for stage configuration. + """ + if stage_configuration is None: + return {} + if not isinstance(stage_configuration, Mapping): + raise StageConfigurationError( + "Stage configuration must be a mapping keyed by stage name." + ) + + stage_configs: dict[str, StageConfig] = {} + for stage_name, stage_payload in stage_configuration.items(): + if stage_payload is not None and not isinstance(stage_payload, Mapping): + raise StageConfigurationError( + f"Stage configuration for {stage_name} must be provided as a mapping." + ) + stage_configs[str(stage_name)] = StageConfig.from_mapping( + str(stage_name), stage_payload + ) + return stage_configs + + @staticmethod + def _resolve_stage_source(stage_name: str, location: Any, work_dir: Path) -> Path: + """ + Resolve the source path for a configured stage. + + Empty locations default to ``work_dir / "scripts" / ".py"``. Relative + paths are first interpreted as given and then relative to ``work_dir``. + """ + if location in (None, ""): + return work_dir / "scripts" / f"{stage_name}.py" + + candidate = Path(location).expanduser() + if candidate.is_absolute() or candidate.exists(): + return candidate + + work_dir_candidate = work_dir / candidate + if work_dir_candidate.exists(): + return work_dir_candidate + + return candidate @staticmethod def _dependencies_for_stage( stage_name: str, - path: Path, - dependencies: Mapping[str, Sequence[str]] | None, + path: Path | Callable[..., Any] | None = None, + dependencies: Mapping[str, Sequence[str]] | None = None, ) -> tuple[str, ...]: + """ + Extracts a tuple of ``dependencies`` for the requested stage. + + Will return a blank tuple if there are no ``dependencies`` for the requested + stage. Allows for ``dependencies`` to be found regardless of how the stage is + referenced in the ``dependencies`` mapping. + + Parameters + ---------- + ``stage_name`` : str + The name of the stage that you are extracting the ``dependencies`` for. + ``path`` : Path + The filepath for the stage source. + ``dependencies`` : Mapping[str, Sequence[str]] or None + Mapping of the stage source name to their relevant ``dependencies`` (stages + required to run before the ``stage_name`` Stage). + """ if not dependencies: return () - - candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + candidates: tuple[str, ...] + if isinstance(path, Path): + candidates = (stage_name, path.name, path.stem, str(path), path.as_posix()) + elif callable(path): + candidates = (stage_name, str(getattr(path, "__name__", stage_name))) + else: + candidates = (stage_name,) for candidate in candidates: if candidate in dependencies: - return tuple(str(dependency) for dependency in dependencies[candidate]) + return Pipeline._normalize_dependency_values(dependencies[candidate]) + + return () - return () \ No newline at end of file + @staticmethod + def _check_stage_configs( + stages: list[Stage], stage_configs: Mapping[str, StageConfig] + ) -> None: + """ + Check that all stages have a corresponding stage configuration. + + Warns + ------ + ``StageConfigurationWarning`` + If any stage does not have a corresponding stage configuration. Handled in one warning instance for all stages without a configuration. + """ + stage_no_config = [] + for stage in stages: + if stage.name not in stage_configs: + stage_no_config.append(stage.name) + if stage_no_config: + warnings.warn( + f"Stage(s) {', '.join(stage_no_config)} added to Pipeline without a corresponding StageConfig. ", + StageConfigurationWarning, + ) diff --git a/onsrap/py.typed b/onsrap/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/onsrap/run_pipeline.py b/onsrap/run_pipeline.py index 842be5f..c9f2567 100644 --- a/onsrap/run_pipeline.py +++ b/onsrap/run_pipeline.py @@ -2,6 +2,10 @@ from .runner import main - +""" +This line of code establishes the function that must be called +for the Pipeline to run. main() is parsed to SystemExit as once +the main() function is run, this will then exit the system. +""" if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/onsrap/runner.py b/onsrap/runner.py index bbbb244..a850376 100644 --- a/onsrap/runner.py +++ b/onsrap/runner.py @@ -7,26 +7,95 @@ from .errors import StageExecutionError from .execution import ExecutionContext from .logger import Logger -from .models import PipelineRun, PipelineStatus, now +from .models import ( + PipelineRun, + PipelineStatus, + RunManifest, + StageResult, + _yaml_safe_encode, + now, +) if TYPE_CHECKING: from .pipeline import Pipeline class PipelineRunner: + """ + Represents the information required to run the Pipeline. + + Parameters + ---------- + ``logger`` : Logger class type + Information used to log progress throughout the Pipeline. + """ + def __init__(self, logger: Logger | None = None): self.logger = logger or Logger() - def run(self, pipeline: "Pipeline") -> PipelineRun: + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``PipelineRunner`` class. + + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return ( + f"PipelineRunner Instance Attributes\n" + f"--------------------------\n" + f"Logger: {self.logger} \n" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``PipelineRunner`` class. + This method is structured to be more concise than the ``__str__`` method and is + intended for debugging purposes. + + Returns + ------- + str + A string representation of the ``PipelineRunner`` class with its attributes. + """ + return f"PipelineRunner(logger={self.logger})" + + def run(self, pipeline: Pipeline) -> PipelineRun: + """ + Method that runs a ``Pipeline`` instance. + + This method validates the source information, establishes the directories and + the context to run the pipeline within, sets out the manifest for the run, attempts + to run the stages in the order outlined by the ``StageGraph`` instance and logs all + progress alongside relevant statuses. Before each stage executes, the runner binds + the current stage name onto the ``ExecutionContext`` so ``context.stage_config`` + resolves to the correct stage-specific configuration. + + It returns a PipelineRun instance containing metadata and logging information for the + specific run of the whole Pipeline. + + Parameters + ---------- + ``pipeline`` : Pipeline + A Pipeline instance that this method will run. + + Raises + ------ + ``StageExecutionError`` + If the stage is unable to be run. Logs will be created to show a failed stage. + """ + # Initial Pipeline steps - validate, create run ID and any relevant directories. pipeline.validate() runtime_id = pipeline._create_runtime_id() pipeline.id = runtime_id - project_root = Path(pipeline.config.project_root or pipeline.config.work_dir) - run_dir = project_root / "runs" / runtime_id.get_id() + run_dir = pipeline.run_output / runtime_id.get_id() run_dir.mkdir(parents=True, exist_ok=True) + # Initialise the ExecutionContext which will be passed to each stage as it runs. This + # context will hold the configuration for the pipeline and for each stage. started_at = now() context = ExecutionContext( pipeline_name=pipeline.name, @@ -36,14 +105,20 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: run_dir=run_dir, started_at=started_at, working_directory=pipeline.config.work_dir, + stage_configs=dict(pipeline.stage_configs), + global_config=pipeline.global_config, ) + # Ensure the stages are in order and create a manifest that explains the run. + ordered_stages = pipeline.ordered_stages() manifest = pipeline._construct_manifest(runtime_id=runtime_id) manifest.stages_run = [] manifest.outputs = {} pipeline.manifest = manifest + _log_config(run_dir, context, manifest) + self.logger.event( "Pipeline started", name=pipeline.name, @@ -51,22 +126,32 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: stages=[stage.name for stage in ordered_stages], ) - stage_results = [] + # Execution of the stages in the dependency-driven order. + stage_results: list[StageResult] = [] try: for stage in ordered_stages: - self.logger.event("Executing stage", name=stage.name, source=stage.source_label) - result = stage.run(context, pipeline.executor) + self.logger.event( + "Executing stage", name=stage.name, source=stage.source_label + ) + context.set_active_stage(stage.name) + try: + result = stage.run(context, pipeline.executor) + finally: + context.set_active_stage(None) context.record(result) stage_results.append(result) manifest.stages_run.append(result.name) manifest.outputs[result.name] = result.outputs + except StageExecutionError as exc: + # Handle recording of execution errors. if exc.result is not None and context.result_for(exc.result.name) is None: context.record(exc.result) stage_results.append(exc.result) manifest.stages_run.append(exc.result.name) manifest.outputs[exc.result.name] = exc.result.outputs + # Log the failure and raise the exception to indicate the pipeline has failed. completed_at = now() run = PipelineRun( manifest=manifest, @@ -78,6 +163,11 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: ) pipeline.manifest = manifest pipeline.last_run = run + + # Creates attributes file in the run_directory to log information for later + # analysis of pipeline runs + _log_pipeline_attributes(pipeline_run=run, run_dir=run_dir, context=context) + self.logger.event( "Pipeline failed", name=pipeline.name, @@ -86,6 +176,7 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: ) raise + # If the pipeline has completed successfully, record the completion and return the run information. completed_at = now() run = PipelineRun( manifest=manifest, @@ -98,26 +189,265 @@ def run(self, pipeline: "Pipeline") -> PipelineRun: pipeline.manifest = manifest pipeline.last_run = run + # Creates attributes file in the run_directory to log information for later + # analysis of pipeline runs + _log_pipeline_attributes(pipeline_run=run, run_dir=run_dir, context=context) + self.logger.event( "Pipeline completed", name=pipeline.name, run_id=runtime_id.get_id(), stages=len(stage_results), ) + return run def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run an onsrap pipeline from Python files.") - parser.add_argument("stages", nargs="+", help="One or more Python stage files to run.") + """ + Determines what arguments are needed when running a Pipeline from the command line. + + Enables stages to be input, followed by a name if provided. + """ + parser = argparse.ArgumentParser( + description="Run an onsrap pipeline from Python files." + ) + parser.add_argument( + "stages", nargs="+", help="One or more Python stage files to run." + ) parser.add_argument("--name", default=None, help="Optional pipeline name.") return parser def main(argv: list[str] | None = None) -> int: + """ + Entrypoint to the pipeline. + + This function can be called from the command line. It builds a parser which enables + the arguments to be held before using those arguments to build a Pipeline instance. + The pipeline.run() method is then run which runs the entire pipeline. If this runs + successfully, a 0 is returned which is the success code. + + Parameters + ---------- + ``argv`` : list[str] or None + Command line arguments to parse. + + Returns + ------- + int + Success code for completion of the run. + """ from .pipeline import Pipeline args = build_parser().parse_args(argv) pipeline = Pipeline.from_files(args.stages, name=args.name) pipeline.run() return 0 + + +def _log_pipeline_attributes( + pipeline_run: PipelineRun, run_dir: Path, context: ExecutionContext +) -> None: + """ + Creates a YAML file within the run directory that contains information + regarding PipelineRun and StageResult instances for the run. This is + later used to extract information about previous runs which are not + currently stored in memory. + + Parameters + ---------- + ``pipeline_run`` : PipelineRun + The PipelineRun instance for the current run of the pipeline. + ``stage_results`` : list[StageResult] + A list of StageResult instances for the current run of the pipeline. + ``run_dir`` : Path + The directory where the pipeline run is being currently being executed. + ``context`` : ExecutionContext + The context of the current pipeline run, containing configuration and + state information. + """ + attributes_file = ( + run_dir + / f"pipeline_attributes_for_{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + import yaml + + with open(attributes_file, "w", encoding="utf-8") as f: + yaml.safe_dump( + pipeline_run._pipeline_run_to_dict(), f, default_flow_style=False + ) + + +def _log_config( + run_dir: Path, context: ExecutionContext, manifest: RunManifest +) -> None: + """ + Outputs the configurations used in an instance of a pipeline to a YAML file in the run directory. + + The file is kept in the block flow style typically expected of a YAML file. + + Parameters + ---------- + ``run_dir`` : Path + The directory where the pipeline run is being executed. + ``context`` : ExecutionContext + The context of the current pipeline run, containing configuration and state information. + ``manifest`` : RunManifest + The manifest of the current pipeline run, containing metadata and outputs. + """ + date = context.started_at.date() + + config_file = ( + run_dir + / f"configuration_for_{context.pipeline_name}_{date}_{context.run_id[-8:]}.yaml" + ) + import yaml + + config_to_dump = _yaml_safe_encode(manifest.config) + + with open(config_file, "w", encoding="utf-8") as f: + yaml.safe_dump(config_to_dump or {}, f, default_flow_style=False) + + +def _flatten(obj: dict | list, prefix: str = "", sep: str = ".") -> dict: + """ + Converts nested dictionaries or lists into flat object using dot notation for keys. + Each key in the resulting dictionary represents the nested branching to get to the value + in the original dictionary. + + Parameters + ---------- + ``obj`` : dict or list + The object to flatten, which can be a dictionary or a list. + ``prefix`` : str + The prefix to use for the keys in the flattened dictionary. Defaults to an empty string. + ``sep`` : str + The separator to use between keys in the flattened dictionary. Defaults to a dot ("."). + + Returns + ------- + dict + A flattened dictionary where each key represents the path to the value in the original object. + """ + items = {} + if isinstance(obj, dict): + for k, v in obj.items(): + items.update(_flatten(v, f"{prefix}{sep}{k}" if prefix else k, sep)) + elif isinstance(obj, list): + for i, v in enumerate(obj): + items.update(_flatten(v, f"{prefix}[{i}]", sep)) + else: + items[prefix] = obj + return items + + +def _diff_yaml_files(path_a: Path, path_b: Path) -> dict: + """ + Calculates the differences between two YAML files and returns a programming oriented dictionary + describing the changes. + + This function calls the ``_flatten`` function to the loaded in dictionaries from the YAML files. + These are then differenced to account for whether a value has changed between the two files, + been added to the second file and was not present in the first, or removed from the second file + and is only present in the first. This output is structured as {changed: {}, added: {}, removed: {}}. + + Parameters + ---------- + ``path_a`` : Path + The path to the first YAML file to compare. + ``path_b`` : Path + The path to the second YAML file to compare. + + Returns + ------- + dict + A dictionary describing the differences between the two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + """ + import yaml + + with open(path_a, encoding="utf-8") as f: + doc_a = yaml.safe_load(f) or {} + with open(path_b, encoding="utf-8") as f: + doc_b = yaml.safe_load(f) or {} + + flat_a = _flatten(doc_a) + flat_b = _flatten(doc_b) + keys_a, keys_b = set(flat_a), set(flat_b) + + return { + "changed": { + k: (flat_a[k], flat_b[k]) for k in keys_a & keys_b if flat_a[k] != flat_b[k] + }, + "added": {k: flat_b[k] for k in keys_b - keys_a}, + "removed": {k: flat_a[k] for k in keys_a - keys_b}, + } + + +def _print_diff(diff: dict) -> dict: + """ + Prints the differences between two YAML files in a human-readable format and returns + the computer-readable dictionary so that it could be used for logging processes if + required. + + Parameters + ---------- + diff : dict + A dictionary describing the differences between two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + + Returns + ------- + dict + The same dictionary that was passed in as the ``diff`` parameter. + """ + changed = diff["changed"] + added = diff["added"] + removed = diff["removed"] + + if changed: + print(f"\nCHANGED ({len(changed)})") + for key, (val_a, val_b) in sorted(changed.items()): + print(f" {key}: {val_a!r} → {val_b!r}") + + if added: + print(f"\nADDED in second configuration ({len(added)})") + for key, val in sorted(added.items()): + print(f" {key}: {val!r}") + + if removed: + print(f"\nREMOVED in second configuration ({len(removed)})") + for key, val in sorted(removed.items()): + print(f" {key}: {val!r}") + + if not any([changed, added, removed]): + print("Files are identical.") + + return diff + + +def print_config_diffs(file_1, file_2) -> dict: + """ + A combining function that calculates the differences between two YAML files + and then prints the outputs to the terminal as well as returning the computer-readable + dictionary of the differences. + + This works by calling the ``_diff_yaml_files`` function to calculate the differences and + then calling the ``_print_diff`` function to print the differences. + + Parameters + ---------- + ``file_1`` : Path + The path to the first YAML file to compare. + ``file_2`` : Path + The path to the second YAML file to compare. + + Returns + ------- + dict + A dictionary describing the differences between the two YAML files, structured as + {changed: {}, added: {}, removed: {}}. + """ + diff_dict = _diff_yaml_files(file_1, file_2) + return _print_diff(diff_dict) diff --git a/onsrap/stage.py b/onsrap/stage.py index 92da700..b64cfdb 100644 --- a/onsrap/stage.py +++ b/onsrap/stage.py @@ -1,27 +1,49 @@ - from __future__ import annotations from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, Optional, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, Optional -from .errors import StageConfigurationError +from .errors import StageConfigurationError, StageDependencyError if TYPE_CHECKING: from .execution import ExecutionContext, StageExecutor from .models import StageResult -def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[str, ...]: +def _normalize_dependencies( + dependencies: Iterable[str] | str | None, +) -> tuple[str, ...]: + """ + Standardise the names of any stages dependant on other stages/processes. + + Removes trailing or leading white space from the name of any stage/process dependant + on another and turns it into a tuple of strings. + + Parameters + ---------- + dependencies : Iterable[str] | str | None + Dependency names to standardise. ``None`` returns an empty tuple. A + string is treated as a single entry in the tuple. Any iterable is converted + into a sequence of names. + Returns + ------- + normalized : tuple + A tuple of cleaned dependency names. + """ if dependencies is None: return () - + if isinstance(dependencies, list) and dependencies == []: + return () if isinstance(dependencies, str): + candidate_items: Iterable[str] candidate_items = [dependencies] + else: - candidate_items = list(dependencies) + candidate_items = dependencies normalized: list[str] = [] + for dependency in candidate_items: dependency_name = str(dependency).strip() if dependency_name and dependency_name not in normalized: @@ -32,8 +54,38 @@ def _normalize_dependencies(dependencies: Iterable[str] | str | None) -> tuple[s @dataclass class Stage: + """ + Represents a single unit of work within a pipeline. + + Can be defined by a data source process or a Python script/callable item. + Stages may be dependant on other stages and can hold metadata for themselves. + + Parameters + ---------- + ``name`` : str + The name of the Stage being run. + ``source`` : Path, Callable, or None + Item being implemented in this Stage. E.g. a file path to a Python script + or a function being executed directly. The full file path is gathered if + a path is used. + ``dependencies`` : tuple of strings + Names of stages that must be completed before this stage is attempted. These + are cleaned post initialisation to remove leading/trailing whitespace. + ``metadata`` : dictionary with string:Any key/value pairs + Location to store any summary information about the stage being run. + ``entrypoint``: str, optional + Name of the starting script to the pipeline. + ``backend`` : str, default = "python" + The name of the system that the code runs on. + + Raises + ------ + ``StageConfigurationError`` + If the stage ``name`` is empty or if the source is not a supported type. + """ + name: str - source: Union[Path, Callable[..., Any], None] = None + source: Path | Callable[..., Any] | None = None dependencies: tuple[str, ...] = field(default_factory=tuple) metadata: dict[str, Any] = field(default_factory=dict) entrypoint: Optional[str] = None @@ -49,12 +101,46 @@ def __post_init__(self) -> None: elif isinstance(self.source, Path): self.source = self.source.expanduser() elif self.source is not None and not callable(self.source): - raise StageConfigurationError("Stage source must be a path, callable, or None.") + raise StageConfigurationError( + "Stage source must be a path, callable, or None." + ) self.dependencies = _normalize_dependencies(self.dependencies) self.metadata = dict(self.metadata or {}) self.backend = str(self.backend or "python").strip() or "python" + def __str__(self) -> str: + """ + String method that returns a human-readable representation of the ``Stage`` class. + + Returns + ------- + str + A string representation of the ``Stage`` class with its attributes. + """ + return ( + f" Name: {self.name}\n Source: {self.source_label} \n" + f" Dependencies: {self.dependencies}\n Metadata: {self.metadata} \n" + f" Entrypoint: {self.entrypoint} \n Backend: {self.backend}" + ) + + def __repr__(self) -> str: + """ + Representation method that returns a human readable representation of the ``Stage`` class. + This method is structured to be more concise than the ``__str__`` method and is intended for + debugging purposes. + + Returns + ------- + str + A string representation of the ``Stage`` class with its attributes. + """ + return ( + f"Stage(name={self.name}, source={self.source_label}, " + f"dependencies={self.dependencies}, metadata={self.metadata}, " + f"entrypoint={self.entrypoint}, backend={self.backend})" + ) + @classmethod def from_file( cls, @@ -65,7 +151,38 @@ def from_file( metadata: Mapping[str, Any] | None = None, entrypoint: str | None = None, backend: str = "python", - ) -> "Stage": + ) -> Stage: + """ + Class method that checks and cleans the file path for the ``Stage``. + + Expands file path to its full name and checks whether it exists. The method + also cleans other parameters in the Stage class in the return line. + + Parameters + ---------- + ``file_path`` : str or Path + The name or file path for the script that the ``Stage`` will be running. + ``name`` : str + The name of the ``Stage`` + ``dependencies`` : Iterable[str], str, or None + The Stage/s that need to be complete before the ``Stage`` currently attempted. + ``metadata`` : Mapping[str, Any], or None + Any supporting information for the ``Stage`` being run. + ``entrypoint`` : str or None + The name of the first script for the Stage. + ``backend``: str, default = "python" + The system that the ``Stage`` is run on. + + Raises + ------ + StageConfigurationError + If the file path does not exist + + Returns + ------- + Stage + Stage class instance with cleaned/checked file path, dependencies, and metadata + """ path = Path(file_path).expanduser() if not path.exists(): raise StageConfigurationError(f"Stage source file does not exist: {path}") @@ -88,8 +205,32 @@ def from_callable( dependencies: Iterable[str] | str | None = None, metadata: Mapping[str, Any] | None = None, backend: str = "python", - ) -> "Stage": - stage_name = name or getattr(callable_object, "__name__", "stage") + ) -> Stage: + """ + Class method that retrieves the name of the Stage from a Callable item. + + Parameters + ---------- + ``callable_object`` : Callable with any number of arguments of any type + The name or file path for the script that the Stage will be running. + ``name`` : str + The name of the Stage + ``dependencies`` : Iterable[str], str, or None + The Stage/s that need to be complete before the Stage currently attempted. + ``metadata`` : Mapping[str, Any], or None + Any supporting information for the Stage being run. + ``entrypoint`` : str or None + The name of the first script for the Stage. + ``backend``: str, default = "python" + The system that the stage is run on. + + Returns + ------- + ``Stage`` + ``Stage class`` instance with collected Stage ``name``, normalised ``dependencies`` + and ``metadata``, and defined the source as the callable_object. + """ + stage_name = str(name or getattr(callable_object, "__name__", "stage")) return cls( name=stage_name, source=callable_object, @@ -99,7 +240,28 @@ def from_callable( ) @classmethod - def from_dict(cls, data: Mapping[str, Any]) -> "Stage": + def from_dict(cls, data: Mapping[str, Any]) -> Stage: + """ + Class method that converts a dictionary stage into a ``Stage`` class instance. + + Extracts the values from the key/value pairs in the stage and holds them as attributes. + + Parameters + ---------- + ``data`` : any number of key/value pairs of strings + The information to convert into a Stage class. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a suitable type (callable or Path). + + Returns + ------- + ``Stage`` + ``Stage`` class instance with collected ``Stage`` attributes based on the type of ``source`` + provided. + """ payload = dict(data) source = payload.pop("source", payload.pop("path", None)) @@ -108,7 +270,13 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Stage": metadata = payload.pop("metadata", {}) entrypoint = payload.pop("entrypoint", None) backend = payload.pop("backend", "python") - name = payload.pop("name", None) + raw_name = payload.pop("name", None) + name = str(raw_name).strip() if raw_name is not None else None + + if isinstance(metadata, Mapping): + metadata = dict(metadata) + else: + metadata = {"metadata": metadata} if callable(source): return cls.from_callable( @@ -138,29 +306,87 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Stage": backend=backend, ) - raise StageConfigurationError("Stage dictionary must define a source, path, or callable.") + raise StageConfigurationError( + "Stage dictionary must define a source, path, or callable." + ) + + def with_dependencies(self, *dependencies: str) -> Stage: + """ + Method that normalises and adds ``dependencies`` to the ``Stage`` class attributes. + + Parameters + ---------- + ``*dependencies`` : str + Information on which scripts need to run before other scripts for this ``Stage``. + + Returns + ------- + ``Stage`` + ``Stage`` class instance with normalised ``dependencies`` attribute. + """ + unpacked_deps: list[str] = [] + for dependency in dependencies: + if isinstance(dependency, list): + unpacked_deps = unpacked_deps + dependency + else: + unpacked_deps.append(dependency) + + for dependency in unpacked_deps: + if isinstance(dependency, list): + raise StageDependencyError( + "Nested lists are not valid arguments for this method! " + "Please provided single list or individual string values" + ) - def with_dependencies(self, *dependencies: str) -> "Stage": return replace( self, - dependencies=self.dependencies + _normalize_dependencies(dependencies), + dependencies=self.dependencies + _normalize_dependencies(unpacked_deps), ) def validate(self) -> None: - if self.source is None: - raise StageConfigurationError(f"Stage '{self.name}' does not define a source.") + """ + Error checking on source attribute. + + Raises + ------- + ``StageConfigurationError`` + If ``source`` attribute does not define a source or does not exist. + """ + if not (isinstance(self.source, Path) or callable(self.source)): + raise StageConfigurationError( + f"Stage '{self.name}' must have a Path or Callable source." + ) + + if self.source is None or self.source == "": + raise StageConfigurationError( + f"Stage '{self.name}' does not define a source. Source provided: {self.source}" + ) if isinstance(self.source, Path) and not self.source.is_file(): raise StageConfigurationError(f"Stage source does not exist: {self.source}") @property def source_path(self) -> Optional[Path]: + """ + Sets a property for the ``Stage`` class if the ``source`` is a path. + + Returns + ------- + ``source_path`` attribute to the ``Stage`` class if the ``source`` is a path. + """ if isinstance(self.source, Path): return self.source return None @property def source_label(self) -> Optional[str]: + """ + Sets a property for the ``Stage`` class with a human-readable name for the ``source``. + + Returns + ------- + ``source_label`` attribute to the ``Stage`` class if ``source`` is a callable or Path. + """ if callable(self.source): return f"{getattr(self.source, '__module__', '')}.{getattr(self.source, '__name__', self.name)}" @@ -169,6 +395,22 @@ def source_label(self) -> Optional[str]: return None - def run(self, context: "ExecutionContext", executor: "StageExecutor") -> "StageResult": + def run(self, context: ExecutionContext, executor: StageExecutor) -> StageResult: + """ + Checks that the ``source`` is valid and then runs the ``source`` + + Properties + ---------- + context : set value "ExecutionContext" + Uses ``ExecutionContext`` class information to provide required metadata on running ``source``. + Any stage-specific configuration resolved by the ``Pipeline`` is available through + ``context.stage_config`` while this stage is running. + executor : set value "StageExecutor" + Uses ``StageExecutor`` class to extract the ``.execute`` method to actually run the ``source``. + + Returns + ------- + ``execute`` method of the ``StageExecutor`` class stored in the ``StageResult`` class. + """ self.validate() - return executor.execute(self, context) \ No newline at end of file + return executor.execute(self, context) diff --git a/onsrap/warnings.py b/onsrap/warnings.py new file mode 100644 index 0000000..0e67739 --- /dev/null +++ b/onsrap/warnings.py @@ -0,0 +1,26 @@ +from __future__ import annotations + + +class OnsrapWarning(Warning): + """Base warning for onsrap.""" + + +class StageConfigurationWarning(OnsrapWarning): + """ + Raised when the stage configuration is not optimal. + Child class with ``OnsrapWarning`` as the parent class. + """ + + +class PipelineConfigurationWarning(OnsrapWarning): + """ + Raised when the pipeline configuration is not optimal. + Child class with ``OnsrapWarning`` as the parent class. + """ + + +class ConfigurationInjectionWarning(OnsrapWarning): + """ + Raised when the configuration injection is not optimal. + Child class with ``OnsrapWarning`` as the parent class. + """ diff --git a/pyproject.toml b/pyproject.toml index 21b441b..83f0f80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ select = [ # isort "I", ] -ignore = ["D203", "E203"] +ignore = ["B028", "D203", "E203", "E501"] [tool.ruff] line-length = 88 @@ -50,3 +50,11 @@ exclude = [] [tool.ruff.format] line-ending = "auto" + +[tool.mypy] +python_version = "3.10" +files = ["onsrap"] +show_error_codes = true +warn_redundant_casts = true +warn_unused_configs = true +warn_unused_ignores = true diff --git a/setup.cfg b/setup.cfg index d270928..aa8718a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,6 @@ version = 0.1.1 author = ONSDigital platforms = win32 classifiers = - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -14,7 +13,7 @@ classifiers = [options] packages = find: -python_requires = >=3.9 +python_requires = >=3.10 zip_safe = no install_requires = pyyaml @@ -23,14 +22,21 @@ install_requires = include = onsrap* +[options.package_data] +onsrap = + py.typed + [options.extras_require] dev = + bandit[toml] coverage - detect-secrets == 1.0.3 + detect-secrets==1.0.3 + mypy myst-parser pre-commit pytest - detect-secrets python-dotenv + ruff Sphinx - toml \ No newline at end of file + toml + types-PyYAML \ No newline at end of file diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 0000000..fc125a6 --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,611 @@ +from pathlib import Path + +import pytest + +from onsrap.errors import PipelineConfigurationError +from onsrap.execution import ExecutionContext, PythonStageExecutor +from onsrap.logger import Logger +from onsrap.models import ( + GlobalConfig, + PipelineConfig, + StageConfig, + StageResult, + StageStatus, +) +from onsrap.warnings import StageConfigurationWarning + + +@pytest.fixture +def logger() -> Logger: + """ + Logger instance for testing + """ + return Logger() + + +@pytest.fixture +def config() -> PipelineConfig: + """ + Return a PipelineConfig object for testing. + """ + work_dir = Path("tmp/work_dir") + project_root = Path("tmp/project") + log_dir = Path("tmp/log") + data_dir = Path("tmp/config_data") + return PipelineConfig( + "test_pipeline", + {"stage_test": True}, + "python", + work_dir, + project_root, + None, + log_dir, + data_dir, + True, + None, + {}, + ) + + +@pytest.fixture +def stage_config() -> StageConfig: + """ + Return a StageConfig object for testing. + """ + return StageConfig( + name="stage_test", + _variables={"sex": "gender", "dob": "date_of_birth"}, + metadata={}, + ) + + +@pytest.fixture +def execution(config, logger, stageresult, stage_config) -> ExecutionContext: + """ + Create an ExecutionContext object for testing. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ + run_dir = Path("tmp/run") + work_dir = Path("tmp/work_dir") + + return ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + run_dir, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {"stage_test": stage_config}, + {}, + None, + ) + + +@pytest.fixture +def stageresult() -> StageResult: + """ + Test StageResult instance for running ExecutionContext tests. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs="example output", + ) + + +@pytest.fixture +def expected_recorded_stage_result() -> StageResult: + """ + Expected StageResult after recording for assertions. + """ + return StageResult( + name="stage_test", + status="pending", + started_at="2024-05-06 15:45:30", + finished_at="2024-05-07 15:45:30", + outputs="example output", + stdout="", + stderr="", + return_code=None, + metadata={}, + error=None, + source=None, + ) + + +class TestExecutionContext: + def test_executioncontext_creation( + self, execution, logger, config, stageresult + ) -> None: + """ + Test that the ExecutionContext creates the right attributes. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + """ + assert execution.pipeline_name == "test_pipeline" + assert execution.run_id == "run_id_1234" + assert execution.config == config + assert execution.logger == logger + assert execution.run_dir == Path("tmp/run") + assert execution.started_at == "2024-05-06 15:45:30" + assert execution.working_directory == Path("tmp/work_dir") + assert execution.stage_results == {"stage_test": stageresult} + assert execution.variables == {} + + def test_record( + self, stageresult, execution, expected_recorded_stage_result + ) -> None: + """ + Tests that StageResult attributes are attached to stage_results and variables + attributes in the ExecutionContext instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. + """ + execution.record(stageresult) + assert execution.stage_results == {"stage_test": expected_recorded_stage_result} + assert execution.variables == {"stage_test": "example output"} + + def test_result_for( + self, execution, stageresult, expected_recorded_stage_result + ) -> None: + """ + Tests that result_for correctly extracts the results of a requested stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + ``expected_recorded_stage_result`` : StageResult + The expected ``StageResult`` object after recording for assertions. + """ + execution.record(stageresult) + assert execution.result_for("stage_test") == expected_recorded_stage_result + + def test_stage_outputs(self, execution, stageresult) -> None: + """ + Tests that stage_outputs shows the outputs attribute of the StageResult + instance for a requested stage is extracted. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + """ + execution.record(stageresult) + assert execution.stage_outputs == {"stage_test": "example output"} + + @pytest.fixture + def blank_context_with_config_none(self, stageresult) -> ExecutionContext: + """ + Fixture that returns a test ExecutionContext instance with a None config for + testing error handling. + + Parameters + ---------- + ``stageresult`` : StageResult + A ``StageResult`` object for testing. + """ + run_dir = Path("tmp/run") + work_dir = Path("tmp/work_dir") + return ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + run_dir, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + def test_get_data_dir(self, execution, blank_context_with_config_none) -> None: + """ + Tests that get_data_dir method extracts the path from the execution context + or, if the context is None, returns an error to indicate that additional input + is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``blank_context_with_config_none`` : ExecutionContext + An ``ExecutionContext`` object with a None config for testing error + handling. + + Raises + ------ + ``PipelineConfigurationError`` + If the config attribute of the ExecutionContext instance is None. + """ + assert execution.get_data_dir() == Path("tmp/config_data") + + with pytest.raises(PipelineConfigurationError): + blank_context_with_config_none.get_data_dir() + + def test_resolve_output_root(self, execution) -> None: + """ + Tests that resolve_output_root method extracts the path from the given run + directory or, if None are given, raises an error to indicate additional input + is required. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + If the run_dir attribute of the ExecutionContext instance is None. + """ + work_dir = Path("tmp/work_dir") + assert execution.resolve_output_root() == Path("tmp/run") + + execution_blank_config = ExecutionContext( + "test_pipeline", + "run_id_1234", + None, + Logger(), + None, + "2024-05-06 15:45:30", + work_dir, + {"stage_test": stageresult}, + {}, + ) + + with pytest.raises(PipelineConfigurationError): + execution_blank_config.resolve_output_root() + + def test_stage_config_accessors_return_named_and_active_configs( + self, config, logger + ) -> None: + """ + Tests that getter methods to return the stage_config for a named stage + returns correct attributes based on given parameters. + + Parameters + ---------- + ``config`` : PipelineConfig + A ``PipelineConfig`` object for testing. + ``logger`` : Logger + A ``Logger`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. + """ + stage_config = StageConfig(name="stage_test", _variables={"years_to_run": 2017}) + context = ExecutionContext( + "test_pipeline", + "run_id_1234", + config, + logger, + Path("tmp/run"), + stage_configs={"stage_test": stage_config}, + active_stage_name="stage_test", + ) + + assert context.stage_config_for("stage_test") == stage_config + assert context.get_stage_config("stage_test") == {"years_to_run": 2017} + assert context.get_stage_config() == {"years_to_run": 2017} + with pytest.raises(PipelineConfigurationError): + context.get_stage_config(vars_only=False) + assert ( + context.get_stage_config(with_global=False, vars_only=False) == stage_config + ) + + def test_set_active_stage(self, execution, stage_config) -> None: + """ + Tests that set_active_stage correctly sets the active_stage attribute in the + ExecutionContext instance. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ + execution.set_active_stage(stage_config.name) + assert execution.active_stage_name == stage_config.name + execution.set_active_stage(None) + assert execution.active_stage_name is None + + def test_stage_config_for(self, execution, stage_config) -> None: + """ + Tests that stage_config_for returns the StageConfig for a named stage. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ + assert execution.stage_config_for(stage_config.name) == stage_config + assert execution.stage_config_for("missing_stage") is None + + def test_stage_config(self, execution, stage_config) -> None: + """ + Tests that stage_config exposes the currently active stage configuration. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + """ + assert execution.stage_config is None + execution.set_active_stage(stage_config.name) + assert execution.stage_config == stage_config + + def test_get_stage_config(self, execution, stage_config) -> None: + """ + Tests that get_stage_config returns variables by default and the full + StageConfig object when requested. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``stage_config`` : StageConfig + A ``StageConfig`` object for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Requested a StageConfig instance as with_global = True, the output must + be a dictionary however quantifying vars_only as False would demand that + the entire StageConfig instance is returned. + + """ + assert execution.get_stage_config() == {} + with pytest.raises(PipelineConfigurationError): + execution.get_stage_config(vars_only=False) + assert execution.get_stage_config(with_global=False, vars_only=False) is None + + execution.set_active_stage(stage_config.name) + assert execution.get_stage_config() == {"sex": "gender", "dob": "date_of_birth"} + + assert ( + execution.get_stage_config(with_global=False, vars_only=False) + == stage_config + ) + + +class TestResolveGivenPath: + """ + Parameters for testing multiple add_folder options in + test_resolve_given_path_add_folders function. + """ + + @pytest.mark.parametrize( + "add_folder,file_name,expected", + [ + ( + ["interim", "testing_files"], + "clean.py", + Path("tmp/data/interim/testing_files/clean.py"), + ), + ("interim", "clean.py", Path("tmp/data/interim/clean.py")), + (None, "clean.py", Path("tmp/data/clean.py")), + ( + ["interim", "testing_files"], + None, + Path("tmp/data/interim/testing_files"), + ), + ("interim", None, Path("tmp/data/interim")), + (None, None, Path("tmp/data")), + ], + ) + def test_resolve_given_path_add_folders( + self, execution, add_folder, file_name, expected + ) -> None: + """ + Tests the add_folder functionality for lists, single strings, or None type in + the resolve_given_path class method as well as when the file_name is a valid + string or None type. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + ``add_folder`` : Union[str, List[str], None] + A string, list of strings, or None type to specify additional folders to + add to the path. + ``file_name`` : Union[str, None] + A string or None type to specify the file name to append to the path. + ``expected`` : Path + The expected Path object that should be returned by the method. + """ + path_name = "data_path" + root = Path("tmp/data") + + assert ( + execution.resolve_given_path(None, path_name, file_name, root, add_folder) + == expected + ) + + def test_resolve_given_path_norm(self, execution) -> None: + """ + Tests that resolve_given_path returns a file path that has been output in a + StageResult instance. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + """ + execution.record( + StageResult( + "stage_test2", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs={"data_path": "clean.py"}, + ) + ) + stage_name = "stage_test2" + path_name = "data_path" + root = Path("tmp/data") + + assert execution.resolve_given_path( + stage_name, path_name, None, root, None + ) == Path("clean.py") + + +"""TEST NOT RUN FOR StageExecutor AS COVERED UNDER PythonStageExecutor""" + + +@pytest.fixture +def pythonstageexecutor() -> PythonStageExecutor: + return PythonStageExecutor(("main.py", "run.py")) + + +class TestPythonStageExecutor: + def test_pythonstageexecutor_setup(self, pythonstageexecutor) -> None: + """ + Checks that entrypoints are set correctly in the PythonStageExecutor + instance. + + Parameters + ---------- + ``pythonstageexecutor`` : PythonStageExecutor + A ``PythonStageExecutor`` object for testing. + """ + assert pythonstageexecutor.preferred_entrypoints == ("main.py", "run.py") + + +class TestCombineVars: + def test_combine_vars(self, execution) -> None: + """ + Test that checks that a dictionary is returned, combining values from a global + configuration and a stage configuration whilst removing any stage specific + exclusions. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1", + } + + def test_combine_vars_errors(self, execution) -> None: + """ + Test that confirms that a warning is raised if there is a variable defined in + both the global and the stage configurations as well as asserting the correct + values. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + + Raises + ------ + ``StageConfigurationWarning`` + If a variable is defined in both the global and stage configurations, a + warning is raised to indicate that the stage variable will take precedence. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {"stage_1": ["global_var2"]} + stage_vars = {"stage_var1": "value3", "global_var1": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + + with pytest.warns( + StageConfigurationWarning, + match="Stage defines variable\\(s\\) that are also defined in global " + "variables: global_var1\\. Stage variables will take precedence.", + ): + combined_vars = execution._combine_vars() + assert combined_vars == {"stage_var1": "value3", "global_var1": "value4"} + + def test_combine_vars_no_exclusion(self, execution) -> None: + """ + Test confirming that a dictionary is returned, combining values from a global + configuration and a stage configuration when there are no exclusions defined. + + Parameters + ---------- + ``execution`` : ExecutionContext + An ``ExecutionContext`` object for testing. + """ + global_vars = {"global_var1": "value1", "global_var2": "value2"} + exclusions = {} + stage_vars = {"stage_var1": "value3", "stage_var2": "value4"} + execution.global_config = GlobalConfig( + _variables=global_vars, exclusion=exclusions + ) + execution.stage_configs = { + "stage_1": StageConfig(name="stage_1", _variables=stage_vars), + } + execution.active_stage_name = "stage_1" + combined_vars = execution._combine_vars() + assert combined_vars == { + "stage_var1": "value3", + "stage_var2": "value4", + "global_var1": "value1", + "global_var2": "value2", + } diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..768e5f4 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import pytest + +from onsrap.errors import StageLoadError +from onsrap.loader import load_historical_run +from onsrap.pipeline import PipelineRun +from tests.test_pipeline import TestLoadLatestRunIntegration + + +class TestLoadHistoricalRun(TestLoadLatestRunIntegration): + def test_raises_stageloaderror_no_file(self, tmp_path: Path) -> None: + """ + Checks that load_historical_run raises a StageLoadError when the specified + directory does not contain any files matching the expected pattern. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``StageLoadError`` + Raised when the specified directory does not contain any files matching + the expected pattern for historical run YAML files. + """ + run_dir = tmp_path / "empty_run" + run_dir.mkdir(parents=True, exist_ok=True) + with pytest.raises(StageLoadError, match="Historical run file does not exist"): + load_historical_run(run_dir=run_dir) + + def test_returns_valid_pipeline_run_from_yaml( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: + """ + Checks that load_historical_run successfully returns a PipelineRun instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + + run_dir = tmp_path / "valid_run" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "pipeline_attributes_for_test.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id"), encoding="utf-8" + ) + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) + assert result.manifest.run_id == "test_id" + + def test_correct_yaml_file_chosen( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: + """ + Checks that if there are multiple files within the same run directory, + the method will pass successfully and return a PipelineRun. This does + not assert which file is chosen, only that the method does not raise + an error and returns a PipelineRun instance. + + Logically, this should be suitable as we would only ever expect one file + to be present in each run directory. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + """ + run_dir = tmp_path / "multiple_runs" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "pipeline_attributes_for_test1.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id_1"), encoding="utf-8" + ) + (run_dir / "pipeline_attributes_for_test2.yaml").write_text( + minimal_pipeline_yaml(run_id="test_id_2"), encoding="utf-8" + ) + + result = load_historical_run(run_dir=run_dir) + assert isinstance(result, PipelineRun) diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..40bcf88 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,324 @@ +import logging +from pathlib import Path + +import pytest + +from onsrap.errors import HistoricalPipelineLoadError +from onsrap.logger import Logger +from tests.test_pipeline import TestLoadLatestRunIntegration + + +class TestExtractHistoricalRunIds(TestLoadLatestRunIntegration): + def test_logger_no_handler_errors(self, tmp_path: Path) -> None: + """ + Tests that if the logger has no handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir=tmp_path / "logs") + logger._logger.handlers.clear() # Remove all handlers to simulate no file logging + logger._logger.propagate = False # Prevent checking root logger handlers + with pytest.raises(HistoricalPipelineLoadError, match="does not write to a"): + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + + def test_logger_no_file_handler_errors(self, tmp_path: Path) -> None: + """ + Tests that if the logger has no file handlers, an error is raised when + attempting to extract historical ids as the logger is not writing + to a file that can be checked. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``HistoricalPipelineLoadError`` + Raised when the logger does not have any handlers, indicating that + it is not writing to a file path and cannot extract historical run ids. + """ + logger = Logger(log_dir=tmp_path / "logs") + logger._logger.handlers = [logging.StreamHandler()] + with pytest.raises( + HistoricalPipelineLoadError, match="does not have a FileHandler" + ): + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + + def test_logger_does_not_exist(self, tmp_path: Path) -> None: + """ + Checks that the method raises an error if the log doesn't exist at the + location specified. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + file_handler.close() + log_path.unlink( + missing_ok=True + ) # Remove the log file to simulate non-existence + + with pytest.raises( + HistoricalPipelineLoadError, match="does not exist at this location" + ): + logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + + def test_return_blank_list_no_matches_in_log(self, tmp_path) -> None: + """ + Tests that a log file that does not have a record covering "Pipeline started" + will return a blank list from extract_historical_run_ids. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:00,000 Some unrelated log entry\n" + "2026-08-10 10:00:01,000 | Another unrelated log entry\n" + "2026-08-10 10:00:02,000 Pipeline Started\n" + ) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result == [] + + def test_skips_poor_json_in_log(self, tmp_path: Path) -> None: + """ + Tests that if a JSON record in the log file is not valid, it will e skipped + and the next valid entry will be extracted. Assert that the returned list + contains only the valid entry. Confirms that only the incorrect record is + skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + "2026-08-10 10:00:00,000 Pipeline started | not_valid_json\n" + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' + ) + + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result == [ + { + "run_id": "2026-06-23_101719_abc1234", + "timestamp": "2026-08-10 10:00:02,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_abc1234", + }, + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_878fcb33", + }, + ] + + @pytest.mark.parametrize( + "string, expected", [('{"name":"test_pipeline"}', []), ('{"run_id":""}', [])] + ) + def test_run_id_absent_falsy( + self, tmp_path: Path, string: str, expected: list + ) -> None: + """ + Tests that if the JSON record in the log file does not have a run_id, it will be skipped + and the returned list will be empty. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``string`` : str + A dictionary representing a valid JSON record in the log file that excludes + run_id. + ``expected`` : list + The expected output from extract_historical_run_ids when the log file + contains a record without a run_id. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text(f"2026-08-10 10:00:01,000 Pipeline started | {string}\n") + + # creates directory for runs to avoid removal given the directory doesn't exist + (tmp_path / "runs").mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result == expected + + def test_records_only_if_directory_exists(self, tmp_path) -> None: + """ + Checks that a record is only output if the run directory exists. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' + ) + + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result == [ + { + "run_id": "2026-06-23_101719_878fcb33", + "timestamp": "2026-08-10 10:00:01,000", + "run_dir": tmp_path / "runs" / "2026-06-23_101719_878fcb33", + } + ] + + def test_reverse_chronological_order(self, tmp_path) -> None: + """ + Checks that the run_ids are output in reverse chronological order + based on their positioning in the log file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + '2026-08-10 10:00:01,000 Pipeline started | {"name": "test_pipeline",' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + '2026-08-10 10:00:02,000 Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_abc1234"}\n' + ) + + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + create_run_dir_2 = tmp_path / "runs" / "2026-06-23_101719_abc1234" + create_run_dir_2.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result[0]["run_id"] == "2026-06-23_101719_abc1234" + assert result[1]["run_id"] == "2026-06-23_101719_878fcb33" + + def test_skip_poor_timestamps(self, tmp_path) -> None: + """ + Checks that entries with poor timestamps are skipped. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + logger = Logger(log_dir=tmp_path / "logs") + + file_handler = next( + h for h in logger._logger.handlers if isinstance(h, logging.FileHandler) + ) + + log_path = Path(file_handler.baseFilename) + + log_path.write_text( + 'BADTIMESTAMP Pipeline started | {"name": "test_pipeline", ' + '"run_id": "2026-06-23_101719_878fcb33"}\n' + ) + + create_run_dir_1 = tmp_path / "runs" / "2026-06-23_101719_878fcb33" + create_run_dir_1.mkdir(parents=True, exist_ok=True) + + result = logger.extract_historical_run_ids( + run_root=tmp_path / "runs", name="test_pipeline" + ) + assert result == [] diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..5ab8849 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,653 @@ +import datetime +from pathlib import Path +from textwrap import dedent + +import pytest + +from onsrap.models import ( + PipelineConfig, + PipelineRun, + PipelineStatus, + RunManifest, + RuntimeID, + StageResult, + StageStatus, +) + +STARTED_AT = datetime.datetime(2024, 5, 6, 15, 45, 30) +FINISHED_AT = datetime.datetime(2024, 5, 7, 15, 45, 30) + + +class TestStatuses: + def test_stagestatus(self) -> None: + """ + Test that stagestatus outputs the correct values. + """ + assert StageStatus.PENDING == "pending" + assert StageStatus.RUNNING == "running" + assert StageStatus.SUCCEEDED == "succeeded" + assert StageStatus.FAILED == "failed" + assert StageStatus.SKIPPED == "skipped" + + def test_pipeline_status(self) -> None: + """ + Test that pipeline status outputs the correct values. + """ + assert PipelineStatus.PENDING == "pending" + assert PipelineStatus.RUNNING == "running" + assert PipelineStatus.SUCCEEDED == "succeeded" + assert PipelineStatus.FAILED == "failed" + + +@pytest.fixture +def runtimeID() -> RuntimeID: + """ + Example RuntimeID instance for testing of other methods. + """ + return RuntimeID( + id="abc123", + timestamp=datetime.datetime(2026, 7, 7, 13, 5, 46), + hash="fnruw9574893ghkwq234h5kg", + short_hash="4h5kg", + ) + + +class TestRuntimeID: + def test_runtimeID_creation(self, runtimeID) -> None: + """ + Test that a RuntimeID is correctly created. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. + """ + assert runtimeID.id == "abc123" + assert runtimeID.timestamp == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.hash == "fnruw9574893ghkwq234h5kg" + assert runtimeID.short_hash == "4h5kg" + + def test_getter_functions_runtimeID(self, runtimeID) -> None: + """ + Tests all the getter functions for the RuntimeID instance. + + Parameters + ---------- + ``runtimeID`` : RuntimeID + A RuntimeID instance for testing. + """ + assert runtimeID.get_id() == "abc123" + assert runtimeID.get_timestamp() == datetime.datetime(2026, 7, 7, 13, 5, 46) + assert runtimeID.get_hash() == "fnruw9574893ghkwq234h5kg" + assert runtimeID.get_short_hash() == "4h5kg" + + +@pytest.fixture +def blankpipelineconfig() -> PipelineConfig: + """ + Blank PipelineConfig instance for class method testing. + """ + return PipelineConfig() + + +@pytest.fixture +def expected_pipeline_config() -> PipelineConfig: + """ + Example PipelineConfig completed class instance for method testing. + """ + return PipelineConfig( + name="test_rap", + backend="python", + work_dir=Path("tmp/work"), + project_root=Path("project"), + log_dir=Path("tmp/logs"), + data_dir=Path("tmp/data"), + allow_subprocess_fallback=True, + python_executable=None, + metadata={"variables": ["name", "age"], "num_stages": 6}, + ) + + +@pytest.fixture +def pipelineconfig(expected_pipeline_config) -> PipelineConfig: + """ + Returns a PipelineConfig instance for testing that is derived + fromthe expected_pipeline_config fixture. Used as a separate + fixture to ensure that the behaviour of the from_any() method is + tested correctly in the TestPipelineConfig class. + """ + return expected_pipeline_config + + +@pytest.fixture +def mapping() -> dict: + """ + Example mapping dictionary for use in testing from_mapping() method. + """ + return { + "name": "test_rap", + "backend": "python", + "work_dir": Path("tmp/work"), + "project_root": Path("project"), + "log_dir": Path("tmp/logs"), + "data_dir": Path("tmp/data"), + "allow_subprocess_fallback": True, + "python_executable": None, + "metadata": {"variables": ["name", "age"], "num_stages": 6}, + } + + +class TestPipelineConfig: + def test_from_any( + self, mapping, pipelineconfig, blankpipelineconfig, expected_pipeline_config + ) -> None: + """ + Test derivation for a PipelineConfig instance using the from_any() method. This + test checks all methods EXCEPT from_file as this will be covered in another + test due to creation of a mock file being required. + + Parameters + ---------- + ``mapping`` : dict + A dictionary mapping of values for a PipelineConfig instance. + + Raises + ------ + TypeError + If the input to from_any() is not of a supported type. + """ + assert blankpipelineconfig.from_any(None) == PipelineConfig() + assert blankpipelineconfig.from_any(pipelineconfig) == expected_pipeline_config + assert blankpipelineconfig.from_any(mapping) == expected_pipeline_config + + with pytest.raises(TypeError): + blankpipelineconfig.from_any(11) + + def test_from_file_errors(self, tmp_path) -> PipelineConfig: + """ + Checks that a PipelineConfig instance raises the correct exceptions when a + file is not found or the file does not contain a dictionary mapping. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``FileNotFoundError`` + If the file path provided does not exist. + + ``TypeError`` + If the file does not contain a dictionary mapping. + """ + + no_map_pipeline_config = tmp_path / "not_valid.py" + no_map_pipeline_config.write_text( + dedent( + """ + variable = "Hello world" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + fake_file = "path_not_real" + with pytest.raises(FileNotFoundError): + PipelineConfig.from_file(fake_file) + with pytest.raises(TypeError): + PipelineConfig.from_file(no_map_pipeline_config) + + def test_from_file_success( + self, tmp_path, expected_pipeline_config + ) -> PipelineConfig: + """ + Checks that a PipelineConfig instance is created successfully from a mock + file. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``expected_pipeline_config`` : PipelineConfig + A PipelineConfig instance that is expected to be created from the mock + file. + """ + pipeline_config = tmp_path / "configuration.py" + pipeline_config.write_text( + dedent( + """ + {"name":"test_rap", + "backend":"python", + "work_dir":"tmp/work", + "project_root":"project", + "log_dir":"tmp/logs", + "data_dir":"tmp/data", + "allow_subprocess_fallback":True, + "python_executable": null, + "metadata":{"variables":["name","age"], + "num_stages":6} + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + configuration = PipelineConfig.from_file(pipeline_config) + assert configuration == expected_pipeline_config + + def test_to_dict(self, pipelineconfig) -> None: + """ + Test of to_dict() class method for PipelineConfig that it outputs the + PipelineConfig values as a dictionary. + + Parameters + ---------- + ``pipelineconfig`` : PipelineConfig + A PipelineConfig instance for testing. + """ + + assert pipelineconfig.to_dict() == { + "name": "test_rap", + "backend": "python", + "work_dir": str(Path("tmp/work")), + "project_root": "project", + "output_dir": None, + "log_dir": str(Path("tmp/logs")), + "data_dir": str(Path("tmp/data")), + "allow_subprocess_fallback": True, + "python_executable": None, + "variables": ["name", "age"], + "num_stages": 6, + } + + +@pytest.fixture +def runmanifest() -> RunManifest: + """ + Example RunManifest class instance for testing of class method. + """ + return RunManifest( + "pipeline", + "1", + None, + ["stage1", "stage2"], + {"uniqueID": "example"}, + {"input_path": "input/data/example.csv"}, + {"output_path": "output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None, + ) + + +@pytest.fixture +def stageresult() -> StageResult: + """ + Example StageResult instance for tests that use the module-level fixture. + """ + return StageResult( + "stage_test", + StageStatus.PENDING, + "2024-05-06 15:45:30", + "2024-05-07 15:45:30", + metadata={}, + outputs="example output", + ) + + +class TestStageResult: + def test_stage_result(self, stageresult: StageResult) -> None: + """ + Uses a StageResult instance created in test_execution to ensure that + the class instance is created suitably with required defaults. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + """ + assert stageresult.name == "stage_test" + assert stageresult.status == "pending" + assert stageresult.started_at == "2024-05-06 15:45:30" + assert stageresult.finished_at == "2024-05-07 15:45:30" + assert stageresult.outputs == "example output" + assert stageresult.stdout == "" + assert stageresult.stderr == "" + assert stageresult.return_code is None + assert stageresult.metadata == {} + assert stageresult.error is None + assert stageresult.source is None + + @pytest.mark.parametrize( + "status_stage,expected_stage", + [ + (StageStatus.PENDING, False), + (StageStatus.RUNNING, False), + (StageStatus.FAILED, False), + (StageStatus.SUCCEEDED, True), + (StageStatus.SKIPPED, False), + ], + ) + def test_succeeded(self, stageresult, status_stage, expected_stage) -> None: + """ + Tests succeeded() method for StageResult which outputs True or False depending + on the status of the StageResult. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``status_stage`` : StageStatus + A StageStatus value to set the status of the StageResult instance. + ``expected_stage`` : bool + The expected boolean output from the succeeded() method based on the + status of the StageResult instance. + """ + stageresult.status = status_stage + assert stageresult.succeeded == expected_stage + + def test_duration_seconds(self, stageresult) -> None: + """ + Tests that duration_seconds() method calculates the correct duration in seconds + between the started_at and finished_at attributes of the StageResult instance. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + """ + stageresult.started_at = STARTED_AT + stageresult.finished_at = FINISHED_AT + seconds_value = (FINISHED_AT - STARTED_AT).total_seconds() + assert stageresult.duration_seconds == seconds_value + + +@pytest.fixture +def pipelinerun(stageresult, runmanifest) -> PipelineRun: + """ + Creates a PipelineRun instance for testing that is used in the TestPipelineRun + class. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + """ + return PipelineRun( + runmanifest, + PipelineStatus.SUCCEEDED, + STARTED_AT, + FINISHED_AT, + [stageresult], + {"stage_test": "example output"}, + ) + + +class TestPipelineRun: + def test_pipelinerun_configuration( + self, pipelinerun, runmanifest, stageresult + ) -> None: + """ + Checks that the PipelineRun instance is created successfully with the correct + attributes and values. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``runmanifest`` : RunManifest + A RunManifest instance for testing. + ``stageresult`` : StageResult + A StageResult instance for testing. + """ + assert pipelinerun.manifest == runmanifest + assert pipelinerun.status == PipelineStatus.SUCCEEDED + assert pipelinerun.started_at == STARTED_AT + assert pipelinerun.completed_at == FINISHED_AT + assert pipelinerun.stage_results == [stageresult] + assert pipelinerun.stage_outputs == {"stage_test": "example output"} + + def test_result_for(self, pipelinerun, stageresult) -> None: + """ + Checks that the result_for() method of the PipelineRun instance returns the + correct StageResult instance when provided with a valid stage name, and returns + None when the stage name is not found. + """ + assert pipelinerun.result_for("stage_test") == stageresult + assert pipelinerun.result_for("not_a_stage") is None + + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.PENDING, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.FAILED, False), + (PipelineStatus.SUCCEEDED, True), + ], + ) + def test_succeeded_pipeline(self, pipelinerun, status, expected) -> None: + """ + Checks that the succeeded() method of the PipelineRun instance returns the + correct boolean value based on its status. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance for testing. + ``status`` : PipelineStatus + A PipelineStatus value to set the status of the PipelineRun instance. + ``expected`` : bool + The expected boolean output from the succeeded() method based on the + status of the PipelineRun instance. + """ + pipelinerun.status = status + assert pipelinerun.succeeded == expected + + +# TODO: Test _extract_stages_run and all methods in StageConfig class + + +class TestToFromDictMethods: + """ + Class to store testing methods for to_dict and from_dict, specifically + for PipelineRun, RunManifest, and StageResult classes. + """ + + @pytest.fixture + def runmanifest(self) -> RunManifest: + return RunManifest( + "pipeline", + "1", + None, + ["stage1", "stage2"], + {"uniqueID": "example"}, + {"input_path": "input/data/example.csv"}, + {"output_path": "output/data/example.csv"}, + "python", + ["1.3.2"], + "", + None, + None, + ) + + @pytest.fixture + def stageresult(self) -> StageResult: + return StageResult( + "stage_test", + StageStatus.SUCCEEDED, + datetime.datetime(2024, 5, 6, 15, 45, 30), + datetime.datetime(2024, 5, 7, 15, 45, 30), + "example output", + "", + "", + None, + {}, + None, + None, + ) + + @pytest.fixture + def pipelinerun(self, runmanifest, stageresult) -> PipelineRun: + return PipelineRun( + runmanifest, + PipelineStatus.SUCCEEDED, + datetime.datetime(2024, 5, 6, 15, 45, 30), + datetime.datetime(2024, 5, 7, 15, 45, 30), + [stageresult], + {"stage_test": "example output"}, + ) + + def test_runmanifest_to_dict(self, runmanifest) -> None: + """ + Test that the to_dict method for RunManifest outputs the correct dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + expected_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config": None, + } + assert runmanifest._runmanifest_to_dict() == expected_dict + + def test_runmanifest_from_dict(self, runmanifest) -> None: + """ + Test that the from_dict method for RunManifest correctly creates a RunManifest instance from a dictionary representation. + + Parameters + ---------- + ``runmanifest`` : RunManifest + A RunManifest instance provided by the pytest fixture. + """ + runmanifest_dict = { + "rap_name": "pipeline", + "run_id": "1", + "git_commit": None, + "stages_run": ["stage1", "stage2"], + "parameters": {"uniqueID": "example"}, + "inputs": {"input_path": "input/data/example.csv"}, + "outputs": {"output_path": "output/data/example.csv"}, + "backend": "python", + "package_versions": ["1.3.2"], + "timestamp": "", + "reason": None, + "user": None, + "config": None, + } + new_runmanifest = RunManifest._runmanifest_from_dict(runmanifest_dict) + assert new_runmanifest == runmanifest + + def test_stageresult_to_dict(self, stageresult) -> None: + """ + Test that the to_dict method for StageResult outputs the correct dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + expected_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None, + } + assert stageresult._stage_result_to_dict() == expected_dict + + def test_stageresult_from_dict(self, stageresult) -> None: + """ + Test that the from_dict method for StageResult correctly creates a StageResult instance from a dictionary representation. + + Parameters + ---------- + ``stageresult`` : StageResult + A StageResult instance provided by the pytest fixture. + """ + stageresult_dict = { + "name": "stage_test", + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "finished_at": "2024-05-07T15:45:30", + "outputs": "example output", + "stdout": "", + "stderr": "", + "return_code": None, + "metadata": {}, + "error": None, + "source": None, + } + new_stageresult = StageResult._stage_result_from_dict(stageresult_dict) + assert new_stageresult == stageresult + + def test_pipelinerun_to_dict(self, pipelinerun) -> None: + """ + Test that the to_dict method for PipelineRun outputs the correct dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + expected_dict = { + "manifest": pipelinerun.manifest._runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": { + result.name: result._stage_result_to_dict() + for result in pipelinerun.stage_results + }, + "stage_outputs": {"stage_test": "example output"}, + } + assert pipelinerun._pipeline_run_to_dict() == expected_dict + + def test_pipelinerun_from_dict(self, pipelinerun) -> None: + """ + Test that the from_dict method for PipelineRun correctly creates a PipelineRun instance from a dictionary representation. + + Parameters + ---------- + ``pipelinerun`` : PipelineRun + A PipelineRun instance provided by the pytest fixture. + """ + pipelinerun_dict = { + "manifest": pipelinerun.manifest._runmanifest_to_dict(), + "status": "succeeded", + "started_at": "2024-05-06T15:45:30", + "completed_at": "2024-05-07T15:45:30", + "stage_results": { + result.name: result._stage_result_to_dict() + for result in pipelinerun.stage_results + }, + "stage_outputs": {"stage_test": "example output"}, + } + new_pipelinerun = PipelineRun._pipeline_run_from_dict(pipelinerun_dict) + assert new_pipelinerun == pipelinerun diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..980bb97 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,1512 @@ +import warnings +from pathlib import Path +from unittest import mock + +import pytest + +from onsrap.errors import ( + HistoricalPipelineLoadError, + PipelineConfigurationError, + PipelineInitialisationError, + StageLoadError, +) +from onsrap.execution import PythonStageExecutor +from onsrap.models import PipelineRun, StageConfig +from onsrap.pipeline import Pipeline, PipelineConfig +from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning + +NO_STAGES_WARNING = "No stages specified to run. All stages running by default." + + +@pytest.fixture +def stage_factory(): + def _build_stage(name: str, dependencies=(), source: Path | None = None) -> Stage: + """ + Function that builds a Stage object with a given name, dependencies, and a + source file path that's built out of the name if it is not provided. This + standardises the creation of Stage objects for testing. + + Parameters + ---------- + ``name`` : str + The name of the stage to be created. + ``dependencies`` : tuple + A tuple of stage names that the created stage depends on. + ``source`` : Path | None + A Path object representing the source file for the stage. If None, a default + source file path is created based on the stage name. + """ + resolved_source = source if source is not None else Path(f"{name}.py") + return Stage(name, source=resolved_source, dependencies=dependencies) + + return _build_stage + + +class TestPipelineNamingAndInit: + def test_pipeline_name(self): + """ + Test to confirm that Pipeline instance uses either defined name from + instance creation (shown in pipeline_named), utilises name from PipelineConfig + if no name was given (shown in pipeline_config), or defaults to "pipeline" if + no name is provided through Pipeline instance creation or through the + PipelineConfig (shown through pipeline_no_name) + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + + """ + pipeline_config = PipelineConfig(name="test_pipeline_config") + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_named = Pipeline( + name="test_pipeline_name", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + pipeline_config = Pipeline( + name=None, + config=pipeline_config, + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + pipeline_no_name = Pipeline( + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())] + ) + + assert pipeline_named.name == "test_pipeline_name" + assert pipeline_config.name == "test_pipeline_config" + assert pipeline_no_name.name == "pipeline" + + def test_assign_dependencies(self, tmp_path): + """ + Test to ensure that different formats of dependencies can be parsed to the + Pipeline creation and appropriately assigned to each stage within the + Pipeline. Will also check for error raise if the dependencies are defined + but there are no defined stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as there are no stages defined in the Pipeline + but there are dependencies. + """ + + def example_function(): + pass + + path_1 = tmp_path / "Stage_1.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_single = {"Stage_2": ("Stage_1",)} + dependencies_multiple = { + "Stage_1": ["Stage_0"], + "Stage_2": ("Stage_1", "Stage_0"), + } + dependencies_non_stage_name = { + "Stage_1.py": ("Stage_0",), + "example_function": ("Stage_1.py",), + } + + with ( + pytest.raises(PipelineInitialisationError), + pytest.warns(PipelineConfigurationWarning), + ): + Pipeline(stages=None, dependencies=dependencies_single) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_1 = Pipeline( + name="pipeline_1", + stages=[ + Stage("Stage_1", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_multiple, + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_2 = Pipeline( + name="pipeline_2", + stages=[ + Stage("Stage_1.py", path_1, None, {}), + Stage("Stage_2", example_function, None, {}), + Stage("Stage_0", path_0, None, {}), + ], + dependencies=dependencies_non_stage_name, + ) + + assert pipeline_1.stages[0].dependencies == ("Stage_0",) + assert pipeline_1.stages[1].dependencies == ("Stage_1", "Stage_0") + + assert pipeline_2.stages[0].dependencies == ("Stage_0",) + assert pipeline_2.stages[1].dependencies == ("Stage_1.py",) + + def test_assign_dependencies_with_config_defined_stages(self, tmp_path): + """ + Test to ensure dependencies can be assigned when stages are loaded from + the pipeline configuration rather than passed directly to the constructor. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + """ + + first_stage = tmp_path / "first_stage.py" + first_stage.write_text( + "def run(context):\n return 'alpha'\n", encoding="utf-8" + ) + + second_stage = tmp_path / "second_stage.py" + second_stage.write_text( + "def run(context):\n return 'beta'\n", encoding="utf-8" + ) + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + "\n".join( + [ + "pipeline_variables:", + f' work_dir: "{tmp_path.as_posix()}"', + f' project_root: "{tmp_path.as_posix()}"', + f' log_dir: "{(tmp_path / "logs").as_posix()}"', + " stages:", + " - first_stage:", + f' location: "{first_stage.as_posix()}"', + " - second_stage:", + f' location: "{second_stage.as_posix()}"', + "stage_configuration: {}", + ] + ) + + "\n", + encoding="utf-8", + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + config=config_file, + dependencies={"second_stage": ("first_stage",)}, + ) + + assert [stage.name for stage in pipeline.stages] == [ + "first_stage", + "second_stage", + ] + assert pipeline.stages[1].dependencies == ("first_stage",) + assert pipeline.dependencies == {"second_stage": ("first_stage",)} + + def test_add_dependencies_single_dict(self, tmp_path): + """ + Tests that a dictionary correctly assigns dependencies to + individual stages and the Pipeline instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'PipelineInitialisationError' + Expected and asserted as dependencies are specified for stages that do not + exist in the Pipeline instance. + """ + + path_1 = tmp_path / "Stage_1.py" + path_2 = tmp_path / "Stage_2.py" + path_0 = tmp_path / "Stage_0.py" + + dependencies_multiple = {"Stage_0": (), "Stage_1": (), "Stage_2": ("Stage_1",)} + dep_dict = {"Stage_1": ("Stage_0",), "Stage_2": ("Stage_0", "Stage_1")} + dep_tuple = ("Stage_0.25",) + stage_1 = Stage("Stage_1", source=path_1, dependencies={}) + stage_2 = Stage("Stage_2", source=path_2, dependencies={}) + stage_0 = Stage("Stage_0", source=path_0, dependencies={}) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline_dict = Pipeline( + stages=[stage_0, stage_1, stage_2], + dependencies=dependencies_multiple, + ) + + with pytest.raises(PipelineInitialisationError): + pipeline_dict.add_dependencies(dep_tuple) + + with pytest.warns(PipelineConfigurationWarning): + pipeline_dict.add_dependencies(dep_dict) + assert stage_1.dependencies == ("Stage_0",) + assert stage_2.dependencies == ( + "Stage_1", + "Stage_0", + ) + assert stage_0.dependencies == () + assert pipeline_dict.dependencies == { + "Stage_0": (), + "Stage_1": ("Stage_0",), + "Stage_2": ( + "Stage_1", + "Stage_0", + ), + } + + +class TestPipelineStageConfigHandling: + def test_add_stage_parses_stage_configs_keyword(self, stage_factory) -> None: + """ + Tests that when a stage is added after a Pipeline has been initialised, + the stage and the stage_configurations are correctly added to the + Pipeline instance and the stage_configurations are correctly associated + with the stage. + + Parameter + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())] + ) + + stage = stage_factory("Stage_1") + stage_config = StageConfig(name="Stage_1", _variables={"years_to_run": 2017}) + with pytest.warns(PipelineConfigurationWarning): + pipeline.add_stage(stage, stage_configs=[stage_config]) + + assert pipeline.stages[-1].name == "Stage_1" + assert pipeline.stage_configs["Stage_1"].require("years_to_run") == 2017 + + def test_add_stage_warns_when_stage_config_count_mismatches( + self, stage_factory + ) -> None: + """ + Tests that when a stage is added but there is not the correct number of + stage_configs provided, a warning is raised and the stage_configuration + for that stage is added as a blank StageConfig object. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[ + Stage("Stage_0_5", source=Path("Stage_0_5.py"), dependencies=()) + ] + ) + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + + with pytest.warns(StageConfigurationWarning) as recorded_warnings: + pipeline.add_stage( + stage_0, stage_1, stage_configs=[{"years_to_run": 2017}] + ) + + assert any( + "does not match the number of stages" in str(recorded_warning.message) + for recorded_warning in recorded_warnings + ) + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + assert pipeline.stage_configs["Stage_1"].to_dict() == {} + + def test_add_stage_config_coerces_mapping_payload_for_named_stage( + self, stage_factory + ) -> None: + """ + Tests that when a stage_configuration is added to a Pipeline instance, + the configuration is correctly associated with the named stage and that + the configuration is coerced into a StageConfig object if it is provided. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_factory("Stage_0")]) + + pipeline.add_stage_config({"years_to_run": 2017}, name="Stage_0") + + assert pipeline.stage_configs["Stage_0"].require("years_to_run") == 2017 + + +class TestPipelineStageSelectionAndGraph: + def test_resolve_stages_to_run_includes_transitive_dependencies( + self, stage_factory + ) -> None: + """ + Tests that when resolving stages_to_run, the Pipeline instance correctly + includes all dependent stages required in the StageGraph even if these + are not explicitly called out in the configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + stage_2 = stage_factory("Stage_2", dependencies=("Stage_1",)) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1, stage_2], + config=PipelineConfig(stages_to_run={"Stage_2": True}), + ) + + assert [stage.name for stage in pipeline.graph.stages] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + assert [stage.name for stage in pipeline.ordered_stages()] == [ + "Stage_0", + "Stage_1", + "Stage_2", + ] + + def test_resolve_stages_to_run_rejects_disabled_dependencies( + self, stage_factory + ) -> None: + """ + Checks that when resolving stages_to_run, the Pipeline init raises an + error if a stage is enabled but one of its dependencies is disabled. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationError`` + Raised when a stage is enabled but one of its dependencies is disabled. + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + + with pytest.raises(PipelineConfigurationError): + Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": False, "Stage_1": True} + ), + ) + + def test_self_stages_is_full_registry_after_disable(self, stage_factory) -> None: + """ + Pipeline.stages always holds all stages; only graph.stages is the effective run + set. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_0, stage_1]) + + pipeline.disable_stage("Stage_1") + + assert [stage.name for stage in pipeline.stages] == ["Stage_0", "Stage_1"] + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + assert [stage.name for stage in pipeline.ordered_stages()] == ["Stage_0"] + + def test_disable_stage_in_implicit_mode_creates_explicit_selection( + self, stage_factory + ) -> None: + """ + Tests that when a stage is manually disabled in a Pipeline instance, it + is initialised in the stages_to_run configuration. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline(stages=[stage_0, stage_1]) + + pipeline.disable_stage("Stage_1") + + assert pipeline.config.stages_to_run == {"Stage_0": True, "Stage_1": False} + + def test_enable_stage_restores_stage_in_explicit_mode(self, stage_factory) -> None: + """ + Tests that when a stage is manually enabled in a Pipeline instance, it + is correctly reflected in the stages_to_run configuration and the stage + is included in the execution graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1") + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) + + pipeline.enable_stage("Stage_1") + + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + + def test_add_stage_keeps_new_stage_out_of_explicit_selection( + self, stage_factory + ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance, it is + kept out of the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + """ + stage_0 = stage_factory("Stage_0") + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) + + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + ) + + assert pipeline.config.stages_to_run["Stage_1"] is False + assert [stage.name for stage in pipeline.graph.stages] == ["Stage_0"] + + def test_add_stage_adds_new_stage_to_explicit_selection_when_enable_stages_is_true( + self, + stage_factory, + ) -> None: + """ + Tests that when a new stage is added to a Pipeline instance with + enable_stages=True, it is included in the explicit selection. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + + """ + stage_0 = stage_factory("Stage_0") + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0], + config=PipelineConfig(stages_to_run={"Stage_0": True}), + ) + + pipeline.add_stage( + stage_factory("Stage_1"), + stage_configs=[StageConfig(name="Stage_1")], + enable_stages=True, + ) + + assert pipeline.config.stages_to_run["Stage_1"] is True + assert {stage.name for stage in pipeline.graph.stages} == {"Stage_0", "Stage_1"} + + +class TestPipelineValidationAndManifest: + def test_validate_skips_source_check_for_disabled_stages( + self, tmp_path: Path + ) -> None: + """ + Disabled stages' source files need not exist - validate() only checks the + effective run set. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. + """ + enabled_file = tmp_path / "Stage_0.py" + enabled_file.write_text("def run(ctx): pass\n", encoding="utf-8") + + stage_0 = Stage("Stage_0", source=enabled_file) + stage_1 = Stage( + "Stage_1", source=tmp_path / "missing.py" + ) # file intentionally absent + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) + + pipeline.validate() # must not raise + + def test_construct_manifest_inputs_contains_only_effective_stages( + self, stage_factory + ) -> None: + """ + Manifest inputs should list only the stages that are part of the execution + graph. + + Parameters + ---------- + ``stage_factory`` : Callable + A factory function that creates Stage objects for testing. + """ + stage_0 = stage_factory("Stage_0") + stage_1 = stage_factory("Stage_1", dependencies=("Stage_0",)) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + stages=[stage_0, stage_1], + config=PipelineConfig( + stages_to_run={"Stage_0": True, "Stage_1": False} + ), + ) + + runtime_id = pipeline._create_runtime_id() + manifest = pipeline._construct_manifest(runtime_id=runtime_id) + + assert list(manifest.inputs.keys()) == ["Stage_0"] + + def test_generate_context_correctly_assigns_executor( + self, + ) -> None: + """ + Test that the correct executor class is assigned to the Pipeline instance + based on the backend specified. If the backend does not have a compatible + executor, an error is raised. + """ + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline( + backend="python", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + assert isinstance(pipeline.executor, PythonStageExecutor) + + with pytest.raises(PipelineInitialisationError): + Pipeline( + backend="nonexistent_backend", + stages=[Stage("Stage_0", source=Path("Stage_0.py"), dependencies=())], + ) + + def test_validate_stage_backends_errors(self) -> None: + """ + Test that the _validate_stage_backends method correctly raises an error if + the backends for a stage do not match the Pipeline backend or if there are + multiple backends across the stages. + + Successful runs not tested here as they are covered in test_generate_context_ + correctly_assigns_executor(). + """ + + with pytest.raises(PipelineInitialisationError): + Pipeline( + backend="python", + stages=[ + Stage( + "Stage_0", + source=Path("Stage_0.py"), + dependencies=(), + backend="nonexistent_backend", + ) + ], + ) + + with pytest.raises(PipelineInitialisationError): + Pipeline( + backend="python", + stages=[ + Stage( + "Stage_0", + source=Path("Stage_0.py"), + dependencies=(), + backend="nonexistent_backend", + ), + Stage( + "Stage_1", + source=Path("Stage_1.py"), + dependencies=(), + backend="python", + ), + ], + ) + + +class TestLoadLatestRunIntegration: + @pytest.fixture + def pipeline_log_line(self): + def _make(run_id: str, timestamp: str) -> str: + """ + Returns a false log file line to simulate a historical run in the log file. + The line is formatted to match the expected log output. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + ``timestamp`` : str + The timestamp of when the historical run was initiated. + """ + return ( + f"{timestamp} Pipeline started | " + f'{{"run_id": "{run_id}", "run_dir": "/path/to/run"}}' + ) + + return _make + + @pytest.fixture + def minimal_pipeline_yaml(self): + def _make(run_id: str) -> str: + """ + Returns a minimal YAML configuration for a historical run. + + Parameters + ---------- + ``run_id`` : str + The unique identifier for the historical run. + """ + return f""" + manifest: + run_id: {run_id} + status: succeeded + started_at: '2026-08-06T17:03:30.000077' + completed_at: '2026-08-06T17:03:30.031654' + stage_results: {{}} + stage_outputs: {{}} + """ + + return _make + + @pytest.fixture + def pipeline_no_history(self, tmp_path: Path) -> Pipeline: + """ + Sets up a blank pipeline instance for testing that accounts for warnings + in init phase rather than dealing with these in the tests. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + """ + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + name="test_pipeline", + config=PipelineConfig( + output_dir=tmp_path / "outputs", + ), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + pipeline.run_output = tmp_path / "runs" + return pipeline + + +class TestLoadLatestRun(TestLoadLatestRunIntegration): + def test_blank_historical_run_ids( + self, pipeline_no_history: Pipeline, monkeypatch + ) -> None: + """ + Tests that if extract_historical_run_ids returns a blank list, the + _load_latest_run method will return None and raise a warning. Assert + that it will also store None in the last_run attribute of the Pipeline + instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr( + pipeline_no_history.logger, "extract_historical_run_ids", lambda x, y: [] + ) + assert ( + pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output, pipeline_no_history.name + ) + == [] + ) + with pytest.warns( + PipelineConfigurationWarning, + match="No previous runs " + "found for this Pipeline. Last_run attribute will be None.", + ): + assert pipeline_no_history._load_latest_run() is None + assert pipeline_no_history.last_run is None + + def test_blank_run_ids(self, pipeline_no_history: Pipeline, monkeypatch) -> None: + """ + Tests that if the found log record does not have a run_id, _load_latest_run + will return None and raise a warning. Assert that it will also store None + in the last_run attribute of the Pipeline instance. + + Parameters + ---------- + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/"), + } + ], + ) + assert pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output, pipeline_no_history.name + ) == [ + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/"), + } + ] + with pytest.warns( + PipelineConfigurationWarning, + match="No previous runs " + "found for this Pipeline. Last_run attribute will be None.", + ): + assert pipeline_no_history._load_latest_run() is None + assert pipeline_no_history.last_run is None + + def test_load_latest_run_success( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Tests that load_latest_run works successfully with fully mocked data. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run"), + } + ], + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) + + result = pipeline_no_history._load_latest_run() + + assert result is expected_run + + expected_path = pipeline_no_history.run_output / run_id + mock_load_historical_run.assert_called_once_with(run_dir=expected_path) + + def test_which_run_is_selected_load_latest_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Checks that the first item is selected from the list of historical runs + returned by extract_historical_run_ids. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + expected_run = mock.MagicMock(spec=PipelineRun) + run_id_1 = "2026-08-10_100000_abc12345" + run_id_2 = "2026-08-10_100000_def67890" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": run_id_1, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_A", + }, + { + "run_id": run_id_2, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": "run_B", + }, + ], + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) + + pipeline_no_history._load_latest_run() + + expected_path = pipeline_no_history.run_output / run_id_1 + mock_load_historical_run.assert_called_once_with(run_dir=expected_path) + + # does not refer to run_dir in the extract_historical_run_ids list but the + # parameter required in load_historical_run. + assert mock_load_historical_run.call_args.kwargs["run_dir"].name == run_id_1 + + def test_no_errors_raised_success_load_latest_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Tests that no errors are raised when load_latest_run is successful. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + + expected_run = mock.MagicMock(spec=PipelineRun) + run_id = "2026-08-10_100000_abc12345" + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": run_id, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run"), + } + ], + ) + + mock_load_historical_run = mock.MagicMock(return_value=expected_run) + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", mock_load_historical_run + ) + + with warnings.catch_warnings(record=True) as w: + pipeline_no_history._load_latest_run() + + assert not any( + issubclass(warning.category, PipelineConfigurationWarning) for warning in w + ) + + +class TestLoadLatestIntegrationInPipeline(TestLoadLatestRunIntegration): + def test_no_previous_runs_pipeline(self, tmp_path: Path) -> None: + """ + Tests that if a Pipeline instance has no previous runs, the _load_latest_run + method will return None and raise a warning. Assert that it will also store + None in the last_run attribute of the Pipeline instance. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when no previous runs are found for the Pipeline, indicating + that the last_run attribute will be None. + """ + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + config=PipelineConfig(output_dir=tmp_path / "outputs"), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + pipeline.run_output = tmp_path / "runs" + assert pipeline.last_run is None + + def test_last_run_populated_one_run( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: + """ + Tests that if a Pipeline instance has one previous run, this is loaded in + last_run attribute at Pipeline creation. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. + """ + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + '{"run_id": "2026-08-11_100000_abc12345", ' + '"name": "test_pipeline", ' + ' "run_dir": "/path/to/run"}\n' + ) + + temp_attributes = ( + tmp_path + / "outputs" + / "runs" + / "2026-08-11_100000_abc12345" + / "pipeline_attributes_for_test.yaml" + ) + temp_attributes.parent.mkdir(parents=True, exist_ok=True) + temp_attributes.write_text( + minimal_pipeline_yaml(run_id="2026-08-11_100000_abc12345"), encoding="utf-8" + ) + + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + name="test_pipeline", + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "2026-08-11_100000_abc12345" + + def test_last_run_most_recent(self, tmp_path: Path, minimal_pipeline_yaml) -> None: + """ + Tests that if a Pipeline instance has multiple previous runs, the most recent + run is loaded in last_run attribute at Pipeline creation. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. + """ + + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + '{"run_id": "run_older", ' + '"name": "test_pipeline", ' + ' "run_dir": "/path/to/run"}\n' + "2026-08-11 10:01:00,000 Pipeline started | " + '{"run_id": "run_newer", ' + '"name": "test_pipeline", ' + ' "run_dir": "/path/to/run"}' + ) + + temp_attributes_1 = ( + tmp_path + / "outputs" + / "runs" + / "run_older" + / "pipeline_attributes_for_test.yaml" + ) + temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_1.write_text( + minimal_pipeline_yaml(run_id="run_older"), encoding="utf-8" + ) + + temp_attributes_2 = ( + tmp_path + / "outputs" + / "runs" + / "run_newer" + / "pipeline_attributes_for_test.yaml" + ) + temp_attributes_2.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_2.write_text( + minimal_pipeline_yaml(run_id="run_newer"), encoding="utf-8" + ) + + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + name="test_pipeline", + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "run_newer" + + def test_last_run_most_recent_no_directory( + self, tmp_path: Path, minimal_pipeline_yaml + ) -> None: + """ + Checks that only the older run is included when the run directory has been + deleted/removed for the most recent run. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files + and directories. + ``minimal_pipeline_yaml`` : callable + A fixture that returns a minimal YAML configuration for a historical run. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is no stages_to_run parameters to warn the user that + all stages will be run by default. + """ + + logs = tmp_path / "logs" + logs.mkdir(parents=True, exist_ok=True) + (logs / "onsrap.log").write_text( + "2026-08-11 10:00:00,000 Pipeline started | " + '{"name": "test_pipeline", "run_id": "run_older", ' + ' "run_dir": "/path/to/run"}\n' + "2026-08-11 10:01:00,000 Pipeline started | " + '{"name": "test_pipeline", "run_id": "run_newer", ' + ' "run_dir": "/path/to/run"}' + ) + + temp_attributes_1 = ( + tmp_path + / "outputs" + / "runs" + / "run_older" + / "pipeline_attributes_for_test.yaml" + ) + temp_attributes_1.parent.mkdir(parents=True, exist_ok=True) + temp_attributes_1.write_text( + minimal_pipeline_yaml(run_id="run_older"), encoding="utf-8" + ) + + with pytest.warns(PipelineConfigurationWarning): + pipeline = Pipeline( + name="test_pipeline", + config=PipelineConfig(output_dir=tmp_path / "outputs", log_dir=logs), + stages=[ + Stage("Stage_0", source=tmp_path / "Stage_0.py", dependencies=()) + ], + ) + + assert pipeline.last_run is not None + assert pipeline.last_run.manifest.run_id == "run_older" + + +class TestLoadAllRunsIntegration(TestLoadLatestRunIntegration): + def test_returns_none_when_error_in_extract_historical_runs( + self, monkeypatch, pipeline_no_history + ) -> None: + """ + Checks that all_runs attribute is None when extract_historical_run_ids + raises an error. This is to ensure that the Pipeline instance does not break + when there is an issue with extracting historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue with extracting historical runs, indicating + that the all_runs attribute will be None. + """ + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + mock.Mock(side_effect=HistoricalPipelineLoadError("test")), + ) + + with pytest.warns(PipelineConfigurationWarning): + assert pipeline_no_history._load_all_runs() is None + assert pipeline_no_history.all_runs is None + + def test_returns_none_when_blank_extract_historical_runs( + self, monkeypatch, pipeline_no_history + ) -> None: + """ + Checks that all_runs attribute is None when extract_historical_run_ids + returns a blank list. This ensures that the Pipeline instance does not + break when there are no historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there are no historical runs found, indicating that the + all_runs attribute will be None. + """ + monkeypatch.setattr( + pipeline_no_history.logger, "extract_historical_run_ids", lambda x, y: [] + ) + + assert ( + pipeline_no_history.logger.extract_historical_run_ids( + pipeline_no_history.run_output, pipeline_no_history.name + ) + == [] + ) + with pytest.warns(PipelineConfigurationWarning): + assert pipeline_no_history._load_all_runs() is None + assert pipeline_no_history.all_runs is None + + def test_single_entry_dict_single_run( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Checks that all_runs attribute is a dictionary with a single entry when + extract_historical_run_ids returns a list with one historical run. This + ensures that the Pipeline instance correctly loads a single historical run. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is one historical run found, indicating that the + all_runs attribute will contain a single entry. + """ + + mock_loader = mock.MagicMock(return_value=mock.sentinel) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + } + ], + ) + + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 1 + assert "run_A" in result + mock_loader.assert_called_once_with( + run_dir=pipeline_no_history.run_output / "run_A" + ) + + def test_multiple_entries_dict_multiple_runs( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Checks that all_runs attribute is a dictionary with multiple entries when + extract_historical_run_ids returns a list with multiple historical runs. + This ensures that the Pipeline instance correctly loads multiple historical runs. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there are multiple historical runs found, indicating that the + all_runs attribute will contain multiple entries. + """ + + mock_loader = mock.MagicMock( + side_effect=[mock.sentinel.run_A, mock.sentinel.run_B] + ) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": "run_A", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) + + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 2 + assert "run_A" in result and "run_B" in result + mock_loader.assert_any_call(run_dir=pipeline_no_history.run_output / "run_A") + mock_loader.assert_any_call(run_dir=pipeline_no_history.run_output / "run_B") + + def test_warning_if_no_run_id( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Checks that a warning is raised if extract_historical_run_ids returns a + historical run without a run_id. This ensures that the Pipeline instance + correctly handles cases where historical runs are missing identifiers. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + """ + mock_loader = mock.MagicMock(return_value=mock.sentinel) + monkeypatch.setattr("onsrap.pipeline.load_historical_run", mock_loader) + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": "", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "run_B", + "timestamp": "2026-08-10 10:01:00,000", + "run_dir": Path("/path/to/run_B"), + }, + { + "run_id": None, + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + ], + ) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 1 + assert "run_A" not in result and "run_B" in result + mock_loader.assert_called_once_with( + run_dir=pipeline_no_history.run_output / "run_B" + ) + + def test_None_with_stageloaderror( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Checks that all_runs attribute is None when load_historical_run raises a + StageLoadError. This ensures that the Pipeline instance correctly handles + cases where historical runs cannot be loaded due to errors. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue loading a historical run, indicating that + the all_runs attribute will be None. + """ + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": "good_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "bad_run", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) + + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[mock.sentinel.good_run, StageLoadError("test")]), + ) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + assert isinstance(result, dict) + assert len(result) == 1 + assert "good_run" in result and "bad_run" not in result + + def test_none_if_all_stageloaderrors( + self, monkeypatch, pipeline_no_history: Pipeline + ) -> None: + """ + Asserts that all_runs attribute is None when load_historical_run raises a + StageLoadError for all historical runs. This ensures that the Pipeline instance + correctly handles cases where all historical runs cannot be loaded due to errors. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for dynamic modification of attributes, + methods, or classes during testing. + ``pipeline_no_history`` : Pipeline + A Pipeline instance with no historical runs. + + Raises + ------ + ``PipelineConfigurationWarning`` + Raised when there is an issue loading all historical runs, indicating that + the all_runs attribute will be None. + """ + + monkeypatch.setattr( + pipeline_no_history.logger, + "extract_historical_run_ids", + lambda x, y: [ + { + "run_id": "bad_run1", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_A"), + }, + { + "run_id": "bad_run2", + "timestamp": "2026-08-10 10:00:00,000", + "run_dir": Path("/path/to/run_B"), + }, + ], + ) + + monkeypatch.setattr( + "onsrap.pipeline.load_historical_run", + mock.Mock(side_effect=[StageLoadError("test"), StageLoadError("test")]), + ) + + with pytest.warns(PipelineConfigurationWarning): + result = pipeline_no_history._load_all_runs() + + assert result is None diff --git a/tests/test_pipeline_architecture.py b/tests/test_pipeline_architecture.py index 1f5c269..d4f8014 100644 --- a/tests/test_pipeline_architecture.py +++ b/tests/test_pipeline_architecture.py @@ -1,104 +1,701 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from textwrap import dedent +import pytest +import yaml + +from onsrap.errors import StageConfigurationError from onsrap.graph import StageGraph from onsrap.pipeline import Pipeline from onsrap.stage import Stage +from onsrap.warnings import PipelineConfigurationWarning, StageConfigurationWarning + +NO_STAGES_SPECIFIED_WARNING = ( + "No stages specified to run. All stages running by default." +) +OUTPUT_DIRECTORY_WARNING = ( + "Output directory is not specified. Using project root or " + "work directory as the run output." +) + + +def _base_pipeline_config(tmp_path: Path) -> dict: + return { + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_config": {}, + } + + +class TestPipelineFromFiles: + def test_pipeline_from_files_executes_python_entrypoints( + self, tmp_path: Path + ) -> None: + """ + Checks that the pipeline entrypoints are run successfully by reviewing + the outputs of the stages. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the run output. + """ + first_stage = tmp_path / "first_stage.py" + first_stage.write_text( + dedent( + """ + def run(context): + return "alpha" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + second_stage = tmp_path / "second_stage.py" + second_stage.write_text( + dedent( + """ + def main(context): + return context.result_for("first_stage").outputs + "-beta" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [first_stage, second_stage], + dependencies={"second_stage": ("first_stage",)}, + config=_base_pipeline_config(tmp_path), + ) + + run = pipeline.run() + + assert run.succeeded is True + assert [result.name for result in run.stage_results] == [ + "first_stage", + "second_stage", + ] + assert run.stage_outputs == { + "first_stage": "alpha", + "second_stage": "alpha-beta", + } + + def test_pipeline_uses_run_specific_output_location(self, tmp_path: Path) -> None: + """ + Checks that the pipelines produce outputs in unique locations based on runs. + As each run produces a unique run_id, the outputs should be written to unique + directories. This test checks that when the same pipeline is run twice, the + outputs are saved into two locations. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. This means that the Pipeline defaults to using + the project root or working directory as the parent directory for the run + output. This does not affect the test capability. + """ + writer_stage = tmp_path / "writer_stage.py" + writer_stage.write_text( + dedent( + """ + from pathlib import Path + + def main(context): + output_path = Path( + context.run_dir + ) / "data" / "interim" / "artifact.txt" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(context.run_id, encoding="utf-8") + return {"output_path": str(output_path), "run_id": context.run_id} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [writer_stage], + config=_base_pipeline_config(tmp_path), + ) + + first_run = pipeline.run() + second_run = pipeline.run() + + first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) + second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) + + assert first_run.manifest.run_id != second_run.manifest.run_id + assert first_output != second_output + assert first_output.exists() + assert second_output.exists() + assert first_output.parents[2].name == first_run.manifest.run_id + assert second_output.parents[2].name == second_run.manifest.run_id + + def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts( + self, tmp_path: Path + ) -> None: + """ + Checks that a pipeline will run with a non-module based Python script by + running the entire script. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. + """ + script_stage = tmp_path / "script_stage.py" + script_stage.write_text("print('script fallback works')\n", encoding="utf-8") + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [script_stage], + name="script-pipeline", + config=_base_pipeline_config(tmp_path), + ) + + run = pipeline.run() + + assert run.stage_results[0].outputs.strip() == "script fallback works" + assert run.stage_results[0].stdout.strip() == "script fallback works" + + +class TestStageGraph: + def test_stage_graph_detects_cycles(self) -> None: + """ + Checks that the stage graph appropriately detects required orders + based on dependencies in the stages. + """ + first_stage = Stage( + name="first_stage", + source=lambda context: None, + dependencies=("second_stage",), + ) + second_stage = Stage( + name="second_stage", + source=lambda context: None, + dependencies=("first_stage",), + ) + + graph = StageGraph.from_stages([first_stage, second_stage]) + + try: + graph.topological_order() + except Exception as exc: # noqa: BLE001 + assert exc.__class__.__name__ == "DependencyCycleError" + else: + raise AssertionError("Expected a dependency cycle error") + + +class TestPipelineFromConfig: + def test_pipeline_from_config_builds_stages_and_injects_stage_config( + self, tmp_path: Path + ) -> None: + """ + Checks that from_config() method appropriately builds the configurations + for the pipeline and uses the configurations to run the Pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. + """ + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_file = scripts_dir / "0_data_validation.py" + stage_file.write_text( + dedent( + """ + def run(context): + return { + "stage_name": context.stage_config.name, + "years_to_run": context.stage_config.get("years_to_run"), + "target_variable": context.stage_config.require( + "target_variable" + ), + } + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + dedent( + f""" + pipeline_variables: + name: "configured-pipeline" + backend: python + working_dir: "{tmp_path.as_posix()}" + project_root: "{tmp_path.as_posix()}" + log_dir: "{(tmp_path / "logs").as_posix()}" + stages: + - 0_data_validation: + location: "{ + (tmp_path / "scripts" / "0_data_validation.py").as_posix() + }" + run: true + dependencies: [] + + stage_configuration: + 0_data_validation: + years_to_run: 2017 + target_variable: "classification" + + global_configuration: + dry_run: true + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) + + assert [stage.name for stage in pipeline.stages] == ["0_data_validation"] + assert pipeline.stage_configs["0_data_validation"].get("years_to_run") == 2017 + + run = pipeline.run() + + assert run.stage_outputs["0_data_validation"] == { + "stage_name": "0_data_validation", + "years_to_run": 2017, + "target_variable": "classification", + } + + def test_pipeline_rejects_unknown_stage_configuration(self, tmp_path: Path) -> None: + """ + Checks that the pipeline raises an error when a stage configuration is provided + for a stage that is not within the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. + """ + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return "ok" + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config = _base_pipeline_config(tmp_path) + config["stage_configuration"] = { + "missing_stage": {"years_to_run": 2017}, + } + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [stage_file], + config=config, + ) + + with pytest.raises(StageConfigurationError, match="unknown stages"): + pipeline.validate() + + def test_pipeline_from_config_parses_stage_configuration_payloads( + self, tmp_path: Path + ) -> None: + """ + Checks that the correct information from a configuration file is parsed into + the correct attributes of a PipelineConfig, StageConfig, and GlobalConfig + instance. Also covers that the stage configuration is correctly injected into + the stage. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + + """ + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + for stage_name in ("0_extract", "1_transform"): + (scripts_dir / f"{stage_name}.py").write_text( + dedent( + """ + def run(context): + return context.stage_config.to_dict() + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + config_payload = { + "pipeline_variables": { + "name": "parse-test", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "data_dir": (tmp_path / "data").as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "metadata": { + "description": "configuration parsing test", + }, + "stages": [ + { + "0_extract": { + "location": "", + "run": True, + "dependencies": [], + "owner": "analytics", + } + }, + { + "1_transform": { + "location": "", + "run": True, + "dependencies": ["0_extract"], + } + }, + ], + }, + "stage_configuration": { + "0_extract": { + "years_to_run": 2017, + "datasets": { + "orders": { + "path": "data/orders.csv", + } + }, + "metadata": { + "purpose": "extract", + }, + }, + "1_transform": { + "target_variable": "classification", + "metadata": { + "purpose": "transform", + }, + }, + }, + "global_config": {}, + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + yaml.safe_dump(config_payload, sort_keys=False), encoding="utf-8" + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) + + assert pipeline.name == "parse-test" + assert pipeline.config.work_dir == tmp_path + assert pipeline.config.project_root == tmp_path + assert pipeline.config.log_dir == tmp_path / "logs" + assert [stage.name for stage in pipeline.stages] == ["0_extract", "1_transform"] + assert ( + pipeline.stages[0].source_path == (scripts_dir / "0_extract.py").resolve() + ) + assert pipeline.stages[0].metadata["owner"] == "analytics" + assert pipeline.stages[1].dependencies == ("0_extract",) + assert pipeline.stage_configs["0_extract"].variables == { + "years_to_run": 2017, + "datasets": {"orders": {"path": "data/orders.csv"}}, + } + assert pipeline.stage_configs["0_extract"].metadata == {"purpose": "extract"} + assert ( + pipeline.stage_configs["1_transform"].require("target_variable") + == "classification" + ) + + def test_pipeline_from_config_scales_stage_configuration_to_many_stages( + self, tmp_path: Path + ) -> None: + """ + Checks that multiple stage configurations can be parsed from a configuration + file and input in the correct order into the pipeline. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. + """ + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + + stage_count = 6 + stage_names = [f"{index}_stage" for index in range(stage_count)] + + for index, stage_name in enumerate(stage_names): + stage_file = scripts_dir / f"{stage_name}.py" + previous_stage_name = stage_names[index - 1] if index > 0 else None + stage_file.write_text( + dedent( + f""" + def run(context): + previous_ordinal = None + if {index} > 0: + previous_ordinal = context.result_for( + "{previous_stage_name}" + ).outputs["ordinal"] + return {{ + "stage_name": context.stage_config.name, + "ordinal": context.stage_config.require("ordinal"), + "label": context.stage_config.require("label"), + "first_stage_ordinal": context.stage_config_for( + "{stage_names[0]}" + ).require("ordinal"), + "known_stage_configs": sorted(context.stage_configs), + "previous_ordinal": previous_ordinal, + }} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + stage_definitions = [] + stage_configuration = {} + for index, stage_name in enumerate(stage_names): + dependencies = [stage_names[index - 1]] if index > 0 else [] + stage_definitions.append( + { + stage_name: { + "location": "", + "run": True, + "dependencies": dependencies, + } + } + ) + stage_configuration[stage_name] = { + "ordinal": index, + "label": f"label-{index}", + } + + config_file = tmp_path / "conf.yaml" + config_file.write_text( + yaml.safe_dump( + { + "pipeline_variables": { + "name": "many-stage-pipeline", + "backend": "python", + "working_dir": tmp_path.as_posix(), + "project_root": tmp_path.as_posix(), + "log_dir": (tmp_path / "logs").as_posix(), + "stages": stage_definitions, + }, + "stage_configuration": stage_configuration, + "global_config": { + "dry_run": True, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_config(config_file) + + assert [stage.name for stage in pipeline.stages] == stage_names + assert sorted(pipeline.stage_configs) == stage_names + + run = pipeline.run() + + assert run.manifest.stages_run == stage_names + assert sorted(run.manifest.parameters["stage_configuration"]) == stage_names + + for index, stage_name in enumerate(stage_names): + output = run.stage_outputs[stage_name] + assert output["stage_name"] == stage_name + assert output["ordinal"] == index + assert output["label"] == f"label-{index}" + assert output["first_stage_ordinal"] == 0 + assert output["known_stage_configs"] == stage_names + expected_previous = None if index == 0 else index - 1 + assert output["previous_ordinal"] == expected_previous + + +REPO_ROOT = Path(__file__).resolve().parents[1] + +MAIN_SCRIPTS = [ + REPO_ROOT / "examples" / "pipeline_1" / "main.py", + REPO_ROOT / "examples" / "pipeline_2" / "main.py", +] + + +class TestExamples: + @pytest.mark.parametrize("script_path", MAIN_SCRIPTS, ids=lambda p: p.parent.name) + def test_example_main_scripts_run_successfully(self, script_path: Path) -> None: + """ + Checks that a main script in a pipeline is successfully run. + + Parameters + ---------- + ``script_path`` : Path + The path to the main.py script of a pipeline example. + """ + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, ( + f"Script failed: {script_path}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert "completed with" in result.stdout.lower() + + +class TestPipelineRunConfigurationLogging: + def test_pipeline_run_writes_manifest_config_yaml_to_run_directory( + self, tmp_path: Path + ) -> None: + """ + Integration test that checks that the _log_config method is correctly called + within PipelineRunner.run() and that the information is parsed in a suitable + format to a YAML file in the run directory. + + This test also captures that _combine_configs() correctly converts all + configuration information into a single dictionary that can be serialized + to YAML. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for testing. + + Raises + ------ + 'PipelineConfigurationWarning' + Expected and asserted as there is no stage run specification in the + Pipeline configuration. This does not affect the test capability. + 'StageConfigurationWarning' + Expected and asserted as there is no output directory specified + in the configuration. + """ + stage_file = tmp_path / "single_stage.py" + stage_file.write_text( + dedent( + """ + def run(context): + return {"status": "ok"} + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + with pytest.warns((PipelineConfigurationWarning, StageConfigurationWarning)): + pipeline = Pipeline.from_files( + [stage_file], + name="config-export-pipeline", + config={ + "pipeline_config": { + "work_dir": tmp_path, + "project_root": tmp_path, + "log_dir": tmp_path / "logs", + }, + "stage_configuration": {}, + "global_configuration": { + "dry_run": True, + }, + }, + ) + + run = pipeline.run() + + run_dir = tmp_path / "runs" / run.manifest.run_id + config_file = run_dir / ( + f"configuration_for_{pipeline.name}_{run.started_at.date()}_{run.manifest.run_id[-8:]}.yaml" + ) + + assert config_file.exists() + file_text = config_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) -def test_pipeline_from_files_executes_python_entrypoints(tmp_path: Path) -> None: - first_stage = tmp_path / "first_stage.py" - first_stage.write_text( - dedent( - """ - def run(context): - return "alpha" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - second_stage = tmp_path / "second_stage.py" - second_stage.write_text( - dedent( - """ - def main(context): - return context.result_for("first_stage").outputs + "-beta" - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - pipeline = Pipeline.from_files( - [first_stage, second_stage], - dependencies={"second_stage": ("first_stage",)}, - ) - - run = pipeline.run() - - assert run.succeeded is True - assert [result.name for result in run.stage_results] == ["first_stage", "second_stage"] - assert run.stage_outputs == {"first_stage": "alpha", "second_stage": "alpha-beta"} - - -def test_pipeline_uses_run_specific_output_directory(tmp_path: Path) -> None: - writer_stage = tmp_path / "writer_stage.py" - writer_stage.write_text( - dedent( - """ - from pathlib import Path - - def main(context): - output_path = Path(context.run_dir) / "data" / "interim" / "artifact.txt" - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(context.run_id, encoding="utf-8") - return {"output_path": str(output_path), "run_id": context.run_id} - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - pipeline = Pipeline.from_files([writer_stage]) - - first_run = pipeline.run() - second_run = pipeline.run() - - first_output = Path(first_run.stage_outputs["writer_stage"]["output_path"]) - second_output = Path(second_run.stage_outputs["writer_stage"]["output_path"]) - - assert first_run.manifest.run_id != second_run.manifest.run_id - assert first_output != second_output - assert first_output.exists() - assert second_output.exists() - assert first_output.parents[2].name == first_run.manifest.run_id - assert second_output.parents[2].name == second_run.manifest.run_id - - -def test_pipeline_falls_back_to_subprocess_for_plain_python_scripts(tmp_path: Path) -> None: - script_stage = tmp_path / "script_stage.py" - script_stage.write_text("print('script fallback works')\n", encoding="utf-8") - - pipeline = Pipeline.from_files([script_stage], name="script-pipeline") - run = pipeline.run() - - assert run.stage_results[0].outputs.strip() == "script fallback works" - assert run.stage_results[0].stdout.strip() == "script fallback works" - - -def test_stage_graph_detects_cycles() -> None: - first_stage = Stage(name="first_stage", source=lambda context: None, dependencies=("second_stage",)) - second_stage = Stage(name="second_stage", source=lambda context: None, dependencies=("first_stage",)) - - graph = StageGraph.from_stages([first_stage, second_stage]) - - try: - graph.topological_order() - except Exception as exc: # noqa: BLE001 - assert exc.__class__.__name__ == "DependencyCycleError" - else: - raise AssertionError("Expected a dependency cycle error") \ No newline at end of file + assert parsed_yaml == run.manifest.config + assert "pipeline_config:\n" in file_text + assert "stage_configs:\n" in file_text + assert "global_config:\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + assert " dry_run: true" in file_text diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..531f159 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,371 @@ +from datetime import datetime +from pathlib import Path +from textwrap import dedent + +import pytest +import yaml + +from onsrap.execution import ExecutionContext +from onsrap.logger import Logger +from onsrap.models import ( + PipelineConfig, + PipelineRun, + PipelineStatus, + RunManifest, + StageResult, + StageStatus, + now, +) +from onsrap.runner import _log_config, _log_pipeline_attributes, print_config_diffs + + +class TestLogConfig: + def test_writes_manifest_config_as_block_style_yaml(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_config`` function correctly writes the manifest + configuration to a YAML file in block style format. + + Parameters + ---------- + tmp_path : Path + A temporary directory provided by pytest for creating test files. + """ + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True) + + config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + manifest_config = { + "pipeline_config": { + "name": "synthetic_pipeline", + "backend": "python", + "output_dir": str(tmp_path / "outputs"), + }, + "stage_configs": { + "stage_a": { + "years_to_run": 2026, + "target_variable": "classification", + } + }, + "global_config": { + "dry_run": True, + }, + } + + manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + config=manifest_config, + ) + + _log_config(run_dir, context, manifest) + + expected_file = run_dir / ( + "configuration_for_" + f"{context.pipeline_name}_{context.started_at.date()}_" + f"{context.run_id[-8:]}.yaml" + ) + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + + assert parsed_yaml == manifest_config + assert "stage_configs:\n" in file_text + assert " stage_a:\n" in file_text + assert " years_to_run: 2026\n" in file_text + assert "pipeline_config: {" not in file_text + assert "stage_configs: {" not in file_text + assert "global_config: {" not in file_text + + +class TestPrintConfigDiffs: + @staticmethod + def _write_yaml(path: Path, content: str) -> None: + """ + Helper function that writes a YAML file to the specified path with + the provided content. The content is dedented and stripped of + leading/trailing whitespace before being written to the file. A newline is + added at the end of the file. + + Parameters + ---------- + ``path`` : Path + The path where the YAML file will be written. + ``content`` : str + The YAML content to write to the file. + """ + path.write_text(dedent(content).strip() + "\n", encoding="utf-8") + + def test_returns_and_prints_differences( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + """ + Tests that two configuration files are correctly compared and the differences + are both returned in a computer-readable format and printed to the console. + One change for each category (changed, added, removed) is included in the test + to ensure that all cases are handled correctly. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary directory provided by pytest for creating test files. + ``capsys`` : pytest.CaptureFixture[str] + A pytest fixture that captures output to stdout and stderr during the test. + """ + test_file_a = tmp_path / "config_a.yaml" + test_file_b = tmp_path / "config_b.yaml" + + self._write_yaml( + test_file_a, + """ + pipeline_config: + name: synthetic_pipeline + output_dir: outputs + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: classification + global_config: + dry_run: True + """, + ) + + self._write_yaml( + test_file_b, + """ + pipeline_config: + name: synthetic_pipeline + backend: python + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: identification + global_config: + dry_run: True + """, + ) + + assert print_config_diffs(test_file_a, test_file_b) == { + "changed": { + "stage_configs.stage_a.target_variable": ( + "classification", + "identification", + ) + }, + "added": {"pipeline_config.backend": "python"}, + "removed": {"pipeline_config.output_dir": "outputs"}, + } + + test_file_b.write_text( + dedent(""" + pipeline_config: + name: synthetic_pipeline + backend: python + stage_configs: + stage_a: + years_to_run: 2026 + target_variable: identification + global_config: + dry_run: True + """).strip() + + "\n", + encoding="utf-8", + ) + + assert print_config_diffs(test_file_a, test_file_b) == { + "changed": { + "stage_configs.stage_a.target_variable": ( + "classification", + "identification", + ) + }, + "added": {"pipeline_config.backend": "python"}, + "removed": {"pipeline_config.output_dir": "outputs"}, + } + + captured = capsys.readouterr() + assert "CHANGED (1)" in captured.out + assert "ADDED in second configuration (1)" in captured.out + assert "REMOVED in second configuration (1)" in captured.out + assert "stage_configs.stage_a.target_variable" in captured.out + + +class TestRunInfoWriteOut: + def test_log_pipeline_attributes_writes_YAML(self, tmp_path: Path) -> None: + """ + Tests that the ``_log_pipeline_attributes`` function correctly writes the pipeline attributes to a YAML file. + """ + + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True, exist_ok=True) + + pipeline_config = PipelineConfig( + name="synthetic_pipeline", + stages_to_run={"stage_a": True}, + backend="python", + work_dir=tmp_path / "work", + project_root=tmp_path, + output_dir=tmp_path / "outputs", + log_dir=tmp_path / "logs", + data_dir=tmp_path / "data", + allow_subprocess_fallback=True, + python_executable=None, + metadata={"reason": "unit test"}, + ) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=pipeline_config, + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + stage_results = [ + StageResult( + name="stage_a", + status=StageStatus.SUCCEEDED, + started_at=now(), + finished_at=now(), + error=None, + source=None, + ) + ] + + run_manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + ) + + pipeline_run = PipelineRun( + manifest=run_manifest, + status=PipelineStatus.SUCCEEDED, + started_at=context.started_at, + completed_at=now(), + stage_results=stage_results, + stage_outputs={}, + ) + + _log_pipeline_attributes( + pipeline_run=pipeline_run, run_dir=run_dir, context=context + ) + + expected_file = run_dir / ( + "pipeline_attributes_for_" + f"{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + + expected_contents = pipeline_run._pipeline_run_to_dict() + + assert expected_file.exists() + + file_text = expected_file.read_text(encoding="utf-8") + parsed_yaml = yaml.safe_load(file_text) + assert parsed_yaml == expected_contents + + assert PipelineRun._pipeline_run_from_dict(parsed_yaml) == pipeline_run + + def test_log_pipeline_attributes_serializes_arbitrary_stage_outputs( + self, tmp_path: Path + ) -> None: + """ + Tests that stage outputs containing non-YAML-native Python objects are + serialized safely and can be reconstructed for supported types. + """ + + class CustomOutput: + def __repr__(self) -> str: + return "CustomOutput(example)" + + run_dir = tmp_path / "runs" / "synthetic_run" + run_dir.mkdir(parents=True, exist_ok=True) + + context = ExecutionContext( + pipeline_name="synthetic_pipeline", + run_id="run_1234", + config=PipelineConfig(name="synthetic_pipeline"), + logger=Logger(), + run_dir=run_dir, + working_directory=tmp_path, + stage_configs={}, + global_config=None, + ) + + run_manifest = RunManifest( + rap_name="synthetic_pipeline", + run_id="run_1234", + ) + + pipeline_run = PipelineRun( + manifest=run_manifest, + status=PipelineStatus.SUCCEEDED, + started_at=context.started_at, + completed_at=now(), + stage_results=[], + stage_outputs={ + "path_value": Path("data/interim/output.csv"), + "datetime_value": datetime(2026, 8, 17, 12, 30, 45), + "tuple_value": (1, "a"), + "set_value": {1, 2}, + "bytes_value": b"abc", + "bytearray_value": bytearray(b"xyz"), + "custom_value": CustomOutput(), + }, + ) + + _log_pipeline_attributes( + pipeline_run=pipeline_run, run_dir=run_dir, context=context + ) + + expected_file = run_dir / ( + "pipeline_attributes_for_" + f"{context.pipeline_name}_{context.run_id[-8:]}.yaml" + ) + parsed_yaml = yaml.safe_load(expected_file.read_text(encoding="utf-8")) + + assert isinstance(parsed_yaml["stage_outputs"]["path_value"], dict) + loaded_pipeline_run = PipelineRun._pipeline_run_from_dict(parsed_yaml) + + assert loaded_pipeline_run.stage_outputs["path_value"] == Path( + "data/interim/output.csv" + ) + assert loaded_pipeline_run.stage_outputs["datetime_value"] == datetime( + 2026, 8, 17, 12, 30, 45 + ) + assert loaded_pipeline_run.stage_outputs["tuple_value"] == (1, "a") + assert loaded_pipeline_run.stage_outputs["set_value"] == {1, 2} + assert loaded_pipeline_run.stage_outputs["bytes_value"] == b"abc" + assert loaded_pipeline_run.stage_outputs["bytearray_value"] == bytearray(b"xyz") + assert ( + loaded_pipeline_run.stage_outputs["custom_value"] == "CustomOutput(example)" + ) diff --git a/tests/test_stage.py b/tests/test_stage.py new file mode 100644 index 0000000..b300082 --- /dev/null +++ b/tests/test_stage.py @@ -0,0 +1,835 @@ +from pathlib import Path +from textwrap import dedent + +import pytest + +from onsrap.errors import StageDependencyError +from onsrap.stage import Stage, StageConfigurationError, _normalize_dependencies + + +class TestNormalizeDependencies: + def test_normalize_dependencies_none(self) -> None: + """ + Tests that None values return empty tuple. + """ + assert _normalize_dependencies(None) == () + + def test_normalize_dependencies_str(self) -> None: + """ + Tests single string and list of string values including + where whitespace appears before and after main text body + """ + assert _normalize_dependencies("stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies(" stage_1.py") == ("stage_1.py",) + assert _normalize_dependencies( + ["Stage_1.py", " Stage_2.py", "Stage_3.py "] + ) == ("Stage_1.py", "Stage_2.py", "Stage_3.py") + + def test_normalize_dependencies_dedupe(self) -> None: + """ + Tests that duplicate values are removed from the normalized dependencies + whilst preserving first seen order. + """ + assert _normalize_dependencies(["Stage_1.py", "Stage_2.py", "Stage_1.py"]) == ( + "Stage_1.py", + "Stage_2.py", + ) + + assert _normalize_dependencies( + ["Stage_2.py", "Stage_1.py", "Stage_2.py", "Stage_1.py"] + ) == ("Stage_2.py", "Stage_1.py") + + def test_normalize_dependencies_whitespace_handling(self) -> None: + """ + Tests that whitespace only or blank dependency values are removed from + the normalised dependencies. + """ + assert _normalize_dependencies([" ", ""]) == () + + def test_normalize_dependencies_type_check(self) -> None: + """ + Tests that normalize_dependencies works with other iterables such as tuples + and sets, and raises a TypeError for non-iterable types. + """ + assert _normalize_dependencies(("Stage_1.py", "Stage_2.py")) == ( + "Stage_1.py", + "Stage_2.py", + ) + + result = _normalize_dependencies({"Stage_1.py", "Stage_2.py"}) + assert set(result) == {"Stage_1.py", "Stage_2.py"} + + with pytest.raises(TypeError): + _normalize_dependencies(11) + + def test_normalize_dependencies_mixed_types(self) -> None: + """ + Tests that normalize_dependencies stringifies non-string types in a + dependency iterable. + """ + assert _normalize_dependencies(["Stage_1.py", 11, "Stage_2.py"]) == ( + "Stage_1.py", + "11", + "Stage_2.py", + ) + + +@pytest.fixture +def example_function(): + """ + Test function to pass as a callable stage for stage testing. + """ + return example_function + + +@pytest.fixture +def stage_test(example_function) -> Stage: + """ + Stage object for testing Stage class methods and construction. + """ + return Stage("callable_stage", example_function, ["stage_1"], {"info": "example"}) + + +class TestStage: + def test_stage_creation_callable(self, stage_test, example_function) -> None: + """ + Tests that attributes have been appropriately assigned to Stage class. + + Parameter + --------- + stage_test : Stage + A ``Stage`` object created with a callable source for testing. + """ + assert stage_test.name == "callable_stage" + assert stage_test.source == example_function + assert stage_test.dependencies == ("stage_1",) + assert stage_test.metadata == {"info": "example"} + assert stage_test.entrypoint is None + assert stage_test.backend == "python" + + def test_stage_name_error(self, example_function) -> None: + """ + Tests that a StageConfigurationError is raised if the name is left blank + in a Stage class instance. This also includes whitespace only names. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + + Raises + ------ + ``StageConfigurationError`` + If the name is left blank or entirely whitespace in a ``Stage`` class + instance. + """ + with pytest.raises(StageConfigurationError): + Stage("", example_function, ["stage_1"], {"info": "example"}) + + with pytest.raises(StageConfigurationError): + Stage(" ", example_function, ["stage_1"], {"info": "example"}) + + def test_stage_source_type(self) -> None: + """ + Tests that a non-valid source type returns a StageConfigurationError. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. + """ + with pytest.raises(StageConfigurationError): + Stage("callable_stage", 11, ["stage_1"], {"info": "example"}) + + def test_stage_backend(self, example_function) -> None: + """ + Tests that backend can be any string, None, and corrects for whitespace. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage_diff = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="java", + ) + stage = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="", + ) + stage_white_space = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend="python ", + ) + assert stage_diff.backend == "java" + assert stage.backend == "python" + assert stage_white_space.backend == "python" + + def test_stage_backend_irregular_values(self, example_function) -> None: + """ + Tests that backend defaults with a None or whitespace only string to "python" + and converts any non-string type (other than None) to a string. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage_none = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=None, + ) + + stage_blank = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=" ", + ) + stage_non_string = Stage( + "callable_stage", + example_function, + ["stage_1"], + {"info": "example"}, + backend=11, + ) + + assert stage_none.backend == "python" + assert stage_blank.backend == "python" + assert stage_non_string.backend == "11" + + def test_stage_constructor_expands_string_source_with_home( + self, monkeypatch, tmp_path + ): + """ + Check that a string source is converted to a Path and expanded with + expanduser(). This uses fake environmental variables to make sure that the + tests are not dependent on the actual user's home directory. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="string_source_stage", + source="~/scripts/my_stage.py", + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + def test_stage_constructor_expands_path_source_with_home( + self, monkeypatch, tmp_path + ): + """ + Check that a Path source is expanded with expanduser(). This uses fake + environmental variables to make sure that the tests are not dependent on the + actual user's home directory. + + Parameters + ---------- + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + stage = Stage( + name="path_source_stage", + source=Path("~/scripts/my_stage.py"), + dependencies=[], + metadata={}, + ) + + expected = fake_home / "scripts" / "my_stage.py" + assert isinstance(stage.source, Path) + assert stage.source == expected + + def test_normalise_dependencies_within_stage_init(self, example_function) -> None: + """ + Thin smoke test to check that _normalize_dependencies is called within the Stage + post_init method. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage = Stage( + name="test_stage", + source=example_function, + dependencies=["dep1", "dep2", "dep1", " dep3 ", "", " "], + metadata={}, + ) + assert stage.dependencies == ("dep1", "dep2", "dep3") + + def test_source_path(self, stage_test, tmp_path, example_function) -> None: + """ + Tests whether source_path detects a path vs other valid and invalid source + types. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage_test.source = tmp_path / "fake_file.py" + assert stage_test.source_path == tmp_path / "fake_file.py" + stage_test.source = 11 + assert stage_test.source_path is None + stage_test.source = "not a file path" + assert stage_test.source_path is None + stage_test.source = example_function + assert stage_test.source_path is None + + def test_source_label(self, stage_test, tmp_path, example_function) -> None: + """ + Tests that source_label is created if the source is a Path or a callable + and is None if it is another type. This also checks that if the callable + has no name attribute, the stage name is used as the source label. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + stage_test.source = tmp_path / "fake_file.py" + temp_path_str = str(tmp_path / "fake_file.py") + assert stage_test.source_label == temp_path_str + + stage_test.source = example_function + assert stage_test.source_label == "tests.test_stage.example_function" + + stage_test.source = 11 + assert stage_test.source_label is None + + class NoName: + def __call__(self): + pass + + stage_test.source = NoName() + assert stage_test.source_label == f"tests.test_stage.{stage_test.name}" + + def test_metadata_copy_safely(self) -> None: + """ + Checks that if the original metadata dictionary is modified after the Stage + instance is created, the Stage instance's metadata remains unchanged. + """ + # TODO: do we want this to be how it works? Or would the user assume that if + # they modify the original dict, it modifies the Stage instance. + original_metadata = {"info": "example"} + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=[], + metadata=original_metadata, + ) + original_metadata["info"] = "modified" + assert stage.metadata["info"] == "example" + + def test_repr_function(self) -> None: + """ + Tests that the __repr__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_repr = ( + "Stage(name=callable_stage, " + "source=tests.test_stage., " + "dependencies=('stage_1',), " + "metadata={'info': 'example'}, " + "entrypoint=main, " + "backend=python)" + ) + assert repr(stage) == expected_repr + + def test_str_function(self) -> None: + """ + Tests that the __str__ function returns a string representation of the Stage + instance with the correct attributes. + """ + stage = Stage( + name="callable_stage", + source=lambda: None, + dependencies=["stage_1"], + metadata={"info": "example"}, + entrypoint="main", + backend="python", + ) + expected_str = ( + " Name: callable_stage\n" + " Source: tests.test_stage. \n" + " Dependencies: ('stage_1',)\n" + " Metadata: {'info': 'example'} \n" + " Entrypoint: main \n" + " Backend: python" + ) + assert str(stage) == expected_str + + +class TestValidateStage: + def test_validate(self, stage_test, tmp_path) -> None: + """ + Tests whether an error is raised if the source file isn't suitable. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source is not a valid callable or file path in a ``Stage`` class + instance. In this instance, it raises if the source is None, an empty + string, or a Path object that is not a file. + """ + stage_test.source = None + with pytest.raises(StageConfigurationError): + stage_test.validate() + not_file_path = tmp_path + stage_test.source = not_file_path + with pytest.raises(StageConfigurationError): + stage_test.validate() + stage_test.source = "" + with pytest.raises(StageConfigurationError): + stage_test.validate() + + def test_validate_successes( + self, stage_test, example_function, temp_script + ) -> None: + """ + Tests that validate successfully approves of callables and file paths as + sources for a stage instance. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + stage_test.source = example_function + assert stage_test.validate() is None + + stage_test.source = temp_script(filename="valid_script.py") + assert stage_test.validate() is None + + +class TestWithDependencies: + def test_with_dependencies_list(self, stage_test) -> None: + """ + Tests adding different types of dependencies when the original dependency is + a list. Also checks that the original stage_test instance is not modified when + with_dependencies is called. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + new_deps_blank = [] + original_deps = stage_test.dependencies + + stage_test_list = stage_test.with_dependencies(new_deps) + stage_test_blank = stage_test.with_dependencies(new_deps_blank) + assert stage_test_list.dependencies == ("stage_1", "stage2", "stage3") + assert stage_test_blank.dependencies == ("stage_1",) + stage_test_2 = stage_test.with_dependencies("stage2", "stage3") + assert stage_test_2.dependencies == ("stage_1", "stage2", "stage3") + + stage_test.with_dependencies("stage2", "stage3") + assert stage_test.dependencies == original_deps + + def test_with_dependencies_errors(self, stage_test) -> None: + """ + Tests that a StageDependencyError is raised if a nested list is + provided in dependencies. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + with pytest.raises(StageDependencyError): + stage_test.with_dependencies(["stage2", ["nested_stage"]]) + + def test_with_dependencies_list_positional_args(self, stage_test) -> None: + """ + Tests that with_dependencies can accept a list and positional arguments in the + same call and combine them into a single normalized dependencies tuple. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_deps = ["stage2", "stage3"] + stage_test_combined = stage_test.with_dependencies(new_deps, "stage4") + assert stage_test_combined.dependencies == ( + "stage_1", + "stage2", + "stage3", + "stage4", + ) + + def test_with_dependencies_duplicates(self, stage_test) -> None: + """ + Tests that when the same dependency is added through with_dependencies, + it is not duplicated in the dependencies tuple of the new Stage instance. + + Caution that this only deduplicates due to Stage post_init calling + _normalize_dependencies however if that moves, + test_normalise_dependencies_within_stage_init will capture the issue. + + Parameters + ---------- + ``stage_test`` : Stage + A ``Stage`` object created with a callable source for testing. + """ + new_stage = stage_test.with_dependencies(["stage_1"]) + assert new_stage.dependencies == ("stage_1",) + + +# TEST NOT CODED FOR RUN() AS ASSUMED THIS IS COVERED IN PIPELINE_ARCHITECTURE TEST + + +@pytest.fixture +def temp_script(tmp_path): + """ + Fixture factory that creates temporary Python scripts. + + Usage: + script = temp_script("def main(): pass") + script = temp_script("def process(): return 42", "processor.py") + """ + + def _create_script(content="def main(): pass\n", filename="temp_script.py"): + script = tmp_path / filename + script.write_text(content, encoding="utf-8") + return script + + return _create_script + + +class TestStageFactories: + """ + Parent class for tests which create Stage class instances from different methods. + """ + + +class TestStageFromFile(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a file path. + """ + + def test_stage_instance_from_file(self, temp_script) -> None: + """ + Tests that a Stage instance is created from a filepath where name is either + default value from file stem or a user defined name. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + """ + test_stage = temp_script( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + "test_stage.py", + ) + + assert Stage.from_file(test_stage, entrypoint="main") == Stage( + "test_stage", test_stage.resolve(), (), {}, "main", "python" + ) + + assert Stage.from_file(test_stage, name="Stage_1", entrypoint="main") == Stage( + "Stage_1", test_stage.resolve(), (), {}, "main", "python" + ) + + def test_stage_from_files_error(self, tmp_path: Path) -> None: + """ + Tests that if the file doesn't exist, a StageConfigurationError is raised. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + + Raises + ------ + ``StageConfigurationError`` + If the source file doesn't exist when attempting to create a + ``Stage`` instance + """ + source_file = tmp_path / "not_an_actual_file.py" + with pytest.raises(StageConfigurationError): + Stage.from_file(source_file) + + def test_from_file_resolves_relative_to_absolute(self, temp_script, monkeypatch): + """ + Tests that a relative path passed to from_file is resolved to an + absolute path on the Stage source attribute. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + script = temp_script() + + monkeypatch.chdir(script.parent) + + stage = Stage.from_file(script.name) + + assert stage.source.is_absolute() + assert stage.source == script.resolve() + + def test_from_file_expands_source_path(self, tmp_path, monkeypatch): + """ + Tests that a path with a tilde (~) is expanded to the user's home directory + when passed to from_file. Uses monkeypatch to set a fake home directory for + testing purposes. + + Parameters + ---------- + ``tmp_path`` : Path + A temporary path provided by pytest for testing file creation and + manipulation. + ``monkeypatch`` : pytest.MonkeyPatch + A pytest fixture that allows for temporary modification of environment + variables and other attributes during testing. + """ + fake_home = tmp_path / "fake_home" + fake_home.mkdir() + + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + + script = fake_home / "scripts" / "my_stage.py" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("def main(): pass\n", encoding="utf-8") + + stage = Stage.from_file("~/scripts/my_stage.py") + + expected = fake_home / "scripts" / "my_stage.py" + assert stage.source == expected + + +class TestStageFromCallable(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a callable object. + """ + + def test_stage_from_callable_name(self, example_function) -> None: + """ + Tests that a stage name is extracted from a callable object stage. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function) + assert test.name == "example_function" + + def test_from_callable_fallback_name(self): + """ + Tests that a fallback name is assigned to a stage instance if the callable + object does not have a name attribute. + """ + + class NoName: + def __call__(self): + pass + + stage = Stage.from_callable(NoName()) + assert stage.name == "stage" + + def test_from_callable_explicit_name(self, example_function) -> None: + """ + Tests that an explicit name is assigned to a stage instance if provided. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + test = Stage.from_callable(example_function, name="explicit_name") + assert test.name == "explicit_name" + + +class TestStageFromDict(TestStageFactories): + """ + Class which tests the creation of Stage class instances from a dictionary. + """ + + def test_from_dict_callable_sources(self, example_function) -> None: + """ + Tests that callable sources are correctly used to create a stage instance + from a dictionary regardless of whether the key is source or callable. + Also validates that the name is correctly assigned from the dictionary or + derived from the callable. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + + data = {"name": "test_Stage", "callable": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + assert stage.name == "test_Stage" + + data = {"source": example_function} + stage = Stage.from_dict(data) + assert stage.source == example_function + assert stage.name == "example_function" + + def test_from_dict_aliases(self, temp_script) -> None: + """ + Tests that a stage instance is created from a dictionary item with aliases + source and path as source options. + + Parameters + ---------- + ``temp_script`` : callable + A fixture factory that creates temporary Python scripts. + """ + script = temp_script( + dedent( + """ + def main(): + variable = "Hello world" + return variable + """ + ).strip() + + "\n", + "test_stage.py", + ) + + data = { + "name": "test_Stage", + "source": script, + "entrypoint": "main", + } + + data_2 = { + "name": "test_Stage2", + "path": script, + "entrypoint": "main", + } + stage = Stage.from_dict(data) + stage_2 = Stage.from_dict(data_2) + assert stage.source == script.resolve() + assert stage.name == "test_Stage" + assert stage_2.source == script.resolve() + assert stage_2.name == "test_Stage2" + + def test_from_dict_errors(self) -> None: + """ + Tests that a StageConfigurationError is raised if the dictionary does not + contain a valid source or callable key. + + Raises + ------ + ``StageConfigurationError`` + If the dictionary does not contain a valid source or callable key. + """ + data = {"name": "test_Stage"} + with pytest.raises(StageConfigurationError): + Stage.from_dict(data) + + def test_all_keys_from_dict_in_stage(self, example_function) -> None: + """ + Checks that from_dict does not change the originally parsed dictionary so + that if the dictionary is needed later, it is not permanently changed when + creating a stage instance from it. + + Parameters + ---------- + ``example_function`` : callable + A callable function to pass as a source for a ``Stage`` class instance. + """ + data = { + "name": "test_Stage", + "source": example_function, + "dependencies": ["dep1", "dep2"], + "metadata": {"info": "example"}, + "entrypoint": "main", + "backend": "python", + } + original = dict(data) + Stage.from_dict(data) + assert original == data