Merging Configuration implementation into Development

.github/workflows/deploy-docs.yml

@@ -0,0 +1,54 @@
+name: Deploy Documentation
+
+on:
+  push:
+      branches:
+        - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+            python-version: "3.10"
+            cache: pip
+      - name: Install dependencies
+        run: |
+            python -m pip install --upgrade pip
+            python -m pip install -e ".[dev]"
+      - name: Build documentation
+        run: |
+            python -m sphinx -b html docs docs/_build/html
+      - name: Configure GitHub Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload documentation artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: docs/_build/html
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+        name: github-pages
+        url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+        - name: Deploy documentation
+          id: deployment
+          uses: actions/deploy-pages@v4
+

.github/workflows/package-build.yml

@@ -0,0 +1,27 @@
+name: Python Package Builds Successfully
+
+on:
+  pull_request:
+  push:
+      branches:
+          - main
+          - development
+
+jobs:
+  build-package:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v4
+        - uses: actions/setup-python@v5
+          with:
+              python-version: "3.10"
+        - name: Install build tools
+          run: |
+              python -m pip install --upgrade pip
+              python -m pip install build
+        - name: Build Package
+          run: |
+              python -m build --sdist --wheel
+        - name: Install Built Wheel
+          run: |
+              pip install dist/*.whl

.github/workflows/python-package.yml

@@ -1,40 +1,62 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
-name: Python package
+name: Python CI
 
 on:
+  # Keep push validation narrow to the main integration branches.
   push:
     branches: [ "main", "development" ]
+
+  # Run CI for every pull request regardless of target branch so branch-to-branch
+  # work still gets the same quality and test coverage before merge.
   pull_request:
-    branches: [ "main", "development" ]
+    types: [opened, synchronize, reopened, ready_for_review]
+
+  workflow_dispatch:
 
 jobs:
-  build:
+  quality:
+    runs-on: ubuntu-latest
 
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+        cache: pip
+    - name: Install development dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e ".[dev]"
+    - name: Lint with Ruff
+      run: |
+        python -m ruff check onsrap
+        python -m ruff format --check onsrap
+    - name: Run security checks
+      run: |
+        python -m bandit -ll -c pyproject.toml -r onsrap
+    - name: Type check with mypy
+      run: |
+        python -m mypy onsrap
+
+  tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+        cache: pip
+    - name: Install development dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        python -m pip install -e ".[dev]"
+        python -m pip install pandas
     - name: Test with pytest
       run: |
-        pytest
+        python -m pytest

.gitignore

@@ -895,18 +895,24 @@ docs/_linkcheck/
 !examples/
 !examples/**
 
-# Allow logs folders and their contents inside examples
-!examples/**/logs/
-!examples/**/logs/**
-!examples/**/*.log
 
 # Allow scripts folders and their contents inside examples
 !examples/**/scripts/
 !examples/**/scripts/**
+examples/**/scripts/__pycache__/
 
 # Allow example data files
 !examples/**/*.csv
 !examples/**/*.xls
 !examples/**/*.xlsx
 
-examples/**/runs/**/
+examples/**/runs/**/
+
+# Ignore coding playground
+tests/playground/
+tests/sandbox/
+
+# Allow logs folders and their contents inside examples
+examples/**/logs/
+examples/**/logs/**
+examples/**/*.log

.pre-commit-config.yaml

@@ -24,6 +24,15 @@ repos:
         args: [ --fix ]
       # Run ruff formatter.
       - id: ruff-format
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy - Type check onsrap package
+        entry: mypy
+        language: python
+        additional_dependencies: [mypy, types-PyYAML]
+        args: [onsrap]
+        pass_filenames: false
   - repo: https://github.com/Yelp/detect-secrets
     rev: v1.5.0
     hooks:
@@ -36,5 +45,5 @@ repos:
     hooks:
       - id: bandit
         name: bandit - Checks for vulnerabilities
-        args: ["-c", "pyproject.toml"]
+        args: ["-ll", "-c", "pyproject.toml"]
         additional_dependencies: ["bandit[toml]"]

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be tracked here.
 
+## [0.1.1] - 2026-06-24
+
+This release adds run-scoped output handling for the example pipeline and aligns runtime timestamps with local time.
+
+### Added
+
+- Introduced per-run output directories so each pipeline execution writes into its own `runs/<run_id>/` tree.
+- Anchored the example pipeline to its own `main.py` directory so runs stay inside `examples/pipeline_1/` instead of the repository working directory.
+- Routed stage output paths through `ExecutionContext.run_dir` so stages can record artifacts in the active run directory.
+- Switched runtime timestamp generation to local time and kept a compatibility alias for the previous helper.
+- Added regression coverage for repeated runs and run-specific output paths.
+
+
 ## [0.1.0] - 2026-06-22
 
 First tracked version of `onsrap`.

README.md

@@ -8,7 +8,20 @@ located.
 ```
 ## What is `onsrap`?
 
-Add a summary of your project here.
+Reproducible Analytical Pipelines (RAPs) are a cornerstone of high quality statistics. Reproducible refers to the concept that if code is run multiple times with the same inputs, it will produce the same outputs. A pipeline is a series of stages (small chunks of work) which are run in a specified order to produce desired outputs. Pipelines are crucial to reproducible work as they ensure that the code is run consistently. This increases the quality of the outputs by ensuring as little manual input as possible. 
+
+ONSRap is a Python package that automatically orchestrates and runs these RAPs. The goal is to standardise how pipelines are run to reduce developer time required to convert existing code into RAP standards. As well as reducing developer time, this package also supports achieving RAP standards through items 4, 6, and 10. These are accomplished through this package by: 
+
+ - Item 4: Document everything that is needed to write and run the code
+    - This package includes inbuilt logging that records when the pipeline was run as well as the Pipeline configuration used. 
+
+- Item 6: Code modules should run end-to-end without manual intervention
+    - This package is designed whereby once the configuration has been provided by the user and the main.py file is run, no further human input is required. 
+
+- Item 10: Don't reinvent the wheel
+    - Multiple pipelines exist within the ONS and each have the potential to be orchestrated in different ways. This package aims to standardise the orchestration, ensuring consistency across pipelines. This consistency means that developers are more easily able to move between pipelines as they will all be structured in a similar way.  
+
+For more information on the ONS Rap Minimum Standards, please see the full [standards documentation][standards].
 
 ## Getting started
 
@@ -18,11 +31,13 @@ requirements.
 It's suggested that you install this package and its requirements within
 a virtual environment.
 
+Stages should use a functional style. A ``stage`` can be a file or a callable item, such as a function. File stages should define an entrypoint function that runs the stage; files without an entrypoint can use subprocess fallback, but the package has less control over that execution mode.
+
+There should be a parent file that sets out configuration, required directories and file paths, and builds the ``Pipeline`` instance. It is recommended that this is named something similar to ``main.py`` so that it is easy for users to see where the ``Pipeline`` starts. This file will be what is run through the terminal to run the entire pipeline.    
+
 ## Requirements
 
-- Python 3.9+ installed
-- a `.secrets` file with the required [secrets and credentials](#required-secrets-and-credentials)
-- to have [loaded environment variables][docs-loading-environment-variables] from `.env`
+- Python 3.14.6 installed
 
 Contributors have some additional requirements - please see our [contributing guidance][contributing].
 
@@ -53,7 +68,18 @@ Remember to update the setup and requirement files inline with any changes to yo
 package.
 
 ## Running the pipeline (Python only)
+### Running Your Own Pipeline
+To run your own Pipeline, you will need to build a ``Pipeline`` instance. This can be built using the from_files() method which requires a list of strings or Paths for your individual ``stages``. These are then compiled into a ``Pipeline`` instance. A ``Pipeline`` instance can also be created using the from_dict() method which takes a dictionary containing each attribute of the intented ``Pipeline`` instance and converts it.
+
+You will also need to define your ``PipelineConfig`` instance. This contains information regarding the working directory, project root, data directory, and log directory required to run the ``Pipeline`` as well as any metadata that you feel needs to be logged. 
+
+Lastly, you need to define any ``dependencies`` required for the ``stages``. These are whether any stage needs to be run before another stage. These should be structured as a dictionary with the name of the stage as the key and the value is the stage/s that need to run before it as a tuple. 
 
+Both ``PipelineConfig`` and ``dependencies`` should be parsed into the ``Pipeline`` instance. 
+
+Once you have your ``Pipeline`` instance, you can run the ``Pipeline.run()`` method which will run the entire ``Pipeline`` instance that has been created.
+
+### Example Pipeline
 The main runnable example now lives in `examples/pipeline_1/main.py`.
 It builds a three-stage pipeline from numbered scripts under `examples/pipeline_1/scripts/`.
 To run the example, use:
@@ -66,21 +92,12 @@ Alternatively, most Python IDEs allow you to run the code directly using a `run`
 
 ## Required secrets and credentials
 
-To run this project, you need a `.secrets` file with [secrets/credentials as
-environmental variables][docs-loading-environment-variables-secrets]. The
-secrets/credentials should have the following environment variable name(s):
-
-| Secret/credential | Environment variable name | Description                                |
-|-------------------|---------------------------|--------------------------------------------|
-| Secret 1          | `SECRET_VARIABLE_1`       | Plain English description of Secret 1.     |
-| Credential 1      | `CREDENTIAL_VARIABLE_1`   | Plain English description of Credential 1. |
+No secrets or credentials are required for running this package.
 
-Once you've added them, [load these environment variables][docs-loading-environment-variables] using
-`.env`.
 
 ## Project structure layout
 
-The cookiecutter template generated for each project will follow this folder structure:
+The ONSRap repository has the following structure:
 
 ```shell
 .
@@ -89,14 +106,46 @@ The cookiecutter template generated for each project will follow this folder str
 │   │   ├── raw/
 │   │   ├── interim/
 │   │   └── processed/
-│   └── onsrap/
-│       ├── example_modules/
-│       │   ├── __init__.py
-│       │   └── example_module.py
-│       ├── __init__.py
-│       ├── example_config.yml
-│       └── run_pipeline.py
-└── ...
+│   ├── onsrap/
+│   │   ├── example_modules/
+│   │   │   ├── __init__.py
+│   │   │   └── example_module.py
+│   │   ├── __init__.py
+│   │   ├── errors.py
+│   │   ├── execution.py
+│   │   ├── graph.py
+│   │   ├── loader.py
+│   │   ├── models.py
+│   │   ├── pipeline.py
+│   │   ├── run_pipeline.py
+│   │   ├── runner.py
+│   │   └── stage.py
+│   ├── examples/
+│   │   ├── pipeline_1/
+│   │   │   ├── data/
+│   │   │   │   └── orders.csv
+│   │   │   ├── logs/
+│   │   │   │   └── onsrap.log
+│   │   │   ├── runs/
+│   │   │   │   └── README.md
+│   │   │   └── scripts/
+│   │   │   │   ├── 0_data_validation.py
+│   │   │   │   ├── 1_preprocessing.py
+│   │   │   │   └── 2_reporting.py
+│   │   │   ├── Example.md
+│   │   │   └── main.py
+│   │   ├── pipeline_2/
+│   │   └── pipeline_3/
+│   ├── tests/
+│   │   ├── __init__.py
+│   │   ├── repo_tests_README.md
+│   │   ├── test_execution.py
+│   │   ├── test_models.py
+│   │   ├── test_pipeline_architecture.py
+│   │   ├── test_pipeline.py
+│   │   └── test_stage.py
+│   └──
+└── 
 ```
 
 ## Licence
@@ -118,3 +167,4 @@ This project structure is based on the [`govcookiecutter` template project][govc
 [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter
 [docs-loading-environment-variables]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md
 [docs-loading-environment-variables-secrets]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/user_guide/loading_environment_variables.md#storing-secrets-and-credentials
+[standards]: https://best-practice-and-impact.github.io/ONS_minimum_RAP/