Release 0.16.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.15.0
+current_version = 0.16.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)

CHANGELOG.md

@@ -17,6 +17,23 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [0.16.0] - 2025-08-22
+
+### Added
+- Added `sha256_sum`, `parse_pyproject_metadata`, `validate_env_vars`,
+  to `helpers/python.py`.
+- Added `create_s3_uri` & `split_s3_uri` to `cdp/helpers/s3_utils.py`.
+
+### Changed
+- Reordered docstrings to place Returns above Raises in `check_year_range`
+  in `helpers/pyspark.py`
+
+### Deprecated
+
+### Fixed
+
+### Removed
+
 ## [0.15.0] - 2025-08-21
 
 ### Added
@@ -914,6 +931,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0
 > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI
 > and GitHub Releases.
 
+- rdsa-utils v0.16.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.16.0) |
+  [PyPI](https://pypi.org/project/rdsa-utils/0.16.0/)
 - rdsa-utils v0.15.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.15.0) |
   [PyPI](https://pypi.org/project/rdsa-utils/0.15.0/)
 - rdsa-utils v0.14.1: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.14.1) |

rdsa_utils/__init__.py

@@ -1 +1 @@
-__version__ = "0.15.0"
+__version__ = "0.16.0"

rdsa_utils/cdp/helpers/s3_utils.py

@@ -29,7 +29,7 @@
 from datetime import datetime, timedelta, timezone
 from io import BytesIO, StringIO, TextIOWrapper
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import boto3
 import pandas as pd
@@ -1980,3 +1980,67 @@ def zip_s3_directory_to_s3(
     except Exception as e:
         logger.error(f"Error zipping S3 directory and uploading to S3: {e}")
         return False
+
+
+def create_s3_uri(bucket: str, key: str, scheme: str = "s3") -> str:
+    """Create an S3 URI from a bucket, key, and scheme.
+
+    Parameters
+    ----------
+    bucket
+        The S3 bucket name.
+    key
+        The S3 object key.
+    scheme
+        The URI scheme to use ('s3' or 's3a').
+        Default is "s3".
+
+    Returns
+    -------
+    str
+        The formatted S3 URI.
+
+    Examples
+    --------
+    >>> create_s3_uri("my-bucket", "folder/file.txt")
+    's3://my-bucket/folder/file.txt'
+    >>> create_s3_uri("my-bucket", "folder/file.txt", scheme="s3a")
+    's3a://my-bucket/folder/file.txt'
+    """
+    return f"{scheme}://{bucket}/{key}"
+
+
+def split_s3_uri(uri: str) -> Tuple[str, str]:
+    """Split an S3 URI into bucket and key.
+
+    Supports both `s3://` and `s3a://` schemes.
+
+    Parameters
+    ----------
+    uri
+        The S3 URI to split, e.g., "s3://my-bucket/path/to/object.txt".
+
+    Returns
+    -------
+    Tuple[str, str]
+        A tuple containing the bucket name and the object key.
+
+    Raises
+    ------
+    ValueError
+        If the URI is malformed or does not use the s3:// or s3a:// scheme.
+
+    Examples
+    --------
+    >>> split_s3_uri("s3://my-bucket/data/file.csv")
+    ('my-bucket', 'data/file.csv')
+    """
+    if not uri or not (uri.startswith("s3://") or uri.startswith("s3a://")):
+        error_msg = f"Invalid S3 URI scheme in '{uri}'. Expected 's3://' or 's3a://'."
+        raise ValueError(error_msg)
+    no_scheme = uri.split("://", 1)[1]
+    parts = no_scheme.split("/", 1)
+    if len(parts) != 2 or not all(parts):
+        error_msg = f"Malformed S3 URI: '{uri}'"
+        raise ValueError(error_msg)
+    return parts[0], parts[1]

rdsa_utils/helpers/pyspark.py

@@ -1845,6 +1845,12 @@ def check_year_range(
     year_col
         The name of the column containing integer year values.
 
+    Returns
+    -------
+    None
+        The function completes successfully if all years within the specified
+        range are present in the DataFrame.
+
     Raises
     ------
     ValueError
@@ -1853,12 +1859,6 @@ def check_year_range(
         - If one or more years within the specified range are missing from
           the DataFrame's `year_col`.
 
-    Returns
-    -------
-    None
-        The function completes successfully if all years within the specified
-        range are present in the DataFrame.
-
     Examples
     --------
     >>> from pyspark.sql import SparkSession

rdsa_utils/helpers/python.py

@@ -4,14 +4,16 @@
 import itertools
 import json
 import logging
+import os
 import subprocess
 from datetime import datetime, time
 from functools import reduce, wraps
 from itertools import tee
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Mapping, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union
 
 import pandas as pd
+import tomli
 from codetiming import Timer
 from more_itertools import always_iterable
 from pandas.tseries.offsets import MonthEnd
@@ -730,6 +732,45 @@ def md5_sum(
         raise FileNotFoundError(msg)
 
 
+def sha256_sum(
+    filepath: str,
+) -> str:
+    """Get SHA256 hash of a specific file on the local file system.
+
+    Parameters
+    ----------
+    filepath
+        Filepath of file to create SHA256 hash from.
+
+    Returns
+    -------
+    str
+        The SHA256 hash of the file.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file does not exist.
+
+    Example
+    -------
+    >>> sha256_sum("folder/file.txt")
+    "9c56cc51b374c3b6e7b8e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8"
+    >>> sha256_sum("folder/non_existing_file.txt")
+    FileNotFoundError: filepath='../folder/non_existing_file.txt' cannot be found.
+    """
+    if Path(filepath).exists():
+        h = hashlib.sha256()
+        with open(filepath, "rb") as f:
+            for chunk in iter(lambda: f.read(1024 * 1024), b""):
+                h.update(chunk)
+        return h.hexdigest()
+    else:
+        msg = f"{filepath=} cannot be found."
+        logger.error(msg)
+        raise FileNotFoundError(msg)
+
+
 def file_exists(
     filepath: str,
 ) -> bool:
@@ -978,3 +1019,146 @@ def dump_environment_requirements(
         f"with args={args}",
     )
     output_path.write_text(result.stdout)
+
+
+def parse_pyproject_metadata(pyproject_path: Path) -> Dict[str, Optional[str]]:
+    """Parse project metadata from a `pyproject.toml` file.
+
+    This function reads the TOML file at pyproject_path and extracts a subset
+    of fields from the [project] table: the project name, required Python
+    version, and package version.
+
+    Parameters
+    ----------
+    pyproject_path
+        Path to the `pyproject.toml` file.
+
+    Returns
+    -------
+    Dict[str, Optional[str]]
+        A dictionary with the following keys:
+        - name : str or None
+            The project name.
+        - requires_python : str or None
+            The Python version specifier (from requires-python).
+        - package_version : str or None
+            The package version.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file does not exist.
+    tomli.TOMLDecodeError
+        If the file content is not valid TOML.
+
+    Examples
+    --------
+    >>> from pathlib import Path
+    >>> meta = parse_pyproject_metadata(Path("pyproject.toml"))
+    >>> meta["name"]  # doctest: +SKIP
+    'my-package'
+    """
+    try:
+        raw_text = pyproject_path.read_text(encoding="utf-8")
+    except FileNotFoundError as e:
+        msg = f"pyproject_path='{pyproject_path}' cannot be found."
+        logger.error(msg)
+        raise FileNotFoundError(msg) from e
+
+    try:
+        data = tomli.loads(raw_text)
+    except tomli.TOMLDecodeError as e:
+        msg = f"Invalid TOML in '{pyproject_path}': {e}"
+        logger.error(msg)
+        raise
+
+    proj = data.get("project", {})
+    meta = {
+        "name": proj.get("name"),
+        "requires_python": proj.get("requires-python"),
+        "package_version": proj.get("version"),
+    }
+
+    logger.info(f"Parsed pyproject.toml metadata from '{pyproject_path}': {meta}")
+    return meta
+
+
+def validate_env_vars(required_vars: List[str]) -> None:
+    """Validate that required environment variables are present and non-empty.
+
+    This function checks whether each name in `required_vars` exists in the
+    current process environment (`os.environ`) and has a non-empty value.
+    Variable names are stripped of surrounding whitespace and de-duplicated
+    before validation. If any variables are missing (unset or empty), the
+    function logs an error and exits by raising `SystemExit`.
+
+    Parameters
+    ----------
+    required_vars
+        Environment variable names to validate.
+
+    Returns
+    -------
+    None
+        This function is intended for its side effects (validation and logging).
+
+    Raises
+    ------
+    TypeError
+        If `required_vars` is not a list of non-empty strings.
+    SystemExit
+        If one or more required environment variables are missing or empty.
+
+    Examples
+    --------
+    Success case
+    ^^^^^^^^^^^^
+    >>> import os
+    >>> os.environ["DB_HOST"] = "localhost"
+    >>> os.environ["DB_PORT"] = "5432"
+    >>> validate_env_vars(["DB_HOST", "DB_PORT"])  # no exception
+
+    Missing variable
+    ^^^^^^^^^^^^^^^^
+    >>> import os
+    >>> os.environ["DB_HOST"] = "localhost"
+    >>> validate_env_vars(["DB_HOST", "DB_PORT"])
+    Traceback (most recent call last):
+    ...
+    SystemExit: [ERROR] Missing environment variables: DB_PORT
+
+    Empty value counts as missing
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    >>> import os
+    >>> os.environ["API_KEY"] = ""   # empty string -> missing
+    >>> validate_env_vars(["API_KEY"])
+    Traceback (most recent call last):
+    ...
+    SystemExit: [ERROR] Missing environment variables: API_KEY
+    """
+    if not isinstance(required_vars, list):
+        error_msg = "required_vars must be a list of strings."
+        raise TypeError(error_msg)
+    cleaned: list[str] = []
+    for name in required_vars:
+        if not isinstance(name, str):
+            error_msg = "All environment variable names must be strings."
+            raise TypeError(error_msg)
+        stripped = name.strip()
+        if not stripped:
+            error_msg = "Environment variable names must be non-empty strings."
+            raise TypeError(error_msg)
+        cleaned.append(stripped)
+
+    # De-duplicate while preserving readable order in logs
+    unique_names = sorted(set(cleaned))
+
+    # Treat unset or empty-string values as missing
+    missing = [n for n in unique_names if os.environ.get(n, "").strip() == ""]
+
+    if missing:
+        msg = f"[ERROR] Missing environment variables: {', '.join(missing)}"
+        logger.error(msg)
+        raise SystemExit(msg)
+
+    logger.info(f"Environment OK: {', '.join(unique_names)}")