From 6b52816e425b71676294609950959911ff03fd62 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Thu, 21 Aug 2025 10:57:29 +0100 Subject: [PATCH 1/3] Rearrange Docstring: check_year_range (#232) --- CHANGELOG.md | 2 ++ rdsa_utils/helpers/pyspark.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 270220e..d3ec736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed +- Reordered docstrings to place Returns above Raises in `check_year_range` + in `helpers/pyspark.py` ### Deprecated diff --git a/rdsa_utils/helpers/pyspark.py b/rdsa_utils/helpers/pyspark.py index 73bddbc..a961692 100644 --- a/rdsa_utils/helpers/pyspark.py +++ b/rdsa_utils/helpers/pyspark.py @@ -1845,6 +1845,12 @@ def check_year_range( year_col The name of the column containing integer year values. + Returns + ------- + None + The function completes successfully if all years within the specified + range are present in the DataFrame. + Raises ------ ValueError @@ -1853,12 +1859,6 @@ def check_year_range( - If one or more years within the specified range are missing from the DataFrame's `year_col`. - Returns - ------- - None - The function completes successfully if all years within the specified - range are present in the DataFrame. - Examples -------- >>> from pyspark.sql import SparkSession From 03efb428876fb85401e9dc1a709e6268d10a7751 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:06:05 +0100 Subject: [PATCH 2/3] Feature: More Python & S3 Utility Functions (#233) * add function: sha256_sum * add function: parse_pyproject_metadata * add function: validate_env_vars * add function: create_s3_uri & split_s3_uri --- CHANGELOG.md | 3 + rdsa_utils/cdp/helpers/s3_utils.py | 66 +++++++++- rdsa_utils/helpers/python.py | 186 ++++++++++++++++++++++++++++- tests/cdp/helpers/test_s3_utils.py | 75 ++++++++++++ tests/helpers/test_python.py | 131 ++++++++++++++++++++ 5 files changed, 459 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ec736..779a069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added `sha256_sum`, `parse_pyproject_metadata`, `validate_env_vars`, + to `helpers/python.py`. +- Added `create_s3_uri` & `split_s3_uri` to `cdp/helpers/s3_utils.py`. ### Changed - Reordered docstrings to place Returns above Raises in `check_year_range` diff --git a/rdsa_utils/cdp/helpers/s3_utils.py b/rdsa_utils/cdp/helpers/s3_utils.py index afc11b1..ca83c3a 100644 --- a/rdsa_utils/cdp/helpers/s3_utils.py +++ b/rdsa_utils/cdp/helpers/s3_utils.py @@ -29,7 +29,7 @@ from datetime import datetime, timedelta, timezone from io import BytesIO, StringIO, TextIOWrapper from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union import boto3 import pandas as pd @@ -1980,3 +1980,67 @@ def zip_s3_directory_to_s3( except Exception as e: logger.error(f"Error zipping S3 directory and uploading to S3: {e}") return False + + +def create_s3_uri(bucket: str, key: str, scheme: str = "s3") -> str: + """Create an S3 URI from a bucket, key, and scheme. + + Parameters + ---------- + bucket + The S3 bucket name. + key + The S3 object key. + scheme + The URI scheme to use ('s3' or 's3a'). + Default is "s3". + + Returns + ------- + str + The formatted S3 URI. + + Examples + -------- + >>> create_s3_uri("my-bucket", "folder/file.txt") + 's3://my-bucket/folder/file.txt' + >>> create_s3_uri("my-bucket", "folder/file.txt", scheme="s3a") + 's3a://my-bucket/folder/file.txt' + """ + return f"{scheme}://{bucket}/{key}" + + +def split_s3_uri(uri: str) -> Tuple[str, str]: + """Split an S3 URI into bucket and key. + + Supports both `s3://` and `s3a://` schemes. + + Parameters + ---------- + uri + The S3 URI to split, e.g., "s3://my-bucket/path/to/object.txt". + + Returns + ------- + Tuple[str, str] + A tuple containing the bucket name and the object key. + + Raises + ------ + ValueError + If the URI is malformed or does not use the s3:// or s3a:// scheme. + + Examples + -------- + >>> split_s3_uri("s3://my-bucket/data/file.csv") + ('my-bucket', 'data/file.csv') + """ + if not uri or not (uri.startswith("s3://") or uri.startswith("s3a://")): + error_msg = f"Invalid S3 URI scheme in '{uri}'. Expected 's3://' or 's3a://'." + raise ValueError(error_msg) + no_scheme = uri.split("://", 1)[1] + parts = no_scheme.split("/", 1) + if len(parts) != 2 or not all(parts): + error_msg = f"Malformed S3 URI: '{uri}'" + raise ValueError(error_msg) + return parts[0], parts[1] diff --git a/rdsa_utils/helpers/python.py b/rdsa_utils/helpers/python.py index 5bcc8d4..ef929c9 100644 --- a/rdsa_utils/helpers/python.py +++ b/rdsa_utils/helpers/python.py @@ -4,14 +4,16 @@ import itertools import json import logging +import os import subprocess from datetime import datetime, time from functools import reduce, wraps from itertools import tee from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Mapping, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union import pandas as pd +import tomli from codetiming import Timer from more_itertools import always_iterable from pandas.tseries.offsets import MonthEnd @@ -730,6 +732,45 @@ def md5_sum( raise FileNotFoundError(msg) +def sha256_sum( + filepath: str, +) -> str: + """Get SHA256 hash of a specific file on the local file system. + + Parameters + ---------- + filepath + Filepath of file to create SHA256 hash from. + + Returns + ------- + str + The SHA256 hash of the file. + + Raises + ------ + FileNotFoundError + If the file does not exist. + + Example + ------- + >>> sha256_sum("folder/file.txt") + "9c56cc51b374c3b6e7b8e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8b4e1e8" + >>> sha256_sum("folder/non_existing_file.txt") + FileNotFoundError: filepath='../folder/non_existing_file.txt' cannot be found. + """ + if Path(filepath).exists(): + h = hashlib.sha256() + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + else: + msg = f"{filepath=} cannot be found." + logger.error(msg) + raise FileNotFoundError(msg) + + def file_exists( filepath: str, ) -> bool: @@ -978,3 +1019,146 @@ def dump_environment_requirements( f"with args={args}", ) output_path.write_text(result.stdout) + + +def parse_pyproject_metadata(pyproject_path: Path) -> Dict[str, Optional[str]]: + """Parse project metadata from a `pyproject.toml` file. + + This function reads the TOML file at pyproject_path and extracts a subset + of fields from the [project] table: the project name, required Python + version, and package version. + + Parameters + ---------- + pyproject_path + Path to the `pyproject.toml` file. + + Returns + ------- + Dict[str, Optional[str]] + A dictionary with the following keys: + - name : str or None + The project name. + - requires_python : str or None + The Python version specifier (from requires-python). + - package_version : str or None + The package version. + + Raises + ------ + FileNotFoundError + If the file does not exist. + tomli.TOMLDecodeError + If the file content is not valid TOML. + + Examples + -------- + >>> from pathlib import Path + >>> meta = parse_pyproject_metadata(Path("pyproject.toml")) + >>> meta["name"] # doctest: +SKIP + 'my-package' + """ + try: + raw_text = pyproject_path.read_text(encoding="utf-8") + except FileNotFoundError as e: + msg = f"pyproject_path='{pyproject_path}' cannot be found." + logger.error(msg) + raise FileNotFoundError(msg) from e + + try: + data = tomli.loads(raw_text) + except tomli.TOMLDecodeError as e: + msg = f"Invalid TOML in '{pyproject_path}': {e}" + logger.error(msg) + raise + + proj = data.get("project", {}) + meta = { + "name": proj.get("name"), + "requires_python": proj.get("requires-python"), + "package_version": proj.get("version"), + } + + logger.info(f"Parsed pyproject.toml metadata from '{pyproject_path}': {meta}") + return meta + + +def validate_env_vars(required_vars: List[str]) -> None: + """Validate that required environment variables are present and non-empty. + + This function checks whether each name in `required_vars` exists in the + current process environment (`os.environ`) and has a non-empty value. + Variable names are stripped of surrounding whitespace and de-duplicated + before validation. If any variables are missing (unset or empty), the + function logs an error and exits by raising `SystemExit`. + + Parameters + ---------- + required_vars + Environment variable names to validate. + + Returns + ------- + None + This function is intended for its side effects (validation and logging). + + Raises + ------ + TypeError + If `required_vars` is not a list of non-empty strings. + SystemExit + If one or more required environment variables are missing or empty. + + Examples + -------- + Success case + ^^^^^^^^^^^^ + >>> import os + >>> os.environ["DB_HOST"] = "localhost" + >>> os.environ["DB_PORT"] = "5432" + >>> validate_env_vars(["DB_HOST", "DB_PORT"]) # no exception + + Missing variable + ^^^^^^^^^^^^^^^^ + >>> import os + >>> os.environ["DB_HOST"] = "localhost" + >>> validate_env_vars(["DB_HOST", "DB_PORT"]) + Traceback (most recent call last): + ... + SystemExit: [ERROR] Missing environment variables: DB_PORT + + Empty value counts as missing + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + >>> import os + >>> os.environ["API_KEY"] = "" # empty string -> missing + >>> validate_env_vars(["API_KEY"]) + Traceback (most recent call last): + ... + SystemExit: [ERROR] Missing environment variables: API_KEY + """ + if not isinstance(required_vars, list): + error_msg = "required_vars must be a list of strings." + raise TypeError(error_msg) + cleaned: list[str] = [] + for name in required_vars: + if not isinstance(name, str): + error_msg = "All environment variable names must be strings." + raise TypeError(error_msg) + stripped = name.strip() + if not stripped: + error_msg = "Environment variable names must be non-empty strings." + raise TypeError(error_msg) + cleaned.append(stripped) + + # De-duplicate while preserving readable order in logs + unique_names = sorted(set(cleaned)) + + # Treat unset or empty-string values as missing + missing = [n for n in unique_names if os.environ.get(n, "").strip() == ""] + + if missing: + msg = f"[ERROR] Missing environment variables: {', '.join(missing)}" + logger.error(msg) + raise SystemExit(msg) + + logger.info(f"Environment OK: {', '.join(unique_names)}") diff --git a/tests/cdp/helpers/test_s3_utils.py b/tests/cdp/helpers/test_s3_utils.py index d0bf82e..0dd1b26 100644 --- a/tests/cdp/helpers/test_s3_utils.py +++ b/tests/cdp/helpers/test_s3_utils.py @@ -15,6 +15,7 @@ check_file, copy_file, create_folder, + create_s3_uri, delete_file, delete_folder, delete_old_objects_and_folders, @@ -31,6 +32,7 @@ read_header, remove_leading_slash, s3_walk, + split_s3_uri, upload_file, upload_folder, validate_bucket_name, @@ -1808,3 +1810,76 @@ def test_zip_s3_directory_to_s3_no_overwrite(self, s3_client): response = s3_client.get_object(Bucket="destination-bucket", Key="folder1.zip") content = response["Body"].read() assert content == b"existing content" + + +class TestCreateS3Uri: + """Tests for create_s3_uri function.""" + + def test_default_scheme(self): + """Builds s3:// URI by default.""" + assert ( + create_s3_uri("my-bucket", "folder/file.txt") + == "s3://my-bucket/folder/file.txt" + ) + + def test_s3a_scheme(self): + """Builds s3a:// when requested.""" + assert ( + create_s3_uri("my-bucket", "folder/file.txt", scheme="s3a") + == "s3a://my-bucket/folder/file.txt" + ) + + def test_roundtrip_with_split(self): + """Round-trips through split_s3_uri.""" + bucket, key = "bucket-name", "a/b/c.txt" + uri = create_s3_uri(bucket, key) + assert split_s3_uri(uri) == (bucket, key) + + +class TestSplitS3Uri: + """Tests for split_s3_uri function.""" + + def test_valid_s3(self): + """Parses valid s3:// URI.""" + assert split_s3_uri("s3://my-bucket/data/file.csv") == ( + "my-bucket", + "data/file.csv", + ) + + def test_valid_s3a(self): + """Parses valid s3a:// URI.""" + assert split_s3_uri("s3a://bucket-x.y/data/2025/08/22.parquet") == ( + "bucket-x.y", + "data/2025/08/22.parquet", + ) + + @pytest.mark.parametrize( + "uri", + [ + "", # empty + "http://my-bucket/key", # wrong scheme + "s4://my-bucket/key", # wrong scheme + "s3:/my-bucket/key", # malformed scheme + "my-bucket/key", # missing scheme + ], + ) + def test_invalid_scheme_raises(self, uri): + """Raises ValueError for invalid/missing scheme.""" + with pytest.raises(ValueError) as exc: + split_s3_uri(uri) + assert "Expected 's3://' or 's3a://'" in str(exc.value) + + @pytest.mark.parametrize( + "uri", + [ + "s3://bucket", # no slash after bucket + "s3://bucket/", # empty key + "s3:///key", # empty bucket + "s3a:///", # empty bucket and key + ], + ) + def test_malformed_uri_parts_raises(self, uri): + """Raises ValueError for missing bucket or key.""" + with pytest.raises(ValueError) as exc: + split_s3_uri(uri) + assert "Malformed S3 URI" in str(exc.value) diff --git a/tests/helpers/test_python.py b/tests/helpers/test_python.py index 6ba7a77..68d8e79 100644 --- a/tests/helpers/test_python.py +++ b/tests/helpers/test_python.py @@ -812,6 +812,31 @@ def test_file_not_found(self): md5_sum("non_existent_file.txt") +class TestSha256Sum: + """Tests for sha256_sum function.""" + + def test_expected(self, tmp_path): + """Test expected functionality.""" + # Create a temporary file + temp_file = tmp_path / "test_file.txt" + content = "This is a test file." + temp_file.write_text(content) + + # Calculate the expected sha256 sum + expected_sha256 = hashlib.sha256(content.encode()).hexdigest() + + # Get the actual sha256 sum + actual_sha256 = sha256_sum(str(temp_file)) + + # Assert the sha256 sums match + assert actual_sha256 == expected_sha256 + + def test_file_not_found(self): + """Test behavior when file does not exist.""" + with pytest.raises(FileNotFoundError): + sha256_sum("non_existent_file.txt") + + class TestFileExists: """Tests for file_exists function.""" @@ -1048,3 +1073,109 @@ def test_raises_on_subprocess_failure(self, tmp_path: Path) -> None: ): with pytest.raises(subprocess.CalledProcessError): dump_environment_requirements(str(output_file)) + + +class TestParsePyprojectMetadata: + """Tests for parse_pyproject_metadata function.""" + + def test_parses_expected_fields(self, tmp_path): + """Parses expected fields.""" + py = tmp_path / "pyproject.toml" + py.write_text( + "[project]\nname = 'my-package'\nrequires-python = '>=3.10'\nversion = '1.2.3'\n", + encoding="utf-8", + ) + + meta = parse_pyproject_metadata(py) + + assert meta["name"] == "my-package" + assert meta["requires_python"] == ">=3.10" + assert meta["package_version"] == "1.2.3" + + def test_missing_project_table_returns_none_fields(self, tmp_path): + """Returns None fields when [project] missing.""" + py = tmp_path / "pyproject.toml" + py.write_text("# no project table here\n", encoding="utf-8") + + meta = parse_pyproject_metadata(py) + + assert meta["name"] is None + assert meta["requires_python"] is None + assert meta["package_version"] is None + + def test_file_not_found_raises(self, tmp_path): + """Raises FileNotFoundError for missing file.""" + missing = tmp_path / "does_not_exist.toml" + with pytest.raises(FileNotFoundError) as excinfo: + parse_pyproject_metadata(missing) + assert "cannot be found" in str(excinfo.value) + + def test_invalid_toml_raises(self, tmp_path): + """Raises TOMLDecodeError for invalid TOML.""" + py = tmp_path / "pyproject.toml" + py.write_text("[project\nname = 'oops'\n", encoding="utf-8") # broken TOML + + with pytest.raises(tomli.TOMLDecodeError): + parse_pyproject_metadata(py) + + +class TestValidateEnvVars: + """Tests for validate_env_vars function.""" + + def test_success_when_all_present(self, monkeypatch): + """Succeeds when all vars are present.""" + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.setenv("DB_PORT", "5432") + validate_env_vars(["DB_HOST", "DB_PORT"]) # no exception + + def test_missing_var_raises(self, monkeypatch): + """Raises SystemExit when a var is missing.""" + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.delenv("DB_PORT", raising=False) + with pytest.raises(SystemExit) as exc: + validate_env_vars(["DB_HOST", "DB_PORT"]) + msg = str(exc.value) + assert "Missing environment variables" in msg + assert "DB_PORT" in msg + assert "DB_HOST" not in msg # DB_HOST is present + + def test_empty_value_counts_as_missing(self, monkeypatch): + """Treats empty string as missing.""" + monkeypatch.setenv("API_KEY", "") + with pytest.raises(SystemExit) as exc: + validate_env_vars(["API_KEY"]) + assert "API_KEY" in str(exc.value) + + def test_whitespace_value_counts_as_missing(self, monkeypatch): + """Treats whitespace-only value as missing.""" + monkeypatch.setenv("TOKEN", " ") + with pytest.raises(SystemExit) as exc: + validate_env_vars(["TOKEN"]) + assert "TOKEN" in str(exc.value) + + def test_names_are_stripped(self, monkeypatch): + """Strips whitespace around variable names.""" + monkeypatch.setenv("SERVICE_URL", "https://example.com") + validate_env_vars([" SERVICE_URL "]) # no exception + + def test_duplicates_do_not_break(self, monkeypatch): + """Handles duplicate names gracefully.""" + monkeypatch.setenv("REGION", "eu-west-1") + validate_env_vars(["REGION", "REGION"]) # no exception + + def test_type_error_non_list(self): + """Raises TypeError if input is not a list.""" + with pytest.raises(TypeError): + validate_env_vars(("DB_HOST", "DB_PORT")) # type: ignore[arg-type] + + def test_type_error_non_string_item(self): + """Raises TypeError if any name is not a string.""" + with pytest.raises(TypeError): + validate_env_vars(["DB_HOST", 123]) # type: ignore[list-item] + + def test_type_error_empty_name(self): + """Raises TypeError for empty/whitespace-only names.""" + with pytest.raises(TypeError): + validate_env_vars([""]) # empty + with pytest.raises(TypeError): + validate_env_vars([" "]) # whitespace only From f1907b7a115249194d02dbf5d5aa24b4c1c68857 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:08:19 +0100 Subject: [PATCH 3/3] Release v0.16.0 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 14 ++++++++++++++ rdsa_utils/__init__.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7245553..3ba0b21 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.15.0 +current_version = 0.16.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 779a069..2e3dc4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Changed + +### Deprecated + +### Fixed + +### Removed + +## [0.16.0] - 2025-08-22 + ### Added - Added `sha256_sum`, `parse_pyproject_metadata`, `validate_env_vars`, to `helpers/python.py`. @@ -919,6 +931,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI > and GitHub Releases. +- rdsa-utils v0.16.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.16.0) | + [PyPI](https://pypi.org/project/rdsa-utils/0.16.0/) - rdsa-utils v0.15.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.15.0) | [PyPI](https://pypi.org/project/rdsa-utils/0.15.0/) - rdsa-utils v0.14.1: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.14.1) | diff --git a/rdsa_utils/__init__.py b/rdsa_utils/__init__.py index 9da2f8f..5a313cc 100644 --- a/rdsa_utils/__init__.py +++ b/rdsa_utils/__init__.py @@ -1 +1 @@ -__version__ = "0.15.0" +__version__ = "0.16.0"