From fab6d710a14f0585fe6114cfa2ae0205990f11fa Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:27:26 +0100 Subject: [PATCH 1/2] fix logger in load_csv (#223) --- CHANGELOG.md | 2 ++ rdsa_utils/helpers/pyspark.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09257f4..d5178f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ### Deprecated ### Fixed +- Modified `load_csv` in `helpers/pyspark.py` to improve logging with conditional + inclusion of read options. ### Removed diff --git a/rdsa_utils/helpers/pyspark.py b/rdsa_utils/helpers/pyspark.py index 9f0d7f5..7960c2f 100644 --- a/rdsa_utils/helpers/pyspark.py +++ b/rdsa_utils/helpers/pyspark.py @@ -877,7 +877,14 @@ def load_csv( """ try: df = spark.read.csv(filepath, header=True, **kwargs) - logger.info(f"Loaded CSV file {filepath} with parameters {kwargs}") + logger.info( + ( + f"Loaded CSV file: {filepath}. " + f"Keep columns: {keep_columns}, Drop columns: {drop_columns}, " + f"Rename columns: {rename_columns}." + + (f" Additional read options: {kwargs}." if kwargs else "") + ), + ) except Exception as e: error_message = f"Error loading file {filepath}: {e}" logger.error(error_message) From 572bbb05dfd5cc567bb7fa11686505b74985d104 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:29:05 +0100 Subject: [PATCH 2/2] Release v0.13.3 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 14 ++++++++++++++ rdsa_utils/__init__.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c143129..ab1e03d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.13.2 +current_version = 0.13.3 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5178f4..d9d742f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,18 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ### Deprecated +### Fixed + +### Removed + +## [0.13.3] - 2025-08-04 + +### Added + +### Changed + +### Deprecated + ### Fixed - Modified `load_csv` in `helpers/pyspark.py` to improve logging with conditional inclusion of read options. @@ -861,6 +873,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI > and GitHub Releases. +- rdsa-utils v0.13.3: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.13.3) | + [PyPI](https://pypi.org/project/rdsa-utils/0.13.3/) - rdsa-utils v0.13.2: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.13.2) | [PyPI](https://pypi.org/project/rdsa-utils/0.13.2/) - rdsa-utils v0.13.1: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.13.1) | diff --git a/rdsa_utils/__init__.py b/rdsa_utils/__init__.py index 83ce76f..26c36ca 100644 --- a/rdsa_utils/__init__.py +++ b/rdsa_utils/__init__.py @@ -1 +1 @@ -__version__ = "0.13.2" +__version__ = "0.13.3"