ONSdigital · jwestw · Jan 9, 2025 · Jan 9, 2025 · Jan 9, 2025 · Jan 9, 2025
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.4
+current_version = 0.5.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,9 +18,9 @@ repos:
       - id: check-yaml
       - id: check-toml
       - id: debug-statements
-  
+
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
 
@@ -29,15 +29,15 @@ repos:
     hooks:
       - id: isort
         args: ["--profile", "black"]
-  
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.9.1
     hooks:
       - id: ruff
         args: ["--config", ".ruff.toml"]
 
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.18.2
+    rev: v8.23.1
     hooks:
       - id: gitleaks
 

diff --git a/.ruff.toml b/.ruff.toml
@@ -35,5 +35,13 @@ inline-quotes = "double"
 "*/__init__.py" = ["D104"]
 "*/" = ["B006", "PTH123", "B008"]
 "tests/*" = ["ANN", "D100", "E501", "F403", "F405", "PT011", "B017", "D205"]
-"rdsa_utils/rdsa_data_validator/data_validation.py" = ["E501","D409", "D406", "D205", "COM812", "D401"]
-
+"rdsa_utils/rdsa_data_validator/*" = [
+    "E501",
+    "D409",
+    "D406",
+    "D205",
+    "COM812",
+    "D401",
+]
+"rdsa_utils/logging.py" = ["A005"]
+"rdsa_utils/typing.py" = ["A005"]
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,8 +8,32 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Created tests for `_load_validation_schema` in `toml_schema_validator` module.
+
+
+### Changed
+- Ran `ruff check . fix` on the codebase to comply with new PEP rules.
+- Added rules to `ruff.toml` to ignore A005 warnings for `rdsa_utils/logging.py` 
+  and `rdsa_utils/typing.py`.
+- Upgraded `black`, `ruff`, `gitleaks` to the latest version 
+  in `.pre-commit-config.yaml`.
+
+### Deprecated
+
+### Fixed
+
+### Removed
+
+## [0.5.0] - 2025-01-09
+
+### Added
+- Added link and description of `easy_pipeline_run` repo to `README.md`.
 
 ### Changed
+- Modified `list_files` function in `cdp/helpers/s3_utils.py` to use pagination 
+  when listing objects from S3 buckets, improving handling of large buckets.
+- Added test cases for new pagination functionality in `list_files` function 
+  in `tests/cdp/helpers/test_s3_utils.py`.
 
 ### Deprecated
 
@@ -516,6 +540,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0
 > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI
 > and GitHub Releases.
 
+- rdsa-utils v0.5.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.5.0) | 
+  [PyPI](https://pypi.org/project/rdsa-utils/0.5.0/)
 - rdsa-utils v0.4.4: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.4) | 
   [PyPI](https://pypi.org/project/rdsa-utils/0.4.4/)
 - rdsa-utils v0.4.3: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.3) | 

diff --git a/README.md b/README.md
@@ -63,6 +63,8 @@ We highly recommend checking out the following resources to learn more about cre
 
 - [PySpark Introduction and Training Book](https://best-practice-and-impact.github.io/ons-spark/intro.html) - An introduction to using PySpark for large-scale data processing.
 
+Additionally, if you are facing the challenge of repeatedly setting up new developers and new users in local Python, then you may want to consider making a batch file to carry out the setup process for you. The [easypipelinerun](https://github.com/ONSdigital/easy_pipeline_run/) repo has a batch file that can be modified to set your users up for your project, taking care of things like conda and pip set up as well as environment management.  
+
 ## 🛡️ Licence
 
 Unless stated otherwise, the codebase is released under the [MIT License][mit].

diff --git a/rdsa_utils/__init__.py b/rdsa_utils/__init__.py
@@ -1 +1 @@
-__version__ = "0.4.4"
+__version__ = "0.5.0"
diff --git a/rdsa_utils/cdp/helpers/s3_utils.py b/rdsa_utils/cdp/helpers/s3_utils.py
@@ -677,11 +677,12 @@ def list_files(
     prefix = remove_leading_slash(prefix)
 
     try:
-        response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
         files = []
-        if "Contents" in response:
-            for obj in response["Contents"]:
-                files.append(obj["Key"])
+        paginator = client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix):
+            if "Contents" in page:
+                for obj in page["Contents"]:
+                    files.append(obj["Key"])
         return files
     except client.exceptions.ClientError as e:
         logger.error(f"Failed to list files in bucket: {str(e)}")

diff --git a/rdsa_utils/rdsa_data_validator/data_validation.py b/rdsa_utils/rdsa_data_validator/data_validation.py
@@ -46,9 +46,7 @@ def create_expectation_suite_from_toml(self, toml_path, data_asset_name):
         """
         pass
 
-    def validate_dataframe_with_expectation_suite(self,
-                                                  dataframe,
-                                                  expectation_suite):
+    def validate_dataframe_with_expectation_suite(self, dataframe, expectation_suite):
         """Validate a DataFrame against a Great Expectations Expectation Suite.
 
         Args:

diff --git a/...a_validator/example_dataframe_schema.toml → ...validator/example_data_source_schema.toml b/...a_validator/example_dataframe_schema.toml → ...validator/example_data_source_schema.toml
@@ -12,6 +12,7 @@
 
 [data_asset]
 name = "example_survey_results"
+dataframe_library = "pandas" # Currently allows "pandas" "pyspark"
 
 [reference]
 description = "Unique identifier for the record."