From 940782cebc23c8db3be12cdcd66cd315b921b5d8 Mon Sep 17 00:00:00 2001 From: jordantgh <56029559+jordantgh@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:40:06 +0000 Subject: [PATCH 1/8] Improvement: multi page s3 list files (#141) * Improvement: Add paginator to handle large S3 buckets * Chore: Update changelog * Fix: Add back file list return * Test: Add test for multi-page s3 file lists - Checks that buckets w/ >1000 objects are correctly enumerated * Chore: Update changelog w/ testing changes --- CHANGELOG.md | 3 +++ rdsa_utils/cdp/helpers/s3_utils.py | 9 +++++---- tests/cdp/helpers/test_s3_utils.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30732962..9cd5e435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed +- Modified `list_files` function in `cdp/helpers/s3_utils.py` to use pagination +when listing objects from S3 buckets, improving handling of large buckets. +- Added test cases for new pagination functionality in `list_files` function in `tests/cdp/helpers/test_s3_utils.py`. ### Deprecated diff --git a/rdsa_utils/cdp/helpers/s3_utils.py b/rdsa_utils/cdp/helpers/s3_utils.py index c497c1a9..f4f5ba91 100644 --- a/rdsa_utils/cdp/helpers/s3_utils.py +++ b/rdsa_utils/cdp/helpers/s3_utils.py @@ -677,11 +677,12 @@ def list_files( prefix = remove_leading_slash(prefix) try: - response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix) files = [] - if "Contents" in response: - for obj in response["Contents"]: - files.append(obj["Key"]) + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): + if "Contents" in page: + for obj in page["Contents"]: + files.append(obj["Key"]) return files except client.exceptions.ClientError as e: logger.error(f"Failed to list files in bucket: {str(e)}") diff --git a/tests/cdp/helpers/test_s3_utils.py b/tests/cdp/helpers/test_s3_utils.py index 790a6ad5..bff9e9e4 100644 --- a/tests/cdp/helpers/test_s3_utils.py +++ b/tests/cdp/helpers/test_s3_utils.py @@ -495,6 +495,20 @@ def test_list_files_no_match(self, s3_client_for_list_files): ) assert len(files) == 0 + def test_list_files_pagination(self, s3_client_for_list_files): + """Test listing >1000 files to verify pagination works correctly.""" + for i in range(1001): + s3_client_for_list_files.put_object( + Bucket="test-bucket", + Key=f"paginated/file_{i:04d}.txt", + Body=b"Test content", + ) + + files = list_files(s3_client_for_list_files, "test-bucket") + assert len(files) == 1006 + assert "paginated/file_0000.txt" in files + assert "paginated/file_1000.txt" in files + @pytest.fixture() def s3_client_for_delete_and_copy(_aws_credentials): From 6cf9df7824744d905392e824c14c1303e7d8ee59 Mon Sep 17 00:00:00 2001 From: James Westwood <67740306+jwestw@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:49:49 +0000 Subject: [PATCH 2/8] Include link to easypipelinerun (#140) * Include link to easypipelinerun * Update CHANGELOG.md --------- Co-authored-by: dombean <46692370+dombean@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5e435..96582a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added link and description of `easy_pipeline_run` repo to `README.md`. ### Changed - Modified `list_files` function in `cdp/helpers/s3_utils.py` to use pagination diff --git a/README.md b/README.md index 8b997b6f..586c6db5 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ We highly recommend checking out the following resources to learn more about cre - [PySpark Introduction and Training Book](https://best-practice-and-impact.github.io/ons-spark/intro.html) - An introduction to using PySpark for large-scale data processing. +Additionally, if you are facing the challenge of repeatedly setting up new developers and new users in local Python, then you may want to consider making a batch file to carry out the setup process for you. The [easypipelinerun](https://github.com/ONSdigital/easy_pipeline_run/) repo has a batch file that can be modified to set your users up for your project, taking care of things like conda and pip set up as well as environment management. + ## 🛡️ Licence Unless stated otherwise, the codebase is released under the [MIT License][mit]. From 6447709d104a3b86550f6ac42b0517de0e94543c Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:56:16 +0000 Subject: [PATCH 3/8] Release v0.5.0 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 19 +++++++++++++++++-- rdsa_utils/__init__.py | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 17c87859..a8c0a074 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.4 +current_version = 0.5.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96582a9d..baa6191e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,26 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Changed + +### Deprecated + +### Fixed + +### Removed + +## [0.5.0] - 2025-01-09 + ### Added - Added link and description of `easy_pipeline_run` repo to `README.md`. ### Changed - Modified `list_files` function in `cdp/helpers/s3_utils.py` to use pagination -when listing objects from S3 buckets, improving handling of large buckets. -- Added test cases for new pagination functionality in `list_files` function in `tests/cdp/helpers/test_s3_utils.py`. + when listing objects from S3 buckets, improving handling of large buckets. +- Added test cases for new pagination functionality in `list_files` function + in `tests/cdp/helpers/test_s3_utils.py`. ### Deprecated @@ -520,6 +533,8 @@ when listing objects from S3 buckets, improving handling of large buckets. > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI > and GitHub Releases. +- rdsa-utils v0.5.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.5.0) | + [PyPI](https://pypi.org/project/rdsa-utils/0.5.0/) - rdsa-utils v0.4.4: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.4) | [PyPI](https://pypi.org/project/rdsa-utils/0.4.4/) - rdsa-utils v0.4.3: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.3) | diff --git a/rdsa_utils/__init__.py b/rdsa_utils/__init__.py index cd1ee63b..3d187266 100644 --- a/rdsa_utils/__init__.py +++ b/rdsa_utils/__init__.py @@ -1 +1 @@ -__version__ = "0.4.4" +__version__ = "0.5.0" From 28b8531c69ed1db7fc5b0192c4bb3cebea42a84b Mon Sep 17 00:00:00 2001 From: James Westwood Date: Tue, 14 Jan 2025 17:47:33 +0000 Subject: [PATCH 4/8] Testing load_validation_schema --- ...a.toml => example_data_source_schema.toml} | 1 + .../toml_schema_validator.py | 125 +++++++++++------- ...toml => toml_schema_validator_config.toml} | 0 tests/invalid_test_schema.toml | 5 + tests/test_schema.toml | 11 ++ tests/test_toml_schema_validator.py | 71 ++++++++++ tests/toml_schema_validator_config.toml | 20 +++ 7 files changed, 186 insertions(+), 47 deletions(-) rename rdsa_utils/rdsa_data_validator/{example_dataframe_schema.toml => example_data_source_schema.toml} (96%) rename rdsa_utils/rdsa_data_validator/{config_validator_config.toml => toml_schema_validator_config.toml} (100%) create mode 100644 tests/invalid_test_schema.toml create mode 100644 tests/test_schema.toml create mode 100644 tests/test_toml_schema_validator.py create mode 100644 tests/toml_schema_validator_config.toml diff --git a/rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml b/rdsa_utils/rdsa_data_validator/example_data_source_schema.toml similarity index 96% rename from rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml rename to rdsa_utils/rdsa_data_validator/example_data_source_schema.toml index 64a07abc..043b7f25 100644 --- a/rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml +++ b/rdsa_utils/rdsa_data_validator/example_data_source_schema.toml @@ -12,6 +12,7 @@ [data_asset] name = "example_survey_results" +dataframe_library = "pandas" # Currently allows "pandas" "pyspark" [reference] description = "Unique identifier for the record." diff --git a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py index 5eb68565..61d25882 100644 --- a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py +++ b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py @@ -78,11 +78,17 @@ class TOMLSchemaValidator: """ - def __init__(self, config_file_path="config_validator_config.toml"): + def __init__( + self, + schema_file_path=None, + config_file_path="toml_schema_validator_config.toml", + ): self.toml_val_logger = logging.getLogger(__name__) + logging.basicConfig(level=logging.INFO) self._load_config(config_file_path) # should create self.config self.selected_functions = {} - + self.schema = self._load_validation_schema(schema_file_path) + self.dataframe_type = self.schema.get("dataframe_library") if self.config: self.all_data_types = self._get_data_type_names() else: @@ -126,24 +132,47 @@ def __init__(self, config_file_path="config_validator_config.toml"): "DateType", ] - def _load_config(self, config_file_name="config_validator_config.toml"): + def _load_config(self, config_path): + """Loads the TOML config file, handling errors gracefully. + + Args: + config_file_path (str or Path): Path to the config file. Can be a string or a Path object. + """ + try: + with open( + Path(__file__).parent.parent / config_path, "r", encoding="utf-8" + ) as f: + toml_string = f.read() + self.config = tomli.loads(toml_string) + + except FileNotFoundError as e: + self.toml_val_logger.error(f"Config file '{config_path}' not found.") + raise e + + except tomli.TOMLDecodeError as e: + self.toml_val_logger.error(f"Error decoding TOML file '{config_path}': {e}") + raise e( + "Invalid TOML in config file." + ) # Stop validation because config is essential + """Loads the TOML config file, handling errors gracefully. Args: - config_file_name (str): Name of config file (allows for easier - testing) + config_file_path (str or Path): Path to the config file. Can be a string or a Path object. """ try: - config_path = os.path.join(os.path.dirname(__file__), config_file_name) - with open(config_path, "r", encoding="utf-8") as f: - toml_string = f.read() # Read entire file into a string - self.config = tomli.loads(toml_string) # Use loads() for strings + with open( + config_path, "r", encoding="utf-8" + ) as f: # Open directly using config_file_path + toml_string = f.read() + self.config = tomli.loads(toml_string) + except FileNotFoundError: - self.toml_val_logger.error(f"Config file '{config_file_name}' not found.") + self.toml_val_logger.error(f"Config file '{config_path}' not found.") + self.config = None # Explicitly set to None if not found except tomli.TOMLDecodeError as e: - self.toml_val_logger.error( - f"Error decoding TOML file '{config_file_name}': {e}" - ) + self.toml_val_logger.error(f"Error decoding TOML file '{config_path}': {e}") + self.config = None # Explicitly set to None if invalid def _load_validation_schema(self, toml_path: str) -> Dict[str, Any]: """Loads a data validation schema from a TOML file. @@ -161,18 +190,16 @@ def _load_validation_schema(self, toml_path: str) -> Dict[str, Any]: with open(toml_path, "rb") as f: schema = tomli.load(f) - if not isinstance(schema, dict): - self.toml_val_logger.error("Invalid schema: TOML must be a dictionary.") - return None - return schema - except FileNotFoundError: + except FileNotFoundError as e: self.toml_val_logger.error(f"TOML file not found at: {toml_path}") - return None + raise e except tomli.TOMLDecodeError as e: - self.toml_val_logger.error(f"Invalid TOML in {toml_path}: {e}") - return None + self.toml_val_logger.error( + f"Invalid TOML in {toml_path}: {e}. Cannot continue without file." + ) + raise e def _check_required_fields( self, @@ -246,8 +273,13 @@ def _validate_nullable(self, col_config, col_errors, col_name): "None", "NULL", float("nan"), - ] # Removed pd.NA from this list - for val in col_config["possible_values"]: # Iterating to handle pd.NA + ] + for val in col_config["possible_values"]: + # Check if the value is considered "NA" by pandas. This is important because + # "nan", empty strings, and other representations of missing values might be + # present in the possible_values list, and we want to treat them as invalid + # if the column is non-nullable. pd.isna() handles various NA representations + # consistently. if pd.isna(val) or val in invalid_values: col_errors.append( f"Column '{col_name}' is non-nullable but 'possible_values' contains null-like values." @@ -269,12 +301,7 @@ def _get_data_type_names(self) -> List[str]: all_type_names.extend(self.config["datatypes"][cat]["types"]) return all_type_names - def _validate_data_type( - self, - col_config: Dict[str, Any], - col_errors: List[str], - col_name: str, - ) -> Dict[str, Any]: + def _validate_data_type(self, col_config, col_errors, col_name): """Validates the 'data_type' field in the schema. Checks for valid data types and appropriate use of min/max value and length constraints. @@ -282,13 +309,15 @@ def _validate_data_type( data_type = col_config.get("data_type") if not data_type or data_type == "": col_errors.append(f"Column '{col_name}' is missing a data_type.") + return col_errors # Return early if data_type is missing - if data_type not in self.all_data_types: + if data_type not in self.all_data_types: # Check for invalid data type col_errors.append( f"{data_type} in column '{col_name}' is not a valid data type" ) + return col_errors # Return early if it is not a valid type - elif data_type == "category": # possible_values must be present + if data_type == "category": # possible_values must be present if ( "possible_values" not in col_config or col_config["possible_values"] == "nan" @@ -296,10 +325,6 @@ def _validate_data_type( col_errors.append( f"Column '{col_name}' must have 'possible_values' if data_type is 'category'." ) - else: - col_errors.append( - f"Invalid data_type '{data_type}' specified for column '{col_name}'." - ) return col_errors @@ -363,7 +388,8 @@ def _validate_min_max( ) -> List[str]: """Validates 'min_value' and 'max_value' fields. - Checks that both min_value and max_value are numbers if specified for numeric or datetime types. + Checks that both min_value and max_value are numbers if specified for + numeric or datetime types. Checks that min_value is not greater than max_value. """ data_type = col_config.get("data_type") @@ -434,7 +460,7 @@ def _validate_possible_values( ) data_type = col_config.get("data_type") - if data_type != "category": + if data_type != "category" and self.dataframe_type == "python": self.toml_val_logger.warning( # Use warnings.warn for non-categorical types f"Column '{col_name}': Using 'possible_values' with data_type '{data_type}' " f"might not be memory-efficient. Consider using 'category' data_type.", @@ -694,6 +720,8 @@ def _go_no_go(self, errors_dict, stop_on_errors=True, threshold=0): ) elif total_errors > 0: for col, errors in errors_dict.items(): + if not errors: + continue for error in errors: self.toml_val_logger.warning(f"Column '{col}': {error}") else: @@ -720,26 +748,29 @@ def validate_schema(self, schema: Dict[str, Any]) -> Dict[str, List[str]]: return errors - def run_validation(self, toml_path: str) -> None: - """Loads the schema, runs validation, and handles results.""" - schema = self._load_validation_schema(toml_path) + def run_validation(self, stop_on_errors=True, threshold=0) -> None: + """Loads the schema, runs validation, and handles results. - if not schema: # Handle empty schema gracefully + Entrypoint function. + """ + + if not self.schema: # Handle empty schema gracefully self.toml_val_logger.error( "Schema is empty. Cannot proceed with validation." ) return - errors = self.validate_schema(schema) # Call validate_schema method + errors = self.validate_schema(self.schema) # Call validate_schema method self._log_errors(errors) # Log the errors - self._go_no_go(errors) # Make the go/no-go decision + self._go_no_go(errors, stop_on_errors, threshold) # Make the go/no-go decision + + self.toml_val_logger.info("Validation complete.") if __name__ == "__main__": - validator = TOMLSchemaValidator() # Create an instance of the validator - toml_file_path = ( + schema_file_path = ( Path("rdsa_utils") / "rdsa_data_validator" / "example_dataframe_schema.toml" ) - - validator.run_validation(str(toml_file_path)) + validator = TOMLSchemaValidator(schema_file_path=str(schema_file_path)) + validator.run_validation(stop_on_errors=False) diff --git a/rdsa_utils/rdsa_data_validator/config_validator_config.toml b/rdsa_utils/rdsa_data_validator/toml_schema_validator_config.toml similarity index 100% rename from rdsa_utils/rdsa_data_validator/config_validator_config.toml rename to rdsa_utils/rdsa_data_validator/toml_schema_validator_config.toml diff --git a/tests/invalid_test_schema.toml b/tests/invalid_test_schema.toml new file mode 100644 index 00000000..808ac36f --- /dev/null +++ b/tests/invalid_test_schema.toml @@ -0,0 +1,5 @@ +# invalid_test_schema.toml +[column1] + +# Missing closing quote +description] = "Test column 1 diff --git a/tests/test_schema.toml b/tests/test_schema.toml new file mode 100644 index 00000000..bafc2c29 --- /dev/null +++ b/tests/test_schema.toml @@ -0,0 +1,11 @@ +# test_schema.toml +[data_asset] +name = "test_dataframe" + +[column1] +description = "Test column 1" +data_type = "int" + +[column2] +description = "Test column 2" +data_type = "StringType" diff --git a/tests/test_toml_schema_validator.py b/tests/test_toml_schema_validator.py new file mode 100644 index 00000000..79638237 --- /dev/null +++ b/tests/test_toml_schema_validator.py @@ -0,0 +1,71 @@ +import sys +from pathlib import Path +from unittest import mock +from unittest.mock import mock_open + +import pytest + +from rdsa_utils.rdsa_data_validator.toml_schema_validator import TOMLSchemaValidator + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + +# Fixtures to provide TOML file paths (adjust paths as needed) +@pytest.fixture() +def valid_test_toml_path(): + return str(Path(__file__).parent / "test_schema.toml") + + +@pytest.fixture() +def invalid_test_toml_path(): + return str(Path(__file__).parent / "invalid_test_schema.toml") + + +@pytest.fixture() +def toml_schema_validator_config_path(): + return str(Path(__file__).parent / "toml_schema_validator_config.toml") + + +@pytest.fixture() +def non_dict_toml_path(): + return str(Path(__file__).parent / "non_dict.toml") + + +# Tests +def test_load_validation_schema_valid( + valid_test_toml_path, toml_schema_validator_config_path, caplog +): + validator = TOMLSchemaValidator( + schema_file_path=valid_test_toml_path, + config_file_path=toml_schema_validator_config_path, + ) # Schema & config loaded at init + assert validator.schema is not None # Check schema loaded correctly + assert validator.schema["column1"]["description"] == "Test column 1" + assert validator.schema["column2"]["data_type"] == "StringType" + + +def test_load_validation_schema_invalid_toml( + invalid_test_toml_path, toml_schema_validator_config_path, caplog +): + with pytest.raises(tomllib.TOMLDecodeError): + TOMLSchemaValidator( + schema_file_path=invalid_test_toml_path, # Provide path to invalid toml + config_file_path=toml_schema_validator_config_path, + ) + assert "Invalid TOML" in caplog.text # Check the specific message + + +def test_load_validation_schema_nonexistent_schema_file( + toml_schema_validator_config_path, caplog +): + nonexistent_path = "nonexistent_file.toml" # Or use tmp_path to create a guaranteed nonexistent path + + with pytest.raises(FileNotFoundError): + TOMLSchemaValidator( + schema_file_path=nonexistent_path, + config_file_path=toml_schema_validator_config_path, + ) + assert f"TOML file not found at: {nonexistent_path}" in caplog.text diff --git a/tests/toml_schema_validator_config.toml b/tests/toml_schema_validator_config.toml new file mode 100644 index 00000000..40641316 --- /dev/null +++ b/tests/toml_schema_validator_config.toml @@ -0,0 +1,20 @@ +# This is the config for the config_validator + +[paths] +example_schema_path = "rdsa_utils/rdsa_data_validator/config_validator_config.toml" + +log_file_path = "rdsa_utils/rdsa_data_validator/logs/toml_validation.log" + +[datatypes] + +[datatypes.python_types] +types = ["int", "float", "str", "bool", "list", "tuple", "dict", "set", "datetime.datetime"] + +[datatypes.pandas_numpy_types] +types = ["int64", "int32", "int16", "int8", "float64", "float32", "object", "bool_", "datetime64[ns]", "timedelta64[ns]", "category"] + +[datatypes.pyspark_types] +types = ["StringType", "IntegerType", "FloatType", "DoubleType", "BooleanType", "TimestampType", "DateType", "ArrayType", "MapType", "StructType"] + +[required_fields] +fields = ["description", "data_type", "nullable"] From 562d7edd66c8497e660e6947a29efdce49390834 Mon Sep 17 00:00:00 2001 From: James Westwood Date: Tue, 14 Jan 2025 17:51:20 +0000 Subject: [PATCH 5/8] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30732962..e3b6a0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Created tests for `_load_validation_schema` in `toml_schema_validator` module. + ### Changed From 3867b530471ffe9c22669b359f7edefd3b5cd7de Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:06:58 +0000 Subject: [PATCH 6/8] Run ruff on codebase to comply with new PEP rules (#177) --- .pre-commit-config.yaml | 10 +++++----- .ruff.toml | 4 ++++ CHANGELOG.md | 5 +++++ rdsa_utils/test_utils.py | 4 ++-- tests/cdp/helpers/test_hdfs_utils.py | 4 ++-- tests/cdp/helpers/test_s3_utils.py | 12 ++++++------ tests/cdp/io/test_cdsw_input.py | 4 ++-- tests/cdp/io/test_cdsw_output.py | 8 ++++---- tests/gcp/helpers/test_gcp_utils.py | 10 +++++----- tests/helpers/test_pyspark.py | 6 +++--- tests/helpers/test_python.py | 2 +- tests/io/conftest.py | 2 +- tests/methods/test_averaging_methods.py | 2 +- tests/test_logging.py | 2 +- 14 files changed, 42 insertions(+), 33 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63e70082..95f9c289 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,9 +18,9 @@ repos: - id: check-yaml - id: check-toml - id: debug-statements - + - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black @@ -29,15 +29,15 @@ repos: hooks: - id: isort args: ["--profile", "black"] - + - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.9.1 hooks: - id: ruff args: ["--config", ".ruff.toml"] - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.2 + rev: v8.23.1 hooks: - id: gitleaks diff --git a/.ruff.toml b/.ruff.toml index 821128a3..981a90f0 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -35,3 +35,7 @@ inline-quotes = "double" "*/__init__.py" = ["D104"] "*/" = ["B006", "PTH123", "B008"] "tests/*" = ["ANN", "D100", "E501", "F403", "F405", "PT011", "B017", "D205"] + +"rdsa_utils/logging.py" = ["A005"] + +"rdsa_utils/typing.py" = ["A005"] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index baa6191e..45854e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed +- Ran `ruff check . fix` on the codebase to comply with new PEP rules. +- Added rules to `ruff.toml` to ignore A005 warnings for `rdsa_utils/logging.py` + and `rdsa_utils/typing.py`. +- Upgraded `black`, `ruff`, `gitleaks` to the latest version + in `.pre-commit-config.yaml`. ### Deprecated diff --git a/rdsa_utils/test_utils.py b/rdsa_utils/test_utils.py index e58f0f7c..e5db7060 100644 --- a/rdsa_utils/test_utils.py +++ b/rdsa_utils/test_utils.py @@ -172,7 +172,7 @@ def to_datetime(dt: str) -> datetime.datetime: return pd.to_datetime(dt).to_pydatetime() -@pytest.fixture() +@pytest.fixture def create_spark_df(spark_session): """Create Spark DataFrame from tuple data with first row as schema. @@ -196,7 +196,7 @@ def _(data): return _ -@pytest.fixture() +@pytest.fixture def to_spark(spark_session): """Convert pandas df to spark.""" diff --git a/tests/cdp/helpers/test_hdfs_utils.py b/tests/cdp/helpers/test_hdfs_utils.py index c32f797b..847759ed 100644 --- a/tests/cdp/helpers/test_hdfs_utils.py +++ b/tests/cdp/helpers/test_hdfs_utils.py @@ -41,7 +41,7 @@ class BaseTest: subprocess.Popen calls. """ - @pytest.fixture() + @pytest.fixture def mock_subprocess_popen(self, monkeypatch): # noqa: PT004 """Fixture to mock the subprocess.Popen function. @@ -71,7 +71,7 @@ def mock_popen(*args, **kwargs): monkeypatch.setattr(subprocess, "Popen", mock_popen) - @pytest.fixture() + @pytest.fixture def mock_subprocess_popen_date_modifed(self): """Fixture to mock subprocess.Popen for testing get_date_modified. diff --git a/tests/cdp/helpers/test_s3_utils.py b/tests/cdp/helpers/test_s3_utils.py index bff9e9e4..93511e4b 100644 --- a/tests/cdp/helpers/test_s3_utils.py +++ b/tests/cdp/helpers/test_s3_utils.py @@ -196,7 +196,7 @@ def test_invalid_non_s3_path_with_invalid_characters(self): ) -@pytest.fixture() +@pytest.fixture def _aws_credentials(): """Mock AWS Credentials for moto.""" boto3.setup_default_session( @@ -206,7 +206,7 @@ def _aws_credentials(): ) -@pytest.fixture() +@pytest.fixture def s3_client(_aws_credentials): """Provide a mocked AWS S3 client for testing using moto with temporary credentials. @@ -234,7 +234,7 @@ def test_file_exists_false(self, s3_client): assert file_exists(s3_client, "test-bucket", "nonexistent.txt") is False -@pytest.fixture() +@pytest.fixture def setup_files(tmp_path): """ Set up local files for upload and download tests. @@ -351,7 +351,7 @@ def test_download_no_overwrite_local_file(self, s3_client, setup_files): ) -@pytest.fixture() +@pytest.fixture def setup_folder(tmp_path): """ Set up local folder and files for upload tests. @@ -435,7 +435,7 @@ def test_upload_folder_no_overwrite_existing_files( ) -@pytest.fixture() +@pytest.fixture def s3_client_for_list_files(_aws_credentials): """ Provide a mocked AWS S3 client with temporary @@ -510,7 +510,7 @@ def test_list_files_pagination(self, s3_client_for_list_files): assert "paginated/file_1000.txt" in files -@pytest.fixture() +@pytest.fixture def s3_client_for_delete_and_copy(_aws_credentials): """ Provide a mocked AWS S3 client with temporary diff --git a/tests/cdp/io/test_cdsw_input.py b/tests/cdp/io/test_cdsw_input.py index 41f92aef..94ec2bfd 100644 --- a/tests/cdp/io/test_cdsw_input.py +++ b/tests/cdp/io/test_cdsw_input.py @@ -13,7 +13,7 @@ class TestGetCurrentDatabase: """Tests for get_current_database function.""" - @pytest.fixture() + @pytest.fixture def setup_and_teardown_database( # noqa: PT004 self, spark_session: SparkSession, @@ -68,7 +68,7 @@ def test_get_current_database_after_setting( class TestExtractDatabaseName: """Tests for extract_database_name function.""" - @pytest.fixture() + @pytest.fixture def dummy_database_and_table( self, spark_session: SparkSession, diff --git a/tests/cdp/io/test_cdsw_output.py b/tests/cdp/io/test_cdsw_output.py index 2d798852..ecc7982c 100644 --- a/tests/cdp/io/test_cdsw_output.py +++ b/tests/cdp/io/test_cdsw_output.py @@ -14,7 +14,7 @@ class TestInsertDataFrameToHiveTable: """Tests for insert_df_to_hive_table function.""" - @pytest.fixture() + @pytest.fixture def test_df(self, spark_session: SparkSession, create_spark_df: Callable): """Fixture to create a test DataFrame with the help of `create_spark_df` callable. @@ -286,12 +286,12 @@ def test_insert_df_to_hive_table_with_empty_dataframe( class TestWriteAndReadHiveTable: """Tests for write_and_read_hive_table function.""" - @pytest.fixture() + @pytest.fixture def mock_spark(self): """Fixture for mocked SparkSession.""" return Mock(spec=SparkSession) - @pytest.fixture() + @pytest.fixture def mock_df(self): """Fixture for mocked DataFrame with 'run_id' and 'data' columns.""" mock_df = Mock(spec=SparkDF) @@ -381,7 +381,7 @@ def test_df_missing_filter_column(self, mock_spark, mock_df): class TestSaveCSVToHDFS: """Tests for save_csv_to_hdfs function.""" - @pytest.fixture() + @pytest.fixture def mock_df(self) -> Mock: """Fixture for mocked Spark DataFrame.""" return Mock(spec=SparkDF) diff --git a/tests/gcp/helpers/test_gcp_utils.py b/tests/gcp/helpers/test_gcp_utils.py index d4621f44..ae615c84 100644 --- a/tests/gcp/helpers/test_gcp_utils.py +++ b/tests/gcp/helpers/test_gcp_utils.py @@ -57,13 +57,13 @@ def test_expected(self): pass -@pytest.fixture() +@pytest.fixture def mock_client(): """Mock GCS client.""" return mock.Mock(spec=storage.Client) -@pytest.fixture() +@pytest.fixture def mock_bucket(mock_client): """Mock GCS bucket.""" bucket = mock.Mock(spec=storage.Bucket) @@ -71,7 +71,7 @@ def mock_bucket(mock_client): return bucket -@pytest.fixture() +@pytest.fixture def mock_blob(mock_bucket): """Mock GCS blob.""" blob = mock.Mock(spec=storage.Blob) @@ -79,14 +79,14 @@ def mock_blob(mock_bucket): return blob -@pytest.fixture() +@pytest.fixture def mock_list_blobs(mock_client): """Mock list_blobs method.""" mock_client.list_blobs.return_value = iter([mock.Mock()]) return mock_client.list_blobs -@pytest.fixture() +@pytest.fixture def mock_path(): """Mock Path object.""" with mock.patch("rdsa_utils.gcp.helpers.gcp_utils.Path") as mock_path: diff --git a/tests/helpers/test_pyspark.py b/tests/helpers/test_pyspark.py index f8b50645..f7128f70 100644 --- a/tests/helpers/test_pyspark.py +++ b/tests/helpers/test_pyspark.py @@ -633,7 +633,7 @@ class TestConvertColsToStructCol: unusual definition of the expected dataframe in these tests. """ - @pytest.fixture() + @pytest.fixture def input_df_fixture(self, create_spark_df) -> SparkDF: """Provide a basic spark dataframe.""" return create_spark_df( @@ -1225,7 +1225,7 @@ def test_load_csv_with_custom_quote(self, custom_spark_session, tmp_path): class TestTruncateExternalHiveTable: """Tests for truncate_external_hive_table function.""" - @pytest.fixture() + @pytest.fixture def create_external_table(self, spark_session: SparkSession): """Create a mock external Hive table for testing.""" spark = ( @@ -1244,7 +1244,7 @@ def create_external_table(self, spark_session: SparkSession): spark.sql("DROP DATABASE test_db") spark.stop() - @pytest.fixture() + @pytest.fixture def create_partitioned_table(self, spark_session: SparkSession): """Create a mock partitioned external Hive table for testing.""" spark = ( diff --git a/tests/helpers/test_python.py b/tests/helpers/test_python.py index af688bba..5f433020 100644 --- a/tests/helpers/test_python.py +++ b/tests/helpers/test_python.py @@ -89,7 +89,7 @@ def test_expected(self): class TestOverwriteDictionary: """Tests for the overwrite_dictionary function.""" - @pytest.fixture() + @pytest.fixture def base_dict(self): """Create base dictionary used across all tests.""" return { diff --git a/tests/io/conftest.py b/tests/io/conftest.py index 63514359..c61a2b38 100644 --- a/tests/io/conftest.py +++ b/tests/io/conftest.py @@ -67,7 +67,7 @@ def yaml_config_string() -> str: """ -@pytest.fixture() +@pytest.fixture def expected_standard_config() -> Dict[str, Any]: """Fixture providing the loaded config from loading the temp file.""" return { diff --git a/tests/methods/test_averaging_methods.py b/tests/methods/test_averaging_methods.py index 715e1284..96d40110 100644 --- a/tests/methods/test_averaging_methods.py +++ b/tests/methods/test_averaging_methods.py @@ -4,7 +4,7 @@ from rdsa_utils.methods.averaging_methods import * -@pytest.fixture() +@pytest.fixture def input_df(create_spark_df): """Fixture containing input data for tests.""" return create_spark_df( diff --git a/tests/test_logging.py b/tests/test_logging.py index e3a08e0a..1019d1b4 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -74,7 +74,7 @@ def test_expected(self, logger, expected): class TestPrintFullTables: """Tests for the print_full_table_and_raise_error.""" - @pytest.fixture() + @pytest.fixture def input_df(self): """Input pandas dataframe.""" return create_dataframe( From 486947fc86a3231ba0b3c0620192f82efe668434 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:40:17 +0000 Subject: [PATCH 7/8] ruff fixes --- .ruff.toml | 9 +- .../rdsa_data_validator/data_validation.py | 4 +- .../toml_schema_validator.py | 90 +++++++++++-------- tests/invalid_test_schema.toml | 3 +- tests/test_toml_schema_validator.py | 35 +++++--- 5 files changed, 84 insertions(+), 57 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index e2dc4f0d..6e67bcb1 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -35,6 +35,13 @@ inline-quotes = "double" "*/__init__.py" = ["D104"] "*/" = ["B006", "PTH123", "B008"] "tests/*" = ["ANN", "D100", "E501", "F403", "F405", "PT011", "B017", "D205"] -"rdsa_utils/rdsa_data_validator/data_validation.py" = ["E501","D409", "D406", "D205", "COM812", "D401"] +"rdsa_utils/rdsa_data_validator/*" = [ + "E501", + "D409", + "D406", + "D205", + "COM812", + "D401", +] "rdsa_utils/logging.py" = ["A005"] "rdsa_utils/typing.py" = ["A005"] diff --git a/rdsa_utils/rdsa_data_validator/data_validation.py b/rdsa_utils/rdsa_data_validator/data_validation.py index 0f86e2bd..c250e061 100644 --- a/rdsa_utils/rdsa_data_validator/data_validation.py +++ b/rdsa_utils/rdsa_data_validator/data_validation.py @@ -46,9 +46,7 @@ def create_expectation_suite_from_toml(self, toml_path, data_asset_name): """ pass - def validate_dataframe_with_expectation_suite(self, - dataframe, - expectation_suite): + def validate_dataframe_with_expectation_suite(self, dataframe, expectation_suite): """Validate a DataFrame against a Great Expectations Expectation Suite. Args: diff --git a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py index 61d25882..517a85a6 100644 --- a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py +++ b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py @@ -61,7 +61,7 @@ class TOMLSchemaValidator: validation process, and offers a go/no-go mechanism to halt processing if errors exceed a defined threshold. - Attributes: + Attributes ---------- config (dict): The loaded validation configuration from the config TOML file. @@ -140,7 +140,9 @@ def _load_config(self, config_path): """ try: with open( - Path(__file__).parent.parent / config_path, "r", encoding="utf-8" + Path(__file__).parent.parent / config_path, + "r", + encoding="utf-8", ) as f: toml_string = f.read() self.config = tomli.loads(toml_string) @@ -151,9 +153,10 @@ def _load_config(self, config_path): except tomli.TOMLDecodeError as e: self.toml_val_logger.error(f"Error decoding TOML file '{config_path}': {e}") - raise e( - "Invalid TOML in config file." - ) # Stop validation because config is essential + error_str = "Invalid TOML in config file." + raise tomli.TOMLDecodeError( + error_str + ) from e # Stop validation because config is essential """Loads the TOML config file, handling errors gracefully. @@ -162,7 +165,9 @@ def _load_config(self, config_path): """ try: with open( - config_path, "r", encoding="utf-8" + config_path, + "r", + encoding="utf-8", ) as f: # Open directly using config_file_path toml_string = f.read() self.config = tomli.loads(toml_string) @@ -197,7 +202,7 @@ def _load_validation_schema(self, toml_path: str) -> Dict[str, Any]: raise e except tomli.TOMLDecodeError as e: self.toml_val_logger.error( - f"Invalid TOML in {toml_path}: {e}. Cannot continue without file." + f"Invalid TOML in {toml_path}: {e}. Cannot continue without file.", ) raise e @@ -216,11 +221,11 @@ def _check_required_fields( for field in required_fields: if field not in col_config or col_config[field] is None: col_errors.append( - f"Column '{col_name}' is missing required field '{field}'." + f"Column '{col_name}' is missing required field '{field}'.", ) if field == "data_type" and "possible_values" in col_config: col_errors.append( - f"Column '{col_name}' cannot have possible values without a data_type" + f"Column '{col_name}' cannot have possible values without a data_type", ) return col_errors @@ -256,7 +261,7 @@ def _validate_description( col_errors.append(f"Column '{col_name}' has an invalid description.") elif len(col_config["description"].split()) == 0: # Check for at least one word col_errors.append( - f"Column '{col_name}' description must contain at least one word." + f"Column '{col_name}' description must contain at least one word.", ) return col_errors @@ -282,7 +287,7 @@ def _validate_nullable(self, col_config, col_errors, col_name): # consistently. if pd.isna(val) or val in invalid_values: col_errors.append( - f"Column '{col_name}' is non-nullable but 'possible_values' contains null-like values." + f"Column '{col_name}' is non-nullable but 'possible_values' contains null-like values.", ) return col_errors # Return early once an error is found @@ -292,9 +297,10 @@ def _validate_nullable(self, col_config, col_errors, col_name): def _get_data_type_names(self) -> List[str]: """Gets all data type names from the loaded config.""" if "datatypes" not in self.config: - raise MissingDataTypesError( + error_str = ( "The 'datatypes' section is missing from the configuration file.", ) + raise MissingDataTypesError(error_str) all_type_names = [] for cat in self.config["datatypes"]: @@ -313,7 +319,7 @@ def _validate_data_type(self, col_config, col_errors, col_name): if data_type not in self.all_data_types: # Check for invalid data type col_errors.append( - f"{data_type} in column '{col_name}' is not a valid data type" + f"{data_type} in column '{col_name}' is not a valid data type", ) return col_errors # Return early if it is not a valid type @@ -323,7 +329,7 @@ def _validate_data_type(self, col_config, col_errors, col_name): or col_config["possible_values"] == "nan" ): col_errors.append( - f"Column '{col_name}' must have 'possible_values' if data_type is 'category'." + f"Column '{col_name}' must have 'possible_values' if data_type is 'category'.", ) return col_errors @@ -376,7 +382,7 @@ def _validate_length( and col_config["length"] ): col_errors.append( - f"Column '{col_name}' is not a string type, it is a {data_type}. 'length' is not applicable." + f"Column '{col_name}' is not a string type, it is a {data_type}. 'length' is not applicable.", ) return col_errors @@ -397,19 +403,21 @@ def _validate_min_max( if "min_value" in col_config: min_val = col_config["min_value"] if data_type in self.numeric_types and not isinstance( - min_val, (int, float) + min_val, + (int, float), ): col_errors.append( - f"Column '{col_name}' min_value must be a number for data_type '{data_type}'." + f"Column '{col_name}' min_value must be a number for data_type '{data_type}'.", ) if "max_value" in col_config: max_val = col_config["max_value"] if data_type in self.numeric_types and not isinstance( - max_val, (int, float) + max_val, + (int, float), ): col_errors.append( - f"Column '{col_name}' max_value must be a number for data_type '{data_type}'." + f"Column '{col_name}' max_value must be a number for data_type '{data_type}'.", ) if "min_value" in col_config and "max_value" in col_config: @@ -420,7 +428,7 @@ def _validate_min_max( and col_config["min_value"] > col_config["max_value"] ): col_errors.append( - f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}" + f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}", ) elif data_type in self.datetime_types: # Handle datetime comparisons # if a min or max time is specified, this validates that it can be parsed @@ -430,11 +438,11 @@ def _validate_min_max( if min_val > max_val: col_errors.append( - f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}" + f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}", ) except (ValueError, TypeError) as e: # Catch time parsing errors col_errors.append( - f"Error comparing datetime values for column '{col_name}': {e}" + f"Error comparing datetime values for column '{col_name}': {e}", ) return col_errors @@ -518,14 +526,14 @@ def _validate_regex_pattern( re.compile(pattern) # Check if the pattern is valid regex except re.error: errors.append( - f"Column '{col_name}': Invalid regex pattern '{pattern}'." + f"Column '{col_name}': Invalid regex pattern '{pattern}'.", ) data_type = col_config.get("data_type") if data_type not in self.string_types: errors.append( - f"Column '{col_name}': 'regex_pattern' can only be applied to string type columns." + f"Column '{col_name}': 'regex_pattern' can only be applied to string type columns.", ) return errors @@ -547,7 +555,10 @@ def _validate_unique( return col_errors def _validate_date_format( - self, col_config: Dict[str, Any], col_errors: List[str], col_name: str + self, + col_config: Dict[str, Any], + col_errors: List[str], + col_name: str, ) -> List[str]: # """Validates the 'date_format' field. @@ -567,16 +578,16 @@ def _validate_date_format( # Check that data_type matched the existence of date_format if data_type not in datetime_types: col_errors.append( - f"Column '{col_name}': 'date_format' can only be used with datetime types, not '{data_type}'." + f"Column '{col_name}': 'date_format' can only be used with datetime types, not '{data_type}'.", ) return col_errors # Stop further checks if the type is incorrect. # Check for date format useage errors try: - datetime.datetime.strptime("2024-05-03", date_format) # Use a test string. + datetime.datetime.strptime("2024-05-03", date_format) # noqa: DTZ007 except ValueError: col_errors.append( - f"Column '{col_name}': Invalid date format '{date_format}'." + f"Column '{col_name}': Invalid date format '{date_format}'.", ) return col_errors @@ -598,7 +609,7 @@ def _validate_number_str_format( if data_type not in self.numeric_types: col_errors.append( f"""Column '{col_name}': 'number_str_format' can only be - used with numeric types, not '{data_type}'.""" + used with numeric types, not '{data_type}'.""", ) return col_errors # Stop further checks if type is incorrect @@ -632,7 +643,7 @@ def _validate_number_str_format( KeyError, ) as e: # Catch all possible format errors col_errors.append( - f"Column '{col_name}': Invalid number format '{number_str_format}' - {e}" + f"Column '{col_name}': Invalid number format '{number_str_format}' - {e}", ) return col_errors @@ -659,7 +670,7 @@ def _validate_custom_check( check_function = getattr(data_validation, custom_check_val) if not callable(check_function): col_errors.append( - f"Column '{col_name}': '{custom_check_val}' is not a callable in data_validation.py." + f"Column '{col_name}': '{custom_check_val}' is not a callable in data_validation.py.", ) except AttributeError: # 2. Attempt to parse as Python code: @@ -667,11 +678,11 @@ def _validate_custom_check( compile(custom_check_val, "", "exec") except (SyntaxError, TypeError, ValueError) as e: col_errors.append( - f"Column '{col_name}': Invalid Python code or function name in 'custom_check': {e}" + f"Column '{col_name}': Invalid Python code or function name in 'custom_check': {e}", ) elif not callable(custom_check_val): # Handle non-string values col_errors.append( - f"Column '{col_name}': 'custom_check' must be a string or callable." + f"Column '{col_name}': 'custom_check' must be a string or callable.", ) return col_errors @@ -699,10 +710,10 @@ def _go_no_go(self, errors_dict, stop_on_errors=True, threshold=0): Defaults to 0. Raises + ------ ValueError: If the number of errors exceeds the threshold and stop_on_errors is True. """ - total_errors = sum( len(errors) for errors in errors_dict.values() if errors ) # only if errors != None @@ -716,7 +727,7 @@ def _go_no_go(self, errors_dict, stop_on_errors=True, threshold=0): error_messages.append(f"Column '{col}': {error}") raise ValueError( f"Validation failed with {total_errors} errors:\n" - + "\n".join(error_messages) + + "\n".join(error_messages), ) elif total_errors > 0: for col, errors in errors_dict.items(): @@ -739,11 +750,13 @@ def validate_schema(self, schema: Dict[str, Any]) -> Dict[str, List[str]]: val_func = self.validation_functions.get(func_name) if val_func: errors[col_name] = val_func( - schema.get(col_name, {}), errors[col_name], col_name + schema.get(col_name, {}), + errors[col_name], + col_name, ) else: self.toml_val_logger.warning( - f"Validation function '{func_name}' not found. Skipping." + f"Validation function '{func_name}' not found. Skipping.", ) return errors @@ -753,10 +766,9 @@ def run_validation(self, stop_on_errors=True, threshold=0) -> None: Entrypoint function. """ - if not self.schema: # Handle empty schema gracefully self.toml_val_logger.error( - "Schema is empty. Cannot proceed with validation." + "Schema is empty. Cannot proceed with validation.", ) return diff --git a/tests/invalid_test_schema.toml b/tests/invalid_test_schema.toml index 808ac36f..12dba107 100644 --- a/tests/invalid_test_schema.toml +++ b/tests/invalid_test_schema.toml @@ -1,5 +1,4 @@ # invalid_test_schema.toml [column1] -# Missing closing quote -description] = "Test column 1 +description = "Test column 1 # Missing closing quote" diff --git a/tests/test_toml_schema_validator.py b/tests/test_toml_schema_validator.py index 79638237..74ccf36d 100644 --- a/tests/test_toml_schema_validator.py +++ b/tests/test_toml_schema_validator.py @@ -1,7 +1,6 @@ import sys from pathlib import Path -from unittest import mock -from unittest.mock import mock_open +from typing import Any import pytest @@ -14,30 +13,37 @@ # Fixtures to provide TOML file paths (adjust paths as needed) -@pytest.fixture() +@pytest.fixture def valid_test_toml_path(): + """Return the path to a valid TOML file.""" return str(Path(__file__).parent / "test_schema.toml") -@pytest.fixture() +@pytest.fixture def invalid_test_toml_path(): + """Return the path to an invalid TOML file.""" return str(Path(__file__).parent / "invalid_test_schema.toml") -@pytest.fixture() +@pytest.fixture def toml_schema_validator_config_path(): + """Return the path to the TOML schema validator config file.""" return str(Path(__file__).parent / "toml_schema_validator_config.toml") -@pytest.fixture() +@pytest.fixture def non_dict_toml_path(): + """Return the path to a TOML file that is not a dictionary.""" return str(Path(__file__).parent / "non_dict.toml") # Tests def test_load_validation_schema_valid( - valid_test_toml_path, toml_schema_validator_config_path, caplog -): + valid_test_toml_path: str, + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading a valid TOML schema file.""" validator = TOMLSchemaValidator( schema_file_path=valid_test_toml_path, config_file_path=toml_schema_validator_config_path, @@ -48,8 +54,11 @@ def test_load_validation_schema_valid( def test_load_validation_schema_invalid_toml( - invalid_test_toml_path, toml_schema_validator_config_path, caplog -): + invalid_test_toml_path: str, + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading an invalid TOML schema file.""" with pytest.raises(tomllib.TOMLDecodeError): TOMLSchemaValidator( schema_file_path=invalid_test_toml_path, # Provide path to invalid toml @@ -59,8 +68,10 @@ def test_load_validation_schema_invalid_toml( def test_load_validation_schema_nonexistent_schema_file( - toml_schema_validator_config_path, caplog -): + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading a nonexistent TOML schema file.""" nonexistent_path = "nonexistent_file.toml" # Or use tmp_path to create a guaranteed nonexistent path with pytest.raises(FileNotFoundError): From bf632d4f09a33bd2a4603bcfc3c1e1b3273833b4 Mon Sep 17 00:00:00 2001 From: dombean <46692370+dombean@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:45:16 +0000 Subject: [PATCH 8/8] dont pin great expectations --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 81baa4b3..9d2ad235 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = google-cloud-bigquery>=3.17.2 google-cloud-storage>=2.14.0 boto3>=1.34.103 - great-expectations>=1.3.0 + great-expectations [options.packages.find] where = .