diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 17c87859..a8c0a074 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.4 +current_version = 0.5.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63e70082..95f9c289 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,9 +18,9 @@ repos: - id: check-yaml - id: check-toml - id: debug-statements - + - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black @@ -29,15 +29,15 @@ repos: hooks: - id: isort args: ["--profile", "black"] - + - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.9.1 hooks: - id: ruff args: ["--config", ".ruff.toml"] - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.2 + rev: v8.23.1 hooks: - id: gitleaks diff --git a/.ruff.toml b/.ruff.toml index 037c3312..6e67bcb1 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -35,5 +35,13 @@ inline-quotes = "double" "*/__init__.py" = ["D104"] "*/" = ["B006", "PTH123", "B008"] "tests/*" = ["ANN", "D100", "E501", "F403", "F405", "PT011", "B017", "D205"] -"rdsa_utils/rdsa_data_validator/data_validation.py" = ["E501","D409", "D406", "D205", "COM812", "D401"] - +"rdsa_utils/rdsa_data_validator/*" = [ + "E501", + "D409", + "D406", + "D205", + "COM812", + "D401", +] +"rdsa_utils/logging.py" = ["A005"] +"rdsa_utils/typing.py" = ["A005"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 30732962..c37840b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,32 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Created tests for `_load_validation_schema` in `toml_schema_validator` module. + + +### Changed +- Ran `ruff check . fix` on the codebase to comply with new PEP rules. +- Added rules to `ruff.toml` to ignore A005 warnings for `rdsa_utils/logging.py` + and `rdsa_utils/typing.py`. +- Upgraded `black`, `ruff`, `gitleaks` to the latest version + in `.pre-commit-config.yaml`. + +### Deprecated + +### Fixed + +### Removed + +## [0.5.0] - 2025-01-09 + +### Added +- Added link and description of `easy_pipeline_run` repo to `README.md`. ### Changed +- Modified `list_files` function in `cdp/helpers/s3_utils.py` to use pagination + when listing objects from S3 buckets, improving handling of large buckets. +- Added test cases for new pagination functionality in `list_files` function + in `tests/cdp/helpers/test_s3_utils.py`. ### Deprecated @@ -516,6 +540,8 @@ and this project adheres to [semantic versioning](https://semver.org/spec/v2.0.0 > due to bugs in the GitHub Action `deploy_pypi.yaml`, which deploys to PyPI > and GitHub Releases. +- rdsa-utils v0.5.0: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.5.0) | + [PyPI](https://pypi.org/project/rdsa-utils/0.5.0/) - rdsa-utils v0.4.4: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.4) | [PyPI](https://pypi.org/project/rdsa-utils/0.4.4/) - rdsa-utils v0.4.3: [GitHub Release](https://github.com/ONSdigital/rdsa-utils/releases/tag/v0.4.3) | diff --git a/README.md b/README.md index 8b997b6f..586c6db5 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ We highly recommend checking out the following resources to learn more about cre - [PySpark Introduction and Training Book](https://best-practice-and-impact.github.io/ons-spark/intro.html) - An introduction to using PySpark for large-scale data processing. +Additionally, if you are facing the challenge of repeatedly setting up new developers and new users in local Python, then you may want to consider making a batch file to carry out the setup process for you. The [easypipelinerun](https://github.com/ONSdigital/easy_pipeline_run/) repo has a batch file that can be modified to set your users up for your project, taking care of things like conda and pip set up as well as environment management. + ## 🛡️ Licence Unless stated otherwise, the codebase is released under the [MIT License][mit]. diff --git a/rdsa_utils/__init__.py b/rdsa_utils/__init__.py index cd1ee63b..3d187266 100644 --- a/rdsa_utils/__init__.py +++ b/rdsa_utils/__init__.py @@ -1 +1 @@ -__version__ = "0.4.4" +__version__ = "0.5.0" diff --git a/rdsa_utils/cdp/helpers/s3_utils.py b/rdsa_utils/cdp/helpers/s3_utils.py index c497c1a9..f4f5ba91 100644 --- a/rdsa_utils/cdp/helpers/s3_utils.py +++ b/rdsa_utils/cdp/helpers/s3_utils.py @@ -677,11 +677,12 @@ def list_files( prefix = remove_leading_slash(prefix) try: - response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix) files = [] - if "Contents" in response: - for obj in response["Contents"]: - files.append(obj["Key"]) + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): + if "Contents" in page: + for obj in page["Contents"]: + files.append(obj["Key"]) return files except client.exceptions.ClientError as e: logger.error(f"Failed to list files in bucket: {str(e)}") diff --git a/rdsa_utils/rdsa_data_validator/data_validation.py b/rdsa_utils/rdsa_data_validator/data_validation.py index 0f86e2bd..c250e061 100644 --- a/rdsa_utils/rdsa_data_validator/data_validation.py +++ b/rdsa_utils/rdsa_data_validator/data_validation.py @@ -46,9 +46,7 @@ def create_expectation_suite_from_toml(self, toml_path, data_asset_name): """ pass - def validate_dataframe_with_expectation_suite(self, - dataframe, - expectation_suite): + def validate_dataframe_with_expectation_suite(self, dataframe, expectation_suite): """Validate a DataFrame against a Great Expectations Expectation Suite. Args: diff --git a/rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml b/rdsa_utils/rdsa_data_validator/example_data_source_schema.toml similarity index 96% rename from rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml rename to rdsa_utils/rdsa_data_validator/example_data_source_schema.toml index 64a07abc..043b7f25 100644 --- a/rdsa_utils/rdsa_data_validator/example_dataframe_schema.toml +++ b/rdsa_utils/rdsa_data_validator/example_data_source_schema.toml @@ -12,6 +12,7 @@ [data_asset] name = "example_survey_results" +dataframe_library = "pandas" # Currently allows "pandas" "pyspark" [reference] description = "Unique identifier for the record." diff --git a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py index 5eb68565..517a85a6 100644 --- a/rdsa_utils/rdsa_data_validator/toml_schema_validator.py +++ b/rdsa_utils/rdsa_data_validator/toml_schema_validator.py @@ -61,7 +61,7 @@ class TOMLSchemaValidator: validation process, and offers a go/no-go mechanism to halt processing if errors exceed a defined threshold. - Attributes: + Attributes ---------- config (dict): The loaded validation configuration from the config TOML file. @@ -78,11 +78,17 @@ class TOMLSchemaValidator: """ - def __init__(self, config_file_path="config_validator_config.toml"): + def __init__( + self, + schema_file_path=None, + config_file_path="toml_schema_validator_config.toml", + ): self.toml_val_logger = logging.getLogger(__name__) + logging.basicConfig(level=logging.INFO) self._load_config(config_file_path) # should create self.config self.selected_functions = {} - + self.schema = self._load_validation_schema(schema_file_path) + self.dataframe_type = self.schema.get("dataframe_library") if self.config: self.all_data_types = self._get_data_type_names() else: @@ -126,24 +132,52 @@ def __init__(self, config_file_path="config_validator_config.toml"): "DateType", ] - def _load_config(self, config_file_name="config_validator_config.toml"): + def _load_config(self, config_path): """Loads the TOML config file, handling errors gracefully. Args: - config_file_name (str): Name of config file (allows for easier - testing) + config_file_path (str or Path): Path to the config file. Can be a string or a Path object. """ try: - config_path = os.path.join(os.path.dirname(__file__), config_file_name) - with open(config_path, "r", encoding="utf-8") as f: - toml_string = f.read() # Read entire file into a string - self.config = tomli.loads(toml_string) # Use loads() for strings + with open( + Path(__file__).parent.parent / config_path, + "r", + encoding="utf-8", + ) as f: + toml_string = f.read() + self.config = tomli.loads(toml_string) + + except FileNotFoundError as e: + self.toml_val_logger.error(f"Config file '{config_path}' not found.") + raise e + + except tomli.TOMLDecodeError as e: + self.toml_val_logger.error(f"Error decoding TOML file '{config_path}': {e}") + error_str = "Invalid TOML in config file." + raise tomli.TOMLDecodeError( + error_str + ) from e # Stop validation because config is essential + + """Loads the TOML config file, handling errors gracefully. + + Args: + config_file_path (str or Path): Path to the config file. Can be a string or a Path object. + """ + try: + with open( + config_path, + "r", + encoding="utf-8", + ) as f: # Open directly using config_file_path + toml_string = f.read() + self.config = tomli.loads(toml_string) + except FileNotFoundError: - self.toml_val_logger.error(f"Config file '{config_file_name}' not found.") + self.toml_val_logger.error(f"Config file '{config_path}' not found.") + self.config = None # Explicitly set to None if not found except tomli.TOMLDecodeError as e: - self.toml_val_logger.error( - f"Error decoding TOML file '{config_file_name}': {e}" - ) + self.toml_val_logger.error(f"Error decoding TOML file '{config_path}': {e}") + self.config = None # Explicitly set to None if invalid def _load_validation_schema(self, toml_path: str) -> Dict[str, Any]: """Loads a data validation schema from a TOML file. @@ -161,18 +195,16 @@ def _load_validation_schema(self, toml_path: str) -> Dict[str, Any]: with open(toml_path, "rb") as f: schema = tomli.load(f) - if not isinstance(schema, dict): - self.toml_val_logger.error("Invalid schema: TOML must be a dictionary.") - return None - return schema - except FileNotFoundError: + except FileNotFoundError as e: self.toml_val_logger.error(f"TOML file not found at: {toml_path}") - return None + raise e except tomli.TOMLDecodeError as e: - self.toml_val_logger.error(f"Invalid TOML in {toml_path}: {e}") - return None + self.toml_val_logger.error( + f"Invalid TOML in {toml_path}: {e}. Cannot continue without file.", + ) + raise e def _check_required_fields( self, @@ -189,11 +221,11 @@ def _check_required_fields( for field in required_fields: if field not in col_config or col_config[field] is None: col_errors.append( - f"Column '{col_name}' is missing required field '{field}'." + f"Column '{col_name}' is missing required field '{field}'.", ) if field == "data_type" and "possible_values" in col_config: col_errors.append( - f"Column '{col_name}' cannot have possible values without a data_type" + f"Column '{col_name}' cannot have possible values without a data_type", ) return col_errors @@ -229,7 +261,7 @@ def _validate_description( col_errors.append(f"Column '{col_name}' has an invalid description.") elif len(col_config["description"].split()) == 0: # Check for at least one word col_errors.append( - f"Column '{col_name}' description must contain at least one word." + f"Column '{col_name}' description must contain at least one word.", ) return col_errors @@ -246,11 +278,16 @@ def _validate_nullable(self, col_config, col_errors, col_name): "None", "NULL", float("nan"), - ] # Removed pd.NA from this list - for val in col_config["possible_values"]: # Iterating to handle pd.NA + ] + for val in col_config["possible_values"]: + # Check if the value is considered "NA" by pandas. This is important because + # "nan", empty strings, and other representations of missing values might be + # present in the possible_values list, and we want to treat them as invalid + # if the column is non-nullable. pd.isna() handles various NA representations + # consistently. if pd.isna(val) or val in invalid_values: col_errors.append( - f"Column '{col_name}' is non-nullable but 'possible_values' contains null-like values." + f"Column '{col_name}' is non-nullable but 'possible_values' contains null-like values.", ) return col_errors # Return early once an error is found @@ -260,21 +297,17 @@ def _validate_nullable(self, col_config, col_errors, col_name): def _get_data_type_names(self) -> List[str]: """Gets all data type names from the loaded config.""" if "datatypes" not in self.config: - raise MissingDataTypesError( + error_str = ( "The 'datatypes' section is missing from the configuration file.", ) + raise MissingDataTypesError(error_str) all_type_names = [] for cat in self.config["datatypes"]: all_type_names.extend(self.config["datatypes"][cat]["types"]) return all_type_names - def _validate_data_type( - self, - col_config: Dict[str, Any], - col_errors: List[str], - col_name: str, - ) -> Dict[str, Any]: + def _validate_data_type(self, col_config, col_errors, col_name): """Validates the 'data_type' field in the schema. Checks for valid data types and appropriate use of min/max value and length constraints. @@ -282,24 +315,22 @@ def _validate_data_type( data_type = col_config.get("data_type") if not data_type or data_type == "": col_errors.append(f"Column '{col_name}' is missing a data_type.") + return col_errors # Return early if data_type is missing - if data_type not in self.all_data_types: + if data_type not in self.all_data_types: # Check for invalid data type col_errors.append( - f"{data_type} in column '{col_name}' is not a valid data type" + f"{data_type} in column '{col_name}' is not a valid data type", ) + return col_errors # Return early if it is not a valid type - elif data_type == "category": # possible_values must be present + if data_type == "category": # possible_values must be present if ( "possible_values" not in col_config or col_config["possible_values"] == "nan" ): col_errors.append( - f"Column '{col_name}' must have 'possible_values' if data_type is 'category'." + f"Column '{col_name}' must have 'possible_values' if data_type is 'category'.", ) - else: - col_errors.append( - f"Invalid data_type '{data_type}' specified for column '{col_name}'." - ) return col_errors @@ -351,7 +382,7 @@ def _validate_length( and col_config["length"] ): col_errors.append( - f"Column '{col_name}' is not a string type, it is a {data_type}. 'length' is not applicable." + f"Column '{col_name}' is not a string type, it is a {data_type}. 'length' is not applicable.", ) return col_errors @@ -363,7 +394,8 @@ def _validate_min_max( ) -> List[str]: """Validates 'min_value' and 'max_value' fields. - Checks that both min_value and max_value are numbers if specified for numeric or datetime types. + Checks that both min_value and max_value are numbers if specified for + numeric or datetime types. Checks that min_value is not greater than max_value. """ data_type = col_config.get("data_type") @@ -371,19 +403,21 @@ def _validate_min_max( if "min_value" in col_config: min_val = col_config["min_value"] if data_type in self.numeric_types and not isinstance( - min_val, (int, float) + min_val, + (int, float), ): col_errors.append( - f"Column '{col_name}' min_value must be a number for data_type '{data_type}'." + f"Column '{col_name}' min_value must be a number for data_type '{data_type}'.", ) if "max_value" in col_config: max_val = col_config["max_value"] if data_type in self.numeric_types and not isinstance( - max_val, (int, float) + max_val, + (int, float), ): col_errors.append( - f"Column '{col_name}' max_value must be a number for data_type '{data_type}'." + f"Column '{col_name}' max_value must be a number for data_type '{data_type}'.", ) if "min_value" in col_config and "max_value" in col_config: @@ -394,7 +428,7 @@ def _validate_min_max( and col_config["min_value"] > col_config["max_value"] ): col_errors.append( - f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}" + f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}", ) elif data_type in self.datetime_types: # Handle datetime comparisons # if a min or max time is specified, this validates that it can be parsed @@ -404,11 +438,11 @@ def _validate_min_max( if min_val > max_val: col_errors.append( - f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}" + f"Column '{col_name}' min_value cannot be greater than max_value for data_type: {data_type}", ) except (ValueError, TypeError) as e: # Catch time parsing errors col_errors.append( - f"Error comparing datetime values for column '{col_name}': {e}" + f"Error comparing datetime values for column '{col_name}': {e}", ) return col_errors @@ -434,7 +468,7 @@ def _validate_possible_values( ) data_type = col_config.get("data_type") - if data_type != "category": + if data_type != "category" and self.dataframe_type == "python": self.toml_val_logger.warning( # Use warnings.warn for non-categorical types f"Column '{col_name}': Using 'possible_values' with data_type '{data_type}' " f"might not be memory-efficient. Consider using 'category' data_type.", @@ -492,14 +526,14 @@ def _validate_regex_pattern( re.compile(pattern) # Check if the pattern is valid regex except re.error: errors.append( - f"Column '{col_name}': Invalid regex pattern '{pattern}'." + f"Column '{col_name}': Invalid regex pattern '{pattern}'.", ) data_type = col_config.get("data_type") if data_type not in self.string_types: errors.append( - f"Column '{col_name}': 'regex_pattern' can only be applied to string type columns." + f"Column '{col_name}': 'regex_pattern' can only be applied to string type columns.", ) return errors @@ -521,7 +555,10 @@ def _validate_unique( return col_errors def _validate_date_format( - self, col_config: Dict[str, Any], col_errors: List[str], col_name: str + self, + col_config: Dict[str, Any], + col_errors: List[str], + col_name: str, ) -> List[str]: # """Validates the 'date_format' field. @@ -541,16 +578,16 @@ def _validate_date_format( # Check that data_type matched the existence of date_format if data_type not in datetime_types: col_errors.append( - f"Column '{col_name}': 'date_format' can only be used with datetime types, not '{data_type}'." + f"Column '{col_name}': 'date_format' can only be used with datetime types, not '{data_type}'.", ) return col_errors # Stop further checks if the type is incorrect. # Check for date format useage errors try: - datetime.datetime.strptime("2024-05-03", date_format) # Use a test string. + datetime.datetime.strptime("2024-05-03", date_format) # noqa: DTZ007 except ValueError: col_errors.append( - f"Column '{col_name}': Invalid date format '{date_format}'." + f"Column '{col_name}': Invalid date format '{date_format}'.", ) return col_errors @@ -572,7 +609,7 @@ def _validate_number_str_format( if data_type not in self.numeric_types: col_errors.append( f"""Column '{col_name}': 'number_str_format' can only be - used with numeric types, not '{data_type}'.""" + used with numeric types, not '{data_type}'.""", ) return col_errors # Stop further checks if type is incorrect @@ -606,7 +643,7 @@ def _validate_number_str_format( KeyError, ) as e: # Catch all possible format errors col_errors.append( - f"Column '{col_name}': Invalid number format '{number_str_format}' - {e}" + f"Column '{col_name}': Invalid number format '{number_str_format}' - {e}", ) return col_errors @@ -633,7 +670,7 @@ def _validate_custom_check( check_function = getattr(data_validation, custom_check_val) if not callable(check_function): col_errors.append( - f"Column '{col_name}': '{custom_check_val}' is not a callable in data_validation.py." + f"Column '{col_name}': '{custom_check_val}' is not a callable in data_validation.py.", ) except AttributeError: # 2. Attempt to parse as Python code: @@ -641,11 +678,11 @@ def _validate_custom_check( compile(custom_check_val, "", "exec") except (SyntaxError, TypeError, ValueError) as e: col_errors.append( - f"Column '{col_name}': Invalid Python code or function name in 'custom_check': {e}" + f"Column '{col_name}': Invalid Python code or function name in 'custom_check': {e}", ) elif not callable(custom_check_val): # Handle non-string values col_errors.append( - f"Column '{col_name}': 'custom_check' must be a string or callable." + f"Column '{col_name}': 'custom_check' must be a string or callable.", ) return col_errors @@ -673,10 +710,10 @@ def _go_no_go(self, errors_dict, stop_on_errors=True, threshold=0): Defaults to 0. Raises + ------ ValueError: If the number of errors exceeds the threshold and stop_on_errors is True. """ - total_errors = sum( len(errors) for errors in errors_dict.values() if errors ) # only if errors != None @@ -690,10 +727,12 @@ def _go_no_go(self, errors_dict, stop_on_errors=True, threshold=0): error_messages.append(f"Column '{col}': {error}") raise ValueError( f"Validation failed with {total_errors} errors:\n" - + "\n".join(error_messages) + + "\n".join(error_messages), ) elif total_errors > 0: for col, errors in errors_dict.items(): + if not errors: + continue for error in errors: self.toml_val_logger.warning(f"Column '{col}': {error}") else: @@ -711,35 +750,39 @@ def validate_schema(self, schema: Dict[str, Any]) -> Dict[str, List[str]]: val_func = self.validation_functions.get(func_name) if val_func: errors[col_name] = val_func( - schema.get(col_name, {}), errors[col_name], col_name + schema.get(col_name, {}), + errors[col_name], + col_name, ) else: self.toml_val_logger.warning( - f"Validation function '{func_name}' not found. Skipping." + f"Validation function '{func_name}' not found. Skipping.", ) return errors - def run_validation(self, toml_path: str) -> None: - """Loads the schema, runs validation, and handles results.""" - schema = self._load_validation_schema(toml_path) + def run_validation(self, stop_on_errors=True, threshold=0) -> None: + """Loads the schema, runs validation, and handles results. - if not schema: # Handle empty schema gracefully + Entrypoint function. + """ + if not self.schema: # Handle empty schema gracefully self.toml_val_logger.error( - "Schema is empty. Cannot proceed with validation." + "Schema is empty. Cannot proceed with validation.", ) return - errors = self.validate_schema(schema) # Call validate_schema method + errors = self.validate_schema(self.schema) # Call validate_schema method self._log_errors(errors) # Log the errors - self._go_no_go(errors) # Make the go/no-go decision + self._go_no_go(errors, stop_on_errors, threshold) # Make the go/no-go decision + + self.toml_val_logger.info("Validation complete.") if __name__ == "__main__": - validator = TOMLSchemaValidator() # Create an instance of the validator - toml_file_path = ( + schema_file_path = ( Path("rdsa_utils") / "rdsa_data_validator" / "example_dataframe_schema.toml" ) - - validator.run_validation(str(toml_file_path)) + validator = TOMLSchemaValidator(schema_file_path=str(schema_file_path)) + validator.run_validation(stop_on_errors=False) diff --git a/rdsa_utils/rdsa_data_validator/config_validator_config.toml b/rdsa_utils/rdsa_data_validator/toml_schema_validator_config.toml similarity index 100% rename from rdsa_utils/rdsa_data_validator/config_validator_config.toml rename to rdsa_utils/rdsa_data_validator/toml_schema_validator_config.toml diff --git a/setup.cfg b/setup.cfg index 81baa4b3..9d2ad235 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = google-cloud-bigquery>=3.17.2 google-cloud-storage>=2.14.0 boto3>=1.34.103 - great-expectations>=1.3.0 + great-expectations [options.packages.find] where = . diff --git a/tests/cdp/helpers/test_s3_utils.py b/tests/cdp/helpers/test_s3_utils.py index 52b7bbf3..93511e4b 100644 --- a/tests/cdp/helpers/test_s3_utils.py +++ b/tests/cdp/helpers/test_s3_utils.py @@ -495,6 +495,20 @@ def test_list_files_no_match(self, s3_client_for_list_files): ) assert len(files) == 0 + def test_list_files_pagination(self, s3_client_for_list_files): + """Test listing >1000 files to verify pagination works correctly.""" + for i in range(1001): + s3_client_for_list_files.put_object( + Bucket="test-bucket", + Key=f"paginated/file_{i:04d}.txt", + Body=b"Test content", + ) + + files = list_files(s3_client_for_list_files, "test-bucket") + assert len(files) == 1006 + assert "paginated/file_0000.txt" in files + assert "paginated/file_1000.txt" in files + @pytest.fixture def s3_client_for_delete_and_copy(_aws_credentials): diff --git a/tests/invalid_test_schema.toml b/tests/invalid_test_schema.toml new file mode 100644 index 00000000..12dba107 --- /dev/null +++ b/tests/invalid_test_schema.toml @@ -0,0 +1,4 @@ +# invalid_test_schema.toml +[column1] + +description = "Test column 1 # Missing closing quote" diff --git a/tests/test_schema.toml b/tests/test_schema.toml new file mode 100644 index 00000000..bafc2c29 --- /dev/null +++ b/tests/test_schema.toml @@ -0,0 +1,11 @@ +# test_schema.toml +[data_asset] +name = "test_dataframe" + +[column1] +description = "Test column 1" +data_type = "int" + +[column2] +description = "Test column 2" +data_type = "StringType" diff --git a/tests/test_toml_schema_validator.py b/tests/test_toml_schema_validator.py new file mode 100644 index 00000000..74ccf36d --- /dev/null +++ b/tests/test_toml_schema_validator.py @@ -0,0 +1,82 @@ +import sys +from pathlib import Path +from typing import Any + +import pytest + +from rdsa_utils.rdsa_data_validator.toml_schema_validator import TOMLSchemaValidator + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + +# Fixtures to provide TOML file paths (adjust paths as needed) +@pytest.fixture +def valid_test_toml_path(): + """Return the path to a valid TOML file.""" + return str(Path(__file__).parent / "test_schema.toml") + + +@pytest.fixture +def invalid_test_toml_path(): + """Return the path to an invalid TOML file.""" + return str(Path(__file__).parent / "invalid_test_schema.toml") + + +@pytest.fixture +def toml_schema_validator_config_path(): + """Return the path to the TOML schema validator config file.""" + return str(Path(__file__).parent / "toml_schema_validator_config.toml") + + +@pytest.fixture +def non_dict_toml_path(): + """Return the path to a TOML file that is not a dictionary.""" + return str(Path(__file__).parent / "non_dict.toml") + + +# Tests +def test_load_validation_schema_valid( + valid_test_toml_path: str, + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading a valid TOML schema file.""" + validator = TOMLSchemaValidator( + schema_file_path=valid_test_toml_path, + config_file_path=toml_schema_validator_config_path, + ) # Schema & config loaded at init + assert validator.schema is not None # Check schema loaded correctly + assert validator.schema["column1"]["description"] == "Test column 1" + assert validator.schema["column2"]["data_type"] == "StringType" + + +def test_load_validation_schema_invalid_toml( + invalid_test_toml_path: str, + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading an invalid TOML schema file.""" + with pytest.raises(tomllib.TOMLDecodeError): + TOMLSchemaValidator( + schema_file_path=invalid_test_toml_path, # Provide path to invalid toml + config_file_path=toml_schema_validator_config_path, + ) + assert "Invalid TOML" in caplog.text # Check the specific message + + +def test_load_validation_schema_nonexistent_schema_file( + toml_schema_validator_config_path: str, + caplog: Any, +) -> None: + """Test loading a nonexistent TOML schema file.""" + nonexistent_path = "nonexistent_file.toml" # Or use tmp_path to create a guaranteed nonexistent path + + with pytest.raises(FileNotFoundError): + TOMLSchemaValidator( + schema_file_path=nonexistent_path, + config_file_path=toml_schema_validator_config_path, + ) + assert f"TOML file not found at: {nonexistent_path}" in caplog.text diff --git a/tests/toml_schema_validator_config.toml b/tests/toml_schema_validator_config.toml new file mode 100644 index 00000000..40641316 --- /dev/null +++ b/tests/toml_schema_validator_config.toml @@ -0,0 +1,20 @@ +# This is the config for the config_validator + +[paths] +example_schema_path = "rdsa_utils/rdsa_data_validator/config_validator_config.toml" + +log_file_path = "rdsa_utils/rdsa_data_validator/logs/toml_validation.log" + +[datatypes] + +[datatypes.python_types] +types = ["int", "float", "str", "bool", "list", "tuple", "dict", "set", "datetime.datetime"] + +[datatypes.pandas_numpy_types] +types = ["int64", "int32", "int16", "int8", "float64", "float32", "object", "bool_", "datetime64[ns]", "timedelta64[ns]", "category"] + +[datatypes.pyspark_types] +types = ["StringType", "IntegerType", "FloatType", "DoubleType", "BooleanType", "TimestampType", "DateType", "ArrayType", "MapType", "StructType"] + +[required_fields] +fields = ["description", "data_type", "nullable"]