diff --git a/.github/.pre-commit-config.yaml b/.github/.pre-commit-config.yaml index bcce425..bdfc541 100644 --- a/.github/.pre-commit-config.yaml +++ b/.github/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.3.0 hooks: - id: flake8 exclude: docs/conf.py args: ["--config", ".github/configurations/python_linters/.flake8"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 8.0.1 hooks: - id: isort - args: ["--settings-path", ".github/configurations/python_linters/.isort.cfg", "--filter-files", "--skip", "venv"] + args: ["--settings-path", ".github/configurations/python_linters/.isort.cfg", "--profile", "black", "--filter-files", "--skip", "venv"] - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 26.5.1 hooks: - id: black args: ["--config", ".github/configurations/python_linters/.black"] diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f155215..337fc91 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -26,7 +26,7 @@ jobs: id: diff_check run: | git branch -a --list | cat - PYTHON_CODE_CHANGES=$(git diff --compact-summary origin/${{ github.base_ref }} -- routingfilter/* | wc -l) + PYTHON_CODE_CHANGES=$(git diff --compact-summary origin/${{ github.base_ref }} -- routingfilter routing_test.py test_data pyproject.toml .github/workflows/python-app.yml .github/configurations | wc -l) echo "python_code=$PYTHON_CODE_CHANGES" >> "$GITHUB_OUTPUT" build: @@ -40,35 +40,20 @@ jobs: id: setup_python uses: actions/setup-python@v7 with: - python-version: "3.10" - - name: "Cache venv" - id: cache_venv - uses: actions/cache@v6 - with: - path: venv - key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies - if: steps.cache_venv.outputs.cache-hit != 'true' run: | - if [ -d "venv" ]; then rm -rf venv; fi - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip - pip install flake8 black isort pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Show which copy is under test + run: python -c "import pathlib, routingfilter; print(pathlib.Path(routingfilter.__file__).resolve())" - name: "Lint with black" - run: | - source venv/bin/activate - black ./routingfilter --config .github/configurations/python_linters/.black --check --diff + run: black ./routingfilter --config .github/configurations/python_linters/.black --check --diff - name: Lint with flake8 - run: | - source venv/bin/activate - flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source + run: flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source - name: "Isort check" - run: | - source venv/bin/activate - isort ./routingfilter --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff + run: isort ./routingfilter --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff - name: Test with pytest - run: | - source ./venv/bin/activate - pytest routing_test.py + run: python -m pytest routing_test.py diff --git a/.gitignore b/.gitignore index 7196391..20eac76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,35 @@ -.vscode -venv -.ropeproject - -# compiled source -*.com -*.class -*.dll -*.exe -*.o -*.so +# IDE and editors +.idea/ +.vscode/ +.ropeproject/ + +# Virtual environments +venv/ +.venv/ + +# Byte-compiled files +__pycache__/ *.pyc -# packages -*.7z -*.dmg -*.iso -*.jar -*.rar +# Caches +.pytest_cache/ -# logs and databases +# Logs and databases *.log *.log.* -*.sql -*.sqlite + +# Environment files +.env # OS generated files .DS_Store .DS_Store? ._* -# Pycharm -.idea/ - # Documentation -docs/_build +docs/_build/ # Build files build/ dist/ -routingfilter.egg-info/ \ No newline at end of file +*.egg-info/ \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 26d3352..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml diff --git a/.idea/RoutingFilter.iml b/.idea/RoutingFilter.iml deleted file mode 100644 index 3d7f1d8..0000000 --- a/.idea/RoutingFilter.iml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 50fcc30..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 7a4d831..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 37d421b..399fd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,16 @@ +## 2.5.x +### 2.5.0 +#### Changes +* Fixed type hints and improved filter robustness, cleaned `.gitignore` +* Migrated packaging metadata from `setup.py` to `pyproject.toml`: removed `setup.py`, `requirements.txt`, `requirements_dev.txt` and `MANIFEST.in`. Development dependencies are now the `dev` extra, installed with `pip install -e ".[dev]"` +* Python 3.12 is now the minimum required version (was 3.10) +* Updated linters and aligned the pre-commit hook revisions with the `dev` extra pins ## 2.4.x ### 2.4.0 #### Bugfix * Fixed bug in CI about isort execution and cache #### Changes -* Updated linters, required Python version (>3.10) and Github Actions +* Updated linters, required Python version (>3.10) and GitHub Actions * Removed dependency from `IPy` in favor of `ipdaddress` ## 2.3.x ### 2.3.3 @@ -65,7 +72,7 @@ ## 1.6.x ### 1.6.3 #### Bugfix -* Fix overwriting of certego field +* Fix overwriting of `certego` field ### 1.6.2 #### Bugfix * Added filter in routing_filter @@ -147,7 +154,7 @@ * An exception is returned (instead of False) when processing a wrong filter type ### 1.1.0 #### Changes -* Changed match output to to list of dicts, to allow multiple rules with different tags to match +* Changed match output to list of dicts, to allow multiple rules with different tags to match * Now the method laod_from_dicts returns nothing; a new getter has been added to retrieve the currently loaded rules ## 1.0.x diff --git a/CLAUDE.md b/CLAUDE.md index e63c041..df35019 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ All commands are run from the **repository root** (not from the inner `routingfi ```bash # Install for development -pip install -r requirements.txt -r requirements_dev.txt +pip install -e ".[dev]" pre-commit install -c .github/.pre-commit-config.yaml # Run the full test suite @@ -93,4 +93,4 @@ Extensive real examples of both rules and events live in `test_data/` (`test_rul ## Release process -Update `requirements.txt`/`setup.py` if needed, add a `CHANGELOG.md` entry, bump the version in `setup.py`, merge to `master`, then publish a GitHub release tagged with the version — CI (`python-publish.yml`) publishes to PyPI automatically. \ No newline at end of file +Update the dependencies in `pyproject.toml` if needed, add a `CHANGELOG.md` entry, bump `version` in `pyproject.toml`, merge to `master`, then publish a GitHub release tagged with the version — CI (`python-publish.yml`) publishes to PyPI automatically. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 830515a..b78f920 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -i# Contributing to pcapoptikon +# Contributing to RoutingFilter Thank you for your interest in contributing to Certego’s Open Source projects! To protect you, our users, and the integrity of the software, we require all contributors to accept our **Contributor License Agreement (CLA)**. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 291fe9f..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -exclude routingfilter/routing_tests.py \ No newline at end of file diff --git a/README.md b/README.md index c366d35..876513e 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ routing.match(test_event_1) See the [online documentation](https://routingfilter.readthedocs.io/en/latest/) for further details. ### Release steps -* (If needed) Update the requirements in `requirements.txt` and `setup.py` +* (If needed) Update the dependencies in `pyproject.toml` * Add a new entry in `CHANGELOG.md` with the new version number -* Update the version number in `setup.py` +* Update the version number in `pyproject.toml` * Commit and merge the changes into `master` branch -* Publish a new release with the version number as a tag: the CI will automatically publish the new version un PyPI +* Publish a new release with the version number as a tag: the CI will automatically publish the new version on PyPI ### Benchmark tests In order to launch the benchmark tests, run ```python routing_benchmark.py``` ### Development -* Install `pip install -r requirements.txt` and `pip install -r requirements_dev.txt` in your local virtual environment +* Install `pip install -e ".[dev]"` in your local virtual environment * Setup pre-commit: `pre-commit install -c .github/.pre-commit-config.yaml` ### License diff --git a/docs/conf.py b/docs/conf.py index 600053c..03ebe7a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # import os import sys + import sphinx_rtd_theme # sys.path.insert(0, os.path.abspath('.')) diff --git a/pyproject.toml b/pyproject.toml index 3ebf33b..3871719 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,36 @@ [build-system] -requires = [ - "setuptools>=60", - "wheel" +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "routingfilter" +version = "2.5.0" +description = "Generic Business Logic Implementation for Routing objects as python dictionaries" +readme = "README.md" +requires-python = ">=3.12" +license = "LGPL-3.0-or-later" +license-files = ["LICENSE.txt"] +authors = [{ name = "Certego S.r.l.", email = "support@certego.net" }] +dependencies = ["macaddress~=2.0.2"] +classifiers = [ + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", ] -build-backend = "setuptools.build_meta" \ No newline at end of file + +[project.optional-dependencies] +# The linter pins below must stay in sync with the hook revisions in .github/.pre-commit-config.yaml: nothing enforces it automatically. +dev = [ + "black==26.5.1", + "flake8==7.3.0", + "isort==8.0.1", + "pre-commit==4.6.2", + "pytest", +] + +[project.urls] +Homepage = "https://github.com/certego/RoutingFilter" + +[tool.setuptools.packages.find] +include = ["routingfilter*"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e8eb67e..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -macaddress~=2.0.2 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 7d61e9e..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -black==26.5.1 -flake8==7.3.0 -isort==8.0.1 -pre-commit==4.6.0 -setuptools~=83.0 \ No newline at end of file diff --git a/routingfilter/filters/filters.py b/routingfilter/filters/filters.py index eb8d559..d7cef4d 100644 --- a/routingfilter/filters/filters.py +++ b/routingfilter/filters/filters.py @@ -2,7 +2,7 @@ import logging import re from abc import ABC, abstractmethod -from typing import NoReturn, Optional +from typing import Any import macaddress from routingfilter.dictquery import DictQuery @@ -20,12 +20,13 @@ def match(self, event: DictQuery) -> bool: return NotImplemented @abstractmethod - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if values in self._value are correct and raise an exception if they are incorrect. If necessary, it converts value in lower case. :return: no value or raise an exception - :rtype: NoReturn | Exception + :rtype: None + :raises Exception: if the value is not correct """ return NotImplemented @@ -35,7 +36,7 @@ def __init__(self): key = value = [] super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: return def match(self, event: DictQuery) -> bool: @@ -55,7 +56,7 @@ def __init__(self, key): value = [] super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: return def match(self, event: DictQuery) -> bool: @@ -90,7 +91,7 @@ class EqualFilter(AbstractFilter): def __init__(self, key, value): super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: tmp = [] for value in self._value: value = str(value).lower() @@ -130,7 +131,7 @@ def match(self, event: DictQuery) -> bool: class StartswithFilter(AbstractFilter): - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: tmp = [] for prefix in self._value: prefix = str(prefix).lower() @@ -171,7 +172,7 @@ def _check_startswith(self, value: str) -> bool: class EndswithFilter(AbstractFilter): - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: tmp = [] for suffix in self._value: suffix = str(suffix).lower() @@ -212,7 +213,7 @@ def _check_endswith(self, value: str) -> bool: class KeywordFilter(AbstractFilter): - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: tmp = [] for keyword in self._value: keyword = str(keyword).lower() @@ -253,12 +254,13 @@ def _check_keyword(self, value: str) -> bool: class RegexpFilter(AbstractFilter): - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if values in self._value are valid regexes. :return: none or error generated: - :rtype: Optional[Exception] + :rtype: None + :raises ValueError: regex check failed """ tmp = [] for value in self._value: @@ -305,24 +307,26 @@ class NetworkFilter(AbstractFilter): def __init__(self, key, value): super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if the values in self._value are valid IP addresses. :return: none or error generated - :rtype: Optional[Exception] + :rtype: None + :raises ValueError: IP address parsing failed """ tmp = [] for value in self._value: try: - value = ipaddress.ip_network(value) + network = ipaddress.ip_network(value) except ValueError as e: self.logger.error(f"IP address (value error) error, during check of value {value} in list {self._value}. Error was: {e}.") raise ValueError(f"IP address check failed: value error for value {value}.") except TypeError as e: self.logger.error(f"IP address (type error) error, during check of value {value} in list {self._value}. Error was: {e}.") raise ValueError(f"IP address check failed: type error for value {value}.") - tmp.append(value) + else: + tmp.append(network) self._value = tmp def match(self, event: DictQuery) -> bool: @@ -381,7 +385,7 @@ class DomainFilter(AbstractFilter): def __init__(self, key, value): super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if values in self._value are string. @@ -434,12 +438,12 @@ def __init__(self, key, value, comparator_type): self._check_comparator_type() super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if values in self._value are float. :return: none or error generated - :rtype: Exception | NoReturn + :rtype: None """ tmp = [] for value in self._value: @@ -450,12 +454,12 @@ def _check_value(self) -> Exception | NoReturn: raise ValueError(f"Comparator check failed: value {value} is not a float") self._value = tmp - def _check_comparator_type(self) -> Exception | NoReturn: + def _check_comparator_type(self) -> None: """ Check if comparator is valid. :return: none or error generated - :rtype: Exception | NoReturn + :rtype: None """ if self._comparator_type not in ["GREATER", "LESS", "GREATER_EQ", "LESS_EQ"]: self.logger.error(f"Comparator check failed: value {self._comparator_type} is not valid.") @@ -513,12 +517,12 @@ class TypeofFilter(AbstractFilter): def __init__(self, key, value): super().__init__(key, value) - def _check_value(self) -> Exception | NoReturn: + def _check_value(self) -> None: """ Check if value is a correct type. :return: no value or raised an exception - :rtype: NoReturn | Exception + :rtype: None """ valid_type = ["str", "int", "float", "bool", "list", "dict", "ip", "mac"] tmp = [] @@ -542,16 +546,17 @@ def match(self, event: DictQuery) -> bool: for key in self._key: for val_type in self._value: - if self._check_type(event.get(key), val_type): + if TypeofFilter._check_type(event.get(key), val_type): return True return False - def _check_type(self, value: any, val_type: str) -> bool: + @staticmethod + def _check_type(value: str | int | float | list | dict, val_type: str) -> bool: """ Check type of the value. :param value: value to check - :type value: any + :type value: str | int | float | list | dict :param val_type: type :rtype str :return: true or false @@ -570,17 +575,18 @@ def _check_type(self, value: any, val_type: str) -> bool: elif val_type == "dict": return type(value) is dict elif val_type == "ip": - return self._check_ip(value) + return TypeofFilter._check_ip(value) elif val_type == "mac": - return self._check_mac(value) + return TypeofFilter._check_mac(value) return False - def _check_ip(self, value: any) -> bool: + @staticmethod + def _check_ip(value: Any) -> bool: """ Check if value is IP address. :param value: value to check - :type: any + :type: Any :return: true or false :rtype: bool """ @@ -590,18 +596,21 @@ def _check_ip(self, value: any) -> bool: except ValueError: try: ipaddress.ip_address(value) - return True except ValueError: return False except TypeError: return False + else: + return True + return False - def _check_mac(self, value: any) -> bool: + @staticmethod + def _check_mac(value: Any) -> bool: """ Check if value is a MAC address. :param value: value to check - :type value: any + :type value: Any :return: true or false :rtype: bool """ diff --git a/routingfilter/filters/results.py b/routingfilter/filters/results.py index d46c8c5..f134512 100644 --- a/routingfilter/filters/results.py +++ b/routingfilter/filters/results.py @@ -4,11 +4,11 @@ @dataclass class Results: rules: str - output: dict + output: dict | None - def __init__(self, rules, output): + def __init__(self, rules: str, output: dict | None): self.rules = rules self.output = output["customer"] if output is not None and "customer" in output.keys() else output - def to_dict(self): + def to_dict(self) -> dict: return {"output": self.output, "rules": self.rules} diff --git a/setup.py b/setup.py deleted file mode 100644 index 75ae195..0000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import find_packages, setup - -with open("README.md", "r") as readme_file: - long_description = readme_file.read() - -setup( - name="routingfilter", - version="2.4.0", - packages=find_packages(include=["routingfilter", "routingfilter.*"]), - include_package_data=True, - install_requires=["macaddress~=2.0.2"], - url="https://github.com/certego/RoutingFilter", - license="GNU LGPLv3", - author="Certego S.r.l.", - author_email="support@certego.net", - description="Generic Business Logic Implementation for Routing objects as python dictionaries", - long_description=long_description, - long_description_content_type="text/markdown", - classifiers=[ - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.10", -)