CI

.github/workflows/python-package.yml

@@ -8,35 +8,19 @@ jobs:
       fail-fast: true
       matrix:
         os: [ "ubuntu-latest", "macos-latest" ]
-        python-version: [ "3.9",]
+        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.python-version }}
-        id: setup-python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+        uses: actions/checkout@v7
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
         with:
-          # Poetry >= 2.3 requires Python >= 3.10, but the matrix still tests 3.9
-          version: 2.2.1
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v3
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+      - name: Set up python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      - name: Install library
-        run: poetry install --no-interaction
+        run: uv sync --python ${{ matrix.python-version }} --locked
       - name: Run tests
-        run: |
-          source .venv/bin/activate
-          pytest . --cov-report=xml --cov=.
+        run: uv run pytest . --cov-report=xml --cov=.

README.md

@@ -6,6 +6,8 @@ This is a Python wrapper for [CrawlerDetect](https://github.com/JayBizzle/Crawle
 bots, crawlers, and spiders using the user agent and other HTTP headers. Currently, it can detect
 over 3,678 bots, spiders, and crawlers.
 
+> Currently synced with [JayBizzle/Crawler-Detect `v1.4.1`](https://github.com/JayBizzle/Crawler-Detect/releases/tag/v1.4.1).
+
 # How to install
 ```bash
 $ pip install crawlerdetect
@@ -66,14 +68,16 @@ Failing that, just create an issue with the user agent you have found, and we'll
 
 # Development
 
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management and packaging.
+
 ## Setup
 ```bash
-$ poetry install
+$ uv sync
 ```
 
 ## Running tests
 ```bash
-$ poetry run pytest
+$ uv run pytest
 ```
 
 ## Update crawlers from upstream PHP repo
@@ -82,6 +86,49 @@ $ ./update_data.sh
 ```
 
 ## Bump version
+Bumping updates the version in `pyproject.toml` and `crawlerdetect/__init__.py`, commits the change, and creates a matching `vX.Y.Z` git tag.
+
+Pass the part to increment:
+```bash
+# 0.4.1 -> 0.4.2
+$ uv run bump-my-version bump patch
+
+# 0.4.1 -> 0.5.0
+$ uv run bump-my-version bump minor
+
+# 0.4.1 -> 1.0.0
+$ uv run bump-my-version bump major
+```
+
+Or set an explicit version instead of incrementing a part:
+```bash
+$ uv run bump-my-version bump --new-version 0.4.1
+```
+
+Preview the change first with `--dry-run -v` (nothing is written or committed):
+```bash
+$ uv run bump-my-version bump --dry-run -v patch
+```
+
+`bump-my-version bump` on its own (no part, no `--new-version`) fails with `Unable to get the next version.` — one of the two is required.
+
+You may also see a harmless warning:
+```
+Specified version (0.3.3) does not match last tagged version (0.3.2)
+```
+This just means the version in `pyproject.toml` and the latest `vX.Y.Z` git tag are out of sync (e.g. after a manual version edit) — bump-my-version still uses `pyproject.toml` as the source of truth and bumps correctly regardless.
+
+Once bumped, push the commit and tag, then [publish to PyPI](#how-to-publish-a-new-release-to-pypi):
+```bash
+$ git push && git push --tags
+```
+
+## How to publish a new release to PyPI
+```bash
+$ uv build
+$ uv publish
+```
+`uv publish` needs a PyPI token, provided via `UV_PUBLISH_TOKEN` (or `--token`):
 ```bash
-$ poetry run bump-my-version bump [patch|minor|major]
+$ UV_PUBLISH_TOKEN=pypi-... uv publish
 ```

crawlerdetect/__init__.py

@@ -3,4 +3,4 @@
 
 __all__ = ("CrawlerDetect", "providers", "__version__")
 
-__version__ = "0.3.3"
+__version__ = "0.4.1"

crawlerdetect/providers/crawlers.py

@@ -14,11 +14,14 @@
     r"^DHSH",
     r"^docker\/[0-9]",
     r"^Expanse",
+    r"^ExposureScanner",
     r"^FDM ",
     r"^git\/",
     r"^Goose\/",
+    r"^Goosee-Audit",
     r"^Grabber",
     r"^Gradle\/",
+    r"^Hello from Palo Alto Networks",
     r"^HTTPClient\/",
     r"^HTTPing",
     r"^Java\/",
@@ -44,6 +47,7 @@
     r"^TLS tester ",
     r"^twine\/",
     r"^ureq",
+    r"^visionheight\.com\/scan",
     r"^VSE\/[0-9]",
     r"^WordPress\.com",
     r"^XRL\/[0-9]",
@@ -415,6 +419,7 @@
     r"fluffy",
     r"Flunky",
     r"flynxapp",
+    r"foda-scanner\/[0-9]",
     r"forensiq",
     r"ForusP",
     r"FoundSeoTool",
@@ -529,6 +534,7 @@
     r"HappyApps-WebCheck",
     r"Hardenize",
     r"Hatena",
+    r"HaveInListEnricher",
     r"Havij",
     r"HaxerMen",
     r"HEADMasterSEO",
@@ -608,6 +614,7 @@
     r"infohelfer",
     r"InfoTekies",
     r"InfoWizards Reciprocal Link",
+    r"Infrawatch\/[0-9]",
     r"inpwrd\.com",
     r"instabid",
     r"Instapaper",
@@ -674,6 +681,7 @@
     r"KOCMOHABT",
     r"kouio",
     r"krawler\.dk",
+    r"krynos",
     r"kube-probe",
     r"kubectl",
     r"kulturarw3",
@@ -783,6 +791,7 @@
     r"mixed-content-scan",
     r"mixnode",
     r"Mnogosearch",
+    r"ModatScanner\/[0-9]",
     r"mogimogi",
     r"Mojeek",
     r"Mojolicious \(Perl\)",
@@ -810,6 +819,7 @@
     r"myseosnapshot",
     r"nagios",
     r"Najdi\.si",
+    r"holubem\.eu",
     r"Name Intelligence",
     r"NameFo\.com",
     r"Nameprotect",
@@ -916,6 +926,7 @@
     r"panscient",
     r"Papa Foto",
     r"parsijoo",
+    r"pathscan\/[0-9]",
     r"Pavuk",
     r"PayPal IPN",
     r"pcBrowser",
@@ -1449,5 +1460,5 @@
     r"Zoom\.Mac",
     r"ZoteroTranslationServer",
     r"ZyBorg",
-    r"[a-z0-9\-_]*(bot|crawl|headless|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"
+    r"[a-z0-9\-_]*(bot|crawl|headless|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)",
 ]