Feat/uv

.github/workflows/python-package.yml

@@ -13,30 +13,14 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.python-version }}
-        id: setup-python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
         with:
-          # Poetry >= 2.3 requires Python >= 3.10, but the matrix still tests 3.9
-          version: 2.2.1
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v3
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+      - name: Set up python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      - name: Install library
-        run: poetry install --no-interaction
+        run: uv sync --python ${{ matrix.python-version }} --locked
       - name: Run tests
-        run: |
-          source .venv/bin/activate
-          pytest . --cov-report=xml --cov=.
+        run: uv run pytest . --cov-report=xml --cov=.

README.md

@@ -6,6 +6,8 @@ This is a Python wrapper for [CrawlerDetect](https://github.com/JayBizzle/Crawle
 bots, crawlers, and spiders using the user agent and other HTTP headers. Currently, it can detect
 over 3,678 bots, spiders, and crawlers.
 
+> Currently synced with [JayBizzle/Crawler-Detect `v1.4.1`](https://github.com/JayBizzle/Crawler-Detect/releases/tag/v1.4.1).
+
 # How to install
 ```bash
 $ pip install crawlerdetect
@@ -66,14 +68,16 @@ Failing that, just create an issue with the user agent you have found, and we'll
 
 # Development
 
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management and packaging.
+
 ## Setup
 ```bash
-$ poetry install
+$ uv sync
 ```
 
 ## Running tests
 ```bash
-$ poetry run pytest
+$ uv run pytest
 ```
 
 ## Update crawlers from upstream PHP repo
@@ -83,5 +87,5 @@ $ ./update_data.sh
 
 ## Bump version
 ```bash
-$ poetry run bump-my-version bump [patch|minor|major]
+$ uv run bump-my-version bump [patch|minor|major]
 ```

crawlerdetect/providers/crawlers.py

@@ -14,11 +14,14 @@
     r"^DHSH",
     r"^docker\/[0-9]",
     r"^Expanse",
+    r"^ExposureScanner",
     r"^FDM ",
     r"^git\/",
     r"^Goose\/",
+    r"^Goosee-Audit",
     r"^Grabber",
     r"^Gradle\/",
+    r"^Hello from Palo Alto Networks",
     r"^HTTPClient\/",
     r"^HTTPing",
     r"^Java\/",
@@ -44,6 +47,7 @@
     r"^TLS tester ",
     r"^twine\/",
     r"^ureq",
+    r"^visionheight\.com\/scan",
     r"^VSE\/[0-9]",
     r"^WordPress\.com",
     r"^XRL\/[0-9]",
@@ -415,6 +419,7 @@
     r"fluffy",
     r"Flunky",
     r"flynxapp",
+    r"foda-scanner\/[0-9]",
     r"forensiq",
     r"ForusP",
     r"FoundSeoTool",
@@ -529,6 +534,7 @@
     r"HappyApps-WebCheck",
     r"Hardenize",
     r"Hatena",
+    r"HaveInListEnricher",
     r"Havij",
     r"HaxerMen",
     r"HEADMasterSEO",
@@ -608,6 +614,7 @@
     r"infohelfer",
     r"InfoTekies",
     r"InfoWizards Reciprocal Link",
+    r"Infrawatch\/[0-9]",
     r"inpwrd\.com",
     r"instabid",
     r"Instapaper",
@@ -674,6 +681,7 @@
     r"KOCMOHABT",
     r"kouio",
     r"krawler\.dk",
+    r"krynos",
     r"kube-probe",
     r"kubectl",
     r"kulturarw3",
@@ -783,6 +791,7 @@
     r"mixed-content-scan",
     r"mixnode",
     r"Mnogosearch",
+    r"ModatScanner\/[0-9]",
     r"mogimogi",
     r"Mojeek",
     r"Mojolicious \(Perl\)",
@@ -810,6 +819,7 @@
     r"myseosnapshot",
     r"nagios",
     r"Najdi\.si",
+    r"holubem\.eu",
     r"Name Intelligence",
     r"NameFo\.com",
     r"Nameprotect",
@@ -916,6 +926,7 @@
     r"panscient",
     r"Papa Foto",
     r"parsijoo",
+    r"pathscan\/[0-9]",
     r"Pavuk",
     r"PayPal IPN",
     r"pcBrowser",
@@ -1449,5 +1460,5 @@
     r"Zoom\.Mac",
     r"ZoteroTranslationServer",
     r"ZyBorg",
-    r"[a-z0-9\-_]*(bot|crawl|headless|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"
+    r"[a-z0-9\-_]*(bot|crawl|headless|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)",
 ]