fix: harden CrowdSec preset apply and hub data persistence

.docker/docker-entrypoint.sh

@@ -10,6 +10,23 @@ is_root() {
     [ "$(id -u)" -eq 0 ]
 }
 
+# Print (one per line) the data files declared by installed CrowdSec hub items
+# (`data: - dest_file:` entries in their YAML) that are absent from the data dir (an empty file is legitimate, e.g. an empty blocklist).
+# `cscli ... inspect -o json` does not expose declared data files, so the installed item
+# files under the config dir are read directly. Prints nothing when everything is present.
+# Usage: list_missing_hub_data_files <config_dir> <data_dir>
+list_missing_hub_data_files() {
+    _cfg="$1"
+    _data="$2"
+    for _sub in parsers scenarios postoverflows contexts; do
+        [ -d "$_cfg/$_sub" ] || continue
+        find -L "$_cfg/$_sub" -type f -name '*.yaml' -exec grep -h 'dest_file:' {} + 2>/dev/null || true
+    done | sed -e 's/^.*dest_file:[[:space:]]*//' -e 's/[[:space:]]*#.*$//' -e "s/[\"']//g" -e 's/[[:space:]]*$//' | sort -u | while IFS= read -r _file; do
+        [ -n "$_file" ] || continue
+        [ -e "$_data/$_file" ] || echo "$_file"
+    done
+}
+
 run_as_charon() {
     if is_root; then
         gosu charon "$@"
@@ -320,6 +337,44 @@ ACQUIS_EOF
         echo "⚠️  WARNING: Could not verify LAPI db_path redirect — bouncer keys may not survive rebuilds"
     fi
 
+    # Redirect CrowdSec data_dir (hub data files: blocklists, GeoLite2 databases, ...) to the
+    # persistent volume. The default /var/lib/crowdsec/data/ lives in the image layer, so after a
+    # container recreation the hub items (kept under /etc/crowdsec -> volume) are still installed
+    # but their data files are gone, which makes CrowdSec log "unable to init data for file".
+    # Idempotent: a config that already points at the volume is left untouched.
+    if grep -q "data_dir: /var/lib/crowdsec/data" "$CS_CONFIG_DIR/config.yaml"; then
+        sed -i "s|data_dir: /var/lib/crowdsec/data/\?\$|data_dir: ${CS_DATA_DIR}/|" "$CS_CONFIG_DIR/config.yaml"
+        # Existing installs: carry over anything an operator kept at the old location
+        # (e.g. a mounted volume), without overwriting files already on the volume.
+        if [ -d /var/lib/crowdsec/data ] && find /var/lib/crowdsec/data -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null | grep -q .; then
+            cp -R -n /var/lib/crowdsec/data/. "$CS_DATA_DIR/" 2>/dev/null || echo "Warning: could not copy existing CrowdSec data files to $CS_DATA_DIR"
+        fi
+    fi
+    if grep -q "data_dir: ${CS_DATA_DIR}/\?\$" "$CS_CONFIG_DIR/config.yaml"; then
+        echo "✓ CrowdSec data_dir redirected to persistent volume: ${CS_DATA_DIR}/"
+    else
+        echo "⚠️  WARNING: Could not verify CrowdSec data_dir redirect — hub data files may not survive container recreation"
+    fi
+
+    # Pin the hub branch. Without cscli.hub_branch, every hub command first asks
+    # version.crowdsec.net for the latest release and derives the branch from the running version,
+    # which hard-fails offline and 404s once the CDN has no branch named after this version.
+    # A non-empty hub_branch is used as-is (no lookup). Idempotent: an operator-chosen non-empty
+    # value is respected; missing/empty/commented values are set to master.
+    if ! grep -Eq '^[[:space:]]+hub_branch:[[:space:]]*[^[:space:]#]' "$CS_CONFIG_DIR/config.yaml"; then
+        sed -i '/^[[:space:]]\+hub_branch:/d' "$CS_CONFIG_DIR/config.yaml"
+        if grep -q '^cscli:' "$CS_CONFIG_DIR/config.yaml"; then
+            sed -i '/^cscli:[[:space:]]*$/a\  hub_branch: master' "$CS_CONFIG_DIR/config.yaml"
+        else
+            printf '\ncscli:\n  hub_branch: master\n' >>"$CS_CONFIG_DIR/config.yaml"
+        fi
+    fi
+    if grep -Eq '^[[:space:]]+hub_branch:[[:space:]]*[^[:space:]#]' "$CS_CONFIG_DIR/config.yaml"; then
+        echo "✓ CrowdSec hub branch pinned: $(sed -n 's/^[[:space:]]*hub_branch:[[:space:]]*//p' "$CS_CONFIG_DIR/config.yaml" | head -n1)"
+    else
+        echo "⚠️  WARNING: Could not pin cscli.hub_branch — hub commands may depend on version.crowdsec.net"
+    fi
+
     # Verify LAPI configuration was applied correctly
     if grep -q "listen_uri:.*:8085" "$CS_CONFIG_DIR/config.yaml"; then
         echo "✓ CrowdSec LAPI configured for port 8085"
@@ -353,6 +408,26 @@ ACQUIS_EOF
         if [ -x /usr/local/bin/install_hub_items.sh ]; then
             /usr/local/bin/install_hub_items.sh || echo "⚠️ Some hub items may not have installed. CrowdSec can still start."
         fi
+
+        # Hub items live under /etc/crowdsec (volume) but their data files live in data_dir. If the
+        # data files are gone while the items are installed (e.g. the data directory was recreated),
+        # `cscli hub upgrade` is the only command that re-downloads them (`install` is a no-op for
+        # installed items). It upgrades every installed item, so it runs ONLY in this recovery case,
+        # never on a healthy start, is bounded, never forced and never fatal.
+        MISSING_HUB_DATA="$(list_missing_hub_data_files "$CS_CONFIG_DIR" "$CS_DATA_DIR")"
+        if [ -n "$MISSING_HUB_DATA" ]; then
+            echo "⚠️ CrowdSec hub data files missing from $CS_DATA_DIR: $(echo "$MISSING_HUB_DATA" | tr '\n' ' ')"
+            echo "Running 'cscli hub upgrade' to restore them (this also upgrades installed hub items)..."
+            if ! timeout 120s cscli hub upgrade 2>&1; then
+                echo "⚠️ Hub upgrade failed or timed out. CrowdSec will start, but some detection data may be missing until the next restart."
+            fi
+            STILL_MISSING="$(list_missing_hub_data_files "$CS_CONFIG_DIR" "$CS_DATA_DIR")"
+            if [ -n "$STILL_MISSING" ]; then
+                echo "⚠️ CrowdSec hub data files still missing: $(echo "$STILL_MISSING" | tr '\n' ' ')"
+            else
+                echo "✓ CrowdSec hub data files restored"
+            fi
+        fi
     fi
 
     # Fix ownership AFTER cscli commands (they run as root and create root-owned files)

.dockerignore

@@ -168,6 +168,7 @@ scripts/tests/
 scripts/toolchain-key.sh
 scripts/verify-toolchain-pin.sh
 scripts/lib/dockerfile-stage.sh
+scripts/crowdsec_data_persistence_test.sh
 
 # -----------------------------------------------------------------------------
 # Tools (not needed in image)

.github/renovate.json

@@ -415,6 +415,19 @@
       "datasourceTemplate": "npm",
       "versioningTemplate": "semver"
     },
+    {
+      "customType": "regex",
+      "description": "Track golang.org/x/mod version ARG in Dockerfile",
+      "managerFilePatterns": [
+        "/^Dockerfile$/"
+      ],
+      "matchStrings": [
+        "#\\s*renovate:\\s*datasource=go\\s+depName=golang\\.org/x/mod\\s*\\nARG XMOD_VERSION=(?<currentValue>[^\\s]+)"
+      ],
+      "depNameTemplate": "golang.org/x/mod",
+      "datasourceTemplate": "go",
+      "versioningTemplate": "semver"
+    },
     {
       "customType": "regex",
       "description": "Track golang.org/x/crypto version ARG in Dockerfile",

.github/skills/integration-test-crowdsec-data-persistence-scripts/run.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Integration Test CrowdSec Data Persistence - Wrapper Script
+# Verifies hub data files survive container recreation
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+
+exec "${PROJECT_ROOT}/scripts/crowdsec_data_persistence_test.sh" "$@"

.github/skills/integration-test-crowdsec-data-persistence.SKILL.md

@@ -0,0 +1,69 @@
+---
+# agentskills.io specification v1.0
+name: "integration-test-crowdsec-data-persistence"
+version: "1.0.0"
+description: "Verify CrowdSec hub data files persist across container recreation"
+author: "Charon Project"
+license: "MIT"
+tags:
+  - "integration"
+  - "crowdsec"
+  - "persistence"
+  - "hub"
+compatibility:
+  os:
+    - "linux"
+    - "darwin"
+  shells:
+    - "bash"
+requirements:
+  - name: "docker"
+    version: ">=24.0"
+    optional: false
+environment_variables:
+  - name: "CHARON_TEST_IMAGE"
+    description: "Image to test"
+    default: "charon:local"
+    required: false
+  - name: "INIT_TIMEOUT_SECONDS"
+    description: "Maximum wait for CrowdSec hub initialization per container start"
+    default: "300"
+    required: false
+parameters: []
+outputs:
+  - name: "test_results"
+    type: "stdout"
+    description: "PASS/FAIL results per check"
+metadata:
+  category: "integration-test"
+  subcategory: "crowdsec"
+  execution_time: "long"
+  risk_level: "low"
+  ci_cd_safe: true
+  requires_network: true
+  idempotent: true
+---
+
+# Integration Test CrowdSec Data Persistence
+
+## Overview
+
+CrowdSec hub items are stored on the `/app/data` volume, but their data files (blocklists,
+GeoLite2 databases) live in CrowdSec's `data_dir`. This test starts throwaway containers on a
+temporary named volume and proves that:
+
+1. A fresh volume really installs the hub items and populates `data_dir` on the volume.
+2. Recreating the container keeps the data files and does NOT run `cscli hub upgrade`.
+3. Recreating after the data files were lost runs `cscli hub upgrade` once and restores them.
+4. A legacy volume whose `config.yaml` still points at `/var/lib/crowdsec/data` is migrated.
+
+## Usage
+
+```bash
+.github/skills/scripts/skill-runner.sh integration-test-crowdsec-data-persistence
+```
+
+Requires network access to the CrowdSec hub. Uses no published ports and removes its container
+and volume on exit.
+
+**Source**: `scripts/crowdsec_data_persistence_test.sh`

.github/workflows/codecov-upload.yml

@@ -161,13 +161,13 @@ jobs:
           path: backend/test-output.txt
           retention-days: 7
 
-      - name: Upload backend coverage to Codecov
-        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+      - name: Stage backend coverage for upload
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./backend/coverage.txt
-          flags: backend
-          fail_ci_if_error: true
+          name: coverage-backend
+          path: backend/coverage.txt
+          if-no-files-found: error
+          retention-days: 1
 
   frontend-codecov:
     name: Frontend Codecov Upload
@@ -199,13 +199,13 @@ jobs:
           bash scripts/frontend-test-coverage.sh 2>&1 | tee frontend/test-output.txt
           exit "${PIPESTATUS[0]}"
 
-      - name: Upload frontend coverage to Codecov
-        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+      - name: Stage frontend coverage for upload
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./frontend/coverage
-          flags: frontend
-          fail_ci_if_error: true
+          name: coverage-frontend
+          path: frontend/coverage/lcov.info
+          if-no-files-found: error
+          retention-days: 1
 
   agent-codecov:
     name: Agent Codecov Upload
@@ -246,7 +246,84 @@ jobs:
           path: agent/test-output.txt
           retention-days: 7
 
+      - name: Stage agent coverage for upload
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: coverage-agent
+          path: agent/coverage.txt
+          if-no-files-found: error
+          retention-days: 1
+
+  # The test jobs above only stage their coverage file as an artifact. Codecov
+  # re-evaluates statuses on every upload, so uploading from three parallel jobs
+  # (agent in ~2 min, backend in ~15 min) made codecov/patch flap red/green.
+  # This job uploads all reports back to back once every test job is done, so
+  # Codecov never sees a partial report (only a seconds-wide window between the
+  # three uploads remains).
+  #
+  # Runs only if no test job failed/was cancelled and at least one succeeded.
+  # A workflow_dispatch with a job disabled (skipped) still uploads, but only
+  # the reports that exist, so Codecov computes from a partial report. If a test
+  # job fails, nothing is uploaded and no codecov/* status is posted.
+  upload-codecov:
+    name: Codecov Upload
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs: [backend-codecov, frontend-codecov, agent-codecov]
+    if: >-
+      ${{ !cancelled()
+      && !contains(needs.*.result, 'failure')
+      && contains(needs.*.result, 'success') }}
+    steps:
+      # Checkout at the tested SHA is required so Codecov can map report paths.
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          ref: ${{ github.sha }}
+
+      - name: Download backend coverage
+        if: ${{ needs.backend-codecov.result == 'success' }}
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          name: coverage-backend
+          path: backend
+
+      - name: Download frontend coverage
+        if: ${{ needs.frontend-codecov.result == 'success' }}
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          name: coverage-frontend
+          path: frontend/coverage
+
+      - name: Download agent coverage
+        if: ${{ needs.agent-codecov.result == 'success' }}
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          name: coverage-agent
+          path: agent
+
+      - name: Upload backend coverage to Codecov
+        if: ${{ needs.backend-codecov.result == 'success' }}
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./backend/coverage.txt
+          flags: backend
+          fail_ci_if_error: true
+
+      - name: Upload frontend coverage to Codecov
+        if: ${{ needs.frontend-codecov.result == 'success' }}
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./frontend/coverage/lcov.info
+          flags: frontend
+          fail_ci_if_error: true
+
       - name: Upload agent coverage to Codecov
+        if: ${{ needs.agent-codecov.result == 'success' }}
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/integration-tests.yml

@@ -431,7 +431,7 @@ jobs:
     name: CrowdSec Bouncer Integration
     needs: build
     runs-on: ubuntu-latest
-    timeout-minutes: 15 # test-only work (two suites); image is built upstream in `build`.
+    timeout-minutes: 25 # test-only work (three suites, the persistence suite starts four containers); image is built upstream in `build`.
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -459,6 +459,13 @@ jobs:
           .github/skills/scripts/skill-runner.sh integration-test-crowdsec-startup 2>&1 | tee lapi-test-output.txt
           exit "${PIPESTATUS[0]}"
 
+      - name: Run CrowdSec Data Persistence Test
+        id: persistence-test
+        run: |
+          chmod +x .github/skills/scripts/skill-runner.sh
+          .github/skills/scripts/skill-runner.sh integration-test-crowdsec-data-persistence 2>&1 | tee persistence-test-output.txt
+          exit "${PIPESTATUS[0]}"
+
       - name: Dump Debug Info on Failure
         if: failure()
         run: |
@@ -536,12 +543,24 @@ jobs:
             grep -E "✗ FAIL|✗ CRITICAL|Error|failed" lapi-test-output.txt | head -20 || echo "See logs for details"
             echo '```'
           fi
+
+          # CrowdSec Data Persistence Test
+          if [ "${{ steps.persistence-test.outcome }}" == "success" ]; then
+            echo "✅ **CrowdSec Data Persistence: Passed**"
+          else
+            echo "❌ **CrowdSec Data Persistence: Failed**"
+            echo '```'
+            grep -E "✗ FAIL|Timed out|exited" persistence-test-output.txt | head -20 || echo "See logs for details"
+            echo '```'
+          fi
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Cleanup
         if: always()
         run: |
           docker rm -f charon-debug || true
           docker rm -f charon-crowdsec-startup-test || true
+          docker rm -f charon-crowdsec-persist-test || true
+          docker volume rm charon_crowdsec_persist_data 2>/dev/null || true
           docker rm -f crowdsec || true
           docker network rm containers_default || true