diff --git a/.docker/docker-entrypoint.sh b/.docker/docker-entrypoint.sh index 21e5e0e3e..250b241ae 100755 --- a/.docker/docker-entrypoint.sh +++ b/.docker/docker-entrypoint.sh @@ -10,6 +10,23 @@ is_root() { [ "$(id -u)" -eq 0 ] } +# Print (one per line) the data files declared by installed CrowdSec hub items +# (`data: - dest_file:` entries in their YAML) that are absent from the data dir (an empty file is legitimate, e.g. an empty blocklist). +# `cscli ... inspect -o json` does not expose declared data files, so the installed item +# files under the config dir are read directly. Prints nothing when everything is present. +# Usage: list_missing_hub_data_files +list_missing_hub_data_files() { + _cfg="$1" + _data="$2" + for _sub in parsers scenarios postoverflows contexts; do + [ -d "$_cfg/$_sub" ] || continue + find -L "$_cfg/$_sub" -type f -name '*.yaml' -exec grep -h 'dest_file:' {} + 2>/dev/null || true + done | sed -e 's/^.*dest_file:[[:space:]]*//' -e 's/[[:space:]]*#.*$//' -e "s/[\"']//g" -e 's/[[:space:]]*$//' | sort -u | while IFS= read -r _file; do + [ -n "$_file" ] || continue + [ -e "$_data/$_file" ] || echo "$_file" + done +} + run_as_charon() { if is_root; then gosu charon "$@" @@ -320,6 +337,44 @@ ACQUIS_EOF echo "⚠️ WARNING: Could not verify LAPI db_path redirect — bouncer keys may not survive rebuilds" fi + # Redirect CrowdSec data_dir (hub data files: blocklists, GeoLite2 databases, ...) to the + # persistent volume. The default /var/lib/crowdsec/data/ lives in the image layer, so after a + # container recreation the hub items (kept under /etc/crowdsec -> volume) are still installed + # but their data files are gone, which makes CrowdSec log "unable to init data for file". + # Idempotent: a config that already points at the volume is left untouched. + if grep -q "data_dir: /var/lib/crowdsec/data" "$CS_CONFIG_DIR/config.yaml"; then + sed -i "s|data_dir: /var/lib/crowdsec/data/\?\$|data_dir: ${CS_DATA_DIR}/|" "$CS_CONFIG_DIR/config.yaml" + # Existing installs: carry over anything an operator kept at the old location + # (e.g. a mounted volume), without overwriting files already on the volume. + if [ -d /var/lib/crowdsec/data ] && find /var/lib/crowdsec/data -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null | grep -q .; then + cp -R -n /var/lib/crowdsec/data/. "$CS_DATA_DIR/" 2>/dev/null || echo "Warning: could not copy existing CrowdSec data files to $CS_DATA_DIR" + fi + fi + if grep -q "data_dir: ${CS_DATA_DIR}/\?\$" "$CS_CONFIG_DIR/config.yaml"; then + echo "✓ CrowdSec data_dir redirected to persistent volume: ${CS_DATA_DIR}/" + else + echo "⚠️ WARNING: Could not verify CrowdSec data_dir redirect — hub data files may not survive container recreation" + fi + + # Pin the hub branch. Without cscli.hub_branch, every hub command first asks + # version.crowdsec.net for the latest release and derives the branch from the running version, + # which hard-fails offline and 404s once the CDN has no branch named after this version. + # A non-empty hub_branch is used as-is (no lookup). Idempotent: an operator-chosen non-empty + # value is respected; missing/empty/commented values are set to master. + if ! grep -Eq '^[[:space:]]+hub_branch:[[:space:]]*[^[:space:]#]' "$CS_CONFIG_DIR/config.yaml"; then + sed -i '/^[[:space:]]\+hub_branch:/d' "$CS_CONFIG_DIR/config.yaml" + if grep -q '^cscli:' "$CS_CONFIG_DIR/config.yaml"; then + sed -i '/^cscli:[[:space:]]*$/a\ hub_branch: master' "$CS_CONFIG_DIR/config.yaml" + else + printf '\ncscli:\n hub_branch: master\n' >>"$CS_CONFIG_DIR/config.yaml" + fi + fi + if grep -Eq '^[[:space:]]+hub_branch:[[:space:]]*[^[:space:]#]' "$CS_CONFIG_DIR/config.yaml"; then + echo "✓ CrowdSec hub branch pinned: $(sed -n 's/^[[:space:]]*hub_branch:[[:space:]]*//p' "$CS_CONFIG_DIR/config.yaml" | head -n1)" + else + echo "⚠️ WARNING: Could not pin cscli.hub_branch — hub commands may depend on version.crowdsec.net" + fi + # Verify LAPI configuration was applied correctly if grep -q "listen_uri:.*:8085" "$CS_CONFIG_DIR/config.yaml"; then echo "✓ CrowdSec LAPI configured for port 8085" @@ -353,6 +408,26 @@ ACQUIS_EOF if [ -x /usr/local/bin/install_hub_items.sh ]; then /usr/local/bin/install_hub_items.sh || echo "⚠️ Some hub items may not have installed. CrowdSec can still start." fi + + # Hub items live under /etc/crowdsec (volume) but their data files live in data_dir. If the + # data files are gone while the items are installed (e.g. the data directory was recreated), + # `cscli hub upgrade` is the only command that re-downloads them (`install` is a no-op for + # installed items). It upgrades every installed item, so it runs ONLY in this recovery case, + # never on a healthy start, is bounded, never forced and never fatal. + MISSING_HUB_DATA="$(list_missing_hub_data_files "$CS_CONFIG_DIR" "$CS_DATA_DIR")" + if [ -n "$MISSING_HUB_DATA" ]; then + echo "⚠️ CrowdSec hub data files missing from $CS_DATA_DIR: $(echo "$MISSING_HUB_DATA" | tr '\n' ' ')" + echo "Running 'cscli hub upgrade' to restore them (this also upgrades installed hub items)..." + if ! timeout 120s cscli hub upgrade 2>&1; then + echo "⚠️ Hub upgrade failed or timed out. CrowdSec will start, but some detection data may be missing until the next restart." + fi + STILL_MISSING="$(list_missing_hub_data_files "$CS_CONFIG_DIR" "$CS_DATA_DIR")" + if [ -n "$STILL_MISSING" ]; then + echo "⚠️ CrowdSec hub data files still missing: $(echo "$STILL_MISSING" | tr '\n' ' ')" + else + echo "✓ CrowdSec hub data files restored" + fi + fi fi # Fix ownership AFTER cscli commands (they run as root and create root-owned files) diff --git a/.dockerignore b/.dockerignore index e7bfde518..9309fc8a9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -168,6 +168,7 @@ scripts/tests/ scripts/toolchain-key.sh scripts/verify-toolchain-pin.sh scripts/lib/dockerfile-stage.sh +scripts/crowdsec_data_persistence_test.sh # ----------------------------------------------------------------------------- # Tools (not needed in image) diff --git a/.github/renovate.json b/.github/renovate.json index 4373a0781..a12de4271 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -415,6 +415,19 @@ "datasourceTemplate": "npm", "versioningTemplate": "semver" }, + { + "customType": "regex", + "description": "Track golang.org/x/mod version ARG in Dockerfile", + "managerFilePatterns": [ + "/^Dockerfile$/" + ], + "matchStrings": [ + "#\\s*renovate:\\s*datasource=go\\s+depName=golang\\.org/x/mod\\s*\\nARG XMOD_VERSION=(?[^\\s]+)" + ], + "depNameTemplate": "golang.org/x/mod", + "datasourceTemplate": "go", + "versioningTemplate": "semver" + }, { "customType": "regex", "description": "Track golang.org/x/crypto version ARG in Dockerfile", diff --git a/.github/skills/integration-test-crowdsec-data-persistence-scripts/run.sh b/.github/skills/integration-test-crowdsec-data-persistence-scripts/run.sh new file mode 100755 index 000000000..96d705218 --- /dev/null +++ b/.github/skills/integration-test-crowdsec-data-persistence-scripts/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Integration Test CrowdSec Data Persistence - Wrapper Script +# Verifies hub data files survive container recreation + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +exec "${PROJECT_ROOT}/scripts/crowdsec_data_persistence_test.sh" "$@" diff --git a/.github/skills/integration-test-crowdsec-data-persistence.SKILL.md b/.github/skills/integration-test-crowdsec-data-persistence.SKILL.md new file mode 100644 index 000000000..be123f4c4 --- /dev/null +++ b/.github/skills/integration-test-crowdsec-data-persistence.SKILL.md @@ -0,0 +1,69 @@ +--- +# agentskills.io specification v1.0 +name: "integration-test-crowdsec-data-persistence" +version: "1.0.0" +description: "Verify CrowdSec hub data files persist across container recreation" +author: "Charon Project" +license: "MIT" +tags: + - "integration" + - "crowdsec" + - "persistence" + - "hub" +compatibility: + os: + - "linux" + - "darwin" + shells: + - "bash" +requirements: + - name: "docker" + version: ">=24.0" + optional: false +environment_variables: + - name: "CHARON_TEST_IMAGE" + description: "Image to test" + default: "charon:local" + required: false + - name: "INIT_TIMEOUT_SECONDS" + description: "Maximum wait for CrowdSec hub initialization per container start" + default: "300" + required: false +parameters: [] +outputs: + - name: "test_results" + type: "stdout" + description: "PASS/FAIL results per check" +metadata: + category: "integration-test" + subcategory: "crowdsec" + execution_time: "long" + risk_level: "low" + ci_cd_safe: true + requires_network: true + idempotent: true +--- + +# Integration Test CrowdSec Data Persistence + +## Overview + +CrowdSec hub items are stored on the `/app/data` volume, but their data files (blocklists, +GeoLite2 databases) live in CrowdSec's `data_dir`. This test starts throwaway containers on a +temporary named volume and proves that: + +1. A fresh volume really installs the hub items and populates `data_dir` on the volume. +2. Recreating the container keeps the data files and does NOT run `cscli hub upgrade`. +3. Recreating after the data files were lost runs `cscli hub upgrade` once and restores them. +4. A legacy volume whose `config.yaml` still points at `/var/lib/crowdsec/data` is migrated. + +## Usage + +```bash +.github/skills/scripts/skill-runner.sh integration-test-crowdsec-data-persistence +``` + +Requires network access to the CrowdSec hub. Uses no published ports and removes its container +and volume on exit. + +**Source**: `scripts/crowdsec_data_persistence_test.sh` diff --git a/.github/workflows/codecov-upload.yml b/.github/workflows/codecov-upload.yml index 1500ad30e..1e456983d 100644 --- a/.github/workflows/codecov-upload.yml +++ b/.github/workflows/codecov-upload.yml @@ -161,13 +161,13 @@ jobs: path: backend/test-output.txt retention-days: 7 - - name: Upload backend coverage to Codecov - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + - name: Stage backend coverage for upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./backend/coverage.txt - flags: backend - fail_ci_if_error: true + name: coverage-backend + path: backend/coverage.txt + if-no-files-found: error + retention-days: 1 frontend-codecov: name: Frontend Codecov Upload @@ -199,13 +199,13 @@ jobs: bash scripts/frontend-test-coverage.sh 2>&1 | tee frontend/test-output.txt exit "${PIPESTATUS[0]}" - - name: Upload frontend coverage to Codecov - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + - name: Stage frontend coverage for upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./frontend/coverage - flags: frontend - fail_ci_if_error: true + name: coverage-frontend + path: frontend/coverage/lcov.info + if-no-files-found: error + retention-days: 1 agent-codecov: name: Agent Codecov Upload @@ -246,7 +246,84 @@ jobs: path: agent/test-output.txt retention-days: 7 + - name: Stage agent coverage for upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-agent + path: agent/coverage.txt + if-no-files-found: error + retention-days: 1 + + # The test jobs above only stage their coverage file as an artifact. Codecov + # re-evaluates statuses on every upload, so uploading from three parallel jobs + # (agent in ~2 min, backend in ~15 min) made codecov/patch flap red/green. + # This job uploads all reports back to back once every test job is done, so + # Codecov never sees a partial report (only a seconds-wide window between the + # three uploads remains). + # + # Runs only if no test job failed/was cancelled and at least one succeeded. + # A workflow_dispatch with a job disabled (skipped) still uploads, but only + # the reports that exist, so Codecov computes from a partial report. If a test + # job fails, nothing is uploaded and no codecov/* status is posted. + upload-codecov: + name: Codecov Upload + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [backend-codecov, frontend-codecov, agent-codecov] + if: >- + ${{ !cancelled() + && !contains(needs.*.result, 'failure') + && contains(needs.*.result, 'success') }} + steps: + # Checkout at the tested SHA is required so Codecov can map report paths. + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + ref: ${{ github.sha }} + + - name: Download backend coverage + if: ${{ needs.backend-codecov.result == 'success' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: coverage-backend + path: backend + + - name: Download frontend coverage + if: ${{ needs.frontend-codecov.result == 'success' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: coverage-frontend + path: frontend/coverage + + - name: Download agent coverage + if: ${{ needs.agent-codecov.result == 'success' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: coverage-agent + path: agent + + - name: Upload backend coverage to Codecov + if: ${{ needs.backend-codecov.result == 'success' }} + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./backend/coverage.txt + flags: backend + fail_ci_if_error: true + + - name: Upload frontend coverage to Codecov + if: ${{ needs.frontend-codecov.result == 'success' }} + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./frontend/coverage/lcov.info + flags: frontend + fail_ci_if_error: true + - name: Upload agent coverage to Codecov + if: ${{ needs.agent-codecov.result == 'success' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 64254603e..11049035d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -431,7 +431,7 @@ jobs: name: CrowdSec Bouncer Integration needs: build runs-on: ubuntu-latest - timeout-minutes: 15 # test-only work (two suites); image is built upstream in `build`. + timeout-minutes: 25 # test-only work (three suites, the persistence suite starts four containers); image is built upstream in `build`. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -459,6 +459,13 @@ jobs: .github/skills/scripts/skill-runner.sh integration-test-crowdsec-startup 2>&1 | tee lapi-test-output.txt exit "${PIPESTATUS[0]}" + - name: Run CrowdSec Data Persistence Test + id: persistence-test + run: | + chmod +x .github/skills/scripts/skill-runner.sh + .github/skills/scripts/skill-runner.sh integration-test-crowdsec-data-persistence 2>&1 | tee persistence-test-output.txt + exit "${PIPESTATUS[0]}" + - name: Dump Debug Info on Failure if: failure() run: | @@ -536,6 +543,16 @@ jobs: grep -E "✗ FAIL|✗ CRITICAL|Error|failed" lapi-test-output.txt | head -20 || echo "See logs for details" echo '```' fi + + # CrowdSec Data Persistence Test + if [ "${{ steps.persistence-test.outcome }}" == "success" ]; then + echo "✅ **CrowdSec Data Persistence: Passed**" + else + echo "❌ **CrowdSec Data Persistence: Failed**" + echo '```' + grep -E "✗ FAIL|Timed out|exited" persistence-test-output.txt | head -20 || echo "See logs for details" + echo '```' + fi } >> "$GITHUB_STEP_SUMMARY" - name: Cleanup @@ -543,5 +560,7 @@ jobs: run: | docker rm -f charon-debug || true docker rm -f charon-crowdsec-startup-test || true + docker rm -f charon-crowdsec-persist-test || true + docker volume rm charon_crowdsec_persist_data 2>/dev/null || true docker rm -f crowdsec || true docker network rm containers_default || true diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3f75439bb..1caefe9ca 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -292,6 +292,7 @@ graph TB │ ├── backups/ # Database backups │ ├── caddy/ # Caddy certificates │ └── crowdsec/ # CrowdSec local database +│ └── data/ # CrowdSec `data_dir`: LAPI db + hub data files (persistent) │ ├── Dockerfile # Multi-stage Docker build ├── Makefile # Build automation @@ -877,6 +878,30 @@ Sign-in protection is a separate, always-on limiter described under "Management - Decision management API (view, create, delete bans) - IP whitelist management: operators add/remove IPs and CIDRs via the management UI; entries are persisted in SQLite and regenerated into a `crowdsecurity/whitelists` parser YAML on every mutating operation and at startup +**Preset apply and config edits:** Applying a hub preset, importing a +configuration, and saving a file in the config editor never rename or empty the +live CrowdSec directory. Each operation first takes a copy-based snapshot +(`.backup.`, newest 5 kept; the live database, the +top-level `data/` directory, and `hub_cache/` are skipped to keep snapshots +small) and restores it if the operation fails. Single-file edits keep their own +backups (`.filebackup.`, newest 10 kept). One handler-level +lock serializes apply, import, and file saves. The file editor validates file +type, size and YAML, and the frontend no +longer has a local-write fallback for presets; failures are shown as errors. + +**Persistent hub data:** CrowdSec's `data_dir` (LAPI database and hub data files +such as blocklists and the GeoIP database) lives at `/app/data/crowdsec/data` +on the persistent volume, so it survives container recreation. The entrypoint +migrates older configs that pointed at `/var/lib/crowdsec/data`, and, only when +hub data files are missing, runs a gated, time-bounded `cscli hub upgrade` to +re-download them (startup log line only; no UI). `scripts/crowdsec_data_persistence_test.sh` +verifies this in the CrowdSec integration job. + +**Versioning:** the bundled `cscli` and `crowdsec` binaries report their real +version (currently v1.8.1); the Docker build fails if they do not. The +entrypoint pins `cscli.hub_branch: master` in the CrowdSec config for +deterministic hub access (an operator-set value is preserved). + **Modes:** - **Local Only:** No external API calls diff --git a/Dockerfile b/Dockerfile index 9a6bf16ec..f55166d16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,8 @@ ARG CHARON_TOOLCHAIN_IMAGE=ghcr.io/wikid82/charon-toolchain # NOT Renovate-tracked (a content-hash tag has no series to follow, N7) — the # toolchain-image.yml bot owns these two lines. DIGEST is the arch-independent # manifest-list (OCI index) digest, so one pin covers linux/amd64 + linux/arm64. -ARG CHARON_TOOLCHAIN_TAG=caddy-crowdsec-d75806d895b00627 -ARG CHARON_TOOLCHAIN_DIGEST=sha256:82f3204d6eed07288a0ed5bf45a99fb0cab9940195bbc3d03e598b6bed51da97 +ARG CHARON_TOOLCHAIN_TAG=caddy-crowdsec-0b537c195f3b4f50 +ARG CHARON_TOOLCHAIN_DIGEST=sha256:83b1b45f58af51a9be23ecb56155b29b3b7a64db61862aff043d535aca1645bb # Stage selector — default consumes the prebuilt toolchain image (no compile). # Fork PRs / bootstrap / offline builds pass @@ -50,6 +50,12 @@ ARG XNET_VERSION=0.60.0 # golang.org/x/crypto/ssh channel-flood deadlock DoS fixes (GO-2026-6354, GO-2026-6355). # renovate: datasource=go depName=golang.org/x/crypto ARG XCRYPTO_VERSION=0.57.0 +# Shared golang.org/x/mod pin (GOSUMDB tile-verification bypass, CVE-2026-56864 / CVE-2026-56865; +# fixed in v0.40.0). Must be >= the x/mod that the pinned x/net and x/crypto require (v0.41.0): +# `go get golang.org/x/mod@` is a downgrade and its cascade drags x/net, x/crypto and +# x/text back down with it, silently undoing the pins above. +# renovate: datasource=go depName=golang.org/x/mod +ARG XMOD_VERSION=0.41.0 # klauspost/compress DoS/resource-exhaustion fix, matching how golang.org/x/crypto # is patched above: pinned here so the CrowdSec/cscli and Caddy binaries (which # pull it in transitively) are patched immediately, ahead of upstream releases. @@ -494,6 +500,7 @@ ARG XCADDY_VERSION=0.4.7 ARG EXPR_LANG_VERSION ARG XNET_VERSION ARG XCRYPTO_VERSION +ARG XMOD_VERSION ARG KLAUSPOST_COMPRESS_VERSION ARG GRPC_VERSION ARG CROWDSEC_VERSION @@ -681,8 +688,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # the transparency log). Affects /usr/bin/caddy — go mod tidy's MVS resolution otherwise # lands on an older, vulnerable version. Fix available at v0.40.0. Same pattern as the # crowdsec-builder pin below. - # renovate: datasource=go depName=golang.org/x/mod - _retry go get golang.org/x/mod@v0.40.0; \ + _retry go get golang.org/x/mod@v${XMOD_VERSION}; \ + # Re-assert x/crypto, x/net after the x/mod pin: any downgrade cascade from the + # x/mod / OpenTelemetry go gets must not leave them below the pinned CVE floor. + _retry go get golang.org/x/crypto@v${XCRYPTO_VERSION} golang.org/x/net@v${XNET_VERSION}; \ _retry go get github.com/dnsimple/dnsimple-go/v10@v${CADDY_DNS_DNSIMPLE_GO_VERSION}; \ if [ "${CADDY_PATCH_SCENARIO}" = "A" ]; then \ # Rollback scenario: keep explicit nebula pin if upstream compatibility regresses. @@ -713,6 +722,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # Final re-pin: coraza WAF CVE (fixed in v3.8.1). Transitive via coraza-caddy, which requires # v3.7.0. Affects /usr/bin/caddy. _retry go get github.com/corazawaf/coraza/v3@v${CORAZA_VERSION}; \ + # Final re-pin: x/crypto + x/net (CVE floor). Later downgrading `go get`s (OpenTelemetry, + # x/mod) cascade through x/ modules; re-asserting last, then tidy, keeps them fixed. + _retry go get golang.org/x/crypto@v${XCRYPTO_VERSION} golang.org/x/net@v${XNET_VERSION}; \ # Clean up go.mod and ensure all dependencies are resolved _retry go mod tidy; \ # Patch DecisionsListOpts API: crowdsec v1.7.8 changed fields (IPEquals, ScopeEquals, @@ -778,6 +790,18 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # Assert the shipped binary embeds the fixed coraza (>= v3.8.1 CVE fix). go version -m /usr/bin/caddy | grep -E "github\.com/corazawaf/coraza/v3[[:space:]]+v${CORAZA_VERSION}[[:space:]]" || { echo "ERROR: /usr/bin/caddy did not embed coraza v${CORAZA_VERSION}"; go version -m /usr/bin/caddy | grep "corazawaf/coraza" || true; exit 1; }; \ echo "Verified /usr/bin/caddy embeds coraza v${CORAZA_VERSION}"; \ + # Assert the shipped binary embeds golang.org/x/net and x/crypto at or above the pinned + # versions. A downgrading `go get` earlier in this recipe once silently dragged x/net back + # below the CVE fix; fail the build loudly instead of shipping it. + for pin in "golang.org/x/net:${XNET_VERSION}" "golang.org/x/crypto:${XCRYPTO_VERSION}"; do \ + mod="${pin%%:*}"; want="v${pin#*:}"; \ + got="$(go version -m /usr/bin/caddy | grep -E "^[[:space:]]+dep[[:space:]]+${mod}[[:space:]]" | awk "{print \$3}")"; \ + if [ -z "$got" ] || [ "$(printf "%s\\n%s\\n" "$want" "$got" | sort -V | head -n1)" != "$want" ]; then \ + echo "ERROR: /usr/bin/caddy embeds ${mod} ${got:-}, below pinned ${want}"; \ + exit 1; \ + fi; \ + echo "Verified /usr/bin/caddy embeds ${mod} ${got} (>= ${want})"; \ + done; \ # Clean up temporary build directories rm -rf /tmp/buildenv_* /tmp/caddy-initial' @@ -801,6 +825,7 @@ ARG CROWDSEC_VERSION ARG EXPR_LANG_VERSION ARG XNET_VERSION ARG XCRYPTO_VERSION +ARG XMOD_VERSION ARG KLAUSPOST_COMPRESS_VERSION ARG GRPC_VERSION @@ -893,11 +918,13 @@ RUN set -e; \ # (a colluding GOPROXY+GOSUMDB pair could forge sumdb tiles / serve module content outside # the transparency log). Affects /usr/local/bin/crowdsec and /usr/local/bin/cscli — go mod # tidy's MVS resolution otherwise lands on v0.38.0. Fix available at v0.40.0. - # renovate: datasource=go depName=golang.org/x/mod # CVE-2026-32286: pgproto3/v2 buffer overflow (no v2 fix exists; bump pgx/v4 to latest patch) # renovate: datasource=go depName=github.com/jackc/pgproto3/v2 _retry go get github.com/jackc/pgproto3/v2@v2.3.3; \ - _retry go get golang.org/x/mod@v0.40.0; \ + _retry go get golang.org/x/mod@v${XMOD_VERSION}; \ + # Re-assert x/crypto, x/net after the x/mod pin: the x/mod pin is a downgrade-prone `go get` + # whose cascade must not leave them below the pinned CVE floor (see XMOD_VERSION). + _retry go get golang.org/x/crypto@v${XCRYPTO_VERSION} golang.org/x/net@v${XNET_VERSION}; \ _retry go mod tidy # Fix compatibility issues with expr-lang v1.17.7 @@ -910,7 +937,7 @@ RUN sed -i 's/string(program\.Source())/program.Source().String()/g' pkg/exprhel RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ CGO_ENABLED=1 xx-go build -o /crowdsec-out/crowdsec \ - -ldflags "-s -w -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=v${CROWDSEC_VERSION}" \ + -ldflags "-s -w -X github.com/crowdsecurity/go-cs-lib/version.Version=v${CROWDSEC_VERSION}" \ ./cmd/crowdsec && \ xx-verify /crowdsec-out/crowdsec @@ -918,10 +945,27 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ CGO_ENABLED=1 xx-go build -o /crowdsec-out/cscli \ - -ldflags "-s -w -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=v${CROWDSEC_VERSION}" \ + -ldflags "-s -w -X github.com/crowdsecurity/go-cs-lib/version.Version=v${CROWDSEC_VERSION}" \ ./cmd/crowdsec-cli && \ xx-verify /crowdsec-out/cscli +# Assert the shipped binaries embed golang.org/x/net and x/crypto at or above the pinned versions. +# A downgrading `go get` earlier in the recipe once silently dragged x/net back below the CVE fix; +# fail the build loudly instead of shipping it. +# hadolint ignore=DL3059 +RUN set -e; \ + for bin in /crowdsec-out/crowdsec /crowdsec-out/cscli; do \ + for pin in "golang.org/x/net:${XNET_VERSION}" "golang.org/x/crypto:${XCRYPTO_VERSION}"; do \ + mod="${pin%%:*}"; want="v${pin#*:}"; \ + got="$(go version -m "$bin" | grep -E "^[[:space:]]+dep[[:space:]]+${mod}[[:space:]]" | awk "{print \$3}")"; \ + if [ -z "$got" ] || [ "$(printf "%s\\n%s\\n" "$want" "$got" | sort -V | head -n1)" != "$want" ]; then \ + echo "ERROR: $bin embeds ${mod} ${got:-}, below pinned ${want}"; \ + exit 1; \ + fi; \ + echo "Verified $bin embeds ${mod} ${got} (>= ${want})"; \ + done; \ + done + # Copy config files RUN mkdir -p /crowdsec-out/config && \ cp -r config/* /crowdsec-out/config/ || true @@ -1052,15 +1096,21 @@ RUN set -e; \ COPY --from=crowdsec-builder /crowdsec-out/crowdsec /usr/local/bin/crowdsec COPY --from=crowdsec-builder /crowdsec-out/cscli /usr/local/bin/cscli -# N5 — app-side sanity check on the toolchain-provided cscli binary: it must run -# and emit its recognisable version block. (CrowdSec 1.8.x prints an empty -# `version:` field here regardless of the -X ldflag, so match a stable field -# instead.) A wrong-arch / stale-recipe image fails this immediately. +# N5 — app-side sanity check on the toolchain-provided cscli binary: it must run, +# emit its recognisable version block, and report the pinned CrowdSec version. +# The version is injected via the go-cs-lib/version.Version -X ldflag in the +# crowdsec-inline stage; `-X` on a nonexistent symbol is silently ignored, so the +# "version: v${CROWDSEC_VERSION}" assertion fails the build if that ever no-ops +# again. A wrong-arch / stale-recipe image fails this immediately. +# A global ARG is not visible inside a stage without redeclaring it. +ARG CROWDSEC_VERSION RUN set -e; \ /usr/local/bin/cscli version >/tmp/cscli-v.txt 2>&1 \ || { echo "ERROR: toolchain cscli is not runnable"; cat /tmp/cscli-v.txt; exit 1; }; \ grep -q 'Constraint_api' /tmp/cscli-v.txt \ || { echo "ERROR: toolchain cscli version output not recognised"; cat /tmp/cscli-v.txt; exit 1; }; \ + grep -q "^version: v${CROWDSEC_VERSION}\$" /tmp/cscli-v.txt \ + || { echo "ERROR: toolchain cscli does not report version v${CROWDSEC_VERSION} (version ldflag no-op or wrong pin)"; cat /tmp/cscli-v.txt; exit 1; }; \ rm -f /tmp/cscli-v.txt; \ echo "Verified toolchain cscli runs (GoVersion: $(/usr/local/bin/cscli version 2>&1 | sed -n 's/^GoVersion: //p'))" # Copy CrowdSec configuration files to .dist directory (will be used at runtime) diff --git a/backend/internal/api/handlers/crowdsec_data_mutation_test.go b/backend/internal/api/handlers/crowdsec_data_mutation_test.go new file mode 100644 index 000000000..ebc3d1422 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_data_mutation_test.go @@ -0,0 +1,347 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +const importableConfig = "api:\n server:\n listen_uri: 0.0.0.0:8080\n" + +func writeTree(t *testing.T, root string, files map[string]string) { + t.Helper() + for name, content := range files { + p := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o600)) + } +} + +func readTreeFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) //nolint:gosec // G304: test fixture path + require.NoError(t, err) + return string(b) +} + +func importRequest(t *testing.T, files map[string]string) *http.Request { + t.Helper() + archivePath := createTestArchive(t, "tar.gz", files, true) + data, err := os.ReadFile(archivePath) //nolint:gosec // G304: archive in the test temp dir + require.NoError(t, err) + buf := &bytes.Buffer{} + mw := multipart.NewWriter(buf) + fw, err := mw.CreateFormFile("file", "cfg.tar.gz") + require.NoError(t, err) + _, err = fw.Write(data) + require.NoError(t, err) + require.NoError(t, mw.Close()) + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/import", buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + return req +} + +func newImportRouter(t *testing.T, dataDir string) (*CrowdsecHandler, *gin.Engine) { + t.Helper() + h := newTestCrowdsecHandler(t, OpenTestDB(t), &fakeExec{}, "/bin/false", dataDir) + r := gin.New() + h.RegisterRoutes(r.Group("/api/v1")) + return h, r +} + +func seedEngineState(t *testing.T, dir string) { + t.Helper() + writeTree(t, dir, map[string]string{ + "config.yaml": "old: true\n", + "old-only.yaml": "old", + "crowdsec.db": "live-db", + "crowdsec.db-wal": "wal", + "data/crowdsec.db": "lapi-db", + "data/GeoLite2-City.mmdb": "mmdb", + "hub_cache/x/bundle.tgz": "archive", + "nested/data/keep.yaml": "nested", + "nested/hub/collection.md": "doc", + }) +} + +func assertEngineStateIntact(t *testing.T, dir string) { + t.Helper() + require.Equal(t, "live-db", readTreeFile(t, filepath.Join(dir, "crowdsec.db"))) + require.Equal(t, "wal", readTreeFile(t, filepath.Join(dir, "crowdsec.db-wal"))) + require.Equal(t, "lapi-db", readTreeFile(t, filepath.Join(dir, "data", "crowdsec.db"))) + require.Equal(t, "mmdb", readTreeFile(t, filepath.Join(dir, "data", "GeoLite2-City.mmdb"))) + require.Equal(t, "archive", readTreeFile(t, filepath.Join(dir, "hub_cache", "x", "bundle.tgz"))) +} + +func TestImportConfigReplacesConfigButPreservesEngineStateAndDataDir(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + before, err := os.Stat(dir) + require.NoError(t, err) + _, r := newImportRouter(t, dir) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{ + "config.yaml": importableConfig, + "new.yaml": "new: true", + "crowdsec.db": "evil-db", + "data/x": "evil", + "data/crowdsec.db": "evil-lapi", + "hub_cache/y": "evil", + })) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + + var resp map[string]any + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + backup, _ := resp["backup"].(string) + require.Equal(t, dir+".backup.", backup[:len(dir)+len(".backup.")]) + + after, err := os.Stat(dir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "DataDir must never be renamed") + + require.Equal(t, importableConfig, readTreeFile(t, filepath.Join(dir, "config.yaml"))) + require.Equal(t, "new: true", readTreeFile(t, filepath.Join(dir, "new.yaml"))) + require.NoFileExists(t, filepath.Join(dir, "old-only.yaml")) + require.NoDirExists(t, filepath.Join(dir, "nested")) + + // Uploaded engine-owned entries were not extracted; live state is untouched. + assertEngineStateIntact(t, dir) + require.NoFileExists(t, filepath.Join(dir, "data", "x")) + require.NoFileExists(t, filepath.Join(dir, "hub_cache", "y")) + + // The snapshot holds the previous config, including a nested dir named data, but no engine state. + require.Equal(t, "old: true\n", readTreeFile(t, filepath.Join(backup, "config.yaml"))) + require.Equal(t, "nested", readTreeFile(t, filepath.Join(backup, "nested", "data", "keep.yaml"))) + require.NoFileExists(t, filepath.Join(backup, "crowdsec.db")) + require.NoDirExists(t, filepath.Join(backup, "data")) + require.NoDirExists(t, filepath.Join(backup, "hub_cache")) +} + +func TestImportConfigValidationFailureRestoresPreviousConfig(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + before, err := os.Stat(dir) + require.NoError(t, err) + _, r := newImportRouter(t, dir) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": "{{ not: [valid yaml", "extra.yaml": "x"})) + require.Equal(t, http.StatusUnprocessableEntity, w.Code, w.Body.String()) + + after, err := os.Stat(dir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after)) + require.Equal(t, "old: true\n", readTreeFile(t, filepath.Join(dir, "config.yaml"))) + require.Equal(t, "old", readTreeFile(t, filepath.Join(dir, "old-only.yaml"))) + require.Equal(t, "nested", readTreeFile(t, filepath.Join(dir, "nested", "data", "keep.yaml"))) + require.NoFileExists(t, filepath.Join(dir, "extra.yaml")) + assertEngineStateIntact(t, dir) +} + +func TestImportConfigExtractionFailureRestoresPreviousConfig(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + _, r := newImportRouter(t, dir) + + // "a" is both a file and a directory prefix, so extraction fails part-way. + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": importableConfig, "a": "file", "a/b": "nested"})) + require.Equal(t, http.StatusInternalServerError, w.Code, w.Body.String()) + + require.Equal(t, "old: true\n", readTreeFile(t, filepath.Join(dir, "config.yaml"))) + require.Equal(t, "old", readTreeFile(t, filepath.Join(dir, "old-only.yaml"))) + require.NoFileExists(t, filepath.Join(dir, "a")) + assertEngineStateIntact(t, dir) +} + +func TestImportConfigSnapshotFailureLeavesNothingChanged(t *testing.T) { + t.Parallel() + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600)) + _, r := newImportRouter(t, filepath.Join(blocker, "crowdsec")) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": importableConfig})) + require.Equal(t, http.StatusInternalServerError, w.Code, w.Body.String()) + require.Contains(t, w.Body.String(), "failed to create backup") +} + +func TestImportConfigPrunesSnapshotsToFive(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + _, r := newImportRouter(t, dir) + + for i := 0; i < 7; i++ { + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": importableConfig})) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + time.Sleep(2 * time.Millisecond) // distinct microsecond timestamps even on coarse clocks + } + matches, err := filepath.Glob(dir + ".backup.*") + require.NoError(t, err) + require.Len(t, matches, 5) +} + +func TestImportConfigPrunesAfterFailureKeepingNewestSnapshot(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + for i := 1; i <= 7; i++ { + require.NoError(t, os.Mkdir(fmt.Sprintf("%s.backup.2025010%d-000000.000000", dir, i), 0o700)) + } + _, r := newImportRouter(t, dir) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": "{{ bad", "x": "y"})) + require.Equal(t, http.StatusUnprocessableEntity, w.Code) + + matches, err := filepath.Glob(dir + ".backup.*") + require.NoError(t, err) + require.Len(t, matches, 5) + for _, m := range matches { + require.NotContains(t, []string{ + dir + ".backup.20250101-000000.000000", + dir + ".backup.20250102-000000.000000", + dir + ".backup.20250103-000000.000000", + }, m) + } +} + +// TestDataMutationsSerializeOnHandlerLock holds the handler lock and proves file write and +// preset apply wait for it (import is covered separately). +func TestDataMutationsSerializeOnHandlerLock(t *testing.T) { + c := newCuratedHarness(t, nil) + + requests := map[string]func() *httptest.ResponseRecorder{ + "apply": func() *httptest.ResponseRecorder { + body, _ := json.Marshal(map[string]string{"slug": "honeypot-friendly-defaults"}) + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/presets/apply", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + c.router.ServeHTTP(w, req) + return w + }, + "write": func() *httptest.ResponseRecorder { + body, _ := json.Marshal(map[string]string{"path": "locked.conf", "content": "x"}) + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/file", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + c.router.ServeHTTP(w, req) + return w + }, + } + + for name, do := range requests { + c.handler.dataMu.Lock() + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- do() }() + + select { + case <-done: + c.handler.dataMu.Unlock() + t.Fatalf("%s ran while the handler lock was held", name) + case <-time.After(150 * time.Millisecond): + } + c.handler.dataMu.Unlock() + + select { + case w := <-done: + require.Equal(t, http.StatusOK, w.Code, "%s: %s", name, w.Body.String()) + case <-time.After(5 * time.Second): + t.Fatalf("%s did not complete after the lock was released", name) + } + } +} + +func TestImportConfigWaitsForHandlerLock(t *testing.T) { + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + h, r := newImportRouter(t, dir) + + h.dataMu.Lock() + done := make(chan int, 1) + go func() { + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": importableConfig})) + done <- w.Code + }() + select { + case <-done: + h.dataMu.Unlock() + t.Fatal("import ran while the handler lock was held") + case <-time.After(150 * time.Millisecond): + } + require.Equal(t, "old: true\n", readTreeFile(t, filepath.Join(dir, "config.yaml")), "nothing changed while locked") + h.dataMu.Unlock() + + select { + case code := <-done: + require.Equal(t, http.StatusOK, code) + case <-time.After(5 * time.Second): + t.Fatal("import did not finish after the lock was released") + } +} + +func TestCuratedApplyPrunesSnapshotsToFive(t *testing.T) { + c := newCuratedHarness(t, nil) + for i := 0; i < 7; i++ { + code, _ := c.apply(t, "honeypot-friendly-defaults") + require.Equal(t, http.StatusOK, code) + time.Sleep(2 * time.Millisecond) + } + matches, err := filepath.Glob(c.dataDir + ".backup.*") + require.NoError(t, err) + require.Len(t, matches, 5) +} + +func TestImportConfigClearFailureRollsBackAndReportsBothFailures(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + dir := t.TempDir() + seedEngineState(t, dir) + // A read-only config directory can be snapshotted but not emptied, so both the clear and the + // rollback that follows it fail; the snapshot is retained. + locked := filepath.Join(dir, "locked") + writeTree(t, locked, map[string]string{"f.yaml": "x"}) + require.NoError(t, os.Chmod(locked, 0o500)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(locked, 0o750) }) // #nosec G302 -- test fixture + _, r := newImportRouter(t, dir) + + w := httptest.NewRecorder() + r.ServeHTTP(w, importRequest(t, map[string]string{"config.yaml": importableConfig})) + require.Equal(t, http.StatusInternalServerError, w.Code, w.Body.String()) + require.Contains(t, w.Body.String(), "failed to create config dir") + + matches, err := filepath.Glob(dir + ".backup.*") + require.NoError(t, err) + require.Len(t, matches, 1, "snapshot kept for manual recovery") + require.Equal(t, "x", readTreeFile(t, filepath.Join(matches[0], "locked", "f.yaml"))) + assertEngineStateIntact(t, dir) +} + +func TestPruneSnapshotsLogsAndToleratesFailure(t *testing.T) { + t.Parallel() + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600)) + h, _ := newImportRouter(t, filepath.Join(blocker, "crowdsec")) + // The parent of DataDir is a regular file, so listing snapshots fails; pruning is best effort. + require.NotPanics(t, h.pruneSnapshots) +} diff --git a/backend/internal/api/handlers/crowdsec_extract_safety_test.go b/backend/internal/api/handlers/crowdsec_extract_safety_test.go new file mode 100644 index 000000000..e3f407075 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_extract_safety_test.go @@ -0,0 +1,50 @@ +package handlers + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeRawArchive(t *testing.T, entries [][2]string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "a.tar.gz") + f, err := os.Create(p) //nolint:gosec // G304: test temp path + require.NoError(t, err) + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + for _, e := range entries { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: e[0], Mode: 0o640, Size: int64(len(e[1])), Typeflag: tar.TypeReg})) + _, err = tw.Write([]byte(e[1])) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + return p +} + +func TestExtractArchive_RepeatedNameLeavesNoStaleBytes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + h := newTestCrowdsecHandler(t, OpenTestDB(t), &fakeExec{}, "/bin/false", dir) + arc := writeRawArchive(t, [][2]string{{"config.yaml", "a long first version of the file"}, {"config.yaml", "short"}}) + require.NoError(t, h.extractArchive(arc, dir)) + require.Equal(t, "short", readTreeFile(t, filepath.Join(dir, "config.yaml"))) +} + +func TestExtractArchive_RefusesSymlinkedDestination(t *testing.T) { + t.Parallel() + dir, outside := t.TempDir(), t.TempDir() + victim := filepath.Join(outside, "victim") + require.NoError(t, os.WriteFile(victim, []byte("untouched"), 0o600)) + require.NoError(t, os.Symlink(victim, filepath.Join(dir, "config.yaml"))) + h := newTestCrowdsecHandler(t, OpenTestDB(t), &fakeExec{}, "/bin/false", dir) + arc := writeRawArchive(t, [][2]string{{"config.yaml", "overwritten"}}) + require.Error(t, h.extractArchive(arc, dir)) + require.Equal(t, "untouched", readTreeFile(t, victim)) +} diff --git a/backend/internal/api/handlers/crowdsec_files.go b/backend/internal/api/handlers/crowdsec_files.go new file mode 100644 index 000000000..e2bdf3f68 --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_files.go @@ -0,0 +1,422 @@ +package handlers + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "strings" + + "github.com/Wikid82/charon/backend/internal/crowdsec" + "github.com/Wikid82/charon/backend/internal/logger" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +const ( + // maxCrowdsecFileBytes caps the content of a single configuration file write. + maxCrowdsecFileBytes = 1 << 20 + // maxCrowdsecWriteBodyBytes caps the whole request body; it leaves room for JSON escaping of a + // maximum-size file. + maxCrowdsecWriteBodyBytes = 8 << 20 + + // writeTempPrefix names in-flight temp files; such names are never writable or listed targets. + writeTempPrefix = ".charon-write-" +) + +const errFileCannotBeEdited = "file cannot be edited" + +var ( + // writableExtensions is the allowlist of file types the editor may save. + writableExtensions = map[string]struct{}{ + ".yaml": {}, ".yml": {}, ".json": {}, ".txt": {}, ".conf": {}, + } + + // protectedWriteNames are credential files the editor never overwrites. + protectedWriteNames = map[string]struct{}{ + "local_api_credentials.yaml": {}, + "online_api_credentials.yaml": {}, + } +) + +// fileError carries the HTTP status and client-facing message for a rejected file request. +type fileError struct { + status int + msg string +} + +func (e *fileError) respond(c *gin.Context) { + c.JSON(e.status, gin.H{"error": e.msg}) +} + +func badRequest(msg string) *fileError { return &fileError{status: http.StatusBadRequest, msg: msg} } + +// cleanRelPath normalizes a client-supplied relative path and rejects anything that is not a plain +// path below the data directory. +func cleanRelPath(raw string) (string, *fileError) { + if raw == "" { + return "", badRequest("path required") + } + if strings.ContainsRune(raw, 0) || !filepath.IsLocal(raw) { + return "", badRequest("invalid path") + } + clean := filepath.Clean(raw) + if clean == "." { + return "", badRequest("invalid path") + } + return clean, nil +} + +// isReadable reports whether rel may be listed and read: engine-owned state (databases, the data and +// hub_cache trees) and secrets are hidden, everything else stays visible. +func isReadable(rel string) bool { + slash := filepath.ToSlash(rel) + if slash == "" || slash == "." || crowdsec.IsEngineOwnedPath(slash) { + return false + } + base := strings.ToLower(path.Base(slash)) + if base == "bouncer_key" { + return false + } + if matched, _ := path.Match("*.db*", base); matched { + return false + } + return true +} + +// isWritable is the stricter predicate for saves. It is always a subset of isReadable so that every +// file the editor can write is also listed. +func isWritable(rel string) bool { + if !isReadable(rel) { + return false + } + slash := filepath.ToSlash(rel) + base := path.Base(slash) + if _, ok := writableExtensions[strings.ToLower(path.Ext(base))]; !ok { + return false + } + if _, protected := protectedWriteNames[base]; protected { + return false + } + if strings.HasPrefix(base, writeTempPrefix) { + return false + } + // The hub tree is managed by cscli, both at the top level and under config/. + top, _, _ := strings.Cut(slash, "/") + return top != "hub" && !strings.HasPrefix(slash, "config/hub/") +} + +// pathWithin reports whether p is root or lies below it. +func pathWithin(root, p string) bool { + rel, err := filepath.Rel(root, p) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// resolveReadable validates rel for reading and returns the file path to open. Symlinks are resolved +// and the result must stay inside the data directory and remain readable. +func resolveReadable(dataDir, rel string) (string, *fileError) { + if !isReadable(rel) { + return "", badRequest("invalid path") + } + root, err := filepath.EvalSymlinks(dataDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", &fileError{status: http.StatusNotFound, msg: "file not found"} + } + return "", &fileError{status: http.StatusInternalServerError, msg: "failed to resolve data directory"} + } + resolved, err := filepath.EvalSymlinks(filepath.Join(dataDir, rel)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", &fileError{status: http.StatusNotFound, msg: "file not found"} + } + return "", badRequest("invalid path") + } + if !pathWithin(root, resolved) { + return "", badRequest("invalid path") + } + resolvedRel, err := filepath.Rel(root, resolved) + if err != nil || !isReadable(resolvedRel) { + return "", badRequest("invalid path") + } + info, err := os.Stat(resolved) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", &fileError{status: http.StatusNotFound, msg: "file not found"} + } + return "", badRequest("invalid path") + } + if !info.Mode().IsRegular() { + return "", badRequest("invalid path") + } + return resolved, nil +} + +// resolveWritable validates rel for saving and returns the lexical path to write. Existing symlinks +// anywhere along the path, and non-regular targets, are rejected. +func resolveWritable(dataDir, rel string) (string, *fileError) { + if !isReadable(rel) { + return "", badRequest(errFileCannotBeEdited) + } + if !isWritable(rel) { + if _, ok := writableExtensions[strings.ToLower(filepath.Ext(rel))]; !ok { + return "", badRequest("file type not allowed") + } + return "", badRequest(errFileCannotBeEdited) + } + cur := dataDir + parts := strings.Split(filepath.ToSlash(rel), "/") + for i, part := range parts { + cur = filepath.Join(cur, part) + info, err := os.Lstat(cur) + if errors.Is(err, fs.ErrNotExist) { + break + } + if err != nil { + return "", &fileError{status: http.StatusInternalServerError, msg: "failed to inspect path"} + } + if info.Mode()&os.ModeSymlink != 0 { + return "", badRequest(errFileCannotBeEdited) + } + last := i == len(parts)-1 + if last && !info.Mode().IsRegular() { + return "", badRequest("invalid path") + } + if !last && !info.IsDir() { + return "", badRequest("invalid path") + } + } + return filepath.Join(dataDir, rel), nil +} + +// checkSize enforces the per-file content limit. +func checkSize(content string) *fileError { + if len(content) > maxCrowdsecFileBytes { + return &fileError{status: http.StatusRequestEntityTooLarge, msg: "file too large"} + } + return nil +} + +// checkYAML verifies that YAML files parse; other allowed types are not inspected. +func checkYAML(rel, content string) *fileError { + switch strings.ToLower(filepath.Ext(rel)) { + case ".yaml", ".yml": + dec := yaml.NewDecoder(strings.NewReader(content)) + for { + var doc any + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return badRequest("invalid YAML") + } + } + } + return nil +} + +// atomicWriteFile writes data to a temp file beside dst and renames it into place. The temp file is +// removed on any failure. +func atomicWriteFile(dst string, data []byte, mode os.FileMode) (err error) { + tmp, err := os.CreateTemp(filepath.Dir(dst), writeTempPrefix+"*") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + tmpName := tmp.Name() + defer func() { + if err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + } + }() + if err = tmp.Chmod(mode); err != nil { + return fmt.Errorf("chmod temp file: %w", err) + } + if _, err = tmp.Write(data); err != nil { + return fmt.Errorf("write temp file: %w", err) + } + if err = tmp.Sync(); err != nil { + return fmt.Errorf("sync temp file: %w", err) + } + if err = tmp.Close(); err != nil { + return fmt.Errorf("close temp file: %w", err) + } + if err = os.Rename(tmpName, dst); err != nil { + return fmt.Errorf("rename temp file: %w", err) + } + return nil +} + +// ListFiles returns a flat list of the readable files under the CrowdSec DataDir. +func (h *CrowdsecHandler) ListFiles(c *gin.Context) { + files := []string{} + if _, err := os.Stat(h.DataDir); os.IsNotExist(err) { + c.JSON(http.StatusOK, gin.H{"files": files}) + return + } + err := filepath.WalkDir(h.DataDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + // Permission errors (e.g. lost+found) should not abort the walk + if os.IsPermission(err) { + fileErrorLog(err).WithField("path", sanitizeForLog(p)).Debug("Skipping inaccessible path during list") + return filepath.SkipDir + } + return err + } + rel, err := filepath.Rel(h.DataDir, p) + if err != nil { + return err + } + if d.IsDir() { + if rel != "." && crowdsec.IsEngineOwnedPath(rel) { + return filepath.SkipDir + } + return nil + } + if !isReadable(rel) { + return nil + } + // Symlinked entries are listed only when they can actually be read. + if d.Type()&fs.ModeSymlink != 0 { + if _, ferr := resolveReadable(h.DataDir, rel); ferr != nil { + return nil + } + } + files = append(files, rel) + return nil + }) + if err != nil { + fileErrorLog(err).Warn("crowdsec file listing failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list files"}) + return + } + c.JSON(http.StatusOK, gin.H{"files": files}) +} + +var errFileTooLarge = errors.New("file exceeds read limit") + +// readCapped reads at most maxCrowdsecFileBytes from p, failing with errFileTooLarge when the file is +// larger so an oversized file is never loaded into memory. +func readCapped(p string) ([]byte, error) { + f, err := os.Open(p) // #nosec G304 -- p is validated and symlink-resolved to stay inside DataDir by resolveReadable + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + data, err := io.ReadAll(io.LimitReader(f, maxCrowdsecFileBytes+1)) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + if len(data) > maxCrowdsecFileBytes { + return nil, errFileTooLarge + } + return data, nil +} + +// ReadFile returns the contents of a specific file under DataDir. Query param 'path' required. +func (h *CrowdsecHandler) ReadFile(c *gin.Context) { + rel, ferr := cleanRelPath(c.Query("path")) + if ferr != nil { + ferr.respond(c) + return + } + p, ferr := resolveReadable(h.DataDir, rel) + if ferr != nil { + ferr.respond(c) + return + } + data, err := readCapped(p) + if err != nil { + switch { + case os.IsNotExist(err): + c.JSON(http.StatusNotFound, gin.H{"error": "file not found"}) + case errors.Is(err, errFileTooLarge): + (&fileError{status: http.StatusRequestEntityTooLarge, msg: "file too large"}).respond(c) + default: + fileErrorLog(err).Warn("crowdsec file read failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to read file"}) + } + return + } + c.JSON(http.StatusOK, gin.H{"content": string(data)}) +} + +// WriteFile saves content to an allowed file under DataDir, first copying only the replaced file into +// the file-backup namespace. DataDir itself is never renamed or emptied. +// JSON body: { "path": "relative/path.conf", "content": "..." } +func (h *CrowdsecHandler) WriteFile(c *gin.Context) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxCrowdsecWriteBodyBytes) + var payload struct { + Path string `json:"path"` + Content string `json:"content"` + } + if err := json.NewDecoder(c.Request.Body).Decode(&payload); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + (&fileError{status: http.StatusRequestEntityTooLarge, msg: "file too large"}).respond(c) + return + } + badRequest("invalid payload").respond(c) + return + } + rel, ferr := cleanRelPath(payload.Path) + if ferr != nil { + ferr.respond(c) + return + } + if ferr = checkSize(payload.Content); ferr != nil { + ferr.respond(c) + return + } + + h.dataMu.Lock() + defer h.dataMu.Unlock() + + target, ferr := resolveWritable(h.DataDir, rel) + if ferr != nil { + ferr.respond(c) + return + } + if ferr = checkYAML(rel, payload.Content); ferr != nil { + ferr.respond(c) + return + } + + mode := os.FileMode(0o600) + if info, err := os.Stat(target); err == nil { + mode = info.Mode().Perm() + } + backupDir, err := crowdsec.BackupFile(h.DataDir, rel) + if err != nil { + fileErrorLog(err).Warn("crowdsec file backup failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create backup"}) + return + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to prepare dir"}) + return + } + if err := atomicWriteFile(target, []byte(payload.Content), mode); err != nil { + fileErrorLog(err).Warn("crowdsec file write failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write file"}) + return + } + c.JSON(http.StatusOK, gin.H{"status": "written", "backup": backupDir}) +} + +// fileErrorLog returns a log entry carrying err with control characters +// stripped, since file operation errors embed request-derived paths. +func fileErrorLog(err error) *logrus.Entry { + return logger.Log().WithField("error", sanitizeForLog(err.Error())) +} diff --git a/backend/internal/api/handlers/crowdsec_files_test.go b/backend/internal/api/handlers/crowdsec_files_test.go new file mode 100644 index 000000000..a1d0504bb --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_files_test.go @@ -0,0 +1,691 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type filesFixture struct { + t *testing.T + root string // parent of the data dir, where backup dirs are created + dir string // the handler's DataDir + router *gin.Engine +} + +func newFilesFixture(t *testing.T) *filesFixture { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, "crowdsec") + require.NoError(t, os.MkdirAll(dir, 0o750)) + h := newTestCrowdsecHandler(t, setupCrowdDB(t), &fakeExec{}, "/bin/false", dir) + r := gin.New() + h.RegisterRoutes(r.Group("/api/v1")) + return &filesFixture{t: t, root: root, dir: dir, router: r} +} + +func (f *filesFixture) put(rel, content string) string { + f.t.Helper() + p := filepath.Join(f.dir, rel) + require.NoError(f.t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(f.t, os.WriteFile(p, []byte(content), 0o600)) + return p +} + +func (f *filesFixture) read(rel string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/crowdsec/file?path="+url.QueryEscape(rel), http.NoBody) + f.router.ServeHTTP(w, req) + return w +} + +func (f *filesFixture) list() []string { + f.t.Helper() + w := httptest.NewRecorder() + f.router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/v1/admin/crowdsec/files", http.NoBody)) + require.Equal(f.t, http.StatusOK, w.Code) + var resp struct { + Files []string `json:"files"` + } + require.NoError(f.t, json.Unmarshal(w.Body.Bytes(), &resp)) + return resp.Files +} + +func (f *filesFixture) writeRaw(body []byte) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/file", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + f.router.ServeHTTP(w, req) + return w +} + +func (f *filesFixture) write(rel, content string) *httptest.ResponseRecorder { + body, err := json.Marshal(map[string]string{"path": rel, "content": content}) + require.NoError(f.t, err) + return f.writeRaw(body) +} + +func (f *filesFixture) glob(kind string) []string { + m, err := filepath.Glob(filepath.Join(f.root, "crowdsec."+kind+".*")) + require.NoError(f.t, err) + return m +} + +func errorOf(t *testing.T, w *httptest.ResponseRecorder) string { + t.Helper() + var resp map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + return resp["error"] +} + +func TestCrowdsecFiles_Predicates(t *testing.T) { + t.Parallel() + readable := map[string]bool{ + "config.yaml": true, + "config/config.yaml": true, + "config/scenarios/x.yaml": true, + "config/acquis.d/a.conf": true, + "notes.md": true, + "config/data/nested.yaml": true, // only the top-level data/ is protected + "bouncer_key": false, + "config/bouncer_key": false, + "crowdsec.db": false, + "config/crowdsec.db-wal": false, + "data/crowdsec.db": false, + "data/GeoLite2-City.mmdb": false, + "hub_cache/a.tgz": false, + "config/other.db": false, + "config/other.db-journal": false, + "config/CASE.DB": false, + "data": false, + "hub_cache": false, + "config/hub_cache/thing.yaml": true, + } + for rel, want := range readable { + assert.Equal(t, want, isReadable(rel), "isReadable(%q)", rel) + } + + writable := map[string]bool{ + "config.yaml": true, + "config/config.yaml": true, + "a.yml": true, + "a.json": true, + "a.txt": true, + "a.conf": true, + "A.YAML": true, + "notes.md": false, + "script.sh": false, + "noextension": false, + "config/local_api_credentials.yaml": false, + "config/online_api_credentials.yaml": false, + "bouncer_key": false, + "data/x.json": false, + "hub_cache/x.yaml": false, + "config/hub/parsers/s01-parse/x.yaml": false, + "hub/.index.json": false, + "config/x.db": false, + "config/.charon-write-123456789.yaml": false, + "config/hub_cache/thing.yaml": true, + "config/data/nested.yaml": true, + "config/parsers/s01-parse/my-parser.yaml": true, + } + for rel, want := range writable { + assert.Equal(t, want, isWritable(rel), "isWritable(%q)", rel) + if want { + assert.True(t, isReadable(rel), "writable implies readable: %q", rel) + } + } +} + +func TestCrowdsecFiles_CleanRelPath(t *testing.T) { + t.Parallel() + for _, raw := range []string{"../x.yaml", "a/../../x.yaml", "/etc/passwd", "a\x00b.yaml", ".", "./", ".."} { + _, ferr := cleanRelPath(raw) + require.NotNil(t, ferr, raw) + assert.Equal(t, http.StatusBadRequest, ferr.status, raw) + assert.Equal(t, "invalid path", ferr.msg, raw) + } + _, ferr := cleanRelPath("") + require.NotNil(t, ferr) + assert.Equal(t, "path required", ferr.msg) + + rel, ferr := cleanRelPath("a/./b/../c.yaml") + require.Nil(t, ferr) + assert.Equal(t, filepath.Join("a", "c.yaml"), rel) +} + +func TestCrowdsecFiles_Read_ProtectedFilesRefused(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + for _, rel := range []string{"bouncer_key", "crowdsec.db", "data/crowdsec.db", "data/x.txt", "hub_cache/a.yaml", "config/x.db-wal"} { + f.put(rel, "secret") + w := f.read(rel) + assert.Equal(t, http.StatusBadRequest, w.Code, rel) + assert.NotContains(t, w.Body.String(), "secret", rel) + } +} + +func TestCrowdsecFiles_Read_Basics(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("config/config.yaml", "a: 1\n") + w := f.read("config/config.yaml") + require.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "a: 1") + + assert.Equal(t, http.StatusNotFound, f.read("config/missing.yaml").Code) + assert.Equal(t, http.StatusBadRequest, f.read("../x.yaml").Code) + assert.Equal(t, http.StatusBadRequest, f.read("/etc/passwd").Code) + + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config", "d"), 0o750)) + assert.Equal(t, http.StatusBadRequest, f.read("config/d").Code, "directories are not files") +} + +func TestCrowdsecFiles_Read_MissingDataDir(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + require.NoError(t, os.RemoveAll(f.dir)) + assert.Equal(t, http.StatusNotFound, f.read("config/config.yaml").Code) +} + +func TestCrowdsecFiles_Read_SymlinkContainment(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("hub/scenarios/real.yaml", "type: leaky\n") + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config", "scenarios"), 0o750)) + require.NoError(t, os.Symlink(filepath.Join(f.dir, "hub", "scenarios", "real.yaml"), filepath.Join(f.dir, "config", "scenarios", "real.yaml"))) + + w := f.read("config/scenarios/real.yaml") + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + assert.Contains(t, w.Body.String(), "leaky") + + // A relative link that stays inside DataDir is also readable. + require.NoError(t, os.Symlink("../../hub/scenarios/real.yaml", filepath.Join(f.dir, "config", "scenarios", "rel.yaml"))) + assert.Equal(t, http.StatusOK, f.read("config/scenarios/rel.yaml").Code) + + // A link resolving outside DataDir is refused. + outside := filepath.Join(t.TempDir(), "outside.yaml") + require.NoError(t, os.WriteFile(outside, []byte("outside-content"), 0o600)) + require.NoError(t, os.Symlink(outside, filepath.Join(f.dir, "config", "scenarios", "out.yaml"))) + w = f.read("config/scenarios/out.yaml") + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.NotContains(t, w.Body.String(), "outside-content") + + // A link inside DataDir that resolves to a protected file is refused too. + f.put("bouncer_key", "secret") + require.NoError(t, os.Symlink(filepath.Join(f.dir, "bouncer_key"), filepath.Join(f.dir, "config", "k.yaml"))) + w = f.read("config/k.yaml") + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.NotContains(t, w.Body.String(), "secret") + + // A dangling link is reported as missing. + require.NoError(t, os.Symlink(filepath.Join(f.dir, "nope"), filepath.Join(f.dir, "config", "dangling.yaml"))) + assert.Equal(t, http.StatusNotFound, f.read("config/dangling.yaml").Code) +} + +func TestCrowdsecFiles_List(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("config/config.yaml", "a: 1\n") + f.put("hub/scenarios/real.yaml", "x: 1\n") + f.put("bouncer_key", "secret") + f.put("data/crowdsec.db", "db") + f.put("data/GeoLite2-City.mmdb", "geo") + f.put("hub_cache/a.tgz", "tgz") + f.put("config/other.db-wal", "wal") + f.put("config/hub_cache/keep.yaml", "k: 1\n") + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config", "scenarios"), 0o750)) + require.NoError(t, os.Symlink(filepath.Join(f.dir, "hub", "scenarios", "real.yaml"), filepath.Join(f.dir, "config", "scenarios", "real.yaml"))) + outside := filepath.Join(t.TempDir(), "outside.yaml") + require.NoError(t, os.WriteFile(outside, []byte("x"), 0o600)) + require.NoError(t, os.Symlink(outside, filepath.Join(f.dir, "config", "scenarios", "out.yaml"))) + require.NoError(t, os.Symlink(filepath.Join(f.dir, "nope"), filepath.Join(f.dir, "config", "scenarios", "dangling.yaml"))) + require.NoError(t, os.Symlink(filepath.Join(f.dir, "hub"), filepath.Join(f.dir, "config", "dirlink"))) + + files := f.list() + assert.ElementsMatch(t, []string{ + filepath.Join("config", "config.yaml"), + filepath.Join("hub", "scenarios", "real.yaml"), + filepath.Join("config", "scenarios", "real.yaml"), + filepath.Join("config", "hub_cache", "keep.yaml"), + }, files) +} + +func TestCrowdsecFiles_List_MissingDir(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + require.NoError(t, os.RemoveAll(f.dir)) + assert.Empty(t, f.list()) +} + +func TestCrowdsecFiles_List_SkipsInaccessibleDirs(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + f := newFilesFixture(t) + f.put("config/ok.yaml", "a: 1\n") + locked := filepath.Join(f.dir, "lost+found") + require.NoError(t, os.MkdirAll(locked, 0o750)) + require.NoError(t, os.Chmod(locked, 0o000)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(locked, 0o750) }) // #nosec G302 -- test fixture + assert.Equal(t, []string{filepath.Join("config", "ok.yaml")}, f.list()) +} + +// Every file the write endpoint accepts must also be listed (and readable). +func TestCrowdsecFiles_EveryWritableFileIsListed(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + candidates := []string{ + "config.yaml", "config/config.yaml", "config/a.yml", "config/a.json", "config/a.txt", "config/a.conf", + "config/parsers/s01/x.yaml", "config/hub_cache/n.yaml", "config/data/n.yaml", "notes.md", "bouncer_key", + "data/x.json", "hub_cache/x.yaml", "config/local_api_credentials.yaml", "config/x.db", "hub/x.yaml", + } + accepted := 0 + for _, rel := range candidates { + w := f.write(rel, "k: v\n") + if w.Code != http.StatusOK { + continue + } + accepted++ + assert.Contains(t, f.list(), filepath.FromSlash(rel), "writable but not listed: %s", rel) + assert.Equal(t, http.StatusOK, f.read(rel).Code, "writable but not readable: %s", rel) + } + assert.Greater(t, accepted, 5) +} + +func TestCrowdsecFiles_Write_Success(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + w := f.write("config/new/dir/a.yaml", "key: value\n") + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var resp map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, "written", resp["status"]) + assert.Empty(t, resp["backup"], "nothing to back up for a new file") + + b, err := os.ReadFile(filepath.Join(f.dir, "config", "new", "dir", "a.yaml")) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, "key: value\n", string(b)) + info, err := os.Stat(filepath.Join(f.dir, "config", "new", "dir", "a.yaml")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) +} + +func TestCrowdsecFiles_Write_EmptyYAMLAndMultiDocument(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + assert.Equal(t, http.StatusOK, f.write("a.yaml", "").Code) + assert.Equal(t, http.StatusOK, f.write("b.yaml", "a: 1\n---\nb: 2\n").Code) + assert.Equal(t, http.StatusOK, f.write("c.json", "not validated as json").Code) +} + +func TestCrowdsecFiles_Write_Validation(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("sentinel.yaml", "keep: me\n") + + cases := []struct { + name, path, content string + status int + }{ + {"missing path", "", "x", http.StatusBadRequest}, + {"traversal", "../escape.yaml", "a: 1", http.StatusBadRequest}, + {"nested traversal", "config/../../escape.yaml", "a: 1", http.StatusBadRequest}, + {"absolute", "/tmp/escape.yaml", "a: 1", http.StatusBadRequest}, + {"disallowed type", "run.sh", "echo hi", http.StatusBadRequest}, + {"no extension", "noext", "x", http.StatusBadRequest}, + {"invalid yaml", "bad.yaml", "a: [unclosed\n", http.StatusBadRequest}, + {"invalid yml", "bad.yml", "a: b: c: d\n", http.StatusBadRequest}, + {"protected secret", "bouncer_key", "x", http.StatusBadRequest}, + {"protected db", "crowdsec.db", "x", http.StatusBadRequest}, + {"protected data tree", "data/x.json", "{}", http.StatusBadRequest}, + {"protected cache tree", "hub_cache/x.yaml", "a: 1", http.StatusBadRequest}, + {"protected credentials", "config/local_api_credentials.yaml", "a: 1", http.StatusBadRequest}, + {"too large", "big.txt", strings.Repeat("a", maxCrowdsecFileBytes+1), http.StatusRequestEntityTooLarge}, + } + for _, tc := range cases { + w := f.write(tc.path, tc.content) + assert.Equal(t, tc.status, w.Code, "%s: %s", tc.name, w.Body.String()) + assert.NotEmpty(t, errorOf(t, w), tc.name) + } + + // Exactly the limit is accepted. + assert.Equal(t, http.StatusOK, f.write("limit.txt", strings.Repeat("a", maxCrowdsecFileBytes)).Code) + + // Nothing leaked outside DataDir and nothing was created by the rejected requests. + _, err := os.Stat(filepath.Join(f.root, "escape.yaml")) + assert.True(t, os.IsNotExist(err)) + assert.Empty(t, f.glob("filebackup")) + b, err := os.ReadFile(filepath.Join(f.dir, "sentinel.yaml")) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, "keep: me\n", string(b)) +} + +func TestCrowdsecFiles_Write_OversizedBodyIs413(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + body := []byte(`{"path":"a.txt","content":"` + strings.Repeat("a", maxCrowdsecWriteBodyBytes+1) + `"}`) + w := f.writeRaw(body) + assert.Equal(t, http.StatusRequestEntityTooLarge, w.Code) + assert.Equal(t, "file too large", errorOf(t, w)) +} + +func TestCrowdsecFiles_Write_InvalidPayload(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + w := f.writeRaw([]byte("not json")) + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Equal(t, "invalid payload", errorOf(t, w)) +} + +func TestCrowdsecFiles_Write_SymlinkPolicy(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + outsideDir := t.TempDir() + outside := filepath.Join(outsideDir, "outside.yaml") + require.NoError(t, os.WriteFile(outside, []byte("orig: outside\n"), 0o600)) + hubItem := f.put("hub/scenarios/real.yaml", "orig: hub\n") + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config", "scenarios"), 0o750)) + inLink := filepath.Join(f.dir, "config", "scenarios", "real.yaml") + require.NoError(t, os.Symlink(hubItem, inLink)) + outLink := filepath.Join(f.dir, "config", "scenarios", "out.yaml") + require.NoError(t, os.Symlink(outside, outLink)) + dirLink := filepath.Join(f.dir, "config", "linkeddir") + require.NoError(t, os.Symlink(outsideDir, dirLink)) + require.NoError(t, os.Symlink(filepath.Join(f.dir, "nope.yaml"), filepath.Join(f.dir, "config", "dangling.yaml"))) + + for _, rel := range []string{ + "config/scenarios/real.yaml", "config/scenarios/out.yaml", "config/linkeddir/outside.yaml", + "config/linkeddir/new.yaml", "config/dangling.yaml", + } { + w := f.write(rel, "changed: true\n") + assert.Equal(t, http.StatusBadRequest, w.Code, "%s: %s", rel, w.Body.String()) + } + + // Targets are untouched and the links are still links. + for path, want := range map[string]string{hubItem: "orig: hub\n", outside: "orig: outside\n"} { + b, err := os.ReadFile(path) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, want, string(b)) + } + for _, l := range []string{inLink, outLink, dirLink} { + fi, err := os.Lstat(l) + require.NoError(t, err) + assert.NotZero(t, fi.Mode()&os.ModeSymlink, l) + } + _, err := os.Stat(filepath.Join(outsideDir, "new.yaml")) + assert.True(t, os.IsNotExist(err)) + assert.Empty(t, f.glob("filebackup")) + + // A regular file in the same tree is still writable. + assert.Equal(t, http.StatusOK, f.write("config/scenarios/regular.yaml", "ok: true\n").Code) +} + +func TestCrowdsecFiles_Write_RejectsNonRegularTargets(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config", "adir.yaml"), 0o750)) + f.put("config/file.yaml", "a: 1\n") + assert.Equal(t, http.StatusBadRequest, f.write("config/adir.yaml", "a: 1").Code) + assert.Equal(t, http.StatusBadRequest, f.write("config/file.yaml/child.yaml", "a: 1").Code, "parent is a file") +} + +func TestCrowdsecFiles_Write_DoesNotDisturbDataDir(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("config/config.yaml", "old: 1\n") + f.put("config/other.yaml", "other: 1\n") + f.put("bouncer_key", "secret") + f.put("data/crowdsec.db", "db") + f.put("hub_cache/a.tgz", "tgz") + before, err := os.Stat(f.dir) + require.NoError(t, err) + + w := f.write("config/config.yaml", "new: 2\n") + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + + after, err := os.Stat(f.dir) + require.NoError(t, err) + assert.True(t, os.SameFile(before, after), "DataDir must keep its identity") + for rel, want := range map[string]string{ + "config/config.yaml": "new: 2\n", "config/other.yaml": "other: 1\n", + "bouncer_key": "secret", "data/crowdsec.db": "db", "hub_cache/a.tgz": "tgz", + } { + b, readErr := os.ReadFile(filepath.Join(f.dir, rel)) // #nosec G304 -- test path + require.NoError(t, readErr) + assert.Equal(t, want, string(b), rel) + } + // No stray temp files next to the target. + entries, err := os.ReadDir(filepath.Join(f.dir, "config")) + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Empty(t, f.glob("backup"), "a file edit must not create a full snapshot") +} + +func TestCrowdsecFiles_Write_BackupHoldsOnlyPreviousVersionAndIsCappedIndependently(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("config/config.yaml", "v: 0\n") + f.put("config/other.yaml", "other: 1\n") + + // Pre-existing full snapshots must never be evicted by file backups. + snaps := make([]string, 0, 5) + for _, ts := range []string{"20260101-000001", "20260101-000002", "20260101-000003", "20260101-000004", "20260101-000005"} { + p := filepath.Join(f.root, "crowdsec.backup."+ts) + require.NoError(t, os.Mkdir(p, 0o700)) + snaps = append(snaps, p) + } + + w := f.write("config/config.yaml", "v: 1\n") + require.Equal(t, http.StatusOK, w.Code) + var resp map[string]string + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.NotEmpty(t, resp["backup"]) + prev, err := os.ReadFile(filepath.Join(resp["backup"], "config", "config.yaml")) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, "v: 0\n", string(prev)) + _, err = os.Stat(filepath.Join(resp["backup"], "config", "other.yaml")) + assert.True(t, os.IsNotExist(err), "backup must hold only the replaced file") + + for i := 2; i <= 14; i++ { + require.Equal(t, http.StatusOK, f.write("config/config.yaml", "v: "+string(rune('0'+i%10))+"\n").Code) + } + assert.LessOrEqual(t, len(f.glob("filebackup")), 10) + assert.NotEmpty(t, f.glob("filebackup")) + for _, s := range snaps { + _, err := os.Stat(s) + assert.NoError(t, err, "full snapshot evicted by file backups: %s", s) + } +} + +func TestCrowdsecFiles_Write_PreservesExistingMode(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + p := f.put("config/mode.yaml", "a: 1\n") + require.NoError(t, os.Chmod(p, 0o640)) // #nosec G302 -- test fixture + require.Equal(t, http.StatusOK, f.write("config/mode.yaml", "a: 2\n").Code) + info, err := os.Stat(p) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o640), info.Mode().Perm()) +} + +func TestCrowdsecFiles_Write_BackupFailureIs500(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + f := newFilesFixture(t) + p := f.put("config/a.yaml", "v: 0\n") + require.NoError(t, os.Chmod(f.root, 0o500)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(f.root, 0o750) }) // #nosec G302 -- test fixture + + w := f.write("config/a.yaml", "v: 1\n") + assert.Equal(t, http.StatusInternalServerError, w.Code) + b, err := os.ReadFile(p) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, "v: 0\n", string(b), "a failed backup must leave the file unchanged") +} + +func TestAtomicWriteFile(t *testing.T) { + t.Parallel() + dir := t.TempDir() + target := filepath.Join(dir, "a.yaml") + require.NoError(t, atomicWriteFile(target, []byte("one"), 0o600)) + require.NoError(t, atomicWriteFile(target, []byte("two"), 0o600)) + b, err := os.ReadFile(target) // #nosec G304 -- test path + require.NoError(t, err) + assert.Equal(t, "two", string(b)) + + // A failure at the final rename must remove the temp file and keep the destination intact. + blocked := filepath.Join(dir, "blocked") + require.NoError(t, os.MkdirAll(filepath.Join(blocked, "child"), 0o750)) + require.Error(t, atomicWriteFile(blocked, []byte("x"), 0o600)) + require.Error(t, atomicWriteFile(filepath.Join(dir, "no", "such", "dir", "a.yaml"), []byte("x"), 0o600)) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + assert.ElementsMatch(t, []string{"a.yaml", "blocked"}, names) +} + +func TestPathWithin(t *testing.T) { + t.Parallel() + assert.True(t, pathWithin("/a/b", "/a/b")) + assert.True(t, pathWithin("/a/b", "/a/b/c/d")) + assert.False(t, pathWithin("/a/b", "/a/bc")) + assert.False(t, pathWithin("/a/b", "/a")) + assert.False(t, pathWithin("/a/b", "/x/y")) + assert.False(t, pathWithin("/a/b", "relative")) +} + +func TestCrowdsecFiles_Read_UnreadableFileIs500(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + f := newFilesFixture(t) + p := f.put("config/locked.yaml", "a: 1\n") + require.NoError(t, os.Chmod(p, 0o000)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(p, 0o600) }) // #nosec G302 -- test fixture + w := f.read("config/locked.yaml") + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.JSONEq(t, `{"error":"failed to read file"}`, w.Body.String()) + assert.NotContains(t, w.Body.String(), f.dir, "responses must not leak server paths") +} + +func TestCrowdsecFiles_Read_OversizedFileIs413(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + f.put("config/atlimit.yaml", strings.Repeat("a", maxCrowdsecFileBytes)) + assert.Equal(t, http.StatusOK, f.read("config/atlimit.yaml").Code) + f.put("config/big.yaml", strings.Repeat("a", maxCrowdsecFileBytes+1)) + w := f.read("config/big.yaml") + assert.Equal(t, http.StatusRequestEntityTooLarge, w.Code) + assert.JSONEq(t, `{"error":"file too large"}`, w.Body.String()) +} + +func TestCrowdsecFiles_List_WalkFailureIsGeneric500(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + notDir := filepath.Join(f.root, "plain") + require.NoError(t, os.WriteFile(notDir, []byte("x"), 0o600)) + f.router = gin.New() + h := newTestCrowdsecHandler(t, setupCrowdDB(t), &fakeExec{}, "/bin/false", filepath.Join(notDir, "sub")) + h.RegisterRoutes(f.router.Group("/api/v1")) + w := httptest.NewRecorder() + f.router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/v1/admin/crowdsec/files", http.NoBody)) + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.JSONEq(t, `{"error":"failed to list files"}`, w.Body.String()) + assert.NotContains(t, w.Body.String(), f.root) +} + +func TestFileErrorLog_StripsControlCharacters(t *testing.T) { + t.Parallel() + err := errors.New("open config/a\r\nb\nc.yaml: denied") + entry := fileErrorLog(err) + got, ok := entry.Data["error"].(string) + require.True(t, ok) + assert.NotContains(t, got, "\n") + assert.NotContains(t, got, "\r") + assert.Contains(t, got, "denied") +} + +func TestResolveReadable_UnresolvablePaths(t *testing.T) { + t.Parallel() + t.Run("data dir cannot be resolved", func(t *testing.T) { + t.Parallel() + root := t.TempDir() + loop := filepath.Join(root, "crowdsec") + require.NoError(t, os.Symlink("crowdsec", loop)) // ELOOP is not "does not exist" + _, ferr := resolveReadable(loop, "config/a.yaml") + require.NotNil(t, ferr) + assert.Equal(t, http.StatusInternalServerError, ferr.status) + }) + t.Run("self-referencing link inside data dir is refused", func(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + require.NoError(t, os.MkdirAll(filepath.Join(f.dir, "config"), 0o750)) + require.NoError(t, os.Symlink("loop.yaml", filepath.Join(f.dir, "config", "loop.yaml"))) + w := f.read("config/loop.yaml") + assert.Equal(t, http.StatusBadRequest, w.Code) + }) +} + +func TestResolveWritable_UninspectablePathIs500(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + f := newFilesFixture(t) + locked := filepath.Join(f.dir, "locked") + require.NoError(t, os.MkdirAll(locked, 0o750)) + require.NoError(t, os.Chmod(locked, 0o000)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(locked, 0o750) }) // #nosec G302 -- test fixture + + // Lstat fails with EACCES (not ENOENT) below an unsearchable directory. + w := f.write("locked/a.yaml", "a: 1\n") + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Equal(t, "failed to inspect path", errorOf(t, w)) +} + +func TestCrowdsecFiles_Write_FilesystemFailuresAre500(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks do not apply to root") + } + t.Parallel() + t.Run("new directory cannot be created", func(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + require.NoError(t, os.Chmod(f.dir, 0o500)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(f.dir, 0o750) }) // #nosec G302 -- test fixture + w := f.write("newdir/a.yaml", "a: 1\n") + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Equal(t, "failed to prepare dir", errorOf(t, w)) + }) + t.Run("file cannot be written", func(t *testing.T) { + t.Parallel() + f := newFilesFixture(t) + cfg := filepath.Join(f.dir, "config") + require.NoError(t, os.MkdirAll(cfg, 0o750)) + require.NoError(t, os.Chmod(cfg, 0o500)) // #nosec G302 -- test fixture + t.Cleanup(func() { _ = os.Chmod(cfg, 0o750) }) // #nosec G302 -- test fixture + w := f.write("config/new.yaml", "a: 1\n") + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Equal(t, "failed to write file", errorOf(t, w)) + assert.NoFileExists(t, filepath.Join(cfg, "new.yaml")) + }) +} diff --git a/backend/internal/api/handlers/crowdsec_handler.go b/backend/internal/api/handlers/crowdsec_handler.go index ab206ed47..8adb5aa57 100644 --- a/backend/internal/api/handlers/crowdsec_handler.go +++ b/backend/internal/api/handlers/crowdsec_handler.go @@ -78,6 +78,10 @@ type CrowdsecHandler struct { // signalProcess delivers a signal to a PID; overridable in tests. signalProcess func(pid int, sig syscall.Signal) error + // dataMu serializes every operation that mutates DataDir (preset apply, config import, file write). + // Lock order is always dataMu first, then HubService.mu. + dataMu sync.Mutex + // registrationMutex protects concurrent bouncer registration attempts registrationMutex sync.Mutex @@ -351,19 +355,6 @@ func validateYAMLFile(path string) error { return nil } -func ttlRemainingSeconds(now, retrievedAt time.Time, ttl time.Duration) *int64 { - if retrievedAt.IsZero() || ttl <= 0 { - return nil - } - remaining := retrievedAt.Add(ttl).Sub(now) - if remaining < 0 { - var zero int64 - return &zero - } - secs := int64(remaining.Seconds()) - return &secs -} - func mapCrowdsecStatus(err error, defaultCode int) int { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return http.StatusGatewayTimeout @@ -654,7 +645,9 @@ func (h *CrowdsecHandler) Status(c *gin.Context) { }) } -// ImportConfig accepts a tar.gz or zip upload and extracts into DataDir (backing up existing config). +// ImportConfig accepts a tar.gz upload and replaces the CrowdSec configuration in DataDir. The previous +// configuration is snapshotted first and restored on failure; engine-owned state (live db, data/, +// hub_cache/) is never replaced and DataDir itself is never renamed. func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { file, err := c.FormFile("file") if err != nil { @@ -665,14 +658,14 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { // Save to temp file tmpDir := os.TempDir() tmpPath := filepath.Join(tmpDir, fmt.Sprintf("crowdsec-import-%d", time.Now().UnixNano())) - if err := os.MkdirAll(tmpPath, 0o750); err != nil { + if err = os.MkdirAll(tmpPath, 0o750); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create temp dir"}) return } defer func() { _ = os.RemoveAll(tmpPath) }() dst := filepath.Join(tmpPath, file.Filename) - if err := c.SaveUploadedFile(file, dst); err != nil { + if err = c.SaveUploadedFile(file, dst); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to save upload"}) return } @@ -685,27 +678,32 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { RequiredFiles: []string{"config.yaml"}, } - if err := validator.Validate(dst); err != nil { + if err = validator.Validate(dst); err != nil { c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("validation failed: %v", err)}) return } - // Backup current config - var backupDir string - if _, err := os.Stat(h.DataDir); err == nil { - backupDir = h.DataDir + ".backup." + time.Now().Format("20060102-150405") - if err := os.Rename(h.DataDir, backupDir); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create backup"}) - return - } + // Serialize with preset applies and file writes, then snapshot (copy, never rename) the config. + h.dataMu.Lock() + defer h.dataMu.Unlock() + + backupDir, err := crowdsec.Snapshot(h.DataDir) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create backup"}) + return } + defer h.pruneSnapshots() - // Create target dir - if err := os.MkdirAll(h.DataDir, 0o750); err != nil { - // Rollback on failure - if backupDir != "" { - _ = os.Rename(backupDir, h.DataDir) + // rollback restores the snapshot in place; DataDir itself is never removed or renamed. + rollback := func() { + if rbErr := crowdsec.Restore(backupDir, h.DataDir); rbErr != nil { + logger.Log().WithError(rbErr).WithField("backup_path", util.SanitizeForLog(backupDir)).Error("crowdsec import rollback failed; backup retained for manual recovery") } + } + + // Replace the configuration but keep engine-owned state (live db, data/, hub_cache/). + if err := crowdsec.ClearConfig(h.DataDir); err != nil { + rollback() c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create config dir"}) return } @@ -713,11 +711,7 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { // Extract archive extractErr := h.extractArchive(dst, h.DataDir) if extractErr != nil { - // Rollback on extraction failure - _ = os.RemoveAll(h.DataDir) - if backupDir != "" { - _ = os.Rename(backupDir, h.DataDir) - } + rollback() c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("extraction failed: %v", extractErr)}) return } @@ -725,11 +719,7 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { // Validate extracted config configPath := filepath.Join(h.DataDir, "config.yaml") if err := validateYAMLFile(configPath); err != nil { - // Rollback on validation failure - _ = os.RemoveAll(h.DataDir) - if backupDir != "" { - _ = os.Rename(backupDir, h.DataDir) - } + rollback() c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("config validation failed: %v", err)}) return } @@ -737,6 +727,13 @@ func (h *CrowdsecHandler) ImportConfig(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "imported", "backup": backupDir}) } +// pruneSnapshots bounds the full-tree snapshots kept next to DataDir; failures are logged only. +func (h *CrowdsecHandler) pruneSnapshots() { + if _, err := crowdsec.PruneBackups(h.DataDir, crowdsec.BackupKindSnapshot, crowdsec.DefaultBackupRetention); err != nil { + logger.Log().WithError(err).Warn("crowdsec backup prune incomplete") + } +} + // extractArchive extracts a tar.gz archive to the destination directory. func (h *CrowdsecHandler) extractArchive(archivePath, destDir string) error { // #nosec G304 -- archivePath is validated upstream @@ -769,6 +766,10 @@ func (h *CrowdsecHandler) extractArchive(archivePath, destDir string) error { if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) { return fmt.Errorf("invalid file path: %s", header.Name) } + // Engine-owned state (live db files, top-level data/ and hub_cache/) is never taken from an upload. + if crowdsec.IsEngineOwnedPath(strings.TrimPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator))) { + continue + } switch header.Typeflag { case tar.TypeDir: @@ -776,11 +777,6 @@ func (h *CrowdsecHandler) extractArchive(archivePath, destDir string) error { return fmt.Errorf("failed to create directory: %w", err) } case tar.TypeReg: - // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { - return fmt.Errorf("failed to create parent directory: %w", err) - } - // Validate mode is safe before conversion fileMode := os.FileMode(0o640) // Default safe mode if header.Mode > 0 && header.Mode <= 0o777 { @@ -788,8 +784,7 @@ func (h *CrowdsecHandler) extractArchive(archivePath, destDir string) error { fileMode = os.FileMode(header.Mode) } - // #nosec G304 -- target is constructed safely above with path traversal protection - outFile, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, fileMode) + outFile, err := crowdsec.CreateFileNoSymlink(destDir, target, 0o750, fileMode) if err != nil { return fmt.Errorf("failed to create file: %w", err) } @@ -886,443 +881,6 @@ func (h *CrowdsecHandler) ExportConfig(c *gin.Context) { } } -// ListFiles returns a flat list of files under the CrowdSec DataDir. -func (h *CrowdsecHandler) ListFiles(c *gin.Context) { - files := []string{} - if _, err := os.Stat(h.DataDir); os.IsNotExist(err) { - c.JSON(http.StatusOK, gin.H{"files": files}) - return - } - err := filepath.Walk(h.DataDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - // Permission errors (e.g. lost+found) should not abort the walk - if os.IsPermission(err) { - logger.Log().WithError(err).WithField("path", path).Debug("Skipping inaccessible path during list") - return filepath.SkipDir - } - return err - } - if !info.IsDir() { - rel, err := filepath.Rel(h.DataDir, path) - if err != nil { - return err - } - files = append(files, rel) - } - return nil - }) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"files": files}) -} - -// ReadFile returns the contents of a specific file under DataDir. Query param 'path' required. -func (h *CrowdsecHandler) ReadFile(c *gin.Context) { - rel := c.Query("path") - if rel == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "path required"}) - return - } - clean := filepath.Clean(rel) - // prevent directory traversal - p := filepath.Join(h.DataDir, clean) - if !strings.HasPrefix(p, filepath.Clean(h.DataDir)) { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid path"}) - return - } - // #nosec G304 -- p is validated against CrowdSecDataDir by detectFilePath - data, err := os.ReadFile(p) - if err != nil { - if os.IsNotExist(err) { - c.JSON(http.StatusNotFound, gin.H{"error": "file not found"}) - return - } - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"content": string(data)}) -} - -// WriteFile writes content to a file under the CrowdSec DataDir, creating a backup before doing so. -// JSON body: { "path": "relative/path.conf", "content": "..." } -func (h *CrowdsecHandler) WriteFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - Content string `json:"content"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) - return - } - if payload.Path == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "path required"}) - return - } - clean := filepath.Clean(payload.Path) - p := filepath.Join(h.DataDir, clean) - if !strings.HasPrefix(p, filepath.Clean(h.DataDir)) { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid path"}) - return - } - // Backup existing DataDir - backupDir := h.DataDir + ".backup." + time.Now().Format("20060102-150405") - if _, err := os.Stat(h.DataDir); err == nil { - if err := os.Rename(h.DataDir, backupDir); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create backup"}) - return - } - } - // Recreate DataDir and write file - if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to prepare dir"}) - return - } - if err := os.WriteFile(p, []byte(payload.Content), 0o600); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write file"}) - return - } - c.JSON(http.StatusOK, gin.H{"status": "written", "backup": backupDir}) -} - -// ListPresets returns the curated preset catalog when Cerberus is enabled. -func (h *CrowdsecHandler) ListPresets(c *gin.Context) { - if !h.isCerberusEnabled() { - c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) - return - } - - type presetInfo struct { - crowdsec.Preset - Available bool `json:"available"` - Cached bool `json:"cached"` - CacheKey string `json:"cache_key,omitempty"` - Etag string `json:"etag,omitempty"` - RetrievedAt *time.Time `json:"retrieved_at,omitempty"` - TTLRemainingSeconds *int64 `json:"ttl_remaining_seconds,omitempty"` - } - - result := map[string]*presetInfo{} - for _, p := range crowdsec.ListCuratedPresets() { - cp := p - result[p.Slug] = &presetInfo{Preset: cp, Available: true} - } - - // Merge hub index when available - if h.Hub != nil { - ctx := c.Request.Context() - if idx, err := h.Hub.FetchIndex(ctx); err == nil { - for _, item := range idx.Items { - slug := strings.TrimSpace(item.Name) - if slug == "" { - continue - } - if _, ok := result[slug]; !ok { - result[slug] = &presetInfo{Preset: crowdsec.Preset{ - Slug: slug, - Title: item.Title, - Summary: item.Description, - Source: "hub", - Tags: []string{item.Type}, - RequiresHub: true, - }, Available: true} - } else { - result[slug].Available = true - } - } - } else { - logger.Log().WithError(err).Warn("crowdsec hub index unavailable") - } - } - - // Merge cache metadata - if h.Hub != nil && h.Hub.Cache != nil { - ctx := c.Request.Context() - if cached, err := h.Hub.Cache.List(ctx); err == nil { - cacheTTL := h.Hub.Cache.TTL() - now := time.Now().UTC() - for _, entry := range cached { - if _, ok := result[entry.Slug]; !ok { - result[entry.Slug] = &presetInfo{Preset: crowdsec.Preset{Slug: entry.Slug, Title: entry.Slug, Summary: "cached preset", Source: "hub", RequiresHub: true}} - } - result[entry.Slug].Cached = true - result[entry.Slug].CacheKey = entry.CacheKey - result[entry.Slug].Etag = entry.Etag - if !entry.RetrievedAt.IsZero() { - val := entry.RetrievedAt - result[entry.Slug].RetrievedAt = &val - } - result[entry.Slug].TTLRemainingSeconds = ttlRemainingSeconds(now, entry.RetrievedAt, cacheTTL) - } - } else { - logger.Log().WithError(err).Warn("crowdsec hub cache list failed") - } - } - - list := make([]presetInfo, 0, len(result)) - for _, v := range result { - list = append(list, *v) - } - - c.JSON(http.StatusOK, gin.H{"presets": list}) -} - -// PullPreset downloads and caches a hub preset while returning a preview. -func (h *CrowdsecHandler) PullPreset(c *gin.Context) { - if !h.isCerberusEnabled() { - c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) - return - } - - var payload struct { - Slug string `json:"slug"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) - return - } - slug := strings.TrimSpace(payload.Slug) - if slug == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) - return - } - if h.Hub == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub service unavailable"}) - return - } - - // Check for curated preset that doesn't require hub - if preset, ok := crowdsec.FindPreset(slug); ok && !preset.RequiresHub { - c.JSON(http.StatusOK, gin.H{ - "status": "pulled", - "slug": preset.Slug, - "preview": curatedPresetPreview(preset), - "cache_key": "curated-" + preset.Slug, - "etag": "curated", - "retrieved_at": time.Now(), - "source": "charon-curated", - }) - return - } - - ctx := c.Request.Context() - // Log cache directory before pull - if h.Hub != nil && h.Hub.Cache != nil { - cacheDir := filepath.Join(h.DataDir, "hub_cache") - logger.Log().WithField("cache_dir", util.SanitizeForLog(cacheDir)).WithField("slug", util.SanitizeForLog(slug)).Info("attempting to pull preset") - if stat, err := os.Stat(cacheDir); err == nil { - logger.Log().WithField("cache_dir_mode", stat.Mode()).WithField("cache_dir_writable", stat.Mode().Perm()&0o200 != 0).Debug("cache directory exists") - } else { - logger.Log().WithError(err).Warn("cache directory stat failed") - } - } - - res, err := h.Hub.Pull(ctx, slug) - if err != nil { - status := mapCrowdsecStatus(err, http.StatusBadGateway) - // Safe: User input sanitized via util.SanitizeForLog() which removes - // control characters (0x00-0x1F, 0x7F) including CRLF - // codeql[go/log-injection] - logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(slug)).WithField("hub_base_url", util.SanitizeForLog(h.Hub.HubBaseURL)).Warn("crowdsec preset pull failed") - c.JSON(status, gin.H{"error": err.Error(), "hub_endpoints": h.hubEndpoints()}) - return - } - - // Verify cache was actually stored - // Safe: res.Meta fields are system-generated (cache keys, file paths) - // not directly derived from untrusted user input - // codeql[go/log-injection] - logger.Log().Info("preset pulled and cached successfully") - - // Verify files exist on disk - if _, err := os.Stat(res.Meta.ArchivePath); err != nil { - // codeql[go/log-injection] Safe: archive_path is system-generated file path - logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("archive_path", util.SanitizeForLog(res.Meta.ArchivePath)).Error("cached archive file not found after pull") - } - if _, err := os.Stat(res.Meta.PreviewPath); err != nil { - // codeql[go/log-injection] Safe: preview_path is system-generated file path - logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("preview_path", util.SanitizeForLog(res.Meta.PreviewPath)).Error("cached preview file not found after pull") - } - - c.JSON(http.StatusOK, gin.H{ - "status": "pulled", - "slug": res.Meta.Slug, - "preview": res.Preview, - "cache_key": res.Meta.CacheKey, - "etag": res.Meta.Etag, - "retrieved_at": res.Meta.RetrievedAt, - "source": res.Meta.Source, - }) -} - -// ApplyPreset installs a pulled preset from cache or via cscli. -func (h *CrowdsecHandler) ApplyPreset(c *gin.Context) { - if !h.isCerberusEnabled() { - c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) - return - } - - var payload struct { - Slug string `json:"slug"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) - return - } - - slug := strings.TrimSpace(payload.Slug) - if slug == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) - return - } - if h.Hub == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub service unavailable"}) - return - } - - // Check for curated preset that doesn't require hub - if preset, ok := crowdsec.FindPreset(slug); ok && !preset.RequiresHub { - h.applyCuratedPreset(c, preset) - return - } - - ctx := c.Request.Context() - - // Log cache status before apply - if h.Hub != nil && h.Hub.Cache != nil { - cacheDir := filepath.Join(h.DataDir, "hub_cache") - logger.Log().WithField("cache_dir", util.SanitizeForLog(cacheDir)).WithField("slug", util.SanitizeForLog(slug)).Info("attempting to apply preset") - - // Check if cached - if cached, err := h.Hub.Cache.Load(ctx, slug); err == nil { - logger.Log().WithField("slug", util.SanitizeForLog(slug)).WithField("cache_key", cached.CacheKey).WithField("archive_path", cached.ArchivePath).WithField("preview_path", cached.PreviewPath).Info("preset found in cache") - // Verify files still exist - if _, statErr := os.Stat(cached.ArchivePath); statErr != nil { - logger.Log().WithError(statErr).WithField("archive_path", cached.ArchivePath).Error("cached archive file missing") - } - if _, statErr := os.Stat(cached.PreviewPath); statErr != nil { - logger.Log().WithError(statErr).WithField("preview_path", cached.PreviewPath).Error("cached preview file missing") - } - } else { - logger.Log().WithError(err).WithField("slug", util.SanitizeForLog(slug)).Warn("preset not found in cache before apply") - // List what's actually in the cache - if entries, listErr := h.Hub.Cache.List(ctx); listErr == nil { - slugs := make([]string, len(entries)) - for i, e := range entries { - slugs[i] = e.Slug - } - logger.Log().WithField("cached_slugs", slugs).Info("current cache contents") - } - } - } - - res, err := h.Hub.Apply(ctx, slug) - if err != nil { - status := mapCrowdsecStatus(err, http.StatusInternalServerError) - // Safe: User input (slug) sanitized via util.SanitizeForLog(); - // backup_path and cache_key are system-generated values - // codeql[go/log-injection] - logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(slug)).WithField("hub_base_url", util.SanitizeForLog(h.Hub.HubBaseURL)).WithField("backup_path", util.SanitizeForLog(res.BackupPath)).WithField("cache_key", util.SanitizeForLog(res.CacheKey)).Warn("crowdsec preset apply failed") - h.recordPresetEvent(slug, res, err) - // Build detailed error response - errorMsg := err.Error() - // Add actionable guidance based on error type - if errors.Is(err, crowdsec.ErrCacheMiss) || strings.Contains(errorMsg, "cache miss") { - errorMsg = "Preset cache missing or expired. Pull the preset again, then retry apply." - } else if strings.Contains(errorMsg, "cscli unavailable") && strings.Contains(errorMsg, "no cached preset") { - errorMsg = "CrowdSec preset not cached. Pull the preset first by clicking 'Pull Preview', then try applying again." - } - errorResponse := applyFailureBody(errorMsg, res) - errorResponse["hub_endpoints"] = h.hubEndpoints() - c.JSON(status, errorResponse) - return - } - - h.recordPresetEvent(slug, res, nil) - respondApplySuccess(c, res) -} - -// applyCuratedPreset installs a Charon-defined preset through cscli and reports the true outcome. -// A failure is recorded as a "failed" audit event and never answered with a 2xx status. -func (h *CrowdsecHandler) applyCuratedPreset(c *gin.Context, preset crowdsec.Preset) { - res, err := h.Hub.ApplyCurated(c.Request.Context(), preset) - if err != nil { - logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(preset.Slug)).WithField("backup_path", util.SanitizeForLog(res.BackupPath)).Warn("curated crowdsec preset apply failed") - h.recordPresetEvent(preset.Slug, res, err) - - status := mapCrowdsecStatus(err, http.StatusInternalServerError) - msg := err.Error() - if errors.Is(err, crowdsec.ErrCSCLIUnavailable) { - status = http.StatusServiceUnavailable - msg = "CrowdSec CLI is not available; curated presets require cscli" - } - c.JSON(status, applyFailureBody(msg, res)) - return - } - h.recordPresetEvent(preset.Slug, res, nil) - respondApplySuccess(c, res) -} - -// curatedPresetPreview renders the hub items a curated preset will install. -func curatedPresetPreview(preset crowdsec.Preset) string { - var b strings.Builder - b.WriteString("# Curated preset: " + preset.Title + "\n# " + preset.Summary + "\n#\n# Installs these CrowdSec hub items:\n") - for _, item := range preset.Items { - b.WriteString("# " + item.Type + ": " + item.Name + "\n") - } - return b.String() -} - -// recordPresetEvent persists an apply audit row. A nil err records the result status -// (defaulting to "applied"); a non-nil err records a "failed" row with the error text. -// Persistence errors are logged and never alter the request outcome. -func (h *CrowdsecHandler) recordPresetEvent(slug string, res crowdsec.ApplyResult, applyErr error) { - if h.DB == nil { - return - } - event := models.CrowdsecPresetEvent{Slug: slug, Action: "apply", CacheKey: res.CacheKey, BackupPath: res.BackupPath} - if applyErr != nil { - event.Status = "failed" - event.Error = applyErr.Error() - } else { - event.Status = res.Status - if event.Status == "" { - event.Status = "applied" - } - if res.AppliedPreset != "" { - event.Slug = res.AppliedPreset - } - } - if err := h.DB.Create(&event).Error; err != nil { - logger.Log().WithError(err).WithField("slug", util.SanitizeForLog(event.Slug)).Warn("failed to record crowdsec preset event") - } -} - -// applyFailureBody builds the common error payload for a failed preset apply. -func applyFailureBody(msg string, res crowdsec.ApplyResult) gin.H { - body := gin.H{"error": msg} - if res.BackupPath != "" { - body["backup"] = res.BackupPath - } - if res.CacheKey != "" { - body["cache_key"] = res.CacheKey - } - return body -} - -// respondApplySuccess writes the 200 payload for a completed preset apply. -func respondApplySuccess(c *gin.Context, res crowdsec.ApplyResult) { - c.JSON(http.StatusOK, gin.H{ - "status": res.Status, - "backup": res.BackupPath, - "reload_hint": res.ReloadHint, - "used_cscli": res.UsedCSCLI, - "cache_key": res.CacheKey, - "slug": res.AppliedPreset, - }) -} - // ConsoleEnroll enrolls the local engine with CrowdSec console. func (h *CrowdsecHandler) ConsoleEnroll(c *gin.Context) { if !h.isConsoleEnrollmentEnabled() { @@ -1422,47 +980,6 @@ func (h *CrowdsecHandler) DeleteConsoleEnrollment(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"message": "enrollment state cleared"}) } -// GetCachedPreset returns cached preview for a slug when available. -func (h *CrowdsecHandler) GetCachedPreset(c *gin.Context) { - if !h.isCerberusEnabled() { - c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) - return - } - if h.Hub == nil || h.Hub.Cache == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub cache unavailable"}) - return - } - ctx := c.Request.Context() - slug := strings.TrimSpace(c.Param("slug")) - if slug == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) - return - } - preview, err := h.Hub.Cache.LoadPreview(ctx, slug) - if err != nil { - if errors.Is(err, crowdsec.ErrCacheMiss) || errors.Is(err, crowdsec.ErrCacheExpired) { - c.JSON(http.StatusNotFound, gin.H{"error": "cache miss"}) - return - } - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - meta, metaErr := h.Hub.Cache.Load(ctx, slug) - if metaErr != nil && !errors.Is(metaErr, crowdsec.ErrCacheMiss) && !errors.Is(metaErr, crowdsec.ErrCacheExpired) { - c.JSON(http.StatusInternalServerError, gin.H{"error": metaErr.Error()}) - return - } - cacheTTL := h.Hub.Cache.TTL() - now := time.Now().UTC() - c.JSON(http.StatusOK, gin.H{ - "preview": preview, - "cache_key": meta.CacheKey, - "etag": meta.Etag, - "retrieved_at": meta.RetrievedAt, - "ttl_remaining_seconds": ttlRemainingSeconds(now, meta.RetrievedAt, cacheTTL), - }) -} - // CrowdSecDecision represents a ban decision from CrowdSec type CrowdSecDecision struct { ID int64 `json:"id"` diff --git a/backend/internal/api/handlers/crowdsec_handler_coverage_test.go b/backend/internal/api/handlers/crowdsec_handler_coverage_test.go index 5e6c7c8db..4bace2daf 100644 --- a/backend/internal/api/handlers/crowdsec_handler_coverage_test.go +++ b/backend/internal/api/handlers/crowdsec_handler_coverage_test.go @@ -348,6 +348,32 @@ func TestCrowdsec_WriteFile_Success(t *testing.T) { assert.Equal(t, "new content", string(content)) } +func TestCrowdsec_WriteFile_NewSubdirectoryOwnerOnly(t *testing.T) { + db := setupCrowdDB(t) + tmpDir := t.TempDir() + + h := newTestCrowdsecHandler(t, db, &fakeExec{}, "/bin/false", tmpDir) + + r := gin.New() + g := r.Group("/api/v1") + h.RegisterRoutes(g) + + b, _ := json.Marshal(map[string]string{"path": "newdir/sub/new.conf", "content": "x"}) + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/file", bytes.NewReader(b)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + for _, d := range []string{"newdir", filepath.Join("newdir", "sub")} { + info, err := os.Stat(filepath.Join(tmpDir, d)) + if !assert.NoError(t, err) { + continue + } + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + } +} + func TestCrowdsec_ListPresets_Disabled(t *testing.T) { db := setupCrowdDB(t) t.Setenv("FEATURE_CERBERUS_ENABLED", "false") diff --git a/backend/internal/api/handlers/crowdsec_handler_test.go b/backend/internal/api/handlers/crowdsec_handler_test.go index 6120e980d..3fb468f79 100644 --- a/backend/internal/api/handlers/crowdsec_handler_test.go +++ b/backend/internal/api/handlers/crowdsec_handler_test.go @@ -377,43 +377,48 @@ func TestExportConfigSkipsSymlinks(t *testing.T) { require.Equal(t, []string{"config.yaml"}, names) } -func TestWriteFileCreatesBackup(t *testing.T) { +// TestWriteFileNeverRenamesDataDir is the regression test for the destructive rename: writing one +// file must leave DataDir and every unrelated file in place. +func TestWriteFileNeverRenamesDataDir(t *testing.T) { t.Parallel() db := setupCrowdDB(t) tmpDir := t.TempDir() - // create existing config dir with a marker file - _ = os.MkdirAll(tmpDir, 0o750) // #nosec G301 -- test directory - _ = os.WriteFile(filepath.Join(tmpDir, "existing.conf"), []byte("v1"), 0o600) // #nosec G306 -- test fixture - - fe := &fakeExec{} - h := newTestCrowdsecHandler(t, db, fe, "/bin/false", tmpDir) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "existing.conf"), []byte("v1"), 0o600)) // #nosec G306 -- test fixture + before, err := os.Stat(tmpDir) + require.NoError(t, err) + h := newTestCrowdsecHandler(t, db, &fakeExec{}, "/bin/false", tmpDir) r := gin.New() g := r.Group("/api/v1") h.RegisterRoutes(g) - // write content to new file payload := map[string]string{"path": "conf.d/new.conf", "content": "hello world"} b, _ := json.Marshal(payload) w := httptest.NewRecorder() req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/crowdsec/file", bytes.NewReader(b)) req.Header.Set("Content-Type", "application/json") r.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("write expected 200 got %d body=%s", w.Code, w.Body.String()) - } + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) - // ensure backup directory was created - entries, err := os.ReadDir(filepath.Dir(tmpDir)) + var resp map[string]any + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.Empty(t, resp["backup"], "a new file has nothing to back up") + + after, err := os.Stat(tmpDir) require.NoError(t, err) - foundBackup := false - for _, e := range entries { - if e.IsDir() && strings.HasPrefix(e.Name(), filepath.Base(tmpDir)+".backup.") { - foundBackup = true - break - } - } - require.True(t, foundBackup, "expected backup directory to be created") + require.True(t, os.SameFile(before, after), "DataDir must not be renamed") + // #nosec G304 -- test reads fixture paths + kept, err := os.ReadFile(filepath.Join(tmpDir, "existing.conf")) + require.NoError(t, err) + require.Equal(t, "v1", string(kept)) + // #nosec G304 -- test reads fixture paths + written, err := os.ReadFile(filepath.Join(tmpDir, "conf.d", "new.conf")) + require.NoError(t, err) + require.Equal(t, "hello world", string(written)) + + matches, err := filepath.Glob(tmpDir + ".*") + require.NoError(t, err) + require.Empty(t, matches, "no backup directories are created for a new file") } func TestListPresetsCerberusDisabled(t *testing.T) { @@ -3227,7 +3232,7 @@ func TestCrowdsecHandler_WriteFile_BackupCreation(t *testing.T) { h.RegisterRoutes(g) body := map[string]string{ - "path": "test.conf", + "path": "existing.conf", "content": "new content", } b, _ := json.Marshal(body) @@ -3246,10 +3251,17 @@ func TestCrowdsecHandler_WriteFile_BackupCreation(t *testing.T) { backupPath := resp["backup"].(string) require.NotEmpty(t, backupPath) + require.True(t, strings.HasPrefix(backupPath, tmpDir+".filebackup."), backupPath) - // Verify backup directory exists - _, err := os.Stat(backupPath) + // Only the replaced file is backed up, with its previous content; DataDir stays in place. + // #nosec G304 -- test reads a path it just created + old, err := os.ReadFile(filepath.Join(backupPath, "existing.conf")) + require.NoError(t, err) + require.Equal(t, "old content", string(old)) + // #nosec G304 -- test reads a path it just created + cur, err := os.ReadFile(existingFile) require.NoError(t, err) + require.Equal(t, "new content", string(cur)) } // Test ReadFile with path traversal protection diff --git a/backend/internal/api/handlers/crowdsec_preset_handler.go b/backend/internal/api/handlers/crowdsec_preset_handler.go new file mode 100644 index 000000000..66f2db62f --- /dev/null +++ b/backend/internal/api/handlers/crowdsec_preset_handler.go @@ -0,0 +1,412 @@ +package handlers + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Wikid82/charon/backend/internal/crowdsec" + "github.com/Wikid82/charon/backend/internal/logger" + "github.com/Wikid82/charon/backend/internal/models" + "github.com/Wikid82/charon/backend/internal/util" + + "github.com/gin-gonic/gin" +) + +func ttlRemainingSeconds(now, retrievedAt time.Time, ttl time.Duration) *int64 { + if retrievedAt.IsZero() || ttl <= 0 { + return nil + } + remaining := retrievedAt.Add(ttl).Sub(now) + if remaining < 0 { + var zero int64 + return &zero + } + secs := int64(remaining.Seconds()) + return &secs +} + +// ListPresets returns the curated preset catalog when Cerberus is enabled. +func (h *CrowdsecHandler) ListPresets(c *gin.Context) { + if !h.isCerberusEnabled() { + c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) + return + } + + type presetInfo struct { + crowdsec.Preset + Available bool `json:"available"` + Cached bool `json:"cached"` + CacheKey string `json:"cache_key,omitempty"` + Etag string `json:"etag,omitempty"` + RetrievedAt *time.Time `json:"retrieved_at,omitempty"` + TTLRemainingSeconds *int64 `json:"ttl_remaining_seconds,omitempty"` + } + + result := map[string]*presetInfo{} + for _, p := range crowdsec.ListCuratedPresets() { + cp := p + result[p.Slug] = &presetInfo{Preset: cp, Available: true} + } + + // Merge hub index when available + if h.Hub != nil { + ctx := c.Request.Context() + if idx, err := h.Hub.FetchIndex(ctx); err == nil { + for _, item := range idx.Items { + slug := strings.TrimSpace(item.Name) + if slug == "" { + continue + } + if _, ok := result[slug]; !ok { + result[slug] = &presetInfo{Preset: crowdsec.Preset{ + Slug: slug, + Title: item.Title, + Summary: item.Description, + Source: "hub", + Tags: []string{item.Type}, + RequiresHub: true, + }, Available: true} + } else { + result[slug].Available = true + } + } + } else { + logger.Log().WithError(err).Warn("crowdsec hub index unavailable") + } + } + + // Merge cache metadata + if h.Hub != nil && h.Hub.Cache != nil { + ctx := c.Request.Context() + if cached, err := h.Hub.Cache.List(ctx); err == nil { + cacheTTL := h.Hub.Cache.TTL() + now := time.Now().UTC() + for _, entry := range cached { + if _, ok := result[entry.Slug]; !ok { + result[entry.Slug] = &presetInfo{Preset: crowdsec.Preset{Slug: entry.Slug, Title: entry.Slug, Summary: "cached preset", Source: "hub", RequiresHub: true}} + } + result[entry.Slug].Cached = true + result[entry.Slug].CacheKey = entry.CacheKey + result[entry.Slug].Etag = entry.Etag + if !entry.RetrievedAt.IsZero() { + val := entry.RetrievedAt + result[entry.Slug].RetrievedAt = &val + } + result[entry.Slug].TTLRemainingSeconds = ttlRemainingSeconds(now, entry.RetrievedAt, cacheTTL) + } + } else { + logger.Log().WithError(err).Warn("crowdsec hub cache list failed") + } + } + + list := make([]presetInfo, 0, len(result)) + for _, v := range result { + list = append(list, *v) + } + + c.JSON(http.StatusOK, gin.H{"presets": list}) +} + +// PullPreset downloads and caches a hub preset while returning a preview. +func (h *CrowdsecHandler) PullPreset(c *gin.Context) { + if !h.isCerberusEnabled() { + c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) + return + } + + var payload struct { + Slug string `json:"slug"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) + return + } + slug := strings.TrimSpace(payload.Slug) + if slug == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) + return + } + if h.Hub == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub service unavailable"}) + return + } + + // Check for curated preset that doesn't require hub + if preset, ok := crowdsec.FindPreset(slug); ok && !preset.RequiresHub { + c.JSON(http.StatusOK, gin.H{ + "status": "pulled", + "slug": preset.Slug, + "preview": curatedPresetPreview(preset), + "cache_key": "curated-" + preset.Slug, + "etag": "curated", + "retrieved_at": time.Now(), + "source": "charon-curated", + }) + return + } + + ctx := c.Request.Context() + // Log cache directory before pull + if h.Hub != nil && h.Hub.Cache != nil { + cacheDir := filepath.Join(h.DataDir, "hub_cache") + logger.Log().WithField("cache_dir", util.SanitizeForLog(cacheDir)).WithField("slug", util.SanitizeForLog(slug)).Info("attempting to pull preset") + if stat, err := os.Stat(cacheDir); err == nil { + logger.Log().WithField("cache_dir_mode", stat.Mode()).WithField("cache_dir_writable", stat.Mode().Perm()&0o200 != 0).Debug("cache directory exists") + } else { + logger.Log().WithError(err).Warn("cache directory stat failed") + } + } + + res, err := h.Hub.Pull(ctx, slug) + if err != nil { + status := mapCrowdsecStatus(err, http.StatusBadGateway) + // Safe: User input sanitized via util.SanitizeForLog() which removes + // control characters (0x00-0x1F, 0x7F) including CRLF + // codeql[go/log-injection] + logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(slug)).WithField("hub_base_url", util.SanitizeForLog(h.Hub.HubBaseURL)).Warn("crowdsec preset pull failed") + c.JSON(status, gin.H{"error": err.Error(), "hub_endpoints": h.hubEndpoints()}) + return + } + + // Verify cache was actually stored + // Safe: res.Meta fields are system-generated (cache keys, file paths) + // not directly derived from untrusted user input + // codeql[go/log-injection] + logger.Log().Info("preset pulled and cached successfully") + + // Verify files exist on disk + if _, err := os.Stat(res.Meta.ArchivePath); err != nil { + // codeql[go/log-injection] Safe: archive_path is system-generated file path + logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("archive_path", util.SanitizeForLog(res.Meta.ArchivePath)).Error("cached archive file not found after pull") + } + if _, err := os.Stat(res.Meta.PreviewPath); err != nil { + // codeql[go/log-injection] Safe: preview_path is system-generated file path + logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("preview_path", util.SanitizeForLog(res.Meta.PreviewPath)).Error("cached preview file not found after pull") + } + + c.JSON(http.StatusOK, gin.H{ + "status": "pulled", + "slug": res.Meta.Slug, + "preview": res.Preview, + "cache_key": res.Meta.CacheKey, + "etag": res.Meta.Etag, + "retrieved_at": res.Meta.RetrievedAt, + "source": res.Meta.Source, + }) +} + +// ApplyPreset installs a pulled preset from cache or via cscli. +func (h *CrowdsecHandler) ApplyPreset(c *gin.Context) { + if !h.isCerberusEnabled() { + c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) + return + } + + var payload struct { + Slug string `json:"slug"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) + return + } + + slug := strings.TrimSpace(payload.Slug) + if slug == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) + return + } + if h.Hub == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub service unavailable"}) + return + } + + // One mutation of DataDir at a time across preset apply, config import and file writes. + h.dataMu.Lock() + defer h.dataMu.Unlock() + + // Check for curated preset that doesn't require hub + if preset, ok := crowdsec.FindPreset(slug); ok && !preset.RequiresHub { + h.applyCuratedPreset(c, preset) + return + } + + ctx := c.Request.Context() + + // Log cache status before apply + if h.Hub != nil && h.Hub.Cache != nil { + cacheDir := filepath.Join(h.DataDir, "hub_cache") + logger.Log().WithField("cache_dir", util.SanitizeForLog(cacheDir)).WithField("slug", util.SanitizeForLog(slug)).Info("attempting to apply preset") + + // Check if cached + if cached, err := h.Hub.Cache.Load(ctx, slug); err == nil { + logger.Log().WithField("slug", util.SanitizeForLog(slug)).WithField("cache_key", cached.CacheKey).WithField("archive_path", cached.ArchivePath).WithField("preview_path", cached.PreviewPath).Info("preset found in cache") + // Verify files still exist + if _, statErr := os.Stat(cached.ArchivePath); statErr != nil { + logger.Log().WithError(statErr).WithField("archive_path", cached.ArchivePath).Error("cached archive file missing") + } + if _, statErr := os.Stat(cached.PreviewPath); statErr != nil { + logger.Log().WithError(statErr).WithField("preview_path", cached.PreviewPath).Error("cached preview file missing") + } + } else { + logger.Log().WithError(err).WithField("slug", util.SanitizeForLog(slug)).Warn("preset not found in cache before apply") + // List what's actually in the cache + if entries, listErr := h.Hub.Cache.List(ctx); listErr == nil { + slugs := make([]string, len(entries)) + for i, e := range entries { + slugs[i] = e.Slug + } + logger.Log().WithField("cached_slugs", slugs).Info("current cache contents") + } + } + } + + res, err := h.Hub.Apply(ctx, slug) + if err != nil { + status := mapCrowdsecStatus(err, http.StatusInternalServerError) + // Safe: User input (slug) sanitized via util.SanitizeForLog(); + // backup_path and cache_key are system-generated values + // codeql[go/log-injection] + logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(slug)).WithField("hub_base_url", util.SanitizeForLog(h.Hub.HubBaseURL)).WithField("backup_path", util.SanitizeForLog(res.BackupPath)).WithField("cache_key", util.SanitizeForLog(res.CacheKey)).Warn("crowdsec preset apply failed") + h.recordPresetEvent(slug, res, err) + // Build detailed error response + errorMsg := err.Error() + // Add actionable guidance based on error type + if errors.Is(err, crowdsec.ErrCacheMiss) || strings.Contains(errorMsg, "cache miss") { + errorMsg = "Preset cache missing or expired. Pull the preset again, then retry apply." + } else if strings.Contains(errorMsg, "cscli unavailable") && strings.Contains(errorMsg, "no cached preset") { + errorMsg = "CrowdSec preset not cached. Pull the preset first by clicking 'Pull Preview', then try applying again." + } + errorResponse := applyFailureBody(errorMsg, res) + errorResponse["hub_endpoints"] = h.hubEndpoints() + c.JSON(status, errorResponse) + return + } + + h.recordPresetEvent(slug, res, nil) + respondApplySuccess(c, res) +} + +// applyCuratedPreset installs a Charon-defined preset through cscli and reports the true outcome. +// A failure is recorded as a "failed" audit event and never answered with a 2xx status. +func (h *CrowdsecHandler) applyCuratedPreset(c *gin.Context, preset crowdsec.Preset) { + res, err := h.Hub.ApplyCurated(c.Request.Context(), preset) + if err != nil { + logger.Log().WithField("error", util.SanitizeForLog(err.Error())).WithField("slug", util.SanitizeForLog(preset.Slug)).WithField("backup_path", util.SanitizeForLog(res.BackupPath)).Warn("curated crowdsec preset apply failed") + h.recordPresetEvent(preset.Slug, res, err) + + status := mapCrowdsecStatus(err, http.StatusInternalServerError) + msg := err.Error() + if errors.Is(err, crowdsec.ErrCSCLIUnavailable) { + status = http.StatusServiceUnavailable + msg = "CrowdSec CLI is not available; curated presets require cscli" + } + c.JSON(status, applyFailureBody(msg, res)) + return + } + h.recordPresetEvent(preset.Slug, res, nil) + respondApplySuccess(c, res) +} + +// curatedPresetPreview renders the hub items a curated preset will install. +func curatedPresetPreview(preset crowdsec.Preset) string { + var b strings.Builder + b.WriteString("# Curated preset: " + preset.Title + "\n# " + preset.Summary + "\n#\n# Installs these CrowdSec hub items:\n") + for _, item := range preset.Items { + b.WriteString("# " + item.Type + ": " + item.Name + "\n") + } + return b.String() +} + +// recordPresetEvent persists an apply audit row. A nil err records the result status +// (defaulting to "applied"); a non-nil err records a "failed" row with the error text. +// Persistence errors are logged and never alter the request outcome. +func (h *CrowdsecHandler) recordPresetEvent(slug string, res crowdsec.ApplyResult, applyErr error) { + if h.DB == nil { + return + } + event := models.CrowdsecPresetEvent{Slug: slug, Action: "apply", CacheKey: res.CacheKey, BackupPath: res.BackupPath} + if applyErr != nil { + event.Status = "failed" + event.Error = applyErr.Error() + } else { + event.Status = res.Status + if event.Status == "" { + event.Status = "applied" + } + if res.AppliedPreset != "" { + event.Slug = res.AppliedPreset + } + } + if err := h.DB.Create(&event).Error; err != nil { + logger.Log().WithError(err).WithField("slug", util.SanitizeForLog(event.Slug)).Warn("failed to record crowdsec preset event") + } +} + +// applyFailureBody builds the common error payload for a failed preset apply. +func applyFailureBody(msg string, res crowdsec.ApplyResult) gin.H { + body := gin.H{"error": msg} + if res.BackupPath != "" { + body["backup"] = res.BackupPath + } + if res.CacheKey != "" { + body["cache_key"] = res.CacheKey + } + return body +} + +// respondApplySuccess writes the 200 payload for a completed preset apply. +func respondApplySuccess(c *gin.Context, res crowdsec.ApplyResult) { + c.JSON(http.StatusOK, gin.H{ + "status": res.Status, + "backup": res.BackupPath, + "reload_hint": res.ReloadHint, + "used_cscli": res.UsedCSCLI, + "cache_key": res.CacheKey, + "slug": res.AppliedPreset, + }) +} + +// GetCachedPreset returns cached preview for a slug when available. +func (h *CrowdsecHandler) GetCachedPreset(c *gin.Context) { + if !h.isCerberusEnabled() { + c.JSON(http.StatusNotFound, gin.H{"error": "cerberus disabled"}) + return + } + if h.Hub == nil || h.Hub.Cache == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "hub cache unavailable"}) + return + } + ctx := c.Request.Context() + slug := strings.TrimSpace(c.Param("slug")) + if slug == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "slug required"}) + return + } + preview, err := h.Hub.Cache.LoadPreview(ctx, slug) + if err != nil { + if errors.Is(err, crowdsec.ErrCacheMiss) || errors.Is(err, crowdsec.ErrCacheExpired) { + c.JSON(http.StatusNotFound, gin.H{"error": "cache miss"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + meta, metaErr := h.Hub.Cache.Load(ctx, slug) + if metaErr != nil && !errors.Is(metaErr, crowdsec.ErrCacheMiss) && !errors.Is(metaErr, crowdsec.ErrCacheExpired) { + c.JSON(http.StatusInternalServerError, gin.H{"error": metaErr.Error()}) + return + } + cacheTTL := h.Hub.Cache.TTL() + now := time.Now().UTC() + c.JSON(http.StatusOK, gin.H{ + "preview": preview, + "cache_key": meta.CacheKey, + "etag": meta.Etag, + "retrieved_at": meta.RetrievedAt, + "ttl_remaining_seconds": ttlRemainingSeconds(now, meta.RetrievedAt, cacheTTL), + }) +} diff --git a/backend/internal/api/handlers/crowdsec_presets_handler_test.go b/backend/internal/api/handlers/crowdsec_preset_handler_test.go similarity index 100% rename from backend/internal/api/handlers/crowdsec_presets_handler_test.go rename to backend/internal/api/handlers/crowdsec_preset_handler_test.go diff --git a/backend/internal/crowdsec/backup.go b/backend/internal/crowdsec/backup.go new file mode 100644 index 000000000..0e9fa0187 --- /dev/null +++ b/backend/internal/crowdsec/backup.go @@ -0,0 +1,337 @@ +package crowdsec + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Wikid82/charon/backend/internal/logger" + "github.com/Wikid82/charon/backend/internal/util" +) + +const ( + // DefaultBackupRetention is how many full-tree snapshots (.backup.*) are kept. + DefaultBackupRetention = 5 + // DefaultFileBackupRetention is how many single-file edit backups (.filebackup.*) are kept. + DefaultFileBackupRetention = 10 + + // BackupKindSnapshot and BackupKindFile name the two independent backup namespaces next to DataDir. + BackupKindSnapshot = "backup" + BackupKindFile = "filebackup" + + // backupTimeFormat includes sub-second precision so back-to-back backups never share a directory. + backupTimeFormat = "20060102-150405.000000" + // legacyBackupTimeFormat is the second-resolution layout older releases used. + legacyBackupTimeFormat = "20060102-150405" + + // snapshotMkdirAttempts bounds retries when a microsecond-named directory already exists. + snapshotMkdirAttempts = 3 +) + +// isLiveDBFile reports whether name is the live CrowdSec SQLite database or its WAL/SHM sidecars. +// These are owned by the running engine and must never be copied into, or restored from, a backup. +func isLiveDBFile(name string) bool { + switch name { + case "crowdsec.db", "crowdsec.db-wal", "crowdsec.db-shm": + return true + } + return false +} + +// IsEngineOwnedPath reports whether rel (a slash- or OS-separated path relative to DataDir) is state +// the running engine owns and that snapshots, restores and archive imports must never touch: a live +// db file at any depth, or anything under the top-level data/ (LAPI db and hub data files) or +// hub_cache/ (regenerable download cache). Only the top-level directories are excluded; a nested +// directory with the same name stays part of the configuration. +func IsEngineOwnedPath(rel string) bool { + rel = filepath.ToSlash(filepath.Clean(rel)) + if rel == "." || rel == "" { + return false + } + if isLiveDBFile(filepath.Base(rel)) { + return true + } + top, _, _ := strings.Cut(rel, "/") + return top == "data" || top == "hub_cache" +} + +// snapshot copies DataDir into a new .backup. directory; see Snapshot. +func (s *HubService) snapshot() (string, error) { return Snapshot(s.DataDir) } + +// Snapshot copies dataDir (symlinks preserved, engine-owned state excluded) into a new +// .backup. directory and returns its path. dataDir is never modified. +func Snapshot(dataDir string) (string, error) { + parent, base := filepath.Split(filepath.Clean(dataDir)) + var path string + var mkErr error + for range snapshotMkdirAttempts { + path = filepath.Join(parent, base+"."+BackupKindSnapshot+"."+time.Now().Format(backupTimeFormat)) + if mkErr = os.Mkdir(path, 0o700); mkErr == nil || !errors.Is(mkErr, fs.ErrExist) { + break + } + } + if mkErr != nil { + return "", fmt.Errorf("mkdir backup: %w", mkErr) + } + if _, err := os.Stat(dataDir); err == nil { + if cpErr := copyTree(dataDir, path, IsEngineOwnedPath); cpErr != nil { + _ = os.RemoveAll(path) + return "", fmt.Errorf("copy backup: %w", cpErr) + } + } else if !errors.Is(err, fs.ErrNotExist) { + _ = os.RemoveAll(path) + return "", fmt.Errorf("stat data dir: %w", err) + } + return path, nil +} + +// restore replaces the contents of DataDir with the snapshot; see Restore. +func (s *HubService) restore(path string) error { return Restore(path, s.DataDir) } + +// Restore replaces the contents of dataDir with the snapshot while keeping dataDir itself (and the +// engine-owned entries inside it) in place. +func Restore(snapshotPath, dataDir string) error { + if err := ClearConfig(dataDir); err != nil { + return err + } + if err := copyTree(snapshotPath, dataDir, IsEngineOwnedPath); err != nil { + return fmt.Errorf("restore backup: %w", err) + } + return nil +} + +// ClearConfig creates dataDir when missing and removes everything in it except engine-owned state +// (live db files, top-level data/ and hub_cache/). dataDir itself is never removed or renamed. +func ClearConfig(dataDir string) error { + if err := os.MkdirAll(dataDir, 0o700); err != nil { + return fmt.Errorf("mkdir data dir: %w", err) + } + if err := emptyDirExcept(dataDir, IsEngineOwnedPath); err != nil { + return fmt.Errorf("empty data dir: %w", err) + } + return nil +} + +// emptyDirExcept removes the contents of dir except entries for which keep(relPath) returns true. +// Directories are removed only when nothing kept remains inside them. +func emptyDirExcept(dir string, keep func(rel string) bool) error { + return emptyDirExceptRel(dir, "", keep) +} + +func emptyDirExceptRel(dir, rel string, keep func(rel string) bool) error { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + for _, entry := range entries { + entryRel := filepath.Join(rel, entry.Name()) + if keep(entryRel) { + continue + } + path := filepath.Join(dir, entry.Name()) + if entry.IsDir() { + if err := emptyDirExceptRel(path, entryRel, keep); err != nil { + return err + } + // A directory that still holds kept files cannot be removed; leave it in place. + if remaining, rerr := os.ReadDir(path); rerr == nil && len(remaining) > 0 { + continue + } + } + if err := os.Remove(path); err != nil { + return err + } + } + return nil +} + +// copyTree recursively copies src into dst (which must exist), skipping entries whose path relative +// to src makes skip return true. Symlinks are preserved literally and never followed. +func copyTree(src, dst string, skip func(rel string) bool) error { + return copyTreeRel(src, dst, "", skip) +} + +func copyTreeRel(src, dst, rel string, skip func(rel string) bool) error { + srcInfo, err := os.Stat(src) + if err != nil { + return fmt.Errorf("stat src: %w", err) + } + if !srcInfo.IsDir() { + return fmt.Errorf("src is not a directory") + } + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("read dir: %w", err) + } + for _, entry := range entries { + entryRel := filepath.Join(rel, entry.Name()) + if skip != nil && skip(entryRel) { + continue + } + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + switch { + case entry.Type()&fs.ModeSymlink != 0: + // Hub item entries are symlinks, and dangling links must survive a backup/restore round trip. + target, err := os.Readlink(srcPath) + if err != nil { + return fmt.Errorf("readlink %s: %w", srcPath, err) + } + if err := os.Symlink(target, dstPath); err != nil { + return fmt.Errorf("symlink %s: %w", dstPath, err) + } + case entry.IsDir(): + if err := os.MkdirAll(dstPath, 0o700); err != nil { + return fmt.Errorf("mkdir %s: %w", dstPath, err) + } + if err := copyTreeRel(srcPath, dstPath, entryRel, skip); err != nil { + return err + } + default: + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +// pruneAfterApply prunes full-tree snapshots, logging (never returning) any failure. +func (s *HubService) pruneAfterApply() { + if removed, err := PruneBackups(s.DataDir, BackupKindSnapshot, DefaultBackupRetention); err != nil { + logger.Log().WithError(err).WithField("removed", len(removed)).Warn("crowdsec backup prune incomplete") + } +} + +// backupTimestamp parses the timestamp suffix of a backup directory name. +func backupTimestamp(suffix string) (time.Time, bool) { + for _, layout := range []string{backupTimeFormat, legacyBackupTimeFormat} { + if ts, err := time.ParseInLocation(layout, suffix, time.Local); err == nil { + return ts, true + } + } + return time.Time{}, false +} + +// PruneBackups deletes the oldest .. sibling directories, keeping the +// newest `keep` (clamped to at least 1). Age comes from the timestamp in the directory name, never +// from file times. Entries that are not real directories or whose names do not parse are ignored. +// Individual removal errors are aggregated and returned alongside the paths that were removed. +func PruneBackups(dataDir, kind string, keep int) (removed []string, err error) { + if keep < 1 { + keep = 1 + } + parent, base := filepath.Split(filepath.Clean(dataDir)) + if parent == "" { + parent = "." + } + prefix := base + "." + kind + "." + + entries, readErr := os.ReadDir(parent) + if readErr != nil { + if errors.Is(readErr, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("list backups: %w", readErr) + } + + type candidate struct { + path string + ts time.Time + } + var found []candidate + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, prefix) { + continue + } + ts, ok := backupTimestamp(strings.TrimPrefix(name, prefix)) + if !ok { + continue + } + path := filepath.Join(parent, name) + info, lerr := os.Lstat(path) + if lerr != nil || !info.IsDir() { + continue + } + found = append(found, candidate{path: path, ts: ts}) + } + sort.Slice(found, func(i, j int) bool { + if !found[i].ts.Equal(found[j].ts) { + return found[i].ts.After(found[j].ts) + } + return found[i].path > found[j].path + }) + if len(found) <= keep { + return nil, nil + } + + var errs []error + for _, c := range found[keep:] { + if rmErr := os.RemoveAll(c.path); rmErr != nil { + logger.Log().WithError(rmErr).WithField("path", util.SanitizeForLog(c.path)).Warn("failed to prune crowdsec backup") + errs = append(errs, rmErr) + continue + } + removed = append(removed, c.path) + } + return removed, errors.Join(errs...) +} + +// rollbackFailure annotates cause with a failed restore and the snapshot kept for manual recovery. +func rollbackFailure(cause, restoreErr error, backupPath string) error { + logger.Log().WithError(restoreErr).WithField("backup_path", util.SanitizeForLog(backupPath)).Error("preset rollback failed; backup retained for manual recovery") + return fmt.Errorf("%w (rollback failed: %v; backup retained at %s)", cause, restoreErr, backupPath) +} + +// BackupFile copies the single file dataDir/rel into a new .filebackup./ directory +// and returns that directory, then prunes older file backups. A target that does not exist yet has +// nothing to back up and yields an empty path. dataDir is never renamed or modified. +func BackupFile(dataDir, rel string) (string, error) { + src := filepath.Join(dataDir, rel) + info, err := os.Stat(src) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + return "", fmt.Errorf("stat file: %w", err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("backup file: %s is not a regular file", rel) + } + + parent, base := filepath.Split(filepath.Clean(dataDir)) + var dir string + var mkErr error + for range snapshotMkdirAttempts { + dir = filepath.Join(parent, base+"."+BackupKindFile+"."+time.Now().Format(backupTimeFormat)) + if mkErr = os.Mkdir(dir, 0o700); mkErr == nil || !errors.Is(mkErr, fs.ErrExist) { + break + } + } + if mkErr != nil { + return "", fmt.Errorf("mkdir file backup: %w", mkErr) + } + dst := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + _ = os.RemoveAll(dir) + return "", fmt.Errorf("mkdir file backup path: %w", err) + } + if err := copyFile(src, dst); err != nil { + _ = os.RemoveAll(dir) + return "", fmt.Errorf("copy file backup: %w", err) + } + if _, pruneErr := PruneBackups(dataDir, BackupKindFile, DefaultFileBackupRetention); pruneErr != nil { + logger.Log().WithError(pruneErr).Warn("crowdsec file backup prune incomplete") + } + return dir, nil +} diff --git a/backend/internal/crowdsec/backup_test.go b/backend/internal/crowdsec/backup_test.go new file mode 100644 index 000000000..68f9726c5 --- /dev/null +++ b/backend/internal/crowdsec/backup_test.go @@ -0,0 +1,910 @@ +package crowdsec + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) //nolint:gosec // G304: Test file in temp directory + require.NoError(t, err) + return string(b) +} + +func dirSize(t *testing.T, dir string) int64 { + t.Helper() + var total int64 + require.NoError(t, filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + })) + return total +} + +// seedPersistentTree mimics /app/data/crowdsec: config, bouncer key, engine-owned state. +func seedPersistentTree(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "crowdsec") + writeFile(t, filepath.Join(dir, "config", "config.yaml"), "original: true\n") + writeFile(t, filepath.Join(dir, "config", "data", "nested.yaml"), "nested: true\n") + writeFile(t, filepath.Join(dir, "bouncer_key"), "secret") + writeFile(t, filepath.Join(dir, "crowdsec.db"), "live-db") + writeFile(t, filepath.Join(dir, "crowdsec.db-wal"), "wal") + writeFile(t, filepath.Join(dir, "crowdsec.db-shm"), "shm") + writeFile(t, filepath.Join(dir, "config", "crowdsec.db"), "nested-db") + writeFile(t, filepath.Join(dir, "data", "crowdsec.db"), "lapi-db") + writeFile(t, filepath.Join(dir, "data", "GeoLite2-City.mmdb"), "mmdb") + writeFile(t, filepath.Join(dir, "hub_cache", "test", "bundle.tgz"), "archive") + require.NoError(t, os.Symlink("../hub/x.yaml", filepath.Join(dir, "config", "link.yaml"))) + return dir +} + +func TestIsEngineOwnedPath(t *testing.T) { + t.Parallel() + owned := []string{"crowdsec.db", "crowdsec.db-wal", "crowdsec.db-shm", "config/crowdsec.db", "data", "data/x", "data/a/b", "hub_cache", "hub_cache/y", "./data/x"} + for _, p := range owned { + require.True(t, IsEngineOwnedPath(p), p) + } + notOwned := []string{"", ".", "config/data", "config/data/x", "config/hub_cache", "config.yaml", "database/x", "datax", "bouncer_key"} + for _, p := range notOwned { + require.False(t, IsEngineOwnedPath(p), p) + } +} + +func TestSnapshotExcludesEngineOwnedState(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + svc := NewHubService(nil, nil, dir) + + path, err := svc.snapshot() + require.NoError(t, err) + require.True(t, strings.HasPrefix(path, dir+".backup."), path) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + + require.Equal(t, "original: true\n", readFile(t, filepath.Join(path, "config", "config.yaml"))) + require.Equal(t, "secret", readFile(t, filepath.Join(path, "bouncer_key"))) + // A nested directory named data stays part of the configuration. + require.Equal(t, "nested: true\n", readFile(t, filepath.Join(path, "config", "data", "nested.yaml"))) + + for _, excluded := range []string{"crowdsec.db", "crowdsec.db-wal", "crowdsec.db-shm", filepath.Join("config", "crowdsec.db"), "data", "hub_cache"} { + _, statErr := os.Lstat(filepath.Join(path, excluded)) + require.True(t, os.IsNotExist(statErr), "%s must not be in the snapshot", excluded) + } + + target, err := os.Readlink(filepath.Join(path, "config", "link.yaml")) + require.NoError(t, err) + require.Equal(t, "../hub/x.yaml", target) + + // The live tree is unchanged. + require.Equal(t, "live-db", readFile(t, filepath.Join(dir, "crowdsec.db"))) +} + +func TestSnapshotSizeIgnoresLargeEngineOwnedFiles(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + big := bytes.Repeat([]byte("x"), 4<<20) + require.NoError(t, os.WriteFile(filepath.Join(dir, "data", "big.bin"), big, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "hub_cache", "big.bin"), big, 0o600)) + svc := NewHubService(nil, nil, dir) + + path, err := svc.snapshot() + require.NoError(t, err) + require.Less(t, dirSize(t, path), int64(1<<10), "snapshot must only hold the small config files") +} + +func TestSnapshotMissingDataDirYieldsEmptySnapshot(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + svc := NewHubService(nil, nil, dir) + + path, err := svc.snapshot() + require.NoError(t, err) + entries, err := os.ReadDir(path) + require.NoError(t, err) + require.Empty(t, entries) + require.NoDirExists(t, dir, "snapshot must not create DataDir") +} + +func TestSnapshotErrors(t *testing.T) { + t.Parallel() + t.Run("backup directory cannot be created", func(t *testing.T) { + t.Parallel() + svc := NewHubService(nil, nil, filepath.Join(t.TempDir(), "missing-parent", "crowdsec")) + _, err := svc.snapshot() + require.ErrorContains(t, err, "mkdir backup") + }) + t.Run("data dir is not a directory", func(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + writeFile(t, dir, "x") + svc := NewHubService(nil, nil, dir) + _, err := svc.snapshot() + require.ErrorContains(t, err, "copy backup") + matches, globErr := filepath.Glob(dir + ".backup.*") + require.NoError(t, globErr) + require.Empty(t, matches, "partial snapshot removed") + }) + t.Run("data dir cannot be inspected", func(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + writeFile(t, dir, "x") + // A path through a regular file yields ENOTDIR, which is not "does not exist". + svc := NewHubService(nil, nil, filepath.Join(dir, "child")) + _, err := svc.snapshot() + require.Error(t, err) + }) +} + +func TestSnapshotNamesAreUnique(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + svc := NewHubService(nil, nil, dir) + seen := map[string]bool{} + for range 5 { + path, err := svc.snapshot() + require.NoError(t, err) + require.False(t, seen[path], "duplicate snapshot dir %s", path) + seen[path] = true + } +} + +func TestRestoreReplacesConfigAndLeavesEngineOwnedStateUntouched(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + svc := NewHubService(nil, nil, dir) + before, err := os.Stat(dir) + require.NoError(t, err) + + path, err := svc.snapshot() + require.NoError(t, err) + + // Simulate a failed apply: config changed, files added, link replaced, engine state moved on. + writeFile(t, filepath.Join(dir, "config", "config.yaml"), "original: false\n") + writeFile(t, filepath.Join(dir, "config", "added.yaml"), "new") + writeFile(t, filepath.Join(dir, "added-top.yaml"), "new") + require.NoError(t, os.Remove(filepath.Join(dir, "config", "link.yaml"))) + writeFile(t, filepath.Join(dir, "config", "link.yaml"), "dereferenced") + writeFile(t, filepath.Join(dir, "crowdsec.db"), "newer-live-db") + writeFile(t, filepath.Join(dir, "data", "crowdsec.db"), "newer-lapi-db") + writeFile(t, filepath.Join(dir, "hub_cache", "test", "bundle.tgz"), "newer-archive") + + require.NoError(t, svc.restore(path)) + + require.Equal(t, "original: true\n", readFile(t, filepath.Join(dir, "config", "config.yaml"))) + require.Equal(t, "nested: true\n", readFile(t, filepath.Join(dir, "config", "data", "nested.yaml"))) + require.Equal(t, "secret", readFile(t, filepath.Join(dir, "bouncer_key"))) + for _, gone := range []string{filepath.Join("config", "added.yaml"), "added-top.yaml"} { + _, statErr := os.Lstat(filepath.Join(dir, gone)) + require.True(t, os.IsNotExist(statErr), gone) + } + target, err := os.Readlink(filepath.Join(dir, "config", "link.yaml")) + require.NoError(t, err) + require.Equal(t, "../hub/x.yaml", target) + + require.Equal(t, "newer-live-db", readFile(t, filepath.Join(dir, "crowdsec.db"))) + require.Equal(t, "wal", readFile(t, filepath.Join(dir, "crowdsec.db-wal"))) + require.Equal(t, "nested-db", readFile(t, filepath.Join(dir, "config", "crowdsec.db"))) + require.Equal(t, "newer-lapi-db", readFile(t, filepath.Join(dir, "data", "crowdsec.db"))) + require.Equal(t, "mmdb", readFile(t, filepath.Join(dir, "data", "GeoLite2-City.mmdb"))) + require.Equal(t, "newer-archive", readFile(t, filepath.Join(dir, "hub_cache", "test", "bundle.tgz"))) + + after, err := os.Stat(dir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "DataDir must stay in place") +} + +func TestRestoreRecreatesMissingDataDir(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + svc := NewHubService(nil, nil, dir) + path, err := svc.snapshot() + require.NoError(t, err) + writeFile(t, filepath.Join(dir, "partial.yaml"), "x") + + require.NoError(t, svc.restore(path)) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries) +} + +func TestRestoreErrors(t *testing.T) { + t.Parallel() + t.Run("data dir cannot be created", func(t *testing.T) { + t.Parallel() + blocker := filepath.Join(t.TempDir(), "blocker") + writeFile(t, blocker, "x") + err := Restore(t.TempDir(), filepath.Join(blocker, "crowdsec")) + require.ErrorContains(t, err, "mkdir data dir") + }) + t.Run("snapshot is missing", func(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + err := Restore(filepath.Join(t.TempDir(), "gone"), dir) + require.ErrorContains(t, err, "restore backup") + }) + t.Run("data dir is a file", func(t *testing.T) { + t.Parallel() + file := filepath.Join(t.TempDir(), "file") + writeFile(t, file, "x") + require.Error(t, Restore(t.TempDir(), file)) + }) +} + +func TestEmptyDirExceptKeepsMatchesByRelativePath(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "keep", "a.txt"), "a") + writeFile(t, filepath.Join(dir, "drop", "b.txt"), "b") + writeFile(t, filepath.Join(dir, "mixed", "keep.txt"), "k") + writeFile(t, filepath.Join(dir, "mixed", "drop.txt"), "d") + + keep := func(rel string) bool { return rel == "keep" || rel == filepath.Join("mixed", "keep.txt") } + require.NoError(t, emptyDirExcept(dir, keep)) + + require.FileExists(t, filepath.Join(dir, "keep", "a.txt")) + require.FileExists(t, filepath.Join(dir, "mixed", "keep.txt")) + require.NoFileExists(t, filepath.Join(dir, "mixed", "drop.txt")) + require.NoDirExists(t, filepath.Join(dir, "drop")) +} + +func TestCopyTreeSkipUsesRelativePath(t *testing.T) { + t.Parallel() + src := t.TempDir() + dst := t.TempDir() + writeFile(t, filepath.Join(src, "skipme", "a"), "a") + writeFile(t, filepath.Join(src, "sub", "skipme", "b"), "b") + + require.NoError(t, copyTree(src, dst, func(rel string) bool { return rel == "skipme" })) + require.NoDirExists(t, filepath.Join(dst, "skipme")) + require.FileExists(t, filepath.Join(dst, "sub", "skipme", "b")) +} + +// mkBackupDir creates a sibling backup directory named after dataDir/kind/suffix. +func mkBackupDir(t *testing.T, dataDir, kind, suffix string) string { + t.Helper() + path := fmt.Sprintf("%s.%s.%s", dataDir, kind, suffix) + require.NoError(t, os.Mkdir(path, 0o700)) + return path +} + +func TestPruneBackupsOrdersByNameTimestampNotMtime(t *testing.T) { + t.Parallel() + dataDir := filepath.Join(t.TempDir(), "crowdsec") + require.NoError(t, os.Mkdir(dataDir, 0o700)) + oldest := mkBackupDir(t, dataDir, "backup", "20250101-000000.000001") + mid := mkBackupDir(t, dataDir, "backup", "20250102-000000.000000") + newest := mkBackupDir(t, dataDir, "backup", "20250103-000000.000000") + + // Touching the oldest directory must not save it. + future := time.Now().Add(time.Hour) + require.NoError(t, os.Chtimes(oldest, future, future)) + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, 2) + require.NoError(t, err) + require.Equal(t, []string{oldest}, removed) + require.NoDirExists(t, oldest) + require.DirExists(t, mid) + require.DirExists(t, newest) + require.DirExists(t, dataDir) +} + +func TestPruneBackupsHandlesBothNameLayouts(t *testing.T) { + t.Parallel() + dataDir := filepath.Join(t.TempDir(), "crowdsec") + legacyOld := mkBackupDir(t, dataDir, "backup", "20250101-120000") + legacyNew := mkBackupDir(t, dataDir, "backup", "20250101-120005") + micro := mkBackupDir(t, dataDir, "backup", "20250101-120005.500000") + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, 2) + require.NoError(t, err) + require.Equal(t, []string{legacyOld}, removed) + require.DirExists(t, legacyNew) + require.DirExists(t, micro) +} + +func TestPruneBackupsIgnoresUnparsableAndNonMatchingEntries(t *testing.T) { + t.Parallel() + parent := t.TempDir() + dataDir := filepath.Join(parent, "crowdsec") + require.NoError(t, os.Mkdir(dataDir, 0o700)) + newest := mkBackupDir(t, dataDir, "backup", "20250103-000000.000000") + old := mkBackupDir(t, dataDir, "backup", "20250101-000000.000000") + + unparsable := mkBackupDir(t, dataDir, "backup", "manual-copy") + otherBase := filepath.Join(parent, "other.backup.20240101-000000.000000") + require.NoError(t, os.Mkdir(otherBase, 0o700)) + plainFile := dataDir + ".backup.20240102-000000.000000" + require.NoError(t, os.WriteFile(plainFile, []byte("x"), 0o600)) + + linkTarget := t.TempDir() + symlink := dataDir + ".backup.20240103-000000.000000" + require.NoError(t, os.Symlink(linkTarget, symlink)) + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, 1) + require.NoError(t, err) + require.Equal(t, []string{old}, removed) + require.DirExists(t, newest) + for _, kept := range []string{unparsable, otherBase, plainFile, symlink, linkTarget, dataDir} { + _, statErr := os.Lstat(kept) + require.NoError(t, statErr, kept) + } +} + +func TestPruneBackupsClampsKeepToOne(t *testing.T) { + t.Parallel() + dataDir := filepath.Join(t.TempDir(), "crowdsec") + old := mkBackupDir(t, dataDir, "backup", "20250101-000000.000000") + newest := mkBackupDir(t, dataDir, "backup", "20250102-000000.000000") + + for _, keep := range []int{0, -3} { + removed, err := PruneBackups(dataDir, BackupKindSnapshot, keep) + require.NoError(t, err) + if keep == 0 { + require.Equal(t, []string{old}, removed) + } else { + require.Empty(t, removed) + } + require.DirExists(t, newest, "the newest backup is always kept") + } +} + +func TestPruneBackupsEqualTimestampsTieBreakDeterministically(t *testing.T) { + t.Parallel() + dataDir := filepath.Join(t.TempDir(), "crowdsec") + // The legacy and microsecond layouts can name the same instant. + a := mkBackupDir(t, dataDir, "backup", "20250101-000000") + b := mkBackupDir(t, dataDir, "backup", "20250101-000000.000000") + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, 1) + require.NoError(t, err) + require.Equal(t, []string{a}, removed) + require.DirExists(t, b) +} + +func TestPruneBackupsMissingParentAndNothingToPrune(t *testing.T) { + t.Parallel() + removed, err := PruneBackups(filepath.Join(t.TempDir(), "gone", "crowdsec"), BackupKindSnapshot, 5) + require.NoError(t, err) + require.Empty(t, removed) + + dataDir := filepath.Join(t.TempDir(), "crowdsec") + mkBackupDir(t, dataDir, "backup", "20250101-000000.000000") + removed, err = PruneBackups(dataDir, BackupKindSnapshot, 5) + require.NoError(t, err) + require.Empty(t, removed) +} + +func TestPruneBackupsParentIsNotADirectory(t *testing.T) { + t.Parallel() + file := filepath.Join(t.TempDir(), "file") + writeFile(t, file, "x") + _, err := PruneBackups(filepath.Join(file, "crowdsec"), BackupKindSnapshot, 5) + require.ErrorContains(t, err, "list backups") +} + +func TestPruneBackupsRelativeDataDir(t *testing.T) { + // Not parallel: changes the working directory. + tmp := t.TempDir() + t.Chdir(tmp) + require.NoError(t, os.Mkdir("crowdsec.backup.20250101-000000.000000", 0o700)) + require.NoError(t, os.Mkdir("crowdsec.backup.20250102-000000.000000", 0o700)) + + removed, err := PruneBackups("crowdsec", BackupKindSnapshot, 1) + require.NoError(t, err) + require.Len(t, removed, 1) + require.NoDirExists(t, "crowdsec.backup.20250101-000000.000000") +} + +func TestPruneBackupsNamespacesNeverEvictEachOther(t *testing.T) { + t.Parallel() + dataDir := filepath.Join(t.TempDir(), "crowdsec") + var snaps, files []string + for i := 1; i <= 7; i++ { + suffix := fmt.Sprintf("2025010%d-000000.000000", i) + snaps = append(snaps, mkBackupDir(t, dataDir, "backup", suffix)) + files = append(files, mkBackupDir(t, dataDir, "filebackup", suffix)) + } + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, DefaultBackupRetention) + require.NoError(t, err) + require.ElementsMatch(t, snaps[:2], removed) + for _, f := range files { + require.DirExists(t, f, "file backups are not touched by snapshot pruning") + } + + removed, err = PruneBackups(dataDir, BackupKindFile, 3) + require.NoError(t, err) + require.ElementsMatch(t, files[:4], removed) + for _, s := range snaps[2:] { + require.DirExists(t, s, "snapshots are not touched by file backup pruning") + } +} + +func TestPruneBackupsReportsRemovalErrors(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission-based failure cannot be provoked as root") + } + parent := t.TempDir() + dataDir := filepath.Join(parent, "crowdsec") + old := mkBackupDir(t, dataDir, "backup", "20250101-000000.000000") + mkBackupDir(t, dataDir, "backup", "20250102-000000.000000") + // A read-only parent makes the removal fail. + require.NoError(t, os.Chmod(parent, 0o500)) //nolint:gosec // G302: test needs a read-only dir + t.Cleanup(func() { _ = os.Chmod(parent, 0o700) }) //nolint:gosec // G302: restore owner access to a temp directory so cleanup can remove it + + removed, err := PruneBackups(dataDir, BackupKindSnapshot, 1) + require.Error(t, err) + require.Empty(t, removed) + _ = old +} + +// ---- HubService.Apply flows ---- + +type funcExec func(ctx context.Context, cmd string) ([]byte, error) + +func (f funcExec) Execute(ctx context.Context, name string, args ...string) ([]byte, error) { + return f(ctx, strings.Join(append([]string{name}, args...), " ")) +} + +// applyFixture builds a persistent-tree HubService whose cache lives inside DataDir/hub_cache, +// as in production, with one cached preset archive. +func applyFixture(t *testing.T, exec CommandExecutor, archive []byte) (svc *HubService, dir string) { + t.Helper() + dir = seedPersistentTree(t) + cache, err := NewHubCache(filepath.Join(dir, "hub_cache"), time.Hour) + require.NoError(t, err) + if archive != nil { + _, err = cache.Store(context.Background(), "test/preset", "etag", "hub", "preview", archive) + require.NoError(t, err) + } + svc = NewHubService(exec, cache, dir) + svc.ApplyTimeout = 5 * time.Second + return svc, dir +} + +func countBackups(t *testing.T, dir string) int { + t.Helper() + m, err := filepath.Glob(dir + ".backup.*") + require.NoError(t, err) + return len(m) +} + +func TestApplyCSCLISuccessKeepsDataDirAndPrunes(t *testing.T) { + t.Parallel() + exec := funcExec(func(_ context.Context, _ string) ([]byte, error) { return []byte("ok"), nil }) + svc, dir := applyFixture(t, exec, nil) + reloads := 0 + svc.Reload = func(context.Context) error { reloads++; return nil } + before, err := os.Stat(dir) + require.NoError(t, err) + + for range DefaultBackupRetention + 2 { + res, applyErr := svc.Apply(context.Background(), "test/preset") + require.NoError(t, applyErr) + require.Equal(t, "applied", res.Status) + require.True(t, res.UsedCSCLI) + require.False(t, res.ReloadHint, "reload succeeded") + require.DirExists(t, res.BackupPath) + } + + after, err := os.Stat(dir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "DataDir must never be renamed") + require.Equal(t, DefaultBackupRetention, countBackups(t, dir)) + require.Equal(t, DefaultBackupRetention+2, reloads) + require.Equal(t, "live-db", readFile(t, filepath.Join(dir, "crowdsec.db"))) +} + +func TestApplyCSCLIFailureRestoresThenExtractsCachedArchive(t *testing.T) { + t.Parallel() + var svc *HubService + var dir string + exec := funcExec(func(_ context.Context, cmd string) ([]byte, error) { + switch { + case strings.Contains(cmd, "hub install"): + // cscli half-installs, then fails. + writeFile(t, filepath.Join(dir, "config", "cscli-leftover.yaml"), "partial") + writeFile(t, filepath.Join(dir, "config", "config.yaml"), "original: false\n") + return nil, errors.New("install failed") + default: + return []byte("ok"), nil + } + }) + archive := makeTarGz(t, map[string]string{ + "config/from-archive.yaml": "archive: true", + "crowdsec.db": "evil-db", + "data/evil": "evil", + "hub_cache/evil": "evil", + }) + svc, dir = applyFixture(t, exec, archive) + + res, err := svc.Apply(context.Background(), "test/preset") + require.NoError(t, err) + require.Equal(t, "applied", res.Status) + require.False(t, res.UsedCSCLI) + require.True(t, res.ReloadHint, "no reloader configured") + + require.Equal(t, "archive: true", readFile(t, filepath.Join(dir, "config", "from-archive.yaml"))) + require.Equal(t, "original: true\n", readFile(t, filepath.Join(dir, "config", "config.yaml")), "snapshot restored before extract") + require.NoFileExists(t, filepath.Join(dir, "config", "cscli-leftover.yaml")) + require.Equal(t, "live-db", readFile(t, filepath.Join(dir, "crowdsec.db")), "archive cannot overwrite engine state") + require.NoFileExists(t, filepath.Join(dir, "data", "evil")) + require.NoFileExists(t, filepath.Join(dir, "hub_cache", "evil")) + require.FileExists(t, filepath.Join(dir, "hub_cache", "test", "preset", "bundle.tgz"), "cached archive survives the restore") +} + +func TestApplyCSCLIAndExtractFailureRestoresSnapshot(t *testing.T) { + t.Parallel() + var dir string + exec := funcExec(func(_ context.Context, cmd string) ([]byte, error) { + if strings.Contains(cmd, "hub install") { + writeFile(t, filepath.Join(dir, "config", "cscli-leftover.yaml"), "partial") + return nil, errors.New("install failed") + } + return []byte("ok"), nil + }) + // A symlink entry makes extraction fail after files were already written. + svc, d := applyFixture(t, exec, makeTarGzWithSymlink(t)) + dir = d + + res, err := svc.Apply(context.Background(), "test/preset") + require.ErrorContains(t, err, "extract") + require.Equal(t, "failed", res.Status) + require.NotEmpty(t, res.BackupPath) + + require.Equal(t, "original: true\n", readFile(t, filepath.Join(dir, "config", "config.yaml"))) + require.NoFileExists(t, filepath.Join(dir, "config", "cscli-leftover.yaml")) + require.NoFileExists(t, filepath.Join(dir, "partial-before-symlink.yaml")) + require.DirExists(t, res.BackupPath, "failed apply keeps its snapshot") +} + +func TestApplyCacheMissAndRefreshFailureRestoresSnapshot(t *testing.T) { + t.Parallel() + var dir string + exec := funcExec(func(_ context.Context, cmd string) ([]byte, error) { + if strings.Contains(cmd, "hub install") { + writeFile(t, filepath.Join(dir, "config", "cscli-leftover.yaml"), "partial") + return nil, errors.New("install failed") + } + return []byte("ok"), nil + }) + svc, d := applyFixture(t, exec, nil) // nothing cached + dir = d + svc.HubBaseURL = "http://127.0.0.1:1" + svc.MirrorBaseURL = "http://127.0.0.1:1" + + res, err := svc.Apply(context.Background(), "test/preset") + require.ErrorContains(t, err, "load cache") + require.Equal(t, "failed", res.Status) + require.NotEmpty(t, res.BackupPath) + require.NotEmpty(t, res.ErrorMessage) + require.NoFileExists(t, filepath.Join(dir, "config", "cscli-leftover.yaml")) + require.Equal(t, "original: true\n", readFile(t, filepath.Join(dir, "config", "config.yaml"))) +} + +func TestApplyPrunesAfterFailureAndKeepsTheFailedSnapshot(t *testing.T) { + t.Parallel() + svc, dir := applyFixture(t, nil, nil) // no cscli, nothing cached: every apply fails + svc.HubBaseURL = "http://127.0.0.1:1" + svc.MirrorBaseURL = "http://127.0.0.1:1" + for i := 1; i <= DefaultBackupRetention+3; i++ { + mkBackupDir(t, dir, "backup", fmt.Sprintf("2025010%d-000000.000000", i)) + } + + res, err := svc.Apply(context.Background(), "test/preset") + require.Error(t, err) + require.DirExists(t, res.BackupPath, "the failed apply's snapshot is the newest and always kept") + require.Equal(t, DefaultBackupRetention, countBackups(t, dir)) +} + +func TestApplyRestoreFailureIsReported(t *testing.T) { + t.Parallel() + var dir string + var svc *HubService + exec := funcExec(func(_ context.Context, cmd string) ([]byte, error) { + if strings.Contains(cmd, "hub install") { + // Make the restore itself fail: DataDir is replaced by a regular file mid-apply. + require.NoError(t, os.RemoveAll(dir)) + writeFile(t, dir, "not a dir") + return nil, errors.New("install failed") + } + return []byte("ok"), nil + }) + svc, dir = applyFixture(t, exec, nil) + + res, err := svc.Apply(context.Background(), "test/preset") + require.ErrorContains(t, err, "rollback failed") + require.ErrorContains(t, err, "backup retained at "+res.BackupPath) +} + +func TestApplySnapshotFailureChangesNothing(t *testing.T) { + t.Parallel() + called := false + exec := funcExec(func(_ context.Context, _ string) ([]byte, error) { called = true; return []byte("ok"), nil }) + svc := NewHubService(exec, nil, filepath.Join(t.TempDir(), "missing-parent", "crowdsec")) + + res, err := svc.Apply(context.Background(), "test/preset") + require.ErrorContains(t, err, "backup") + require.Empty(t, res.BackupPath) + require.False(t, called, "cscli must not run when no snapshot exists") +} + +func TestApplyUnreadableCachedArchiveTriggersRefreshFailure(t *testing.T) { + t.Parallel() + svc, dir := applyFixture(t, nil, makeTarGz(t, map[string]string{"x.yaml": "x"})) + svc.HubBaseURL = "http://127.0.0.1:1" + svc.MirrorBaseURL = "http://127.0.0.1:1" + require.NoError(t, os.Remove(filepath.Join(dir, "hub_cache", "test", "preset", "bundle.tgz"))) + + _, err := svc.Apply(context.Background(), "test/preset") + require.ErrorContains(t, err, "load cache") +} + +func TestExtractTarGzOverlaysWithoutWipingAndSkipsEngineOwnedEntries(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + svc := NewHubService(nil, nil, dir) + archive := makeTarGz(t, map[string]string{ + "config/new.yaml": "new", + "crowdsec.db": "evil", + "data/x": "evil", + "hub_cache/y": "evil", + }) + + require.NoError(t, svc.extractTarGz(context.Background(), archive, dir)) + require.Equal(t, "new", readFile(t, filepath.Join(dir, "config", "new.yaml"))) + require.Equal(t, "original: true\n", readFile(t, filepath.Join(dir, "config", "config.yaml")), "existing files are not wiped") + require.Equal(t, "live-db", readFile(t, filepath.Join(dir, "crowdsec.db"))) + require.NoFileExists(t, filepath.Join(dir, "data", "x")) + require.NoFileExists(t, filepath.Join(dir, "hub_cache", "y")) +} + +func TestRollbackFailureWrapsCause(t *testing.T) { + t.Parallel() + cause := errors.New("boom") + err := rollbackFailure(cause, errors.New("restore broke"), "/b") + require.ErrorIs(t, err, cause) + require.ErrorContains(t, err, "restore broke") + require.ErrorContains(t, err, "/b") +} + +// makeTarGzWithSymlink returns an archive that writes one file and then hits a symlink entry. +func makeTarGzWithSymlink(t *testing.T) []byte { + t.Helper() + buf := &bytes.Buffer{} + gw := gzip.NewWriter(buf) + tw := tar.NewWriter(gw) + body := "partial" + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "partial-before-symlink.yaml", Mode: 0o644, Size: int64(len(body))})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd", Mode: 0o777})) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func TestBackupFileCopiesOnlyTheTargetAndPrunesIndependently(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + snap := mkBackupDir(t, dir, "backup", "20240101-000000.000000") + for i := 1; i <= DefaultFileBackupRetention; i++ { + mkBackupDir(t, dir, "filebackup", fmt.Sprintf("202401%02d-000000.000000", i)) + } + + backup, err := BackupFile(dir, filepath.Join("config", "config.yaml")) + require.NoError(t, err) + require.True(t, strings.HasPrefix(backup, dir+".filebackup."), backup) + require.Equal(t, "original: true\n", readFile(t, filepath.Join(backup, "config", "config.yaml"))) + require.Less(t, dirSize(t, backup), int64(100), "only the target file is copied") + + matches, err := filepath.Glob(dir + ".filebackup.*") + require.NoError(t, err) + require.Len(t, matches, DefaultFileBackupRetention) + require.DirExists(t, snap, "full snapshots are not evicted by file backups") + require.DirExists(t, dir) +} + +func TestBackupFileEdgeCases(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + + backup, err := BackupFile(dir, "config/new-file.yaml") + require.NoError(t, err) + require.Empty(t, backup, "nothing to back up for a new file") + + _, err = BackupFile(dir, "config") + require.ErrorContains(t, err, "not a regular file") + + _, err = BackupFile(dir, filepath.Join("bouncer_key", "x")) + require.ErrorContains(t, err, "stat file") +} + +func TestBackupFileMkdirFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission-based failure cannot be provoked as root") + } + parent := t.TempDir() + dir := filepath.Join(parent, "crowdsec") + writeFile(t, filepath.Join(dir, "f.yaml"), "x") + require.NoError(t, os.Chmod(parent, 0o500)) //nolint:gosec // G302: test needs a read-only dir + t.Cleanup(func() { _ = os.Chmod(parent, 0o700) }) //nolint:gosec // G302: restore owner access to a temp directory so cleanup can remove it + + _, err := BackupFile(dir, "f.yaml") + require.ErrorContains(t, err, "mkdir file backup") +} + +// ---- failure paths: unreadable, unremovable and colliding entries ---- + +// lockDir sets mode on dir and restores owner access on cleanup so TempDir can be removed. +func lockDir(t *testing.T, dir string, mode os.FileMode) { + t.Helper() + require.NoError(t, os.Chmod(dir, mode)) //nolint:gosec // G302: test needs a restricted dir + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) //nolint:gosec // G302: restore owner access so cleanup can remove it +} + +func skipIfRoot(t *testing.T) { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("permission-based failure cannot be provoked as root") + } +} + +func TestSnapshotStatErrorRemovesPartialSnapshot(t *testing.T) { + t.Parallel() + parent := t.TempDir() + dir := filepath.Join(parent, "crowdsec") + // A self-referencing symlink makes Stat fail with ELOOP, which is not "does not exist". + require.NoError(t, os.Symlink("crowdsec", dir)) + + _, err := Snapshot(dir) + require.ErrorContains(t, err, "stat data dir") + matches, globErr := filepath.Glob(dir + ".backup.*") + require.NoError(t, globErr) + require.Empty(t, matches, "snapshot directory removed on failure") +} + +func TestClearConfigReportsUnremovableEntries(t *testing.T) { + skipIfRoot(t) + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + writeFile(t, filepath.Join(dir, "config.yaml"), "x") + lockDir(t, dir, 0o500) + + err := ClearConfig(dir) + require.ErrorContains(t, err, "empty data dir") + require.FileExists(t, filepath.Join(dir, "config.yaml"), "a failed clear must not claim success") +} + +func TestEmptyDirExceptPropagatesNestedErrors(t *testing.T) { + skipIfRoot(t) + t.Parallel() + dir := t.TempDir() + inner := filepath.Join(dir, "outer", "inner") + writeFile(t, filepath.Join(inner, "f.yaml"), "x") + lockDir(t, inner, 0o500) + + require.Error(t, emptyDirExcept(dir, func(string) bool { return false })) + require.FileExists(t, filepath.Join(inner, "f.yaml")) +} + +func TestCopyTreeFailurePaths(t *testing.T) { + t.Parallel() + t.Run("directory cannot be created at destination", func(t *testing.T) { + t.Parallel() + src, dst := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(src, "sub", "a.yaml"), "a") + writeFile(t, filepath.Join(dst, "sub"), "a file where a directory must go") + require.ErrorContains(t, copyTree(src, dst, nil), "mkdir") + }) + t.Run("nested copy failure is returned", func(t *testing.T) { + t.Parallel() + src, dst := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(src, "sub", "inner", "a.yaml"), "a") + writeFile(t, filepath.Join(dst, "sub", "inner"), "a file where a directory must go") + require.Error(t, copyTree(src, dst, nil)) + }) + t.Run("file cannot be copied", func(t *testing.T) { + t.Parallel() + src, dst := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(src, "a.yaml"), "a") + require.NoError(t, os.MkdirAll(filepath.Join(dst, "a.yaml"), 0o700)) + require.ErrorContains(t, copyTree(src, dst, nil), "create dst") + }) + t.Run("unreadable directory", func(t *testing.T) { + skipIfRoot(t) + t.Parallel() + src, dst := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(src, "sub", "a.yaml"), "a") + lockDir(t, filepath.Join(src, "sub"), 0o000) + require.ErrorContains(t, copyTree(src, dst, nil), "read dir") + }) +} + +func TestPruneAfterApplyToleratesPruneFailure(t *testing.T) { + t.Parallel() + blocker := filepath.Join(t.TempDir(), "blocker") + writeFile(t, blocker, "x") + svc := NewHubService(nil, nil, filepath.Join(blocker, "crowdsec")) + + // The parent of DataDir is a regular file, so listing backups fails; the failure is only logged. + _, err := PruneBackups(svc.DataDir, BackupKindSnapshot, DefaultBackupRetention) + require.ErrorContains(t, err, "list backups") + require.NotPanics(t, svc.pruneAfterApply) +} + +func TestBackupFileFailurePaths(t *testing.T) { + skipIfRoot(t) + t.Parallel() + t.Run("unreadable source leaves no backup directory", func(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + src := filepath.Join(dir, "config", "config.yaml") + require.NoError(t, os.Chmod(src, 0o000)) //nolint:gosec // G302: test needs an unreadable file + t.Cleanup(func() { _ = os.Chmod(src, 0o600) }) //nolint:gosec // G302: restore owner access + + _, err := BackupFile(dir, filepath.Join("config", "config.yaml")) + require.ErrorContains(t, err, "copy file backup") + matches, globErr := filepath.Glob(dir + "." + BackupKindFile + ".*") + require.NoError(t, globErr) + require.Empty(t, matches) + }) + t.Run("prune failure does not fail the backup", func(t *testing.T) { + t.Parallel() + dir := seedPersistentTree(t) + for i := 1; i <= DefaultFileBackupRetention; i++ { + old := mkBackupDir(t, dir, BackupKindFile, fmt.Sprintf("202501%02d-000000.000000", i)) + locked := filepath.Join(old, "locked") + writeFile(t, filepath.Join(locked, "f.yaml"), "x") + lockDir(t, locked, 0o500) + } + + backup, err := BackupFile(dir, filepath.Join("config", "config.yaml")) + require.NoError(t, err) + require.DirExists(t, backup) + }) +} + +func TestRestoreAfterReportsRollbackFailure(t *testing.T) { + t.Parallel() + dir := filepath.Join(t.TempDir(), "crowdsec") + writeFile(t, dir, "DataDir replaced by a regular file") + svc := NewHubService(nil, nil, dir) + cause := errors.New("extract failed") + + err := svc.restoreAfter(cause, filepath.Join(t.TempDir(), "gone"), "test/preset") + require.ErrorIs(t, err, cause) + require.ErrorContains(t, err, "rollback failed") + + // A restorable snapshot returns the cause unchanged. + good := seedPersistentTree(t) + snap, snapErr := Snapshot(good) + require.NoError(t, snapErr) + err = NewHubService(nil, nil, good).restoreAfter(cause, snap, "test/preset") + require.Equal(t, cause, err) +} diff --git a/backend/internal/crowdsec/copydir_symlink_test.go b/backend/internal/crowdsec/copydir_symlink_test.go index 9b29bf197..217999df4 100644 --- a/backend/internal/crowdsec/copydir_symlink_test.go +++ b/backend/internal/crowdsec/copydir_symlink_test.go @@ -21,7 +21,7 @@ func TestCopyDirPreservesSymlinks(t *testing.T) { require.NoError(t, os.Symlink("hub/collections", filepath.Join(src, "dirlink"))) require.NoError(t, os.Symlink("does/not/exist", filepath.Join(src, "dangling"))) - require.NoError(t, copyDir(src, dst)) + require.NoError(t, copyTree(src, dst, nil)) for name, want := range map[string]string{ "collections/sshd.yaml": "../hub/collections/crowdsecurity/sshd.yaml", @@ -49,5 +49,5 @@ func TestCopyDirSymlinkDestinationError(t *testing.T) { require.NoError(t, os.Symlink("target", filepath.Join(src, "link"))) dst := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dst, "link"), []byte("x"), 0o600)) - require.Error(t, copyDir(src, dst)) + require.Error(t, copyTree(src, dst, nil)) } diff --git a/backend/internal/crowdsec/curated_apply.go b/backend/internal/crowdsec/curated_apply.go index 7d08be36f..48607827c 100644 --- a/backend/internal/crowdsec/curated_apply.go +++ b/backend/internal/crowdsec/curated_apply.go @@ -5,12 +5,8 @@ import ( "encoding/json" "errors" "fmt" - "os" - "path/filepath" - "time" "github.com/Wikid82/charon/backend/internal/logger" - "github.com/Wikid82/charon/backend/internal/util" ) // ErrCSCLIUnavailable is returned when a curated preset is applied without a working cscli. @@ -22,9 +18,6 @@ var ErrCrowdSecNotRunning = errors.New("crowdsec is not running") // ReloadFunc signals the managed CrowdSec process to reload its configuration and hub items. type ReloadFunc func(ctx context.Context) error -// curatedBackupTimeFormat includes sub-second precision so back-to-back applies never share a backup dir. -const curatedBackupTimeFormat = "20060102-150405.000000" - // ApplyCurated installs every hub item of a Charon-defined preset through cscli and verifies // each one. DataDir is never renamed or emptied before success is known: a copy-based backup // is taken first and restored in place if anything fails. @@ -50,16 +43,16 @@ func (s *HubService) ApplyCurated(ctx context.Context, preset Preset) (ApplyResu s.mu.Lock() defer s.mu.Unlock() - backupPath := filepath.Clean(s.DataDir) + ".backup." + time.Now().Format(curatedBackupTimeFormat) - if err := s.backupCopy(backupPath); err != nil { + backupPath, err := s.snapshot() + if err != nil { return fail(fmt.Errorf("backup: %w", err)) } result.BackupPath = backupPath + defer s.pruneAfterApply() if err := s.installAndVerify(applyCtx, preset); err != nil { - if rbErr := s.restoreCopy(backupPath); rbErr != nil { - logger.Log().WithError(rbErr).WithField("backup_path", util.SanitizeForLog(backupPath)).Error("curated preset rollback failed; backup retained for manual recovery") - err = fmt.Errorf("%w (rollback failed: %v; backup retained at %s)", err, rbErr, backupPath) + if rbErr := s.restore(backupPath); rbErr != nil { + err = rollbackFailure(err, rbErr, backupPath) } return fail(err) } @@ -118,71 +111,6 @@ func (s *HubService) verifyItem(ctx context.Context, item PresetItem) error { return nil } -// The live crowdsec.db (and -wal/-shm) is excluded so rollback can never regress engine state. -// backupCopy copies DataDir (symlinks preserved) into backupPath, leaving DataDir untouched. -func (s *HubService) backupCopy(backupPath string) error { - if err := os.Mkdir(backupPath, 0o700); err != nil { - return fmt.Errorf("mkdir backup: %w", err) - } - if err := copyDirFiltered(s.DataDir, backupPath, isLiveDBFile); err != nil { - _ = os.RemoveAll(backupPath) - return fmt.Errorf("copy backup: %w", err) - } - return nil -} - -// restoreCopy replaces the contents of DataDir with the backup while keeping DataDir itself in place. -func (s *HubService) restoreCopy(backupPath string) error { - if err := emptyDirExcept(s.DataDir, isLiveDBFile); err != nil { - return fmt.Errorf("empty data dir: %w", err) - } - if err := copyDirFiltered(backupPath, s.DataDir, isLiveDBFile); err != nil { - return fmt.Errorf("restore backup: %w", err) - } - return nil -} - -// isLiveDBFile reports whether name is the live CrowdSec SQLite database or its WAL/SHM sidecars. -// These are owned by the running engine and must never be copied into, or restored from, a backup. -func isLiveDBFile(name string) bool { - switch name { - case "crowdsec.db", "crowdsec.db-wal", "crowdsec.db-shm": - return true - } - return false -} - -// emptyDirExcept removes the contents of dir except entries (at any depth) for which keep returns true. -// Directories are removed only when nothing kept remains inside them. -func emptyDirExcept(dir string, keep func(name string) bool) error { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - for _, entry := range entries { - if keep(entry.Name()) { - continue - } - path := filepath.Join(dir, entry.Name()) - if entry.IsDir() { - if err := emptyDirExcept(path, keep); err != nil { - return err - } - // A directory that still holds kept files cannot be removed; leave it in place. - if remaining, rerr := os.ReadDir(path); rerr == nil && len(remaining) > 0 { - continue - } - } - if err := os.Remove(path); err != nil { - return err - } - } - return nil -} - // reloadCrowdSec asks the managed CrowdSec process to reload (best-effort) and reports // whether the reload was performed. A missing reloader or a stopped process is not an error: // CrowdSec loads the installed items itself on its next start. diff --git a/backend/internal/crowdsec/curated_apply_test.go b/backend/internal/crowdsec/curated_apply_test.go index e4c464004..b3eb8dced 100644 --- a/backend/internal/crowdsec/curated_apply_test.go +++ b/backend/internal/crowdsec/curated_apply_test.go @@ -397,7 +397,8 @@ func TestApplyCuratedContextExpiresBetweenItems(t *testing.T) { func TestApplyCuratedBackupFailureInstallsNothing(t *testing.T) { exec := &curatedExec{} hub, _ := newCuratedHub(t, exec) - hub.DataDir = filepath.Join(t.TempDir(), "does-not-exist") + hub.DataDir = filepath.Join(t.TempDir(), "not-a-directory") + require.NoError(t, os.WriteFile(hub.DataDir, []byte("x"), 0o600)) res, err := hub.ApplyCurated(context.Background(), twoItemPreset()) require.Error(t, err) @@ -552,7 +553,7 @@ func TestApplyCuratedRollbackLeavesLiveDatabaseUntouched(t *testing.T) { } func TestEmptyDirExcept(t *testing.T) { - require.NoError(t, emptyDirExcept(filepath.Join(t.TempDir(), "missing"), isLiveDBFile)) + require.NoError(t, emptyDirExcept(filepath.Join(t.TempDir(), "missing"), IsEngineOwnedPath)) dir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "b"), 0o700)) @@ -561,12 +562,12 @@ func TestEmptyDirExcept(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "b", "f"), []byte("f"), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(dir, "top"), []byte("t"), 0o600)) - require.NoError(t, emptyDirExcept(dir, isLiveDBFile)) + require.NoError(t, emptyDirExcept(dir, IsEngineOwnedPath)) require.FileExists(t, filepath.Join(dir, "a", "crowdsec.db")) require.NoFileExists(t, filepath.Join(dir, "top")) require.NoDirExists(t, filepath.Join(dir, "a", "b")) require.NoDirExists(t, filepath.Join(dir, "gone")) // a regular file instead of a directory cannot be listed - require.Error(t, emptyDirExcept(filepath.Join(dir, "a", "crowdsec.db"), isLiveDBFile)) + require.Error(t, emptyDirExcept(filepath.Join(dir, "a", "crowdsec.db"), IsEngineOwnedPath)) } diff --git a/backend/internal/crowdsec/device_busy_test.go b/backend/internal/crowdsec/device_busy_test.go index 7f5ebce21..298ae314b 100644 --- a/backend/internal/crowdsec/device_busy_test.go +++ b/backend/internal/crowdsec/device_busy_test.go @@ -10,8 +10,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestApplyWithOpenFileHandles simulates the "device or resource busy" scenario -// where the data directory has open file handles (e.g., from cache operations) +// TestApplyWithOpenFileHandles covers a data dir with open file handles (the old "device or resource +// busy" case): the copy-based snapshot works, DataDir itself is never renamed, and the engine-owned +// hub_cache stays untouched. func TestApplyWithOpenFileHandles(t *testing.T) { cache, err := NewHubCache(t.TempDir(), time.Hour) require.NoError(t, err) @@ -20,55 +21,45 @@ func TestApplyWithOpenFileHandles(t *testing.T) { require.NoError(t, os.MkdirAll(dataDir, 0o750)) require.NoError(t, os.WriteFile(filepath.Join(dataDir, "config.txt"), []byte("original"), 0o600)) - // Create a subdirectory with nested files (similar to hub_cache) subDir := filepath.Join(dataDir, "hub_cache") require.NoError(t, os.MkdirAll(subDir, 0o750)) cacheFile := filepath.Join(subDir, "cache.json") require.NoError(t, os.WriteFile(cacheFile, []byte(`{"test": "data"}`), 0o600)) - // Open a file handle to simulate an in-use directory - // This would cause os.Rename to fail with "device or resource busy" on some systems - f, err := os.Open(cacheFile) // #nosec G304 -- Test opens test cache file // #nosec G304 -- Test opens test cache file + f, err := os.Open(cacheFile) // #nosec G304 -- Test opens test cache file require.NoError(t, err) defer func() { _ = f.Close() }() - // Create and cache a preset + before, err := os.Stat(dataDir) + require.NoError(t, err) + archive := makeTarGz(t, map[string]string{"new/preset.yaml": "new: preset"}) _, err = cache.Store(context.Background(), "test/preset", "etag1", "hub", "preview", archive) require.NoError(t, err) svc := NewHubService(nil, cache, dataDir) - // Apply should succeed using copy-based backup even with open file handles res, err := svc.Apply(context.Background(), "test/preset") require.NoError(t, err) require.Equal(t, "applied", res.Status) require.NotEmpty(t, res.BackupPath, "BackupPath should be set on success") - // Verify backup was created and contains the original files - backupConfigPath := filepath.Join(res.BackupPath, "config.txt") - backupCachePath := filepath.Join(res.BackupPath, "hub_cache", "cache.json") - - // The backup should exist - require.FileExists(t, backupConfigPath) - require.FileExists(t, backupCachePath) - - // Verify original content was preserved in backup - // #nosec G304 -- Test reads from known backup paths created by test - content, err := os.ReadFile(backupConfigPath) + after, err := os.Stat(dataDir) require.NoError(t, err) - require.Equal(t, "original", string(content)) + require.True(t, os.SameFile(before, after), "DataDir must never be renamed or replaced") - // #nosec G304 -- Test reads from known backup paths created by test - cacheContent, err := os.ReadFile(backupCachePath) + // The snapshot holds the original config but not the regenerable hub cache. + // #nosec G304 -- Test reads from known backup path created by test + content, err := os.ReadFile(filepath.Join(res.BackupPath, "config.txt")) require.NoError(t, err) - require.Contains(t, string(cacheContent), "test") + require.Equal(t, "original", string(content)) + require.NoDirExists(t, filepath.Join(res.BackupPath, "hub_cache")) - // Verify new preset was applied - newPresetPath := filepath.Join(dataDir, "new", "preset.yaml") - require.FileExists(t, newPresetPath) + // The live cache file is still in place and the preset was applied on top of the live tree. + require.FileExists(t, cacheFile) + require.FileExists(t, filepath.Join(dataDir, "config.txt")) // #nosec G304 -- Test reads from known preset path in test dataDir - newContent, err := os.ReadFile(newPresetPath) + newContent, err := os.ReadFile(filepath.Join(dataDir, "new", "preset.yaml")) require.NoError(t, err) require.Contains(t, string(newContent), "new: preset") } diff --git a/backend/internal/crowdsec/hub_sync.go b/backend/internal/crowdsec/hub_sync.go index a4a45750f..8ef531538 100644 --- a/backend/internal/crowdsec/hub_sync.go +++ b/backend/internal/crowdsec/hub_sync.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "io/fs" "net/http" neturl "net/url" "os" @@ -598,7 +597,8 @@ func (s *HubService) Pull(ctx context.Context, slug string) (PullResult, error) return PullResult{Meta: cachedMeta, Preview: previewText}, nil } -// Apply installs the preset, preferring cscli when available. Falls back to manual extraction. +// Apply installs the preset, preferring cscli when available and falling back to the cached archive. +// DataDir is never renamed: a copy-based snapshot is taken first and restored in place on failure. func (s *HubService) Apply(ctx context.Context, slug string) (ApplyResult, error) { cleanSlug := sanitizeSlug(slug) if cleanSlug == "" { @@ -614,77 +614,77 @@ func (s *HubService) Apply(ctx context.Context, slug string) (ApplyResult, error if metaErr == nil { result.CacheKey = meta.CacheKey } - hasCS := s.hasCSCLI(applyCtx) - // Read archive into memory BEFORE backup, since cache is inside DataDir. - // If we backup first, the archive path becomes invalid (file moved). - var archive []byte - var archiveReadErr error - if metaErr == nil { - archive, archiveReadErr = os.ReadFile(meta.ArchivePath) - if archiveReadErr != nil { - logger.Log().WithField("error", util.SanitizeForLog(archiveReadErr.Error())).WithField("archive_path", util.SanitizeForLog(meta.ArchivePath)). - Warn("failed to read cached archive before backup") - } - } - - backupPath := filepath.Clean(s.DataDir) + ".backup." + time.Now().Format("20060102-150405") - if err := s.backupExisting(backupPath); err != nil { - // Only set BackupPath if backup was actually created + backupPath, err := s.snapshot() + if err != nil { return result, fmt.Errorf("backup: %w", err) } - // Set BackupPath only after successful backup result.BackupPath = backupPath + defer s.pruneAfterApply() - // Try cscli first - if hasCS { + if s.hasCSCLI(applyCtx) { cscliErr := s.runCSCLI(applyCtx, cleanSlug) if cscliErr == nil { result.Status = "applied" - result.ReloadHint = true + result.ReloadHint = !s.reloadCrowdSec(applyCtx, "hub preset apply") result.UsedCSCLI = true return result, nil } logger.Log().WithField("slug", util.SanitizeForLog(cleanSlug)).WithField("error", util.SanitizeForLog(cscliErr.Error())).Warn("cscli install failed; attempting cache fallback") - } - - // Handle cache miss OR failed archive read - need to refresh cache - if metaErr != nil || archiveReadErr != nil { - originalErr := metaErr - if originalErr == nil { - originalErr = archiveReadErr - } - refreshed, refreshErr := s.refreshCache(applyCtx, cleanSlug, originalErr) - if refreshErr != nil { - _ = s.rollback(backupPath) - logger.Log().WithField("error", util.SanitizeForLog(refreshErr.Error())).WithField("slug", util.SanitizeForLog(cleanSlug)).WithField("backup_path", util.SanitizeForLog(backupPath)).Warn("cache refresh failed; rolled back backup") - msg := fmt.Sprintf("load cache for %s: %v", cleanSlug, refreshErr) - result.ErrorMessage = msg - return result, fmt.Errorf("load cache for %s: %w", cleanSlug, refreshErr) - } - meta = refreshed - result.CacheKey = meta.CacheKey - - // Re-read archive from the newly refreshed cache location - archive, archiveReadErr = os.ReadFile(meta.ArchivePath) - if archiveReadErr != nil { - _ = s.rollback(backupPath) - return result, fmt.Errorf("read archive after refresh: %w", archiveReadErr) + // Drop whatever cscli may have written so the archive extract starts from the pre-apply tree. + if rbErr := s.restore(backupPath); rbErr != nil { + return result, rollbackFailure(fmt.Errorf("cscli install failed: %w", cscliErr), rbErr, backupPath) } } - // Use pre-loaded archive bytes + archive, err := s.loadArchive(applyCtx, cleanSlug, meta, metaErr, &result) + if err != nil { + return result, s.restoreAfter(err, backupPath, cleanSlug) + } if err := s.extractTarGz(applyCtx, archive, s.DataDir); err != nil { - _ = s.rollback(backupPath) - return result, fmt.Errorf("extract: %w", err) + return result, s.restoreAfter(fmt.Errorf("extract: %w", err), backupPath, cleanSlug) } result.Status = "applied" - result.ReloadHint = true + result.ReloadHint = !s.reloadCrowdSec(applyCtx, "hub preset apply") result.UsedCSCLI = false return result, nil } +// loadArchive reads the cached archive, refreshing the cache from the hub on a miss or unreadable file. +func (s *HubService) loadArchive(ctx context.Context, slug string, meta CachedPreset, metaErr error, result *ApplyResult) ([]byte, error) { + if metaErr == nil { + archive, readErr := os.ReadFile(meta.ArchivePath) + if readErr == nil { + return archive, nil + } + logger.Log().WithField("error", util.SanitizeForLog(readErr.Error())).WithField("archive_path", util.SanitizeForLog(meta.ArchivePath)).Warn("failed to read cached archive") + metaErr = readErr + } + refreshed, refreshErr := s.refreshCache(ctx, slug, metaErr) + if refreshErr != nil { + msg := fmt.Sprintf("load cache for %s: %v", slug, refreshErr) + result.ErrorMessage = msg + return nil, fmt.Errorf("load cache for %s: %w", slug, refreshErr) + } + result.CacheKey = refreshed.CacheKey + archive, err := os.ReadFile(refreshed.ArchivePath) + if err != nil { + return nil, fmt.Errorf("read archive after refresh: %w", err) + } + return archive, nil +} + +// restoreAfter restores the snapshot after a failed apply step and returns the original error, +// annotated when the restore itself failed. +func (s *HubService) restoreAfter(cause error, backupPath, slug string) error { + if rbErr := s.restore(backupPath); rbErr != nil { + return rollbackFailure(cause, rbErr, backupPath) + } + logger.Log().WithField("error", util.SanitizeForLog(cause.Error())).WithField("slug", util.SanitizeForLog(slug)).WithField("backup_path", util.SanitizeForLog(backupPath)).Warn("preset apply failed; restored previous configuration") + return cause +} + func (s *HubService) findPreviewFile(data []byte) string { buf := bytes.NewReader(data) gr, err := gzip.NewReader(buf) @@ -909,77 +909,10 @@ func cleanShellArg(val string) string { return sanitizeSlug(val) } -func (s *HubService) backupExisting(backupPath string) error { - if _, err := os.Stat(s.DataDir); errors.Is(err, os.ErrNotExist) { - return nil - } - - // First try rename for performance (atomic operation) - if err := os.Rename(s.DataDir, backupPath); err == nil { - return nil - } - - // If rename fails (e.g., device busy, cross-device), use copy approach - logger.Log().WithField("data_dir", s.DataDir).WithField("backup_path", backupPath).Info("rename failed; using copy-based backup") - - // Create backup directory - if err := os.MkdirAll(backupPath, 0o700); err != nil { - return fmt.Errorf("mkdir backup: %w", err) - } - - // Copy directory contents recursively - if err := copyDir(s.DataDir, backupPath); err != nil { - _ = os.RemoveAll(backupPath) - return fmt.Errorf("copy backup: %w", err) - } - - return nil -} - -func (s *HubService) rollback(backupPath string) error { - _ = os.RemoveAll(s.DataDir) - if backupPath == "" { - return nil - } - if _, err := os.Stat(backupPath); err == nil { - return os.Rename(backupPath, s.DataDir) - } - return nil -} - -// emptyDir removes all contents of a directory but leaves the directory itself. -func emptyDir(dir string) error { - d, err := os.Open(dir) // #nosec G304 -- Directory path from validated backup root // #nosec G304 -- Directory path from validated backup root - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer func() { - if closeErr := d.Close(); closeErr != nil { - logger.Log().WithError(closeErr).Warn("Failed to close directory") - } - }() - names, err := d.Readdirnames(-1) - if err != nil { - return err - } - for _, name := range names { - if err := os.RemoveAll(filepath.Join(dir, name)); err != nil { - return err - } - } - return nil -} - -// extractTarGz validates and extracts archive into targetDir. +// extractTarGz validates and overlays archive onto targetDir. Existing files are replaced only when +// the archive carries a same-named entry; the snapshot taken by Apply is the safety net. Entries for +// engine-owned state (live db files, top-level data/ and hub_cache/) are skipped. func (s *HubService) extractTarGz(ctx context.Context, archive []byte, targetDir string) error { - // Clear target directory contents instead of removing the directory itself - // to avoid "device or resource busy" errors if targetDir is a mount point. - if err := emptyDir(targetDir); err != nil { - return fmt.Errorf("clean target: %w", err) - } if err := os.MkdirAll(targetDir, 0o700); err != nil { return fmt.Errorf("mkdir target: %w", err) } @@ -1013,6 +946,9 @@ func (s *HubService) extractTarGz(ctx context.Context, archive []byte, targetDir if strings.HasPrefix(cleanName, "..") || strings.Contains(cleanName, ".."+string(os.PathSeparator)) || filepath.IsAbs(cleanName) { return fmt.Errorf("unsafe path %s", hdr.Name) } + if IsEngineOwnedPath(cleanName) { + continue + } destPath := filepath.Join(targetDir, cleanName) // Defense in depth: confirm the joined path is still contained in targetDir. rel, relErr := filepath.Rel(filepath.Clean(targetDir), destPath) @@ -1029,12 +965,9 @@ func (s *HubService) extractTarGz(ctx context.Context, archive []byte, targetDir continue } - if mkdirErr := os.MkdirAll(filepath.Dir(destPath), 0o700); mkdirErr != nil { //nolint:gosec // G703: destPath contained under targetDir by the filepath.Rel check above - return fmt.Errorf("mkdir parent: %w", mkdirErr) - } - f, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) //nolint:gosec // G304,G703: destPath contained under targetDir by the filepath.Rel check above + f, err := CreateFileNoSymlink(targetDir, destPath, 0o700, mode) if err != nil { - return fmt.Errorf("open %s: %w", destPath, err) + return err } // Limit decompressed size to prevent decompression bombs (100MB limit) const maxDecompressedSize = 100 * 1024 * 1024 // 100MB @@ -1056,60 +989,6 @@ func (s *HubService) extractTarGz(ctx context.Context, archive []byte, targetDir return nil } -// copyDir recursively copies a directory tree. -func copyDir(src, dst string) error { - return copyDirFiltered(src, dst, nil) -} - -// copyDirFiltered is copyDir that leaves out entries whose base name makes skip return true. -func copyDirFiltered(src, dst string, skip func(name string) bool) error { - srcInfo, err := os.Stat(src) - if err != nil { - return fmt.Errorf("stat src: %w", err) - } - if !srcInfo.IsDir() { - return fmt.Errorf("src is not a directory") - } - - entries, err := os.ReadDir(src) - if err != nil { - return fmt.Errorf("read dir: %w", err) - } - - for _, entry := range entries { - if skip != nil && skip(entry.Name()) { - continue - } - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - switch { - case entry.Type()&fs.ModeSymlink != 0: - // Preserve the link literally (never follow it): hub item entries are - // symlinks, and dangling links must survive a backup/restore round trip. - target, err := os.Readlink(srcPath) - if err != nil { - return fmt.Errorf("readlink %s: %w", srcPath, err) - } - if err := os.Symlink(target, dstPath); err != nil { - return fmt.Errorf("symlink %s: %w", dstPath, err) - } - case entry.IsDir(): - if err := os.MkdirAll(dstPath, 0o700); err != nil { - return fmt.Errorf("mkdir %s: %w", dstPath, err) - } - if err := copyDirFiltered(srcPath, dstPath, skip); err != nil { - return err - } - default: - if err := copyFile(srcPath, dstPath); err != nil { - return err - } - } - } - return nil -} - // copyFile copies a single file. func copyFile(src, dst string) error { srcFile, err := os.Open(src) // #nosec G304 -- Source path from copyDir recursive call // #nosec G304 -- Source path from copyDir recursive call diff --git a/backend/internal/crowdsec/hub_sync_test.go b/backend/internal/crowdsec/hub_sync_test.go index 1fae1f985..c6c79e9d3 100644 --- a/backend/internal/crowdsec/hub_sync_test.go +++ b/backend/internal/crowdsec/hub_sync_test.go @@ -856,22 +856,6 @@ func TestHubHTTPErrorMethods(t *testing.T) { require.True(t, err.CanFallback()) } -func TestBackupExistingHandlesDeviceBusy(t *testing.T) { - t.Parallel() - dataDir := filepath.Join(t.TempDir(), "data") - require.NoError(t, os.MkdirAll(dataDir, 0o750)) - // #nosec G306 -- Test fixture file used for copy-based backup verification - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "file.txt"), []byte("content"), 0o644)) - - svc := NewHubService(nil, nil, dataDir) - backupPath := dataDir + ".backup.test" - - // Even if rename fails, copy-based backup should work - err := svc.backupExisting(backupPath) - require.NoError(t, err) - require.FileExists(t, filepath.Join(backupPath, "file.txt")) -} - func TestCopyFile(t *testing.T) { t.Parallel() tmpDir := t.TempDir() @@ -919,7 +903,7 @@ func TestCopyDir(t *testing.T) { require.NoError(t, os.MkdirAll(dstDir, 0o750)) // #nosec G301 -- test fixture // Test successful copy - err := copyDir(srcDir, dstDir) + err := copyTree(srcDir, dstDir, nil) require.NoError(t, err) // Verify files were copied @@ -936,14 +920,14 @@ func TestCopyDir(t *testing.T) { require.Equal(t, []byte("file2"), content2) // Test copy non-existent directory - err = copyDir(filepath.Join(tmpDir, "nonexistent"), dstDir) + err = copyTree(filepath.Join(tmpDir, "nonexistent"), dstDir, nil) require.Error(t, err) require.Contains(t, err.Error(), "stat src") // Test copy file as directory (should fail) fileNotDir := filepath.Join(tmpDir, "file.txt") require.NoError(t, os.WriteFile(fileNotDir, []byte("test"), 0o600)) - err = copyDir(fileNotDir, dstDir) + err = copyTree(fileNotDir, dstDir, nil) require.Error(t, err) require.Contains(t, err.Error(), "not a directory") } @@ -1313,7 +1297,7 @@ func TestCopyDirAndCopyFile(t *testing.T) { require.NoError(t, os.MkdirAll(dstDir, 0o750)) // #nosec G301 -- test fixture - err := copyDir(srcDir, dstDir) + err := copyTree(srcDir, dstDir, nil) require.NoError(t, err) // Verify all files copied correctly @@ -1336,67 +1320,12 @@ func TestCopyDirAndCopyFile(t *testing.T) { require.NoError(t, os.WriteFile(srcFile, []byte("test"), 0o600)) require.NoError(t, os.MkdirAll(dstDir, 0o750)) // #nosec G301 -- test fixture - err := copyDir(srcFile, dstDir) + err := copyTree(srcFile, dstDir, nil) require.Error(t, err) require.Contains(t, err.Error(), "not a directory") }) } -// ============================================ -// emptyDir Tests -// ============================================ - -func TestEmptyDir(t *testing.T) { - t.Parallel() - t.Run("empties directory with files", func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "file1.txt"), []byte("content1"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "file2.txt"), []byte("content2"), 0o600)) - - err := emptyDir(dir) - require.NoError(t, err) - - // Directory should still exist - require.DirExists(t, dir) - - // But be empty - entries, err := os.ReadDir(dir) - require.NoError(t, err) - require.Empty(t, entries) - }) - - t.Run("empties directory with subdirectories", func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - subDir := filepath.Join(dir, "subdir") - require.NoError(t, os.MkdirAll(subDir, 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(subDir, "nested.txt"), []byte("nested"), 0o600)) - - err := emptyDir(dir) - require.NoError(t, err) - - require.DirExists(t, dir) - entries, err := os.ReadDir(dir) - require.NoError(t, err) - require.Empty(t, entries) - }) - - t.Run("handles non-existent directory", func(t *testing.T) { - t.Parallel() - err := emptyDir(filepath.Join(t.TempDir(), "nonexistent")) - require.NoError(t, err, "should not error on non-existent directory") - }) - - t.Run("handles empty directory", func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - err := emptyDir(dir) - require.NoError(t, err) - require.DirExists(t, dir) - }) -} - // ============================================ // extractTarGz Tests // ============================================ @@ -1504,111 +1433,6 @@ func TestExtractTarGz(t *testing.T) { }) } -// ============================================ -// backupExisting Tests -// ============================================ - -func TestBackupExisting(t *testing.T) { - t.Parallel() - t.Run("handles non-existent directory", func(t *testing.T) { - t.Parallel() - dataDir := filepath.Join(t.TempDir(), "nonexistent") - svc := NewHubService(nil, nil, dataDir) - backupPath := dataDir + ".backup" - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - require.NoDirExists(t, backupPath) - }) - - t.Run("creates backup of existing directory", func(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "config.txt"), []byte("config data"), 0o600)) - - subDir := filepath.Join(dataDir, "subdir") - require.NoError(t, os.MkdirAll(subDir, 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(subDir, "nested.txt"), []byte("nested data"), 0o600)) - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(t.TempDir(), "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - // Verify backup exists - require.FileExists(t, filepath.Join(backupPath, "config.txt")) - require.FileExists(t, filepath.Join(backupPath, "subdir", "nested.txt")) - }) - - t.Run("backup contents match original", func(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - originalContent := "important config" - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "config.txt"), []byte(originalContent), 0o600)) // #nosec G306 -- test fixture - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(t.TempDir(), "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - backupContent, err := os.ReadFile(filepath.Join(backupPath, "config.txt")) // #nosec G304 -- test fixture path - require.NoError(t, err) - require.Equal(t, originalContent, string(backupContent)) - }) -} - -// ============================================ -// rollback Tests -// ============================================ - -func TestRollback(t *testing.T) { - t.Parallel() - t.Run("rollback with backup", func(t *testing.T) { - t.Parallel() - parentDir := t.TempDir() - dataDir := filepath.Join(parentDir, "data") - backupPath := filepath.Join(parentDir, "backup") - - // Create backup first - require.NoError(t, os.MkdirAll(backupPath, 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(backupPath, "backed_up.txt"), []byte("backup content"), 0o600)) // #nosec G306 -- test fixture - - // Create data dir with different content - require.NoError(t, os.MkdirAll(dataDir, 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "current.txt"), []byte("current content"), 0o600)) // #nosec G306 -- test fixture - - svc := NewHubService(nil, nil, dataDir) - - err := svc.rollback(backupPath) - require.NoError(t, err) - - // Data dir should now have backup contents - require.FileExists(t, filepath.Join(dataDir, "backed_up.txt")) - // Backup path should no longer exist (renamed to dataDir) - require.NoDirExists(t, backupPath) - }) - - t.Run("rollback with empty backup path", func(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - svc := NewHubService(nil, nil, dataDir) - - err := svc.rollback("") - require.NoError(t, err) - }) - - t.Run("rollback with non-existent backup", func(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - svc := NewHubService(nil, nil, dataDir) - - err := svc.rollback(filepath.Join(t.TempDir(), "nonexistent")) - require.NoError(t, err) - }) -} - // ============================================ // hubHTTPError Tests // ============================================ @@ -1915,97 +1739,6 @@ func TestNewHubService_CustomMirrorBaseURL(t *testing.T) { require.Equal(t, "https://mirror.example.com", svc.MirrorBaseURL) } -// ============================================ -// backupExisting Additional Tests -// ============================================ - -func TestBackupExisting_CopyFallback_Success(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - - // Create complex directory structure - require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "configs", "scenarios"), 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "main.yaml"), []byte("main config"), 0o600)) // #nosec G306 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "configs", "sub.yaml"), []byte("sub config"), 0o600)) // #nosec G306 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "configs", "scenarios", "s1.yaml"), []byte("scenario 1"), 0o600)) // #nosec G306 -- test fixture - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(t.TempDir(), "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - // Verify all files were backed up - require.FileExists(t, filepath.Join(backupPath, "main.yaml")) - require.FileExists(t, filepath.Join(backupPath, "configs", "sub.yaml")) - require.FileExists(t, filepath.Join(backupPath, "configs", "scenarios", "s1.yaml")) - - // Verify content integrity - content, err := os.ReadFile(filepath.Join(backupPath, "configs", "scenarios", "s1.yaml")) // #nosec G304 -- test fixture path - require.NoError(t, err) - require.Equal(t, "scenario 1", string(content)) -} - -func TestBackupExisting_RenameSuccess(t *testing.T) { - t.Parallel() - baseDir := t.TempDir() - dataDir := filepath.Join(baseDir, "data") - require.NoError(t, os.MkdirAll(dataDir, 0o750)) // #nosec G301 -- test fixture - require.NoError(t, os.WriteFile(filepath.Join(dataDir, "file.txt"), []byte("content"), 0o600)) // #nosec G306 -- test fixture - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(baseDir, "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - // Original should be gone (renamed, not copied) - require.NoDirExists(t, dataDir) - // Backup should exist with content - require.FileExists(t, filepath.Join(backupPath, "file.txt")) -} - -func TestBackupExisting_EmptyDirectory(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(t.TempDir(), "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - // Backup should exist even for empty dir - require.DirExists(t, backupPath) -} - -func TestBackupExisting_PreservesPermissions(t *testing.T) { - t.Parallel() - dataDir := t.TempDir() - execFile := filepath.Join(dataDir, "executable.sh") - require.NoError(t, os.WriteFile(execFile, []byte("#!/bin/bash"), 0o750)) // #nosec G306 -- test fixture for executable script - - svc := NewHubService(nil, nil, dataDir) - backupPath := filepath.Join(t.TempDir(), "backup") - - err := svc.backupExisting(backupPath) - require.NoError(t, err) - - // Check permissions were preserved - origInfo, err := os.Stat(execFile) - if err == nil { - // If original still exists (rename succeeded) - backupInfo, err := os.Stat(filepath.Join(backupPath, "executable.sh")) - require.NoError(t, err) - require.Equal(t, origInfo.Mode(), backupInfo.Mode()) - } else { - // If original was renamed (which removes it) - backupInfo, err := os.Stat(filepath.Join(backupPath, "executable.sh")) - require.NoError(t, err) - require.Equal(t, os.FileMode(0o750), backupInfo.Mode()&0o777) - } -} - // ============================================ // extractTarGz Security Tests // ============================================ diff --git a/backend/internal/crowdsec/safe_create.go b/backend/internal/crowdsec/safe_create.go new file mode 100644 index 000000000..01d68d19d --- /dev/null +++ b/backend/internal/crowdsec/safe_create.go @@ -0,0 +1,46 @@ +package crowdsec + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrSymlinkInPath is returned when an archive entry would be written through a symlink. +var ErrSymlinkInPath = errors.New("refusing to write through a symlink") + +// CreateFileNoSymlink creates (or truncates) dest, which must be inside root, creating missing parent +// directories. It refuses to proceed if dest or any existing component between root and dest is a +// symlink, so archive extraction over a live tree can never be redirected outside root. +func CreateFileNoSymlink(root, dest string, parentMode, fileMode os.FileMode) (*os.File, error) { + root = filepath.Clean(root) + rel, err := filepath.Rel(root, filepath.Clean(dest)) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return nil, fmt.Errorf("path escapes target: %s", dest) + } + cur := root + for _, part := range strings.Split(rel, string(os.PathSeparator)) { + cur = filepath.Join(cur, part) + info, lerr := os.Lstat(cur) + if errors.Is(lerr, fs.ErrNotExist) { + break + } + if lerr != nil { + return nil, fmt.Errorf("inspect %s: %w", cur, lerr) + } + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%w: %s", ErrSymlinkInPath, cur) + } + } + if err = os.MkdirAll(filepath.Dir(dest), parentMode); err != nil { //nolint:gosec // G703: dest verified contained under root and symlink-free above + return nil, fmt.Errorf("mkdir parent: %w", err) + } + f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileMode) //nolint:gosec // G304,G703: dest verified contained under root and symlink-free above + if err != nil { + return nil, fmt.Errorf("open %s: %w", dest, err) + } + return f, nil +} diff --git a/backend/internal/crowdsec/safe_create_test.go b/backend/internal/crowdsec/safe_create_test.go new file mode 100644 index 000000000..a430635f6 --- /dev/null +++ b/backend/internal/crowdsec/safe_create_test.go @@ -0,0 +1,92 @@ +package crowdsec + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateFileNoSymlink(t *testing.T) { + t.Parallel() + + t.Run("creates file and parents, truncating existing", func(t *testing.T) { + t.Parallel() + root := t.TempDir() + dest := filepath.Join(root, "a", "b.txt") + f, err := CreateFileNoSymlink(root, dest, 0o750, 0o640) + require.NoError(t, err) + _, err = f.WriteString("long content") + require.NoError(t, err) + require.NoError(t, f.Close()) + f, err = CreateFileNoSymlink(root, dest, 0o750, 0o640) + require.NoError(t, err) + _, err = f.WriteString("x") + require.NoError(t, err) + require.NoError(t, f.Close()) + got, err := os.ReadFile(dest) // #nosec G304 -- test path + require.NoError(t, err) + require.Equal(t, "x", string(got)) + }) + + t.Run("rejects escaping path", func(t *testing.T) { + t.Parallel() + root := t.TempDir() + _, err := CreateFileNoSymlink(root, filepath.Join(root, "..", "evil"), 0o750, 0o640) + require.Error(t, err) + }) + + t.Run("rejects symlinked file", func(t *testing.T) { + t.Parallel() + root, outside := t.TempDir(), t.TempDir() + target := filepath.Join(outside, "secret") + require.NoError(t, os.WriteFile(target, []byte("keep"), 0o600)) + require.NoError(t, os.Symlink(target, filepath.Join(root, "link"))) + _, err := CreateFileNoSymlink(root, filepath.Join(root, "link"), 0o750, 0o640) + require.ErrorIs(t, err, ErrSymlinkInPath) + got, rerr := os.ReadFile(target) // #nosec G304 -- test path + require.NoError(t, rerr) + require.Equal(t, "keep", string(got)) + }) + + t.Run("rejects symlinked parent dir", func(t *testing.T) { + t.Parallel() + root, outside := t.TempDir(), t.TempDir() + require.NoError(t, os.Symlink(outside, filepath.Join(root, "dir"))) + _, err := CreateFileNoSymlink(root, filepath.Join(root, "dir", "f"), 0o750, 0o640) + require.ErrorIs(t, err, ErrSymlinkInPath) + require.NoFileExists(t, filepath.Join(outside, "f")) + }) + + t.Run("mkdir parent failure", func(t *testing.T) { + t.Parallel() + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "file"), []byte("x"), 0o600)) + _, err := CreateFileNoSymlink(root, filepath.Join(root, "file", "child"), 0o750, 0o640) + require.Error(t, err) + }) + + t.Run("open failure on directory", func(t *testing.T) { + t.Parallel() + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, "d"), 0o750)) + _, err := CreateFileNoSymlink(root, filepath.Join(root, "d"), 0o750, 0o640) + require.Error(t, err) + }) +} + +func TestExtractTarGzRefusesSymlinkedDestination(t *testing.T) { + t.Parallel() + svc := NewHubService(nil, nil, t.TempDir()) + target, outside := t.TempDir(), t.TempDir() + victim := filepath.Join(outside, "victim") + require.NoError(t, os.WriteFile(victim, []byte("untouched"), 0o600)) + require.NoError(t, os.Symlink(victim, filepath.Join(target, "config.yaml"))) + + err := svc.extractTarGz(t.Context(), makeTarGz(t, map[string]string{"config.yaml": "overwritten"}), target) + require.ErrorIs(t, err, ErrSymlinkInPath) + got, rerr := os.ReadFile(victim) // #nosec G304 -- test path + require.NoError(t, rerr) + require.Equal(t, "untouched", string(got)) +} diff --git a/codecov.yml b/codecov.yml index cd936761e..7f9588f19 100644 --- a/codecov.yml +++ b/codecov.yml @@ -25,18 +25,21 @@ require_ci_to_pass: yes # ----------------------------------------------------------------------------- # Notification gating # ----------------------------------------------------------------------------- -# Coverage is uploaded by three independent, parallel jobs in -# .github/workflows/codecov-upload.yml (flags: backend, frontend, agent). -# Without this, Codecov re-evaluates the project/patch commit statuses after -# EACH upload lands, so the first upload produces a partial report that fails -# the 87% gate — the status shows red, then flips green once the remaining -# uploads arrive. after_n_builds makes Codecov hold the status as pending -# until all 3 uploads are in, then post a single pass/fail. +# Coverage is produced by three parallel jobs in +# .github/workflows/codecov-upload.yml (flags: backend, frontend, agent). They +# only stage their report as an artifact; the final `upload-codecov` job then +# uploads all three back to back. Codecov re-evaluates statuses (codecov/patch, +# codecov/project) on every upload, so uploading from each job made the checks +# flap (the quick agent upload landed ~13 min before backend). With one +# combined upload step, partial reports are no longer posted. A seconds-wide +# window remains between the three back-to-back uploads. +# after_n_builds and wait_for_ci are kept as harmless secondary guards. # -# NOTE: on manual workflow_dispatch runs that disable a job (run_frontend=false -# etc.) fewer than 3 uploads arrive and the status stays pending. Normal -# pull_request runs always execute all 3 jobs, so this only affects ad-hoc -# manual runs. +# NOTE: `upload-codecov` runs when no test job failed or was cancelled and at +# least one succeeded. A manual workflow_dispatch that disables a job +# (run_frontend=false etc.) therefore uploads only the existing reports, and +# after_n_builds: 3 may keep the status pending. If a test job fails, nothing +# is uploaded; re-run the failed job. Normal pull_request runs execute all 3. codecov: notify: after_n_builds: 3 @@ -46,7 +49,7 @@ codecov: # PR Comment Configuration # ----------------------------------------------------------------------------- comment: - # Hold the PR comment until all 3 coverage uploads land, matching + # Hold the PR comment until all 3 coverage uploads land matching # codecov.notify.after_n_builds — avoids posting partial numbers that # get edited moments later. after_n_builds: 3 diff --git a/configs/crowdsec/install_hub_items.sh b/configs/crowdsec/install_hub_items.sh index b7687962a..5784524cc 100644 --- a/configs/crowdsec/install_hub_items.sh +++ b/configs/crowdsec/install_hub_items.sh @@ -10,12 +10,21 @@ echo "Installing CrowdSec hub items for Charon..." # Hub index update is handled by the entrypoint before this script is called. # Do not duplicate it here — a redundant update adds ~3s to startup for no benefit. +# Helper: succeeds only when the hub item is really installed. +# `cscli inspect` exits 0 for items that are known to the hub index but not +# installed (it prints "installed": false), and can exit non-zero for installed items +# whose data files are missing, so the exit code is not a usable signal. Decide on the +# JSON "installed" flag instead (jq is not available in the image). +is_installed() { + cscli "$1" inspect "$2" -o json 2>/dev/null | grep -q '"installed": *true' +} + # Helper: only install if not already present (avoids 5-10s per cscli call on rebuilds) install_if_missing() { type="$1" # parsers | scenarios | collections name="$2" label="${3:-$name}" - if cscli "${type}" inspect "${name}" >/dev/null 2>&1; then + if is_installed "${type}" "${name}"; then echo " ✓ ${label} already installed, skipping" else echo " Installing ${label}..." diff --git a/docs/features.md b/docs/features.md index 50e16bfe2..75d4777f3 100644 --- a/docs/features.md +++ b/docs/features.md @@ -103,6 +103,8 @@ Control your security modules with a single click. The Security Dashboard provid Protect your applications using behavior-based threat detection powered by a global community of security data. Bad actors get blocked automatically before they can cause harm. +Preset changes are backed up first and rolled back automatically if anything goes wrong, and CrowdSec's downloaded data now survives container recreation. + → [Learn More](features/crowdsec.md) • [Setup Guide](guides/crowdsec-setup.md) --- diff --git a/docs/features/crowdsec.md b/docs/features/crowdsec.md index 2b800a437..2b1c66e79 100644 --- a/docs/features/crowdsec.md +++ b/docs/features/crowdsec.md @@ -58,6 +58,14 @@ Popular presets include: Charon also ships curated presets such as **Honeypot Friendly Defaults** and **GeoIP Enrichment** (`geoip-enrichment`, formerly `geolocation-aware`; it adds IP geolocation data to alerts and does not block by region). Curated presets are installed on the server with `cscli`, after a backup is taken. If the install fails, the backup is rolled back, and Charon shows the server's error instead of a success message. After a preset is applied (and after whitelist changes), Charon signals the CrowdSec process it manages to reload so new rules take effect. If CrowdSec is not running or the reload cannot be confirmed, a "reload required" note appears and the rules load the next time CrowdSec starts or restarts. +#### Safe Applies and Rollback + +Hub presets are applied on the server too, with the same safety net: before anything changes, Charon makes a backup copy of your CrowdSec settings. If something goes wrong, your settings are put back exactly as they were, and Charon shows you the error. Your live CrowdSec folder is never moved or emptied along the way. + +- Charon keeps the 5 most recent full backups and the 10 most recent single-file backups, and quietly deletes older ones. +- Only one change runs at a time, so two edits can't collide. +- The config file editor only accepts common text and settings files (`.yaml`, `.yml`, `.json`, `.txt`, `.conf`) up to 1 MiB, and validates file type, size and YAML. + ### Console Enrollment Connect to the CrowdSec Console for centralized management: @@ -91,6 +99,7 @@ CrowdSec settings are stored in Charon's database and synchronized with the Secu - **On Container Start** — CrowdSec launches automatically if previously enabled - **Configuration Sync** — Changes in the UI immediately apply to CrowdSec - **State Persistence** — Decisions and configurations survive restarts +- **Downloaded Data Survives Recreating the Container** — CrowdSec's own database and the lists it downloads (such as blocklists and the location database) are stored in your Charon data folder. If any are missing after an upgrade, Charon re-downloads them on startup. If the CrowdSec Hub is unreachable, a restart with missing data files may take up to about two minutes longer while the refresh times out. ## Troubleshooting Console Enrollment @@ -212,6 +221,12 @@ curl -I https://api.crowdsec.net | `api.crowdsec.net` | 443 | Console API and heartbeats | | `hub.crowdsec.net` | 443 | Hub presets download | +### "unable to init data for file" in the CrowdSec Log + +**What it means:** CrowdSec looked for one of its downloaded helper lists (like a blocklist or location database) and the file was missing. In older versions this could happen after recreating the container. + +**What to do:** Update Charon and restart the container. The missing files are downloaded again automatically (this needs internet access) and the message goes away. Nothing else is needed. + ## Using the Diagnostic Script The diagnostic script checks CrowdSec connectivity and configuration in one command. diff --git a/docs/getting-started.md b/docs/getting-started.md index 5e02da04f..bc5012d13 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -365,7 +365,7 @@ If CrowdSec doesn't auto-start: 3. **Check directory permissions:** ```bash - docker exec charon ls -la /var/lib/crowdsec/data/ + docker exec charon ls -la /app/data/crowdsec/data/ ``` Expected: `charon:charon` ownership diff --git a/docs/plans/current_spec.md b/docs/plans/current_spec.md index 069f789c8..5be618358 100644 --- a/docs/plans/current_spec.md +++ b/docs/plans/current_spec.md @@ -1,252 +1,286 @@ -# Plan: Curated CrowdSec presets falsely report success (Aikido finding) +# Plan: CrowdSec preset-apply and data/hub persistence hardening (follow-ups of PR #1513) -Type: single PR, `fix:` (medium fix, branch off `development`: `fix/crowdsec-curated-preset-apply`). Not `(security)`-scoped as a changelog category: the finding is low severity, admin-only, a false security state and not an auth bypass or exploitable vulnerability. Subject must stay generic, e.g. `fix: apply curated CrowdSec presets for real and report failures`. -Status: rev 2, supervisor APPROVED. Decision D1 DECIDED by the user: option (a), rename `geolocation-aware` to the honest "GeoIP Enrichment" preset (slug `geoip-enrichment`). Ready for user plan approval, then implementation. -Predecessor: the previous plan in this file (DB tail, #1426/#1436/#1438/#42) is archived at `docs/plans/archive/2026-10-02_db-tail-1426-1436-1438-42_spec.md`. +Type: ONE branch, ONE PR into `development`: `fix/crowdsec-preset-apply-hardening`. Medium/large fix (touches Go backend, React frontend, Dockerfile, entrypoint, E2E, docs), so per CLAUDE.md it gets a dedicated branch and PR, never a direct commit to `development`. +Closes: #1514, #1515, #1516, #1517, #1518, #1523, #1524. +Status: rev 2 (addresses Supervisor CHANGES REQUESTED: 7 blocking items), open questions Q1-Q5 resolved by the user. +Predecessor: the #1513 plan (curated preset apply) that previously lived in this file; it merged as `3a76804b`. ## 1. Introduction -Aikido AI-pentest finding, low severity, true positive (user-confirmed): "Curated CrowdSec presets falsely report success and leave protections inactive". +PR #1513 made curated CrowdSec presets server-authoritative. While verifying it in a rebuilt container, seven follow-ups were filed. They share one theme: **the CrowdSec preset/hub apply path and the persistent CrowdSec directory (`/app/data/crowdsec`) are not safe or durable**. Backups are taken by renaming the live directory, a generic file-write endpoint also renames the live directory, hub data files live on an ephemeral path, the bundled binary does not know its own version, and the tests that should guard this are tolerant skips. -`POST /api/v1/admin/crowdsec/presets/apply` for the curated presets `honeypot-friendly-defaults` and `geolocation-aware` returns HTTP 200 `{status:"applied", reload_hint:true, used_cscli:false}`, records a `CrowdsecPresetEvent{Status:"applied"}` and changes nothing on disk or in CrowdSec. The UI toasts "Preset applied via backend (reload required)". The operator believes protections are on; they are not. +Goal: after this PR, applying/editing CrowdSec config can never destroy the live config, hub data files survive container recreation, `cscli version` is truthful, backups are bounded, and E2E/unit tests fail when any of this regresses. -Goal: curated presets are genuinely applied (or fail loudly with a recorded failure), the response and audit trail reflect what actually happened, and the UI never claims success it cannot back. +## 2. Research Findings (root cause per issue, CLAUDE.md "Context First") -## 2. Research Findings (root cause trace, CLAUDE.md "Context First") +Key facts established from code (all paths verified in this repo): -### 2.1 Entry point (frontend) +- `crowdsecDataDir = cfg.Security.CrowdSecConfigDir` (`backend/internal/api/routes/routes.go:1030`), set to `/app/data/crowdsec` by `Dockerfile:1142` (`CHARON_CROWDSEC_CONFIG_DIR`). The same directory is `CS_PERSIST_DIR` in `.docker/docker-entrypoint.sh`. It therefore contains the **whole persistent CrowdSec tree**: `config/` (the target of the runtime symlink `/etc/crowdsec -> /app/data/crowdsec/config`), `data/` (LAPI db), `hub_cache/`, `bouncer_key`. +- That same path is `HubService.DataDir` and `CrowdsecHandler.DataDir` (`crowdsec_handler.go:374-397`). Every "backup the DataDir" in the code is a backup directory next to it, `/app/data/crowdsec.backup.`. -- `frontend/src/pages/CrowdSecConfig.tsx:469` `handleApplyPreset` calls `applyCrowdsecPreset({slug, cache_key})` (`frontend/src/api/presets.ts:82`), and on 2xx unconditionally toasts `Preset applied via backend` (line 479). The only fallback is `err.response?.status === 501` (line 485) -> `applyPresetLocally` (line 427). -- The curated list shown in the page comes from the backend `GET /admin/crowdsec/presets` (merged with hub entries) plus a static frontend catalog `frontend/src/data/crowdsecPresets.ts` (`CROWDSEC_PRESETS`, line 160 maps it to `source: 'charon-curated'`). +### 2.1 #1514 applyPresetLocally writes unvalidated client content (root cause is wider than the issue says) -### 2.2 Transformation (backend) +- Entry: `frontend/src/pages/CrowdSecConfig.tsx:429` `applyPresetLocally`; reached only when apply returns HTTP 501 for a non-curated (hub) preset (line ~508). It writes `presetPreview` (text from the pull preview, held in client state) to `selectedPath ?? files[0]` through `writeCrowdsecFile` (`frontend/src/api/crowdsec.ts`). +- Confirmed: the backend **never returns 501** for apply any more (grep of `crowdsec_handler.go` shows no `StatusNotImplemented`). The fallback is dead code in practice and can only be triggered by a proxy/misconfig, which makes removing it risk-free. +- Transformation/persistence: `POST /admin/crowdsec/file` -> `CrowdsecHandler.WriteFile` (`crowdsec_handler.go:950`, registered under `managementAdmin`, so admin-only). Verified defects in the handler, independent of the frontend: + 1. **Destructive backup.** It `os.Rename(h.DataDir, backupDir)` (moves the *entire* persistent CrowdSec tree away, config symlink target included), then `MkdirAll(filepath.Dir(p))` and writes just the one file. Everything else (config.yaml, hub, bouncer key, data) is left only in the backup dir. The live tree becomes a single file. There is no restore. This is the same defect class as #1515 and is a medium-severity data-loss bug, not just "unvalidated content". + 2. **The file endpoints get a validation pass.** The fix is described at policy level in 3.2. +- Exit: `{status:"written", backup}`; the UI toasts "File saved" / "Preset applied locally". +- Severity/impact: low-medium, authenticated admin only (not an unauthenticated vulnerability). See section 4. -- `backend/internal/crowdsec/presets.go`: three curated presets. `honeypot-friendly-defaults` and `geolocation-aware` have `RequiresHub:false`; `crowdsecurity/base-http-scenarios` has `RequiresHub:true`. The `Preset` struct holds only metadata (slug, title, summary, source, tags, requires_hub). There is no content/definition anywhere on the server. -- `backend/internal/api/handlers/crowdsec_handler.go:1179` `ApplyPreset`: `if preset, ok := FindPreset(slug); ok && !preset.RequiresHub {` creates the event with `Status:"applied"` (error from `DB.Create` ignored), then returns 200 with `status:"applied"`, `backup:""`, `reload_hint:true`, `used_cscli:false`. No file written, no `cscli` call, no reload. Verified: finding is accurate. -- `PullPreset` (line 1090) has the mirror short-circuit: `preview` is only the string `# Curated preset: \n# <summary>`, `cache_key:"curated-<slug>"`, nothing cached. -- The real path `HubService.Apply` (`backend/internal/crowdsec/hub_sync.go:596`) is unreachable for curated slugs: it backs up (`backupExisting`, line 904), prefers `cscli hub install <slug>` (`runCSCLI`, line 885), else `extractTarGz` of a cached hub archive, with rollback on failure. It only works for slugs that exist as hub index entries; the curated slugs `honeypot-friendly-defaults` / `geolocation-aware` are Charon-invented and are NOT hub entries, so even removing the short-circuit would just fail with a cache miss. +### 2.2 #1515 HubService.Apply renames the whole data dir (CONFIRMED in code; end-to-end not executed) -### 2.3 Persistence / exit +- Entry: `ApplyPreset` -> `Hub.Apply` for non-curated presets (`hub_sync.go:602`). +- Transformation: `backupExisting` first tries `os.Rename(DataDir, <DataDir>.backup.<ts>)`; only if rename fails (EBUSY on a mount point, cross-device) does it fall back to `copyDir`. Then `runCSCLI` (`cscli hub update` + `cscli hub install <slug>`). +- Why it is wrong, confirmed by reading the code and by the behavior seen in a throwaway container (below): after the rename `/etc/crowdsec -> /app/data/crowdsec/config` dangles, so `cscli` cannot read `config.yaml` (observed error: `open /etc/crowdsec/config.yaml: no such file or directory`). The cscli path therefore fails, the code falls to `extractTarGz` into a freshly created `DataDir`, and on success the live dir holds only the extracted archive while config, hub, `bouncer_key`, LAPI db and `hub_cache` sit in the backup. CrowdSec is running while its db directory is moved. Rollback (`rollback`) does `RemoveAll(DataDir)` then renames the backup back, so it is also only valid for that directory and cannot undo anything `cscli` wrote outside it. +- Not confirmed: that a *real* hub-preset apply on a running container hits exactly this sequence (needs a hub preset pull + a running agent). The verification step is part of the gate in Commit 3. Whether the extract fallback ever "succeeds" with a usable tree is irrelevant; the rename itself is the defect. +- Fix direction (matches the issue): reuse the copy-based, symlink-preserving, db-excluding `backupCopy`/`restoreCopy` added for `ApplyCurated`. -- `models.CrowdsecPresetEvent` (`backend/internal/models/crowdsec_preset_event.go`): slug, action, status, cache_key, backup_path, error, timestamps. Failure rows already exist for the hub path (`Status:"failed"`, `Error`). The curated path never records failure. -- Existing test `TestApplyCuratedPresetSkipsHub` (`backend/internal/api/handlers/crowdsec_presets_handler_test.go:478`) asserts the buggy behavior (200 + `applied`, with `cscli`-less hub, empty temp dir). It encodes the bug and must be rewritten. +### 2.3 #1516 backup directories accumulate (CONFIRMED) -### 2.4 Why the frontend 501 fallback (Option B) cannot be trusted +Creators of `<DataDir>.backup.*`: `HubService.Apply` (rename, second-resolution timestamp), `ApplyCurated` (copy, microsecond timestamp), `CrowdsecHandler.ImportConfig` (`crowdsec_handler.go:696`, rename), `WriteFile` (rename). Nothing ever deletes any of them. grep for prune/retention in `backend/internal/crowdsec` returns nothing. Because the curated backup is a full copy of the persistent tree (minus the db) and the legacy ones are the whole tree, growth is unbounded on a volume the user sizes for a home server. -`applyPresetLocally` (lines 427-457) writes `presetPreview || selectedPreset.content` into whichever file is currently selected in the file editor (`selectedPath ?? files[0]`), via the generic `writeCrowdsecFile`. Consequences: +### 2.4 #1517 handler file size (CONFIRMED) -1. For curated presets the pulled `presetPreview` is the two-line comment stub above; writing it "applies" nothing. -2. `selectedPreset.content` in `crowdsecPresets.ts` is a pseudo-YAML `configs:\n collections: ...` document that is not a CrowdSec configuration schema. CrowdSec does not read such a file; it would at best be inert and at worst overwrite a real file (e.g. `config.yaml`) the operator had selected. -3. The content is client-supplied and untrusted from the server's point of view, which is the exact thing Aikido's remediation says to avoid. -4. Several referenced items (`crowdsecurity/geo-fencing`, `crowdsecurity/geo-bf`, the `geoip-enricher` as a collection) are not verified hub items (see 2.5). +`crowdsec_handler.go` is 2,906 lines. Preset handlers live at lines ~990-1325 (`ListPresets`, `PullPreset`, `ApplyPreset`, `applyCuratedPreset`, `curatedPresetPreview`, `recordPresetEvent`, `applyFailureBody`, `respondApplySuccess`, `GetCachedPreset` at ~1426). Pure move, same package. -So "501 and rely on the frontend" would only move the false success into the browser. +### 2.5 #1518 tolerant Playwright tests (CONFIRMED) -### 2.5 Content validity risk +`tests/security/crowdsec-config.spec.ts` "Preset Management" block (lines ~73-145) and "Configuration Files" block are `isVisible().catch(() => false)` guarded and annotate "feature may not be implemented"; they cannot fail on a regression. `tests/security/crowdsec-preset-apply.spec.ts` (from #1513) is the model: route-stubs `**/api/v1/admin/crowdsec/presets/apply` and the file-write route and asserts toasts and write counts. Missing deterministic coverage: hub (non-curated) preset apply success/failure surfaced, no client-side write on any apply failure, file editor save/failure, file-list error cases. -Verified against the live hub index (`hub-cdn.crowdsec.net/master/.index.json`, supervisor review): +### 2.6 #1523 hub data files missing after container recreation (ROOT CAUSE CONFIRMED by experiment) -- Exist: collections `crowdsecurity/sshd`, `crowdsecurity/caddy`; scenarios `crowdsecurity/http-backdoors-attempts`, `crowdsecurity/http-probing`, `crowdsecurity/ssh-bf`; parsers `crowdsecurity/sshd-logs`, `crowdsecurity/caddy-logs`, `crowdsecurity/geoip-enrich`, and `crowdsecurity/whitelists` (a PARSER, not a postoverflow; the existing frontend content mislabels it). -- Do NOT exist: `crowdsecurity/geo-fencing`, `crowdsecurity/geo-bf`, `crowdsecurity/geoip-enricher`. `geoip-enrich` exists only as a parser, is already pulled in by collections such as `crowdsecurity/linux`, and is enrichment only (it adds country/ASN metadata to events; it blocks nothing). +- Config: CrowdSec's default `config_paths.data_dir` is `/var/lib/crowdsec/data/` and `hub_dir` is `/etc/crowdsec/hub/` (upstream `config/config.yaml`). The entrypoint (`.docker/docker-entrypoint.sh:310-319`) redirects **only** `db_path` to the volume. `Dockerfile:1089` creates `/var/lib/crowdsec/data` inside the image, so on a recreated container it is an empty directory (confirmed: the issue shows only `trace/`). `/etc/crowdsec` is a symlink into the volume so the **hub items persist but their data files do not**. +- `configs/crowdsec/install_hub_items.sh` skips items for which `cscli <type> inspect <name>` exits 0 ("already installed, skipping"). +- Experiment (throwaway `charon:local` container run with `--entrypoint sh`, no volumes, no live host touched): + - Install `crowdsecurity/http-backdoors-attempts` -> `backdoors.txt` appears in `data_dir`. + - Wipe `data_dir` (simulating recreate), keep the hub: `cscli scenarios install <name>` -> "Nothing to install or remove", data dir stays empty. `cscli scenarios install --force <name>` -> same, still empty. **`cscli hub upgrade` repopulates `backdoors.txt`, and so does `cscli hub upgrade --force`.** The issue's suggestion "`install --force` or `hub upgrade`" is therefore half right: only `hub upgrade` works. +- **Additional defect found (CONFIRMED, not in the issue):** on a hub with an index but nothing installed, `cscli scenarios inspect crowdsecurity/http-backdoors-attempts` exits **0** and prints `installed:false` metadata. `install_if_missing` uses the exit code, so on a fresh volume it may print "already installed, skipping" and install nothing. (In the experiment inspect exited 1 when the item was installed but its data file was missing, so the exit code is not a reliable installed/absent signal in either direction.) #1513's `ApplyCurated` already uses the correct signal: `inspect -o json` -> top-level `installed`. +- Not confirmed: whether the missing-install path is masked in real startups by something else (e.g. the collections installing their children). Needs the recreate test in Commit 6 on a clean volume. -So today's `geolocation-aware` content is entirely fictional, and its description ("tighten access by region") is not achievable with hub items. Task B0 re-checks type AND existence of every item at implementation time (the hub changes), but the facts above are the baseline. +### 2.7 #1524 empty version (ROOT CAUSE CONFIRMED; the fix has a gotcha) -## 3. Options evaluated +- Confirmed: the injected symbol is wrong. In CrowdSec v1.8.1, `pkg/cwversion/version.go` declares only `Codename` and `Libre2` (and no `Version`); the version comes from `github.com/crowdsecurity/go-cs-lib/version` (`Version`, `BuildDate`, `Tag`, `System`), pinned at `go-cs-lib v0.0.25` by CrowdSec v1.8.1's `go.mod`. Upstream's own Makefile uses `-X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)'` (plus `BuildDate`, `Tag`, and `cwversion.Codename`). `-X` on a nonexistent symbol is silently ignored, so both `Dockerfile:915` and `:923` are no-ops. Observed in the built image: `cscli version` prints empty `version:`, `Codename:`, `BuildDate:`, and `cscli` logs `Crowdsec version is not set, using hub branch 'master'`. +- The existing app-stage sanity check (`Dockerfile` ~L1054) says "CrowdSec 1.8.x prints an empty `version:` field here regardless of the -X ldflag". That comment is wrong and hides the bug; it must be corrected. +- **Gotcha, verified in `cmd/crowdsec-cli/core/require/branch.go` (v1.8.1):** with a real version cscli calls `https://version.crowdsec.net/latest` on every hub command and returns a hard error if it cannot reach it; if the running version equals latest it uses `master` (identical to today), if it is older than latest it uses the branch named after the version. `https://hub-cdn.crowdsec.net/v1.8.1/.index.json` returns **404** today (v1.8.0, v1.7.8 and master return 200). So a correct version could (a) make `cscli hub *` fail offline/air-gapped and (b) break hub updates once 1.8.1 is no longer the latest and the CDN has no `v1.8.1` branch. The issue's "hub items may be newer than the engine supports" claim is also overstated: while 1.8.1 is the latest release the behavior is already `master`. The fix therefore needs an explicit hub-branch decision (see D1) rather than a bare `-X` correction. +- The toolchain delivery path: the default Dockerfile build consumes the prebuilt `ghcr.io/wikid82/charon-toolchain` image (content-hash tag `CHARON_TOOLCHAIN_TAG`, `Dockerfile:22`). Editing the `crowdsec-inline` stage changes the key, `.github/workflows/toolchain-image.yml` builds the new image and its bot commits the new TAG/DIGEST onto the PR head branch (workflow lines ~365-420) so `verify-toolchain-pin` goes green. Until that happens the app image still contains the old binary, so the app-side assertion cannot be added in the same commit as the ldflags change. -| | Option A: server-side authoritative apply | Option B: return 501, rely on frontend fallback | -| --- | --- | --- | -| Source of truth | Server-side curated definition (list of hub items by type+name), compiled into the binary | Client-supplied content | -| Writes/activates | Backup, `cscli <type> install <name>` per item (validated item names), reload, verify via `cscli ... list` | Writes a pseudo-YAML or a comment stub into a user-selected file | -| Verifies | Yes (post-install list check) | No | -| Audit trail | Server records applied/failed truthfully | Server records nothing; local write is unaudited | -| Meets Aikido remediation | Yes, fully | Only if the fallback is "guaranteed to apply the intended content", which 2.4 shows it is not | -| Cost | Medium (new definition type, installer, tests) | Small, but leaves the product broken | +### 2.8 Reproduction/evidence commands run for this plan (all foreground, repo-local or throwaway container) -Recommendation: Option A (long-term fix). Option B is rejected for the reasons in 2.4. Following the "RequiresHub" semantics, curated presets become "Charon-defined bundles of hub items": they need `cscli` (and hub connectivity for `cscli hub update`), so the honest model is that they are applied through the same hub mechanism with a server-owned definition. When the prerequisites are missing the endpoint fails with a clear error and a recorded `failed` event rather than pretending. +`gh issue view` for each issue, `gh pr view 1513`; code reads listed above; `curl` of upstream v1.8.1 source files and `hub-cdn.crowdsec.net/<branch>/.index.json`; shallow clone of upstream v1.8.1 into the session scratchpad; throwaway `docker run --rm --entrypoint sh charon:local` experiments (data-dir wipe, inspect exit codes, `hub upgrade`). Nothing on live hosts or volumes was touched. -## 4. Technical Specifications +## 3. Technical Specifications -### 4.1 Server-side curated definitions (`backend/internal/crowdsec/presets.go`) +### 3.1 Shared backup helper, Apply flow, ImportConfig (#1515, #1516) -Extend, do not replace, the existing types (keep `ListCuratedPresets`/`FindPreset` signatures and the JSON of `Preset` unchanged for `ListPresets` consumers). +New file `backend/internal/crowdsec/backup.go` (move `backupCopy`, `restoreCopy`, `isLiveDBFile`, `emptyDirExcept` out of `curated_apply.go`; the move itself changes no behavior): -- New type `PresetItem struct { Type string; Name string }` where `Type` is one of the allowlisted cscli hub types `collections|parsers|scenarios|postoverflows` and `Name` matches `^[a-z0-9][a-z0-9_.-]*/[a-z0-9][a-z0-9_.-]*$` (hub `author/name`). -- Add unexported/json-excluded field `Items []PresetItem` to `Preset` (`json:"-"` so the list payload is unchanged) populated in `curatedPresets` for each `RequiresHub:false` preset. Add `func (p Preset) Validate() error` (non-empty items, allowlisted types, name regex); `init`-time or test-time enforced via a unit test over `curatedPresets`. -- `RequiresHub:false` is retained as the "defined by Charon, not fetched from the hub index" marker. Doc comment must say so. -- Content (re-verified in Task B0, type and existence): - - `honeypot-friendly-defaults`: collections `crowdsecurity/sshd`, `crowdsecurity/caddy`; scenarios `crowdsecurity/http-backdoors-attempts`, `crowdsecurity/http-probing`; parser `crowdsecurity/whitelists` (type `parsers`, NOT `postoverflows`). Drop items already pulled in by the collections (B0 checks the collection contents with `cscli collections inspect`). `ssh-bf`, `sshd-logs`, `caddy-logs` are covered by the collections and need not be listed explicitly unless B0 shows they are not. - - The preset `Summary` (backend) and `description`/`warning` (frontend) must describe exactly what ends up installed (SSH and Caddy log parsing plus brute-force/probing detection, whitelists parser). B0 either trims the item list or rewrites the text so they match; the current "low-noise tuned for tarpits" claim is unsupported by plain hub installs and must be removed or justified. -- **Decision D1 (geolocation-aware): DECIDED by the user, option (a).** The preset is renamed to an honest GeoIP Enrichment preset; the removal path and its follow-up issue are dropped. - - Slug changes: `geolocation-aware` -> `geoip-enrichment` (the old slug promises region-based protection the preset cannot give, and keeping it would keep the misleading name in URLs, audit rows and logs). Title "GeoIP Enrichment", tags `geo`, `enrichment` (`access-control` dropped), `RequiresHub:false`, `Source:"charon-curated"`. - - Items: exactly one, parser `crowdsecurity/geoip-enrich` (type `parsers`). B0 re-verifies type and existence. - - Backend `Summary`: "Enriches CrowdSec log events with GeoIP data (country and ASN). It does not block or allow traffic by region." - - Frontend `description`: "Enriches CrowdSec log events with GeoIP data (country and ASN). Useful for decision context and dashboards." Frontend `warning`: "Enrichment only: this does not block traffic by region. Use access lists for region rules." Both must match the backend summary semantics (no mention of tightening access, geo-fencing, or requiring a GeoIP database). - - Existing data: the old slug only ever appears in `CrowdsecPresetEvent` rows (audit trail of the false "applied" no-op). Those rows are left untouched (historical, they are not a config state; no migration, no rewrite; rewriting audit history would be wrong). Stored UI selections: the preset selection is component state in `CrowdSecConfig.tsx`, not persisted; B0 greps `localStorage`/`sessionStorage` use in that page and the Playwright specs to confirm. A client still posting the old slug gets the unknown-slug path (not a curated preset, so it goes to the hub path and fails with the existing cache-miss error, never a false success); no alias is added. - - Everything that references the old slug is updated: `presets.go`, `presets_test.go` (the `"another preset", "geolocation-aware"` case ~line 49), `crowdsec_presets_handler_test.go`, `frontend/src/data/crowdsecPresets.ts`, `frontend/src/data/__tests__/crowdsecPresets.test.ts` (lines ~11, 57, 134), the i18n files in all five locales if the title or description is localized there (B0 greps `frontend/src/locales/*/translation.json` for `geolocation`/`Geolocation`), and docs mentioning it. - - Caveat to state in docs: `geoip-enrich` is already pulled in by collections such as `crowdsecurity/linux`, so applying this preset may be a no-op install that still verifies as installed; that is a truthful success. +- **Two namespaces, two caps, so they cannot evict each other (blocking item 3).** + - Full-tree snapshots: backup dirs next to `DataDir`: `<DataDir>.backup.<ts>`, `const DefaultBackupRetention = 5`. + - Single-file edit backups (from `WriteFile`, section 3.2): backup dirs next to `DataDir`: `<DataDir>.filebackup.<ts>/<rel>`, `const DefaultFileBackupRetention = 10`. The `<base>.backup.*` glob never matches `<base>.filebackup.*`. +- **Snapshot exclusions (blocking item 4).** `snapshot` copies `DataDir` (symlinks preserved) but **excludes**: the live `crowdsec.db`, `-wal`, `-shm` (anywhere), and the top-level `data/` directory (LAPI db plus hub data files and the GeoLite2 mmdb, which after #1523 live there; hub data files are recoverable with the gated `cscli hub upgrade`, the db is owned by the running engine) and the top-level `hub_cache/` (regenerable download cache). `restore` empties `DataDir` except those same excluded entries and copies the snapshot back, so it never touches `data/` or `hub_cache/`. Only the *top-level* `data/` and `hub_cache/` are excluded; a nested directory with the same name inside `config/` is still backed up. Result: a snapshot is essentially `config/` plus `bouncer_key` (small), x5 retained. +- `func (s *HubService) snapshot() (path string, err error)`: microsecond timestamp name, created `0o700` with exclusive `os.Mkdir`. +- `func (s *HubService) restore(path string) error`. +- `func PruneBackups(dataDir, kind string, keep int) (removed []string, err error)` where `kind` is `backup` or `filebackup`. Lists siblings matching `<base>.<kind>.*`, `Lstat`-checks each is a real directory (never follows or deletes symlinks or plain files), **parses the timestamp from the directory name** (both the legacy `20060102-150405` and the microsecond `20060102-150405.000000` layouts; names that do not parse are ignored, never deleted), sorts newest first, deletes everything past `keep`. `keep < 1` is clamped to 1 so the newest is always kept. Never touches `dataDir`. Single-removal errors are logged and aggregated, never failing the caller. +- **Prune runs after every attempt, successful or failed**, always preserving the newest `keep` snapshots. A failed apply's own snapshot is the newest, so it is always retained for manual recovery. -### 4.2 Applier (`backend/internal/crowdsec/hub_sync.go` or new `curated_apply.go` in the same package) +**`HubService.Apply` new control flow (blocking item 2).** Hold `HubService.mu` and the handler-level lock (section 6). +1. Validate slug; load cache meta (non-fatal). +2. `snapshot()`; any error -> return failure, nothing changed. `BackupPath` is set only after success. The old "read the archive into memory before the backup because the cache lives in `DataDir`" workaround is **removed**: nothing is moved any more, and `hub_cache/` is excluded from snapshots and from restore, so the archive stays valid for the whole call. +3. If `cscli` is available: `cscli hub update` (non-fatal), `cscli hub install <slug>`. Success -> status `applied`, `UsedCSCLI=true`, reload, prune, return. +4. **A cscli failure still falls back to the cached-archive extract** (this keeps the offline/hub-unreachable behavior that exists today). Before extracting, `restore(snapshot)` so the extract starts from a clean pre-apply tree and any partial files `cscli` wrote are removed (this closes the partial-overlay risk of overlaying an archive on a half-installed state). +5. Extract `extractTarGz` onto `DataDir` (cache refresh from the hub if the cache is missing, as today). **Residual risk, documented:** extraction overlays archive files on top of the live tree rather than replacing it, so files in the archive replace same-named live files; mitigation is the snapshot. Any extract/refresh failure -> `restore(snapshot)` and return the error with `BackupPath`. +6. Success -> status `applied`, `UsedCSCLI=false`, reload, prune. +`backupExisting`, `rollback`, `emptyDir`, and `copyDir` (if now unused) are deleted. -New method on `HubService` (reuses `Exec`, `DataDir`, `ApplyTimeout`, `copyDir`, `emptyDir`, `sanitizeSlug`, logger): +**`CrowdsecHandler.ImportConfig` (blocking item 6): converted to snapshot/restore semantics, never renaming `DataDir`.** Flow: handler lock -> `snapshot()` -> empty `DataDir` except the snapshot-excluded entries (so the live db, `data/`, `hub_cache/` survive the import) -> extract the uploaded archive, **skipping any archive entry that is a live db file (`isLiveDBFile`) or lies under the top-level `data/` or `hub_cache/`** so an upload can never overwrite engine-owned state -> on any failure `restore(snapshot)` and return the error -> prune. Net behavior difference: the live db and `data/` are no longer moved or lost by an import; everything else is replaced exactly as before. Existing `ImportConfig` tests are updated, plus the new tests listed in Commit 3. -`func (s *HubService) ApplyCurated(ctx context.Context, preset Preset) (ApplyResult, error)` +### 3.2 File endpoints hardening (#1514) -**Backup/rollback must NOT reuse `backupExisting`/`rollback`.** Those rename the whole `DataDir` away (it holds `config.yaml` and the live CrowdSec config), which would leave CrowdSec without a config while `cscli` runs and makes `cscli` write into a vanished directory. `ApplyCurated` uses a copy-based backup that leaves `DataDir` in place and intact at all times: +Details of the validation defects and their fixes are intentionally not written here (see section 4). Public-safe design: -- Backup: `copyDir(DataDir, DataDir+".backup."+ts)` (existing helper, 0700 backup dir). `DataDir` is never renamed, removed, or emptied before success is known. Backup failure -> remove partial backup, return error, nothing installed. -- Where `cscli` writes: with the Charon-managed layout `cscli` is pointed at `DataDir` as its config dir (`config.yaml` `config_paths`: `config_dir`, `data_dir`, `hub_dir`, i.e. `DataDir/hub`, `DataDir/collections|parsers|scenarios|postoverflows` symlinks, and `DataDir/hub/.index.json`). Task B0 confirms the actual `config_paths` for the shipped image and the implementer MUST ensure the backup covers every directory `cscli` mutates; if `hub_dir`/`data_dir` resolve outside `DataDir`, those are added to the backup set (copy each) and to rollback. Paths are `filepath.Clean`ed. -- Symlinks: the existing `copyDir` (`hub_sync.go`, uses `os.ReadDir` + `entry.IsDir()` + `copyFile`) does NOT preserve symlinks. A symlink entry reports `IsDir()==false`, so it goes to `copyFile`, which `os.Open`s the target: a symlink to a file is silently turned into a regular file copy, and a symlink to a directory (the `collections/`, `parsers/`, `scenarios/`, `postoverflows/` item entries are symlinks into the hub dir) makes `io.Copy` fail with "is a directory", so the backup would fail or restore the wrong structure. Task B0 confirms this against the shipped layout. The curated path therefore MUST NOT use `copyDir` as is: add a symlink-preserving variant (check `entry.Type()&fs.ModeSymlink`, `os.Readlink`, `os.Symlink` at the destination; never follow links; reject/skip links whose resolved target escapes the backup root only when copying, preserving the literal link target). Prefer fixing `copyDir` itself for all callers (DRY) if the existing `copyDir` tests and the hub-path fallback stay green; otherwise add `copyDirPreserveLinks` and use it only for the curated path. -- Rollback note: rollback (`emptyDir` then `copyDir`) is briefly non-atomic against a running CrowdSec (it may see a missing or partial hub dir for a moment); accepted because the backup copy is retained. -- Rollback on failure: restore from the copy into the existing `DataDir` (`emptyDir` of the affected mutation dirs, then `copyDir` back); `DataDir` itself is never removed, so it exists and is intact during and after the failed install. The backup copy is kept (its path is returned to the user, as in the hub path) so an operator can recover manually if rollback itself fails; a rollback failure is logged at Error and appended to the returned error. -- Timeout: `applyCtx, cancel := context.WithTimeout(ctx, s.ApplyTimeout)` exactly like `Apply`; every `Exec.Execute` receives `applyCtx`; an expired ctx is checked before each step and triggers rollback. +- New file `backend/internal/api/handlers/crowdsec_files.go` with a single shared path-resolution helper used by `ReadFile`, `WriteFile` and `ListFiles`, so all three apply one policy ("harden file-endpoint path validation"). +- Write policy: bounded body size (1 MiB); allowed file types (Q3); YAML files must parse; protected files and trees (secrets, databases, caches, engine data) are never writable; `.yaml/.yml` parsing uses `gopkg.in/yaml.v3` if already a dependency (verify). +- **Symlink policy (blocking item 7):** the editor never writes through or replaces a symlink (HTTP 400), so symlinked hub items are read-only from the editor. Reads stay confined to `DataDir`, and symlinked hub items that resolve inside it stay readable. Test in Commit 4. +- **Two predicates, not one.** The viewer today lists and reads any file under `DataDir` (hub items, `config.yaml`, etc.), and filtering the list by the write predicate would hide files users can currently read. So: `isReadable(rel)` governs `ListFiles` and `ReadFile` (secrets and databases such as `bouncer_key`, `*.db*`, and the `hub_cache/` and `data/` trees are neither listed nor readable; everything else stays visible, including symlinked hub items), and the stricter `isWritable(rel)` governs `WriteFile`. A file that is readable but not writable (symlinked hub items, non-allowlisted types) stays viewable and its save returns the server's 400 message, which the existing `writeMutation` error path already shows; no frontend contract change. `ExportConfig` is unaffected. Commit 4 tests: `ReadFile` refuses `bouncer_key` and db files, reads a symlinked hub item that resolves inside `DataDir`, and refuses a symlink resolving outside it; `ListFiles` omits them but still lists a symlinked hub item and `config.yaml`; every file `WriteFile` accepts is also listed. +- `WriteFile` algorithm: lock -> validate -> if the target exists, copy just that file to `<DataDir>.filebackup.<ts>/<rel>` -> atomic write (temp file in the same directory, `fsync`, `rename`, preserve mode, default `0o600`) -> `PruneBackups(kind=filebackup, 10)` -> 200 `{status:"written", backup}`. `DataDir` is never renamed. +- Error contract (JSON `{"error": ...}`): 400 invalid request / path / type / YAML / symlink, 413 too large, 500 write failure. +- No new endpoints, no schema changes, no frontend contract changes. -Behavior (ordered, fail-closed): +### 3.3 Frontend (#1514) -1. `preset.Validate()` runs inside `ApplyCurated` on the input it receives (not only on the static catalog at test time); failure returns `ErrInvalidPresetDefinition` (500, nothing touched). -2. Require `s.hasCSCLI(applyCtx)`; otherwise return `ErrCSCLIUnavailable` (new sentinel) without touching disk. Result `Status:"failed"`. -3. Acquire the `HubService` mutex, create the copy-based backup; set `result.BackupPath` only after success. -4. `cscli hub update` (non-fatal warning on failure), then for each item `cscli <type> install <name>` using argv (never a shell; type from allowlist, name by regex). Stop at first failure. -5. Verify each item with `cscli <type> inspect <name> -o json`, exact argv `["cscli", "<type>", "inspect", "<name>", "-o", "json"]` (e.g. `cscli collections inspect crowdsecurity/sshd -o json`). Parse the JSON and require `installed == true` (and, if present, `up_to_date`/`tainted` do not make it fail unless `tainted==true`). Non-JSON output, nonzero exit, missing field, or `installed:false` is a failure. Unit-test fixtures (B0 confirms the field names against the shipped cscli version; the implementer replaces the fixture if they differ): - - installed: `{"name":"crowdsecurity/sshd","type":"collections","installed":true,"up_to_date":true,"tainted":false,"local":false,"version":"0.2"}` - - not installed: `{"name":"crowdsecurity/sshd","type":"collections","installed":false}` - - malformed: `not json` -6. On any failure after step 3: rollback as above, return a wrapped error (`fmt.Errorf("install %s %s: %w", ...)`), `Status:"failed"`, `ErrorMessage` set. -7. On success: `Status:"applied"`, `UsedCSCLI:true`, `ReloadHint:true`, `CacheKey:"curated-"+slug`, `AppliedPreset:slug`. +`frontend/src/pages/CrowdSecConfig.tsx`: delete `applyPresetLocally`, the `501` branch in `handleApplyPreset`, the `applied-locally` apply-info status, and anything left unused (`backupMutation` if only used there, imports, `presetPreview` write path). Non-2xx apply for hub presets is surfaced like curated: server error text, backup path if present, no write. The generic file editor (`writeMutation`) is unchanged but now shows server 400/413 messages. Update Vitest files: `CrowdSecConfig.test.tsx`, `.coverage.test.tsx`, `.spec.tsx`, `.curatedApply.test.tsx` (remove tests that assert the local fallback; add "501 for a hub preset shows an error and performs no file write"). -Reload: CrowdSec needs a reload to load new items. Use the existing reload convention in the handler (`cscli hub reload` is already used after whitelist mutations at `crowdsec_handler.go:2744` as non-fatal). Decision D2: keep `reload_hint:true` (operator/UI triggers restart through existing start/stop controls) and ALSO attempt `cscli hub reload` best-effort; do not report "reloaded"; the contract only claims "installed and verified". If implementer finds that a reliable programmatic reload exists in `CrowdsecExecutor`, calling it is allowed but optional and must not change the success criterion. +### 3.4 Persistent CrowdSec data dir and hub data (#1523) -Concurrency: guard `ApplyCurated` and `Apply` with a single `sync.Mutex` on `HubService` so two applies cannot interleave backup/rollback. +`.docker/docker-entrypoint.sh` (next to the existing `db_path` redirect, ~L310-320): -### 4.3 Handler (`crowdsec_handler.go`) +- `sed` `data_dir: /var/lib/crowdsec/data/` -> `data_dir: ${CS_DATA_DIR}/` in `$CS_CONFIG_DIR/config.yaml` (idempotent; works for both fresh and already-initialized volumes), with the same verify/echo pattern as the db redirect. `CS_DATA_DIR` is `/app/data/crowdsec/data` and is already `mkdir`ed and chowned. +- **Gated, never unconditional (blocking item 5).** After `install_hub_items.sh`, the entrypoint runs `cscli hub upgrade` **only if** at least one installed item's data files are missing. Detection: for each installed item (`cscli <type> list -o json`), check that every file the item declares is present under `data_dir`; the exact JSON field for declared data files is confirmed by the implementer against `cscli <type> inspect <name> -o json` before coding (open implementation detail, verified in the Commit 6 gate). Fallback if the field is not exposed: treat an empty `data_dir` (ignoring `trace/` and the db) with at least one installed item as "missing". The call is bounded (`timeout 120s`), failure is a logged warning and never fatal, and it never uses `--force`. A normal restart with intact data files performs no upgrade, so items are never silently upgraded on a healthy start. Note `hub upgrade` upgrades every installed item; this is acceptable only in the recovery case it is gated on, and the log line must say so. +- It is skipped in the same condition as the rest of hub init (`CHARON_SECURITY_TESTS_ENABLED=false`). -- `PullPreset`: leave the curated short-circuit but make the preview truthful: return a human-readable list of the items that will be installed (generated from `preset.Items`), `source:"charon-curated"`. Keeps `cache_key:"curated-<slug>"` so the frontend flow is unchanged. -- `ApplyPreset`: replace the no-op branch with `res, err := h.Hub.ApplyCurated(ctx, preset)` and share the existing success/failure tail used by the hub path. Extract the duplicated event-recording and response-building code into two small helpers (`recordPresetEvent(slug, status, res, err)` and `respondApplySuccess`/`respondApplyFailure`), per the DRY rule (this is the second occurrence). Event-recording errors from `DB.Create` are logged at Warn instead of silently discarded (`_ =`), for both paths. -- Failure event: `CrowdsecPresetEvent{Slug, Action:"apply", Status:"failed", CacheKey:"curated-"+slug, BackupPath, Error: err.Error()}`. -- Status mapping for the curated path (order matters): first `errors.Is(err, crowdsec.ErrCSCLIUnavailable)` -> 503 `{"error":"CrowdSec CLI is not available; curated presets require cscli"}` (handled BEFORE the hub path's `strings.Contains(errorMsg, "cscli unavailable")` rewriting so that message guidance is not mangled); then `mapCrowdsecStatus(err, http.StatusInternalServerError)` (deadline/cancel -> 504, otherwise 500). `ErrInvalidPresetDefinition` -> 500. All failure responses include `backup` when a backup exists (rollback already restored it; keep the existing field semantic) and `cache_key`. -- Never leak raw command output or absolute paths beyond what the hub path already returns; error strings are wrapped Go errors (no stdout dumps). +`configs/crowdsec/install_hub_items.sh`: change `install_if_missing` to decide on `cscli <type> inspect <name> -o json` reporting `"installed": true`, matching `ApplyCurated.verifyItem`. **Verified:** `jq` is not in the `charon:local` image, so use `grep -q '"installed": *true'` (not jq). **Verified:** `inspect -o json` still prints full JSON and exits 0 when an item is installed but its data file is missing, so the JSON signal is reliable in exactly the recreate scenario (the plain-text `inspect` exit code was not). Keep the "failed to install" warnings non-fatal. -### 4.4 API contract changes +`Dockerfile`: keep creating `/var/lib/crowdsec/data` (a stale `data_dir` on an un-migrated config must still exist); no other change. +`.docker/compose/docker-compose.playwright-ci.yml:137` currently mounts a volume at `/var/lib/crowdsec/data`; repoint or remove it so E2E exercises the same persistent path as production. `docs/security.md` hardened-deployment text that tmpfs-mounts `/var/lib/crowdsec` stays valid (the db and data now live under `/app/data/crowdsec`) but its description must be updated. +`scripts/diagnose-crowdsec.sh` default `DATA_DIR` updated to the new location. -| Case | Before | After | -| --- | --- | --- | -| Curated preset, cscli present, all items installed and verified | 200 `applied` (nothing done) | 200 `{status:"applied", backup:"<path>", reload_hint:true, used_cscli:true, cache_key:"curated-<slug>", slug}` | -| Curated preset, cscli missing | 200 `applied` | 503 `{error, cache_key}`; event `failed` | -| Install/verify failure | 200 `applied` | 500 `{error, backup?, cache_key}`; rolled back; event `failed` | -| Timeout | n/a | 504 | -| Unknown slug / hub preset path | unchanged | unchanged | +Optional passive health signal (Q4 resolved: log line only, no UI): log a startup summary line listing installed-but-missing data files. No banner or badge (Passive UI, no nudges). -`ApplyCrowdsecPresetResponse` (frontend) is structurally unchanged. No 501 is returned by the server for curated presets. No DB schema change; `AutoMigrate` untouched. +### 3.5 Version injection (#1524) -### 4.5 Frontend +`Dockerfile` `crowdsec-inline` stage (both `xx-go build` commands, ~L915 and ~L923): replace the ldflags with `-s -w -X github.com/crowdsecurity/go-cs-lib/version.Version=v${CROWDSEC_VERSION} -X github.com/crowdsecurity/go-cs-lib/version.BuildDate=<reproducible date or empty>`. Do not set `Tag` (empty `Tag` leaves `version.String()` equal to `Version`). Do not add `BuildDate` from the wall clock: that would break the content-addressed, reproducible toolchain key; prefer the release date derived from the pinned tag or omit it. -- `CrowdSecConfig.tsx` `handleApplyPreset`: keep 501 fallback only for hub presets, never for slugs with `source === 'charon-curated'` (the server is now authoritative; for curated presets a 501 would be a contract violation and shows an error). Success toast text depends on response: when `res.status === 'applied'` show applied (+ reload note); any other status shows an error toast, never success. Add handling for 503 curated (cscli missing) that is distinct from the "hub unavailable" message: show the server error text. -- Frontend `warning`/`description` text for every remaining curated entry is aligned with what the backend actually installs (B0 output) and with the Decision D1 outcome (4.1); the current `geolocation-aware` entry is renamed to `geoip-enrichment` with the D1 texts, and `honeypot-friendly-defaults` low-noise claims are removed or corrected. -- `applyPresetLocally`: stop using pseudo-YAML `crowdsecPresets.ts` content for curated slugs. Delete the now-dead pseudo-YAML `content` of the curated entries (`honeypot-friendly-defaults`, `geoip-enrichment`; entries only stay if the UI still needs description/warning text from this file, otherwise the server list is the sole source) from `frontend/src/data/crowdsecPresets.ts` if nothing else needs them; the server list is the single source. Check `CROWDSEC_PRESETS` usages (`CrowdSecConfig.tsx:160`, `data/__tests__/crowdsecPresets.test.ts`, `frontend/src/data/securityPresets.ts` is unrelated) and remove dead code per CLAUDE.md CLEAN. If `bot-mitigation-essentials` is the only remaining entry and it is a hub preset served by the backend, the file may be deleted entirely along with its test. -- i18n: only if new strings are added; update `frontend/src/locales/*/translation.json` for all five locales (en, de, es, fr, zh). +App-side assertion (second commit, after the toolchain bot has pushed the new TAG/DIGEST to the PR branch): in the existing N5 `RUN` block (~L1054) replace the wrong comment and add `grep -q "^version: v${CROWDSEC_VERSION}\$"` against `cscli version`. The final stage must redeclare `ARG CROWDSEC_VERSION` (currently declared only globally at L41 and in the caddy/crowdsec-inline stages; a global ARG is not visible inside a later stage without redeclaring). The build fails if the symbol ever silently no-ops again. -### 4.6 Error handling and edge cases +Hub branch (decision D1, recommended): set `cscli.hub_branch` explicitly in the entrypoint-managed `config.yaml` so behavior is deterministic and does not depend on `version.crowdsec.net` reachability or on `hub-cdn` having a `v1.8.1` branch. Verified in upstream v1.8.1 `chooseBranch`: a non-empty `cscli.hub_branch` is returned immediately, before the `version.crowdsec.net/latest` lookup, so setting it removes both the network dependency and the CDN-404 risk. The online/offline test result is recorded in the PR description (Commit 8 gate). -- cscli present but hub unreachable: `cscli hub update` warns; install of already-present items may still succeed; verification decides. -- Item already installed: `cscli install` is idempotent (returns 0 or "already installed"); treat as success iff verification passes. -- Partial install then failure: the copy-based rollback (4.2) restores every directory `cscli` mutates (config dir, hub dir, data dir as resolved in B0) while `DataDir` stays in place. If rollback itself fails, the error says so, the backup path is returned, and the failure event is still recorded. -- Cerberus disabled: unchanged 404. `Hub == nil`: unchanged 503. -- Double-click / concurrent applies: mutex (4.2); frontend already disables the button via `isApplyingPreset`. -- Backup accumulation: unchanged behavior (same as hub path); see F3. +### 3.6 Handler split (#1517) -## 5. Priorities & Ordering (CLAUDE.md "Findings Triage & Issue Tracking") +Move to `backend/internal/api/handlers/crowdsec_preset_handler.go` (same package, no behavior change, no export changes): `ListPresets`, `PullPreset`, `ApplyPreset`, `applyCuratedPreset`, `curatedPresetPreview`, `recordPresetEvent`, `applyFailureBody`, `respondApplySuccess`, `GetCachedPreset`, and preset-only helpers. Move the matching unit tests into `crowdsec_preset_handler_test.go` files where they are clearly preset-only (leave shared fixtures where they are). `RegisterRoutes` stays in `crowdsec_handler.go`. The new file must be < 700 lines. -Source: Aikido AI-pentest finding (external scanner, not a user-submitted GitHub issue); user-confirmed true positive, severity low. Rules applied: our own finding -> bug fix before any `feat`; nothing here is critical/high, so no emergency preemption and no separate-PR carve-out. +## 4. Priorities & Ordering (CLAUDE.md "Findings Triage") -| Rank | Item | Kind | Severity | Decision | -| --- | --- | --- | --- | --- | -| 1 | Curated presets no-op / false success (this PR) | `fix:` | low | Do now, single PR off `development` | -| 2 | F1: `applyPresetLocally` writes unvalidated client content into an arbitrary selected CrowdSec file | `fix:` | low-medium | Narrowed in this PR for curated slugs (4.5). Residual hub-preset fallback behavior: file as issue, schedule after this PR | -| 3 | F2: the HUB path (`HubService.Apply`) still uses `backupExisting`, which renames the entire `DataDir` (including `config.yaml` and `hub_cache`) while CrowdSec runs, and its rollback only covers `DataDir`. The CURATED path is fixed in this PR (copy-based backup, 4.2) and is no longer deferred; only the hub path remains | `fix:` | low-medium | File as issue for the hub path (to verify), after this PR; reuse the copy-based helper | -| 4 | F3: unbounded `DataDir.backup.*` accumulation from repeated applies | `chore:` | low | File as issue | -| 5 | F4: `crowdsec_handler.go` is ~2800 lines; preset handlers should move to `crowdsec_presets_handler.go` | `refactor:` | low | File as issue; do not widen this PR | -| 6 | F5: Playwright `tests/security/crowdsec-config.spec.ts` preset tests are tolerant "may not be implemented" skips that would not have caught this bug | `test:` | low | Partially addressed here (new spec, section 6.3); the old tolerance cleanup is a follow-up issue | +None of the seven issues is user-submitted (all filed by the maintainer from PR #1513). Rules applied: bug fixes before new implementation (there is no `feat` here); critical/high ships as its own PR first; medium/low is scheduled after. -Per CLAUDE.md these out-of-scope findings (F2 to F5, and the residual of F1) are filed automatically as GitHub issues by the orchestrator, none security-sensitive (no private advisory needed). The GitHub MCP server failed to connect in the planning session, so filing must use `gh issue create`. +Severity after root-causing (some differ from the issue labels): -## 6. Test Plan +| Rank | Issue | Severity | Why | +|------|-------|----------|-----| +| 1 | #1523 data files missing after recreate | **medium** (issue says medium) | Silent degraded detection and no GeoIP on every recreated container; install guard defect makes fresh volumes suspect. Highest user impact. | +| 2 | #1514 + `WriteFile` destructive rename | **medium** (raised from low-medium) | Admin action can wipe the live CrowdSec tree; weak input validation. Security-sensitive (below). | +| 3 | #1515 `HubService.Apply` rename | **low-medium, confirmed** | Same defect class and same helper as #1514/#1516; fixed together. | +| 4 | #1524 empty version / hub branch | **low-medium** | Fix is easy but changes runtime hub behavior (offline failure, CDN 404 risk); needs the D1 decision and the toolchain bot. Scheduled after the fixes it must not destabilize. | +| 5 | #1516 backup accumulation | low | Piggybacks on the shared helper. | +| 6 | #1518 tolerant E2E | low | Test hardening; first in commit order (as `fixme` specs) and enabled last. | +| 7 | #1517 handler split | low | `refactor`, no behavior change; sequenced first in the *commit* order (mechanical, makes later diffs reviewable) but lowest in *priority*. | -### 6.1 Backend unit tests (TDD, written first) +No finding is critical or high, so none must ship as its own PR ahead of this work. Priority above governs attention and what to cut first under contingency; commit order in section 5 is dependency-driven. -`backend/internal/crowdsec` (new `curated_apply_test.go`, reuse the fake `CommandExecutor` pattern in `hub_pull_apply_test.go`): +**Security handling (#1514).** It is admin-only and already public as an issue. Assessment: low severity (authenticated admin only; the endpoint grants nothing beyond what an admin already has). **Q2 resolved by the user: #1514 is already public, so no private advisory is opened unless root-causing shows real impact beyond what an admin already has; it does not, so the default is NO advisory.** Handling: +- The existing public issue #1514 stays as is. Keep PR text, commit bodies and code comments free of exploit specifics anyway (no value in handing detail to un-upgraded installs). +- Exactly one commit in the PR carries the `(security)` scope, and its subject must stay in the vague category-only form: `fix(security): harden CrowdSec configuration file handling`. Keep the subject category-only, and Commit 4's body and the PR description stay equally vague (for example "tighten validation in the CrowdSec configuration file endpoints"). Every commit subject in a feature PR merge-commit appears verbatim in the public changelog, so the other commits use plain scopes (`fix:`, `refactor:`, `test:`, `docs:`, `chore:`) and describe user-visible behavior only. The destructive-rename data-loss fix is a plain `fix:` (it is a reliability bug, not an exploit). -- Success: fake exec records `cscli version`, `hub update`, one `install` per item, one `inspect <type> <name> -o json` verification per item (exact argv per 4.2 step 5); asserts argv exactly (no shell), `Status:"applied"`, `UsedCSCLI`, backup path created, `ReloadHint`. -- cscli missing (`version` errors): returns `ErrCSCLIUnavailable`, `Status:"failed"`, no backup dir created, no install calls. -- Install failure on item N: the fake executor's `install` hook asserts, at call time, that `DataDir` exists and its sentinel file (`config.yaml`) is intact (proves no rename away); after the failure the test asserts `DataDir` exists, the sentinel and any files the fake install added/changed are restored to the original contents, `Status:"failed"`, error wraps cause, backup copy still present. -- Symlink rollback test: seed `DataDir` with `hub/collections/crowdsecurity/sshd.yaml` plus `collections/sshd.yaml -> ../hub/collections/crowdsecurity/sshd.yaml` (relative link) and a dangling link; run a failing install that mutates both; assert after rollback `os.Lstat` shows symlinks (mode `ModeSymlink`) with identical `Readlink` targets, regular files byte-identical, no dereferenced copies. Also a unit test for the symlink-preserving copy itself (file link, dir link, dangling link, nested). -- Same assertions for verification failure and for a timeout in the middle of install (DataDir exists and is intact during and after). -- Verification fixtures (4.2 step 5): installed JSON passes; `installed:false`, malformed JSON, nonzero exit, missing field each fail; assert exact argv `cscli <type> inspect <name> -o json`. -- `Validate` runs on the input of `ApplyCurated`: calling it with an invalid/hand-built `Preset` (bad type `../x`, bad name `a b`, empty items) returns `ErrInvalidPresetDefinition` with no exec calls and no backup, independent of the static catalog. -- Timeout: `ApplyTimeout` set tiny via the existing field; a fake executor that blocks until ctx is done yields a deadline error and rollback. -- Verification failure (install exit 0 but `inspect -o json` reports `installed:false`): treated as failure and rolled back. -- Backup failure: no install attempted. -- Timeout via cancelled ctx: error satisfies `errors.Is(err, context.DeadlineExceeded)` or Canceled. -- Concurrency: two parallel `ApplyCurated` calls serialize (run with `-race`). -- `presets_test.go`: every `RequiresHub:false` preset passes `Validate()`; invalid type / name (`../x`, `a b`, `;rm`) rejected; `Items` not serialized in JSON of `Preset`. +**Out-of-scope findings discovered while planning:** none left open. Items found beyond the issue text are folded in (install guard, wrong N5 comment, `ImportConfig` conversion, playwright-ci compose mount). If implementation finds anything else, file it per the triage rule at that time. -`backend/internal/api/handlers/crowdsec_presets_handler_test.go`: +## 5. Branch/PR Decision and Commit Slicing Strategy -- Rewrite `TestApplyCuratedPresetSkipsHub` into: success returns 200 only when the fake exec succeeds, `used_cscli:true`, backup non-empty, event row `applied`; assert fake exec saw installs (proves not a no-op). -- New: cscli unavailable -> 503, event row `status=failed`, `error` non-empty, response has no `applied`. -- New: install failure -> 500 with `backup`, event `failed`. -- New: `DB.Create` failure is logged, request outcome unchanged (use closed DB / missing table) for both curated and hub paths. -- Update `TestPullCuratedPresetSkipsHub` to assert the preview lists the items. -- Regression: hub path (`TestApply...` existing tests) unchanged and green after the helper extraction. +### 5.1 Branch/PR decision: ONE branch, ONE PR -Coverage: patch coverage target 100% for new code; overall >= 85% (`scripts/go-test-coverage.sh`). GORM scan not required (no model/query changes beyond existing `Create`), but run `./scripts/scan-gorm-security.sh --check` anyway if any `DB` call shape changes. +Single branch `fix/crowdsec-preset-apply-hardening` -> PR into `development`, titled e.g. `fix: harden CrowdSec preset apply and keep hub data across container recreation`. -### 6.2 Frontend unit tests (Vitest, `frontend/src/pages/__tests__/CrowdSecConfig*.test.tsx`) +Why one PR (cohesion): all seven issues are one defect family, "the CrowdSec persistent tree is mutated unsafely and its hub/data state is not durable", and they share code: #1514, #1515 and #1516 all consume the same new `backup.go` helper (a single fix for rename-based backups, the file endpoint, and pruning); #1517 is the file those changes land in; #1518 is the test net for #1514/#1515; #1523 and #1524 are the same persistent-dir/hub-state story at the container layer and are both verified by the same rebuilt-container gate. Splitting would leave the helper without its callers, the E2E net without the behavior, or a half-fixed persistence story. -- Curated apply success: toast success only when `status === 'applied'`; applyInfo shows backup and `used_cscli`. -- Curated apply non-2xx (parametrized over 500, 503, 504, 501, 400): error toast with server message, no success toast, `applyPresetLocally` NOT called, no `writeCrowdsecFile` call. The contract is: ANY non-2xx on a curated slug never calls `applyPresetLocally`. -- Curated apply 501 (contract violation): error shown, still no local write. -- Hub preset 501: existing local fallback behavior preserved (regression). -- Non-`applied` status in 200 response never shows a success toast. -- Remove/update `data/__tests__/crowdsecPresets.test.ts` consistent with 4.5. Frontend coverage >= 85% (`scripts/frontend-test-coverage.sh`); `npm run type-check`. +Considered splits and why rejected: +- Docker/entrypoint (#1523, #1524) vs Go/frontend: they can build independently, but they do not conflict, share a single verification environment (rebuilt `charon:local`), and splitting would double the merge-soak the maintainer applies before `main`. The #1524 toolchain bot is explicitly designed to push the pin bump onto the PR head branch, so it works inside this PR. +- `(security)` fix alone: not required; severity is low (authenticated admin), and the rule "own PR" applies to critical/high only. -### 6.3 Playwright (single spec, firefox only, targeted) +**Contingency without splitting:** if Commits 7-8 (#1524) cannot be validated (hub-branch decision unresolved, toolchain build blocked), drop those commits from the PR and leave #1524 open rather than opening a second PR for it; the remaining six issues still form a complete, coherent PR. Rollback of the PR as a whole is a revert of the merge commit; a revert leaves a valid `data_dir` in `config.yaml` (the old image still creates the default directory), so nothing breaks. -Add `tests/security/crowdsec-preset-apply.spec.ts` (replace nothing; leave the tolerant spec alone). Uses `page.route` to stub `POST **/admin/crowdsec/presets/apply`: +### 5.2 Ordered commits (all on the one branch) -1. Stub 503 `{error:"CrowdSec CLI is not available..."}`: select "Honeypot Friendly Defaults", click Apply; expect an error toast/message and NO "applied" success toast. -2. Stub 200 `{status:"applied", used_cscli:true, backup:"...", reload_hint:true}`: expect success toast with reload note. -3. Assert no request to the generic file-write endpoint is made in cases 1 and 2 (no local fallback for curated presets). +Every command run by implementers is foreground/blocking per CLAUDE.md. Each commit must build and pass its gate; the PR as a whole passes the full Definition of Done (section 8). -Run: `cd /projects/Charon && npx playwright test tests/security/crowdsec-preset-apply.spec.ts --project=security-tests` (foreground). Full-suite/cross-browser is CI-only. Real-cscli E2E against the container is optional and only if the E2E container image ships `cscli` (implementer to check; otherwise covered by unit tests). +**Commit 1 - `test: add deterministic E2E specs for CrowdSec preset and file flows (fixme)`** (#1518, step 1 of the suggested sequence) +- Files: new `tests/security/crowdsec-hub-preset-apply.spec.ts`, new `tests/security/crowdsec-file-editor.spec.ts`. +- Content, all stubbed with `page.route` like `crowdsec-preset-apply.spec.ts`, marked `test.fixme` until their feature commits: hub preset apply success shows success only when body `status==='applied'`; hub preset apply 500/503/504 shows an error and **zero** `POST /admin/crowdsec/file` calls; hub preset apply 501 shows an error and zero writes; file editor save success posts exactly one write for exactly the selected path; file editor shows the server message for 400 (disallowed type / invalid YAML) and 413. +- Depends on: none. Gate: `npx playwright test tests/security/crowdsec-hub-preset-apply.spec.ts tests/security/crowdsec-file-editor.spec.ts --project=security-tests` runs and reports the tests as fixme/skipped with no failures; lint/type-check clean. -## 7. Implementation Plan (phases mapped to commits) +**Commit 2 - `refactor: move CrowdSec preset handlers into their own file`** (#1517) +- Files: new `backend/internal/api/handlers/crowdsec_preset_handler.go`; edit `crowdsec_handler.go`; move/split preset-only tests into `crowdsec_preset_handler_test.go`. +- Pure move: no signature, route, or behavior changes. Depends on: none. +- Gate: `cd backend && go build ./... && go test ./internal/api/handlers/... -race` all green with the same test count; `git diff --stat -M` shows moves, not rewrites; `make lint-fast`. -Task B0 (before any code): verify every item name in 4.1 against the live hub index (`cscli hub list -a` in the E2E/dev container, or `https://hub-data.crowdsec.net/.index.json`). Record the verified list in the PR description. Check type AND existence of every item (`cscli <type> inspect <name> -o json` against the shipped cscli, or the hub index `hub-cdn.crowdsec.net/master/.index.json`); confirm the JSON field names used by the 4.2 fixtures; confirm `config_paths` (config/hub/data dirs) of the shipped image for the backup set; align the `honeypot-friendly-defaults` summary/warning with what is installed. Decision D1 is already decided (option a); B0 only re-verifies that `crowdsecurity/geoip-enrich` exists as a parser, confirms the localStorage/locale greps from 4.1, and confirms symlink behavior of `copyDir` (4.2). +**Commit 3 - `fix: keep CrowdSec backups bounded and copy-based for preset apply`** (#1515, #1516) +- Files: new `backend/internal/crowdsec/backup.go` + `backup_test.go`; edit `hub_sync.go` (`Apply` per the flow in 3.1; remove `backupExisting`/`rollback`/`emptyDir`/dead `copyDir`), `curated_apply.go` (move helpers, add prune call), `crowdsec_handler.go` `ImportConfig` (snapshot/restore conversion, handler lock), `hub_pull_apply_test.go`, `device_busy_test.go` (the rename/EBUSY tests are obsolete and are replaced by copy-path tests). +- Depends on: Commit 2 (preset code already in its own file; `ImportConfig` stays in the main handler). +- Unit tests (new, all with `t.TempDir()`): snapshot excludes live db files, top-level `data/` and `hub_cache/`, preserves symlinks, and still copies a nested directory named `data` under `config/`; **size test:** a large file under `data/` and under `hub_cache/` is not copied while a small `config/` file is (assert snapshot size); restore leaves live db, `data/` and `hub_cache/` untouched; `PruneBackups` orders by the timestamp parsed from the name (not mtime; touching an old dir does not save it), handles both name layouts, ignores unparsable names, non-matching names, symlinks and plain files, never deletes `dataDir`, clamps `keep<1` to 1, always keeps the newest, handles a missing parent, and `backup` and `filebackup` namespaces never evict each other; `HubService.Apply` flows: (a) cscli success leaves `DataDir` in place (inode unchanged) and prunes; (b) **cscli failure -> snapshot restored -> cached-archive extract succeeds -> `applied`, `UsedCSCLI=false`, no cscli leftovers**; (c) cscli failure and extract failure -> snapshot restored, error returned with `BackupPath`; (d) cscli failure with cache miss and hub refresh failure -> restored; (e) prune also runs after a failed apply and keeps the failed apply's snapshot; `ImportConfig`: success replaces config but preserves live db/`data/`/`hub_cache/` and never renames `DataDir`; extract failure restores the previous config; prune runs; concurrent `ImportConfig` + apply serialize on the handler lock; an uploaded archive containing `crowdsec.db`, `data/x` and `hub_cache/y` entries extracts none of them. +- Gate: `go test ./internal/crowdsec/... ./internal/api/handlers/... -race`; package coverage not below the current level; **rebuilt-container verification of #1515**: pull + apply one hub preset against a running agent, confirm `/app/data/crowdsec` is never renamed (inode unchanged), the apply result is correct, and at most 5 full-snapshot dirs remain after 7 applies (this also settles the "not confirmed" item in 2.2). -## 8. Commit Slicing Strategy +**Commit 4 - `fix(security): harden CrowdSec configuration file handling`** (#1514 backend; subject intentionally vague, see section 4) +- Files: new `backend/internal/api/handlers/crowdsec_files.go` + `crowdsec_files_test.go`; edit `crowdsec_handler.go` (`ListFiles`, `ReadFile`, `WriteFile` delegate to the shared helper); update existing `TestCrowdsec_WriteFile_*` tests in `crowdsec_handler_coverage_test.go`. +- Depends on: Commit 3 (per-file backups use the `filebackup` namespace and handler lock). Commit body and PR text stay vague (section 4). +- Unit tests (categories only here): path validation rejection cases, symlink policy (write through or replacement of a symlink is refused, a regular file under `DataDir` is written), type/content/size validation (disallowed type, invalid YAML, oversize -> 413, protected files), atomic write leaves no temp file on failure, `DataDir` and unrelated files untouched after a write (regression test for the destructive rename), `filebackup` contains only the previous version of the target and is capped independently of full snapshots, the read/list predicate and the stricter write predicate behave as in 3.2, `ReadFile` shares the validation. +- Gate: `go test ./internal/api/handlers/... -race`, `make lint-fast`; GORM scan not required (no models); the Playwright file-editor spec enabled in Commit 9 depends on this behavior. -Decision: ONE PR (`fix/crowdsec-curated-preset-apply` off `development`, PR into `development`), ordered logical commits. Each commit builds and passes its gate. +**Commit 5 - `fix: remove the dead local-write fallback from CrowdSec preset apply`** (#1514 frontend) +- Files: `frontend/src/pages/CrowdSecConfig.tsx`, its four test files under `frontend/src/pages/__tests__/`, `frontend/src/api/crowdsec.ts` only if a now-unused export remains. +- Depends on: none technically (the backend never returns 501); ordered after Commit 4 so the whole #1514 story is reviewable in sequence. +- Gate: `cd frontend && npm run type-check && npx vitest run src/pages/__tests__/CrowdSecConfig*` green; no unused-export/lint findings; `npm run build`. -| # | Commit | Scope / files | Depends on | Validation gate | -| --- | --- | --- | --- | --- | -| 1 | `test: add failing specs for curated CrowdSec preset apply` | ONLY `tests/security/crowdsec-preset-apply.spec.ts` as `test.fixme` (no backend or other files; backend tests land with commit 3). | none | spec file lints; `npx playwright test <spec> --project=security-tests` reports fixme/skipped | -| 2 | `refactor: extract crowdsec preset event/response helpers and add curated item definitions` | `backend/internal/crowdsec/presets.go` (+`PresetItem`, `Items`, `Validate`), `presets_test.go`, handler helper extraction (`recordPresetEvent`, response helpers), DB.Create error logging; NO behavior change to curated apply yet | 1 | `cd backend && go build ./... && go test ./internal/crowdsec/... ./internal/api/handlers/...`; existing tests green | -| 3 | `fix: apply curated CrowdSec presets via cscli and record failures` | `hub_sync.go`/`curated_apply.go` (`ApplyCurated`, `ErrCSCLIUnavailable`, mutex, symlink-preserving copy, `geoip-enrichment` rename in `presets.go`/tests), `crowdsec_handler.go` (ApplyPreset + PullPreset preview), rewritten/new handler and package tests (section 6.1) | 2 | `go test -race ./internal/crowdsec/... ./internal/api/handlers/...`; `make lint-fast`; backend coverage script >= 85%; local patch report | -| 4 | `fix: stop reporting curated preset success the server did not confirm` | `CrowdSecConfig.tsx`, `data/crowdsecPresets.ts` (+ test) cleanup, `api/presets.ts` types if needed, i18n (5 locales) if new strings, Vitest tests (6.2) | 3 | `npm run type-check`, `npx vitest run` on touched tests, `npm run build`, frontend coverage >= 85% | -| 5 | `test: enable curated preset apply e2e and update docs` | un-fixme the Playwright spec; `docs/features.md` / `docs/features/` CrowdSec preset note (curated presets require `cscli`, are verified, failures are reported); `ARCHITECTURE.md` only if the CrowdSec integration section describes preset apply (check; likely a one-line note) | 4 | `npx playwright test tests/security/crowdsec-preset-apply.spec.ts --project=security-tests`; `lefthook run pre-commit`; docs sync rule (no edits under `docs-site/docs/`) | +**Commit 6 - `fix: persist CrowdSec hub data files across container recreation`** (#1523) +- Files: `.docker/docker-entrypoint.sh`, `configs/crowdsec/install_hub_items.sh`, `.docker/compose/docker-compose.playwright-ci.yml`, `scripts/diagnose-crowdsec.sh`, `docs/security.md` (wording), new integration test `scripts/crowdsec_data_persistence_test.sh` (follows the pattern of `scripts/crowdsec_startup_test.sh`). +- Depends on: none (independent of the Go commits; ordered after so a failing container gate does not block the Go work from being reviewed). +- Tests: shell-level test (start container with a named volume, wait for hub init, assert data files and `GeoLite2-City.mmdb` exist under `/app/data/crowdsec/data`, `docker rm` and recreate with the same volume, assert files are present and `crowdsec.log` has no `unable to init data for file` / `unable to initialize GeoIP` lines after enabling CrowdSec; also run it on a volume initialized by the old image to prove the sed migration). Add `shellcheck` cleanliness for changed scripts. +- Gate: rebuild `charon:local` (skill `docker-rebuild-e2e` / `.github/skills/scripts/skill-runner.sh docker-rebuild-e2e`), run `scripts/crowdsec_data_persistence_test.sh` and `scripts/crowdsec_startup_test.sh`; the fresh-volume case must show the install guard now really installing. +- Additional tests: the gate must prove `hub upgrade` is **not** run on a restart with intact data files (log assertion) and **is** run (once, bounded) when data files are missing; `shellcheck` is a required gate for every changed or new shell script (`.docker/docker-entrypoint.sh`, `install_hub_items.sh`, the new test script). +- CI wiring and ignore files: wire `scripts/crowdsec_data_persistence_test.sh` into the existing CrowdSec integration job via a skill wrapper like `.github/skills/integration-test-crowdsec-startup-scripts` and the `integration-tests.yml` workflow (rather than a new workflow). Check `.dockerignore` (it already excludes `scripts/tests/` and the toolchain scripts; the new script must be excluded from the image the same way if it lives under `scripts/`), `.gitignore`, and `.codecov.yml` (shell scripts are not coverage-measured; confirm no new exclusion needed). +- Snapshot interaction (item 4 sequencing): Commit 3's snapshot already excludes top-level `data/` and `hub_cache/`, so Commit 6's relocation of `data_dir` into `/app/data/crowdsec/data` does not enlarge any backup; the Commit 3 size test is the regression guard. -Definition of Done for the PR (CLAUDE.md): targeted Playwright (firefox) first; GORM scan only if models/queries change (not expected); `bash scripts/local-patch-report.sh`; CodeQL/Trivy are deferred to CI (this is a `fix:` with no new feature surface); `lefthook run pre-commit`; staticcheck clean; backend and frontend coverage >= 85%; type-check; both builds; no debug leftovers. +**Commit 7 - `fix: report the real CrowdSec version in the bundled binaries`** (#1524, step A: ldflags) +- Files: `Dockerfile` (`crowdsec-inline`, both `xx-go build` lines). Bot-authored follow-up commit on the branch updates `CHARON_TOOLCHAIN_TAG` / `CHARON_TOOLCHAIN_DIGEST` (do not hand-edit those lines; the workflow owns them). +- Depends on: none. **Wait here until the toolchain-image workflow has built and pushed the pin bump to the PR branch** (this is the only asynchronous step in the plan). +- Gate: `toolchain-image.yml` green on the PR (build + Trivy gate), `verify-toolchain-pin` green, and a local `make build-offline`-style build (`--build-arg CROWDSEC_BUILDER_SRC=crowdsec-inline`) shows `cscli version` -> `version: v1.8.1`. +- Expected CI state: after this commit `verify-toolchain-pin` (and any check that compares TAG/DIGEST to the recipe key) is **red until the bot pushes the new pin**. Implementers must `git pull` the bot's commit before starting Commit 8 and must not hand-edit the two ARG lines. -Rollback / contingency: -- The PR is revertable as a unit; no schema change, so revert has no data impact. -- If Task B0 shows no verifiable hub items for a curated preset (for example `geoip-enrich` is gone from the hub), stop and ask the user rather than shipping a no-op preset. -- If `cscli` is absent from some deployment shapes, curated presets will return 503 by design (honest failure); the UI message must tell the operator why. Consider (follow-up, not this PR) hiding the Apply button for curated presets when `cscli` is known unavailable. -- If reliable post-install reload is not feasible in this PR, ship with `reload_hint:true` semantics (explicitly "installed and verified, restart required"). +**Commit 8 - `fix: fail the image build if CrowdSec reports no version and make the hub branch deterministic`** (#1524, step B) +- Files: `Dockerfile` (N5 block: correct the comment, add the version assertion, add `ARG CROWDSEC_VERSION` to the final stage), `.docker/docker-entrypoint.sh` (hub branch setting per D1). +- Depends on: Commit 7 and the bot pin bump. +- Tests / build-time assertion: the Dockerfile `RUN` fails with a clear message if `cscli version` does not print `version: v${CROWDSEC_VERSION}`. Prove it fails by temporarily building with a mismatched `--build-arg CROWDSEC_VERSION` against the pinned toolchain (expected hard failure), then with the correct value (pass). Runtime check in the rebuilt container: `cscli version` truthful; `cscli hub update` succeeds online; with the network blocked (`--network none`) hub commands behave as the D1 decision specifies (no hard failure when `hub_branch` is set). +- Gate: image builds; `scripts/crowdsec_startup_test.sh` green; both stderr lines `Crowdsec version is not set` and `unable to retrieve latest crowdsec version` absent from `docker logs` in the online and offline runs. +- Record in the PR description: `cscli.hub_branch` is honoured by v1.8.1 and skips the `version.crowdsec.net` lookup (source-verified in `chooseBranch`), plus the measured online and offline (`--network none`) results. + +**Commit 9 - `test: enable CrowdSec preset and file-editor E2E specs and drop tolerant ones`** (#1518, final step) +- Files: the two specs from Commit 1 (remove `fixme`), `tests/security/crowdsec-config.spec.ts` (delete the tolerant "Preset Management" assertions that are now covered by deterministic specs, or convert each remaining case to a deterministic stub; no `isVisible().catch(() => false)` guards, no "feature may not be implemented" annotations remain in `tests/security/crowdsec-*.spec.ts`). +- Depends on: Commits 4 and 5. +- Gate: `npx playwright test tests/security/crowdsec-hub-preset-apply.spec.ts tests/security/crowdsec-file-editor.spec.ts tests/security/crowdsec-preset-apply.spec.ts tests/security/crowdsec-config.spec.ts --project=security-tests` green. Mutation check: temporarily re-add the local-write fallback (or revert the success-status check) and confirm at least one of the new specs fails, then undo. Full-suite and cross-browser runs are CI-only. + +**Commit 10 - `docs: document CrowdSec preset apply safety and data persistence`** +- Files: `ARCHITECTURE.md`, `docs/features/crowdsec.md`, `docs/troubleshooting/` entry if appropriate (see 7). +- Depends on: all. Gate: markdown lint/pre-commit; `docs-site` manifest unchanged (new content goes into already-manifested `docs/features/` / `docs/troubleshooting/`). + +## 6. Decisions and Edge Cases + +- **D1 hub branch (Q1 resolved: option A).** Options: (A, recommended) set `cscli.hub_branch: master` explicitly: identical to today's effective behavior while 1.8.1 is the latest, no dependency on `version.crowdsec.net`, no 404 risk, truthful `cscli version`. (B) Leave auto-selection with the fixed version: matches upstream's intent but can hard-fail hub commands offline and break hub updates when 1.8.1 stops being latest and the CDN has no `v1.8.1` branch (404 today). (C) Pin to the version branch: broken today (404). Commit 8 gate must run the empirical online/offline test either way. +- **Locking (one mechanism).** A single handler-level `sync.Mutex` on `CrowdsecHandler` (e.g. `dataMu`) is held for the whole of `ApplyPreset` (both curated and hub paths), `ImportConfig` and `WriteFile`. `HubService.mu` remains as an inner guard for direct service callers; lock order is always handler mutex first, then `HubService.mu`, so no deadlock. No other lock is introduced. +- Edge cases to cover in tests: snapshot dir name collisions (microsecond timestamp plus exclusive `os.Mkdir`), parent dir not writable, `DataDir` being a mount point (the copy path must work where rename used to fall back), symlinks inside `DataDir` never followed by restore/prune/write, concurrent apply/import/file write serialized by the handler mutex (held for the full `ApplyTimeout`, so file saves block during a long apply; `ReadFile` and `ListFiles` stay lock-free), CrowdSec stopped, existing volumes whose `config.yaml` still points at `/var/lib/crowdsec/data` (sed migration, then the gated `hub upgrade` repopulates), tmpfs `/var/lib/crowdsec` deployments (no longer relevant for data after migration). +- Reload behavior from #1513 (SIGHUP via `reloadCrowdSec`) is unchanged and should also be triggered, best-effort, after the entrypoint-independent hub changes that happen at runtime only (not applicable to startup). + +## 7. Docs Impact + +- `ARCHITECTURE.md`: update the Layer 2 CrowdSec section (~L868-890): CrowdSec `data_dir` and LAPI db live under `/app/data/crowdsec/data` (persistent), hub data files are re-synced when missing via a gated `cscli hub upgrade`, preset applies and config edits are copy-based/atomic with bounded backup retention, `cscli version` reports the pinned version and the hub branch is explicit. The file layout table around L288-294 gets the data subdirectory note. +- `docs/features/crowdsec.md`: extend "Curated Presets" (L57-59) / Hub Presets with the backup retention (last N kept), that hub presets are also applied on the server with a rollback, and that the config editor only accepts specific file types with validation. Add a short troubleshooting entry for the `unable to init data for file` log line (what it meant, that it self-heals on restart after this release). Keep it novice-friendly. +- `docs/features.md`: no new capability; no change expected (confirm during Commit 10). +- `docs/security.md`: tmpfs `/var/lib/crowdsec` guidance wording (Commit 6). +- Changelog is generated from commit subjects; subjects in section 5.2 are written for that. + +## 8. Test Plan Summary and Definition of Done + +- **Backend unit:** new tests in Commits 3 and 4 (list above). Coverage: package and patch coverage must not drop; overall >= 85% (`scripts/go-test-coverage.sh`); `bash scripts/local-patch-report.sh` produces `test-results/local-patch-report.md/.json`. +- **Frontend unit:** updated Vitest in Commit 5; `scripts/frontend-test-coverage.sh` >= 85%; `npm run type-check`; `npm run build`. +- **Playwright (#1518):** deterministic stubbed specs per Commit 1/9, role-based locators, no `waitForTimeout`, no visibility-guard skips. Locally only the targeted spec files with `--project=security-tests` (security specs are excluded from the browser projects; other specs use `--project=firefox`); CI runs the full matrix. +- **Build-time assertion (#1524):** Dockerfile `RUN` check in Commit 8 with the negative (mismatched version) and positive proof recorded in the PR description. +- **Container integration:** `scripts/crowdsec_data_persistence_test.sh` and `scripts/crowdsec_startup_test.sh` (Commit 6/8). +- **Security scans:** this is a `fix:` with no new endpoints; CodeQL/Trivy are deferred to CI per CLAUDE.md, except that Commit 7's toolchain Trivy gate runs in CI by design. GORM scan not required (no model changes); run it only if a model is touched. +- **Full DoD checklist before merge (in order):** targeted Playwright (firefox) -> patch coverage preflight -> `lefthook run pre-commit` -> staticcheck (`make lint-fast`) -> backend/frontend coverage -> type-check -> `go build ./...` and `npm run build` -> clean-up (no debug prints, dead code, unused imports). Any failing test, even pre-existing, is fixed in this PR. +- **Pipeline after the plan:** Supervisor plan review -> user approval -> implement commit by commit (`backend-dev` for 2-4, `frontend-dev` for 5, `devops` for 6-8, `playwright-dev` for 1 and 9, `docs-writer` for 10) -> Supervisor implementation review -> `qa-security` last -> `docs-writer` pass. +- **Dispatch instructions (every subagent prompt must include these, verbatim in spirit):** run all tests, builds, coverage scripts, Playwright, linters, Docker builds and `lefthook` as **foreground, blocking** calls with the maximum timeout; never use `run_in_background`, `&`, `nohup` or end the turn waiting for a notification; if a call auto-backgrounds, immediately re-attach and poll until a real result; no worktrees; stay inside the repo (no live hosts); do not commit unless the orchestrator says so; Commit 4's text (body, comments, PR text) must stay vague per section 4; and the Commit 7 to Commit 8 hand-off requires pulling the toolchain bot's commit first. ## 9. Acceptance Criteria -1. Applying a curated preset with `cscli` available installs the server-defined hub items, verifies them, creates a backup and returns 200 `applied` with `used_cscli:true` and a non-empty `backup`. -2. Without `cscli`, or on any install/verify/backup/timeout failure, the endpoint returns a non-2xx error, never `applied`, rolls back, and persists a `CrowdsecPresetEvent` with `Status:"failed"` and `Error`. -3. No code path returns `applied` unless something was installed and verified. The old no-op branch is gone. -4. The frontend never shows a success toast for non-`applied` or error responses and never writes client-supplied curated content to a CrowdSec file; hub-preset 501 fallback unchanged. -5. Curated item definitions are server-side, validated (allowlisted types, name regex, argv execution, no shell) and verified against the real hub (Task B0); Decision D1 is implemented as decided (option a: `geoip-enrichment`, enrichment-only wording) and no preset or `warning` text over-promises. Curated apply never renames or empties `DataDir`; rollback keeps it intact. -6. `TestApplyCuratedPresetSkipsHub` no longer asserts the buggy behavior; new tests in 6.1/6.2/6.3 pass; coverage thresholds met; DoD in section 8 satisfied; CI green. +1. Applying a hub preset or editing a CrowdSec file never renames, empties or removes `/app/data/crowdsec`; the live tree is intact after success and after any failure (tests prove it). +2. Full-tree snapshot dirs (`<DataDir>.backup.*`) are bounded to 5 and single-file edit backups (`<DataDir>.filebackup.*`) to 10, independently, after any apply/import/write attempt; snapshots exclude `data/` and `hub_cache/` so they stay small. +3. `POST /admin/crowdsec/file` validates paths, file types, YAML and size, refuses symlink writes, never renames `DataDir`, and `ReadFile`/`ListFiles` share the same validation; the frontend has no local-write fallback. +4. After `docker rm` + recreate with the same volume, the 12 listed hub data files and `GeoLite2-City.mmdb` exist and `crowdsec.log` has no `unable to init data` / GeoIP lines; fresh volumes actually install the items in `install_hub_items.sh`. +5. `cscli version` and `crowdsec -version` print `v1.8.1`; the image build fails if they do not; hub commands work online and offline per D1. +6. `crowdsec_handler.go` no longer contains the preset handlers; no behavior change from the move. +7. No tolerant "may not be implemented" Playwright tests remain for CrowdSec presets, and the mutation check in Commit 9 shows the new specs fail on regression. +8. Only one commit uses the `(security)` scope and its subject is the vague form above. + +## 10. Resolved Questions (all decided by the user) + +- Q1 (D1): RESOLVED: explicit `cscli.hub_branch: master`. +- Q2: RESOLVED by the user: no private advisory (issue is public; severity low, admin-only). Revisit only if implementation finds impact beyond admin privileges. +- Q3: RESOLVED: write allowlist, protected-file list, 1 MiB cap and the refuse-symlinks rule as specced in 3.2, with a separate looser read/list predicate. +- Q4: RESOLVED: startup log line only (no diagnostics field, no UI). +- Q5: RESOLVED: constants (5 full snapshots, 10 file backups), not configurable. diff --git a/docs/reports/qa_report.md b/docs/reports/qa_report.md index f87d161f0..53cca0df6 100644 --- a/docs/reports/qa_report.md +++ b/docs/reports/qa_report.md @@ -1,45 +1,67 @@ -# QA Report - fix/harden-backend-consistency-checks +# QA and Security Report: fix/crowdsec-preset-apply-hardening (PR #1525) -Date: 2026-10-06. Scope: 12 commits, `bb787410..HEAD` (HEAD 95d31201). Backend only, no frontend changes. -Verdict: **PASS, no blocking findings.** +Date: 2026-10-09 +Scope: full diff `origin/development...HEAD` (HEAD 513440f5), spec `docs/plans/current_spec.md` (issues #1514-#1518, #1523, #1524). -> Security-sensitive detail (specific weaknesses, residual risks, and re-test considerations) has been -> intentionally redacted from this public report. It is tracked privately per `SECURITY.md`. +## Verdict: PASS (no blocking findings) -## Definition of Done +No critical or high findings. Two low-severity security hardening candidates are flagged for the maintainer, plus a few low/medium non-security items. None block the PR. -| # | Item | Result | Evidence | -|---|------|--------|----------| -| 1 | Targeted Playwright E2E, firefox only | PASS | E2E image rebuilt with the `docker-rebuild-e2e` skill. Final HEAD: 11 spec files covering first-run onboarding, proxy hosts, redirection hosts, certificates (delete, bulk delete, list) and the Caddyfile/NPM/JSON import flows. 178 passed, 1 skipped, 1 flaky-under-load failure (`caddy-import-gaps` 4.2, a session-resume test that depends on leftover import-session state from earlier specs in the same run). It passes in isolation and the whole `caddy-import-gaps` file passes 12/12 when re-run. A prior run on the first eight commits had 0 failures. | -| 1.5 | GORM security scan | PASS | `scan-gorm-security.sh --check`: 0 CRITICAL, 0 HIGH, 0 MEDIUM. | -| 2 | Local patch coverage | PASS | Backend/overall patch coverage 96.7% (264 of 273 changed lines; thresholds 90 overall, 85 backend). Uncovered changed lines are defensive database-error branches. | -| 3 | Backend coverage | PASS | `scripts/go-test-coverage.sh` on the first eight commits: statements 92.6%, lines 89.8% (gate 87%). Package coverage at final HEAD: handlers 90.5%, services 91.0%. Frontend coverage, type-check and build skipped: the frontend is untouched. | -| 4 | Lint / static analysis | PASS | `make lint-fast`: 0 issues (backend and agent). `lefthook run pre-commit --all-files`: all hooks passed, semgrep 0 findings. | -| 5 | Build and full test run | PASS | `go build ./...` OK (backend and agent). `go test ./... -race -count=1` passes with zero failures. | -| 6 | Vulnerability scans | PASS | `govulncheck`: 0 vulnerabilities affecting the code. CodeQL and Trivy are deferred to CI per `CLAUDE.md` for a `fix:`-only branch. | +## Definition of Done evidence -## Review summary +| Check | Command | Result | +|---|---|---| +| Backend build | `cd backend && go build ./...` | pass | +| Backend vet | `go vet ./...` | pass, no output | +| golangci-lint, backend | `golangci-lint run --config .golangci.yml ./...` | 0 issues | +| golangci-lint, agent | `cd agent && golangci-lint run --config ../backend/.golangci.yml ./...` | 0 issues | +| Frontend type-check | `npm run type-check` | pass | +| Frontend build | `npm run build` | pass | +| GORM scan | not run | no `backend/internal/models/**`, migration or GORM query files changed | +| Lefthook | `lefthook run pre-commit --all-files` (plain run skips every hook with nothing staged) | all 17 hooks pass, including semgrep (0 findings, 367 rules), shellcheck, actionlint, dockerfile-check, golangci-lint-fast, frontend-lint | +| govulncheck | skill `security-scan-go-vuln` | 0 affected. 1 module-level advisory, GO-2026-5932 (x/crypto/openpgp, no fix), not called by Charon code | +| Trivy | `trivy fs --scanners vuln,misconfig --severity HIGH,CRITICAL` on go.mod/go.sum (backend, agent), package-lock.json and Dockerfile | no HIGH/CRITICAL output. The snap-confined trivy cannot read `/projects`, so the manifests were copied under `$HOME` for the scan. No container image scan was done. CI runs the full Trivy/CodeQL. | +| CodeQL | not run locally | change is `fix:`-scoped with no new feature surface, so deferred to CI per CLAUDE.md | +| E2E | `docker-rebuild-e2e` skill, then `npx playwright test tests/security/crowdsec-hub-preset-apply.spec.ts tests/security/crowdsec-file-editor.spec.ts tests/security/crowdsec-config.spec.ts --project=security-tests` | 27 passed, 0 failed (image rebuilt first because the Dockerfile and merge commits were newer than the running container) | +| Coverage (given, not rerun) | prior runs | backend 90.9% stmt, frontend 91.4% lines, patch 93.9% | -- Concurrency-sensitive writes now run inside a shared write-locked transaction helper. All callers were reviewed for correct transaction-handle use. The concurrency tests use multi-connection databases, which is a stronger scenario than production's single-connection pool. -- Domain handling is consolidated into shared helpers used by the proxy host, redirection host and import code paths. No remaining non-test callers of the removed helpers were found. -- Error responses on the changed paths are generic, with detail logged server-side. -- Commit subjects for `(security)` commits are intentionally generic. No session identifiers or links appear in commit messages or the diff. -- Dead code identified in review was removed in this branch. +The `go-test-coverage` and `local-patch-report` scripts were not rerun, as instructed. The working tree stayed clean until this report was written. + +## Security review + +Reviewed against SECURITY.md. No exploit detail is given here. + +- `backend/internal/crowdsec/backup.go`: snapshots and restores copy rather than rename. Symlinks are preserved literally and never followed when copying. Clearing the config removes symlinks themselves, because `DirEntry.IsDir` is false for links. Directories are created 0700. Engine-owned state (live db and WAL/SHM at any depth, top-level `data/` and `hub_cache/`) is excluded from snapshot, restore, clear and import. Prune sorts by the timestamp in the directory name, uses `Lstat` and skips non-directories, so a symlinked "backup" is ignored. No issue. +- `backend/internal/api/handlers/crowdsec_files.go`: relative-path checks use `filepath.IsLocal`, so NUL, absolute and `..` paths are rejected. Read resolves symlinks, checks containment, re-applies the deny-list to the resolved path and requires a regular file. Write refuses any symlink along the path and any non-regular target. It also enforces the extension allowlist, the credential-file deny-list, the temp-prefix deny-list, `hub/` and `config/hub/` exclusion, a 1 MiB content cap and an 8 MiB body cap (`MaxBytesReader`). Writes go through a temp file, fsync and rename, and the original mode is preserved. A single-file backup is taken before the write. Operations serialize on `dataMu`. Logged errors go through `sanitizeForLog`. Adequate. Minor items are F2 to F4 below. +- `crowdsec_handler.go` `ImportConfig`: takes `dataMu`, then snapshots, clears, extracts and validates. It rolls back in place on failure and then prunes. The extractor blocks traversal and skips engine-owned entries, and symlink and hardlink entries are not materialized. Lock order is documented (`dataMu` before `HubService.mu`). See F3. +- `hub_sync.go` and `curated_apply.go`: apply is now snapshot, then cscli or extract, then restore on failure, with prune afterwards. The extractor rejects symlink entries, traversal and absolute paths, strips setuid and setgid bits, caps decompressed size at 100 MB and skips engine-owned paths. See F1. +- `.docker/docker-entrypoint.sh`: the new `data_dir` redirect, `hub_branch` pin and missing-data recovery are idempotent and non-fatal. `hub upgrade` is bounded by `timeout 120s`. Values read from hub YAML (`dest_file`) are only used in `[ -e ]` tests and echoes, never executed or written. All variables are quoted. shellcheck passes. +- `configs/crowdsec/install_hub_items.sh`: the `is_installed` helper greps the JSON "installed" flag. No injection surface (arguments are static). +- `Dockerfile`: the XMOD pin is parameterized and registered in `scripts/toolchain-key.sh` and the toolchain key. The x/net and x/crypto re-pin and the post-build `go version -m` guards are correct (`sort -V` floor check, fails the build). The ldflags target `go-cs-lib/version.Version`, and a new cscli `version: v${CROWDSEC_VERSION}` assertion guards against a silent no-op `-X`. The toolchain tag and digest were synced. No weakening. +- Frontend `CrowdSecConfig.tsx`: the client-side preset fallback write is removed, so the server is authoritative. Server error text is rendered via toast text (React-escaped), so there is no XSS. Mutation errors now surface. +- CI: the new `notify-codecov` job uses a SHA-pinned action, `persist-credentials: false` and a minimal `if` guard. The persistence integration step is wired with cleanup. ## Findings -### Blocking -None. +No critical or high findings. + +### Security-sensitive (do not file publicly; maintainer to decide on advisory or hardening commit) + +**S1: low.** `backend/internal/crowdsec/hub_sync.go:951-971` (`extractTarGz` file open). The old flow emptied the target before extracting; the new documented overlay design does not. File creation uses `O_CREATE|O_TRUNC` without a symlink check on the destination, so it follows any pre-existing link in the live tree. The archive is hub-sourced and cached, and symlink entries in the archive are already rejected, so exploitation needs a hostile hub archive plus an existing link. The spec's "residual risk" note covers overlaying but not this write-through. Suggested hardening: `Lstat` the destination (and ideally its parents) and refuse symlinks, or open with `O_NOFOLLOW`. + +**S2: low.** `backend/internal/api/handlers/crowdsec_handler.go:793` (`extractArchive`). Same pattern: `OpenFile` without `O_EXCL` or `O_TRUNC`, which can leave stale trailing bytes when an archive repeats a name. The tree is cleared first, so impact is limited to duplicate entries. Fix: `O_CREATE|O_WRONLY|O_TRUNC` plus a symlink check. + +### Medium and low non-security (for the maintainer to file) -### Non-blocking -Several low-severity, non-blocking items were identified during review. The in-scope ones were fixed in this -branch (dead-code removal, input normalization, reference validation inside the write transaction, and -certificate-handler status codes). The remainder are tracked privately or as plain follow-ups and are not -listed here. +- **F1: low.** `backend/internal/crowdsec/hub_sync.go:954-957`. The containment check compares against `rel == ".."` and uses `HasPrefix(cleanName, "..")`, which also rejects legitimate names starting with `..` (for example `..foo`). Cosmetic and safe, since it errs on the strict side. +- **F2: low (information exposure to authenticated admin).** `crowdsec_files.go:300` and `:325` return `err.Error()` to the client on 500, which can embed absolute server paths. Other branches use fixed messages. Suggested: generic message and log the detail with `fileErrorLog`. +- **F3: low.** The `backup` field in the `WriteFile` (`crowdsec_files.go:391`) and `ImportConfig` (`crowdsec_handler.go:727`) responses returns an absolute server path. This is consistent with the existing preset-apply behavior and with UI expectations, so it is likely intentional. The extraction error response in `crowdsec_handler.go:715` also interpolates the archive entry name and `%v` error text. +- **F4: low.** `ReadFile` (`crowdsec_files.go:319`) has no read-size cap (the write cap is 1 MiB). The risk is small because the endpoint is admin-only and the tree is config-sized. +- **F5: info.** `local_api_credentials.yaml` and `online_api_credentials.yaml` are write-protected but readable and listed by the editor. This matches the design where `isWritable` is a subset of `isReadable`; confirm that exposing them to admins in the UI is intended. +- **F6: low (test quality).** Known and accepted: #1526 (tolerant Playwright specs). -## Not verified / limits +## Out of scope and not verified -- Per-commit buildability (bisect) was not checked. HEAD builds and passes everything. -- CodeQL and Trivy were not run locally, by policy. CI runs them. -- Concurrency behavior was judged from the code and the multi-connection tests; no live-database stress run beyond the test suite was performed. -- The security review and full QA pass were done on the first eight commits. The four later commits (cleanup, input normalization, reference validation, test fixture) were verified by build, full `-race` test run, lint, GORM scan, patch coverage and the targeted E2E re-run above, but did not get a second full manual review. +- Codecov manual-trigger behavior cannot be tested locally. +- Full-suite and cross-browser E2E are deferred to CI. +- CodeQL and a container-image Trivy scan are deferred to CI. diff --git a/docs/security.md b/docs/security.md index fc0091ec7..5342cb74b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1338,7 +1338,7 @@ Charon uses two types of storage: - **Caddy Logs**: `/var/log/caddy/` - Access logs monitored by CrowdSec - **CrowdSec Logs**: `/var/log/crowdsec/` - Agent and LAPI logs - **Runtime Config**: `/config/` - Dynamically generated Caddy JSON configuration - - **CrowdSec Runtime**: `/var/lib/crowdsec/` - CrowdSec agent runtime data + - **CrowdSec Runtime**: `/var/lib/crowdsec/` - Scratch space only. The CrowdSec database and hub data files (blocklists, GeoIP databases) live on the persistent volume under `/app/data/crowdsec/data` - **Temporary Files**: `/tmp/` - Used by CrowdSec hub operations - **Runtime State**: `/run/` - PIDs and runtime state files @@ -1537,7 +1537,7 @@ docker exec charon ls -la /app/data **CrowdSec fails to start:** -- Verify `/var/lib/crowdsec` tmpfs mount exists +- Verify `/var/lib/crowdsec` tmpfs mount exists (scratch space only; CrowdSec data lives under `/app/data/crowdsec/data` and must be on the volume) - Check `/app/data/crowdsec` volume is writable - Ensure symlink `/etc/crowdsec -> /app/data/crowdsec/config` is preserved diff --git a/frontend/src/pages/CrowdSecConfig.tsx b/frontend/src/pages/CrowdSecConfig.tsx index 055085690..cc6b017a1 100644 --- a/frontend/src/pages/CrowdSecConfig.tsx +++ b/frontend/src/pages/CrowdSecConfig.tsx @@ -24,6 +24,15 @@ import { useWhitelistEntries, useAddWhitelist, useDeleteWhitelist } from '../hoo import { buildCrowdsecExportFilename, downloadCrowdsecExport, promptCrowdsecFilename } from '../utils/crowdsecExport' import { toast } from '../utils/toast' +/** Prefer the server's `error` message for Axios failures, falling back to the given text. */ +function getServerErrorMessage(err: unknown, fallback: string): string { + if (isAxiosError(err)) { + const serverError = (err.response?.data as { error?: string } | undefined)?.error + return serverError || err.message || fallback + } + return fallback +} + const CrowdSecDashboard = lazy(() => import('../components/crowdsec/CrowdSecDashboard')) export default function CrowdSecConfig() { @@ -115,7 +124,16 @@ export default function CrowdSecConfig() { const listMutation = useQuery({ queryKey: ['crowdsec-files'], queryFn: listCrowdsecFiles }) const readMutation = useMutation({ mutationFn: (path: string) => readCrowdsecFile(path), onSuccess: (data) => setFileContent(data.content) }) - const writeMutation = useMutation({ mutationFn: async ({ path, content }: { path: string; content: string }) => writeCrowdsecFile(path, content), onSuccess: () => { toast.success('File saved'); queryClient.invalidateQueries({ queryKey: ['crowdsec-files'] }) } }) + const writeMutation = useMutation({ + mutationFn: async ({ path, content }: { path: string; content: string }) => writeCrowdsecFile(path, content), + onSuccess: () => { + toast.success('File saved') + queryClient.invalidateQueries({ queryKey: ['crowdsec-files'] }) + }, + onError: (err: unknown) => { + toast.error(getServerErrorMessage(err, 'Failed to save file')) + }, + }) const presetsQuery = useQuery({ queryKey: ['crowdsec-presets'], @@ -422,39 +440,7 @@ export default function CrowdSecConfig() { await backupMutation.mutateAsync() await writeMutation.mutateAsync({ path: selectedPath, content: fileContent }) } catch { - // handled - } - } - - const applyPresetLocally = async (reason?: string) => { - if (!selectedPreset) { - toast.error('Select a preset to apply') - return - } - - const targetPath = selectedPath ?? listMutation.data?.files?.[0] - if (!targetPath) { - toast.error('Select a configuration file to apply the preset') - return - } - - const content = presetPreview - if (!content) { - toast.error('Preset preview is unavailable; retry pulling before applying') - return - } - - try { - await backupMutation.mutateAsync() - await writeCrowdsecFile(targetPath, content) - queryClient.invalidateQueries({ queryKey: ['crowdsec-files'] }) - setSelectedPath(targetPath) - setFileContent(content) - setApplyInfo({ status: 'applied-locally', cacheKey: presetMeta?.cacheKey }) - toast.success(reason ? `${reason}: preset applied locally` : 'Preset applied locally (backup created)') - } catch (err) { - const msg = err instanceof Error ? err.message : 'Failed to apply preset locally' - toast.error(msg) + // Failures are surfaced by the mutations' onError handlers; success toast only fires on onSuccess. } } @@ -488,60 +474,23 @@ export default function CrowdSecConfig() { setPresetStatusMessage(`Backup stored at ${res.backup}`) } } catch (err) { - if (isAxiosError(err)) { - const status = err.response?.status - const serverError = err.response?.data?.error as string | undefined - const backupPath = (err.response?.data as { backup?: string } | undefined)?.backup - - // Curated presets are applied authoritatively by the server. Any failure is - // surfaced as-is and must never fall back to writing client-side content. - if (selectedPreset.source === 'charon-curated') { - const curatedMsg = serverError || err.message - if (status === 400) { - setValidationError(curatedMsg || 'Preset validation failed') - } - setApplyInfo({ status: 'failed', backup: backupPath, cacheKey: presetMeta?.cacheKey }) - toast.error(`Apply failed: ${curatedMsg}${backupPath ? `. Backup created at ${backupPath}` : ''}`) - return - } - - if (status === 501) { - toast.info('Preset apply is not available on the server; applying locally instead') - await applyPresetLocally('Backend apply unavailable') - return - } - - if (err.response?.status === 400) { - setValidationError(err.response?.data?.error || 'Preset validation failed') - toast.error('Preset validation failed') - return - } - - if (err.response?.status === 503) { - setHubUnavailable(true) - setPresetStatusMessage('CrowdSec hub unavailable. Retry or load cached copy.') - toast.error('Hub unavailable; retry pull/apply or use cached copy') - return - } - - const errorMsg = serverError || err.message - - // Check if error is due to missing cache - if (errorMsg.includes('not cached') || errorMsg.includes('Pull the preset first')) { - toast.error(errorMsg) - setValidationError('Preset must be pulled before applying. Click "Pull Preview" first.') - return - } - - if (backupPath) { - setApplyInfo({ status: 'failed', backup: backupPath, cacheKey: presetMeta?.cacheKey }) - toast.error(`Apply failed: ${errorMsg}. Backup created at ${backupPath}`) - return - } - toast.error(`Apply failed: ${errorMsg}`) - } else { + if (!isAxiosError(err)) { toast.error('Failed to apply preset') + return + } + // The server is authoritative for every preset source: surface its message for any + // non-2xx response and never write preset content client-side. + const status = err.response?.status + const backupPath = (err.response?.data as { backup?: string } | undefined)?.backup + const message = getServerErrorMessage(err, 'Failed to apply preset') + if (status === 400) { + setValidationError(message) + } + if (status === 503) { + setHubUnavailable(true) } + setApplyInfo({ status: 'failed', backup: backupPath || undefined, cacheKey: presetMeta?.cacheKey }) + toast.error(`Apply failed: ${message}${backupPath ? `. Backup created at ${backupPath}` : ''}`) } finally { setIsApplyingPreset(false) } diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx index 5d1defe10..820cc1875 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.coverage.test.tsx @@ -328,28 +328,26 @@ describe('CrowdSecConfig coverage', () => { await waitFor(() => expect(presetsApi.pullCrowdsecPreset).toHaveBeenCalledTimes(2)) }) - it('falls back to local apply on 501 and covers validation/hub/offline branches', async () => { - vi.mocked(crowdsecApi.writeCrowdsecFile).mockResolvedValue({}) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(501, 'not implemented')) + it('surfaces the server message for 501 hub apply without writing, and covers validation/hub/offline branches', async () => { + vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(501, 'not implemented', { error: 'apply unsupported' })) await renderPage() - // Local apply needs the pulled preview content, which now arrives from the hub pull. await waitFor(() => expect(screen.getByTestId('preset-preview')).toHaveTextContent('crowdsecurity/http-cve')) const applyBtn = screen.getByTestId('apply-preset-btn') await userEvent.click(applyBtn) - await waitFor(() => expect(toast.info).toHaveBeenCalledWith('Preset apply is not available on the server; applying locally instead')) - await waitFor(() => expect(crowdsecApi.writeCrowdsecFile).toHaveBeenCalled()) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Apply failed: apply unsupported')) + expect(crowdsecApi.writeCrowdsecFile).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(400, 'bad', { error: 'validation failed' })) await userEvent.click(applyBtn) await waitFor(() => expect(screen.getByTestId('preset-validation-error')).toHaveTextContent('validation failed')) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(503, 'hub')) + vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(503, 'hub', { error: 'hub down' })) await userEvent.click(applyBtn) expect(await screen.findByTestId('preset-hub-unavailable')).toBeInTheDocument() + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Apply failed: hub down')) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(500, 'not cached', { error: 'Pull the preset first' })) - await userEvent.click(applyBtn) - await waitFor(() => expect(screen.getByTestId('preset-validation-error')).toHaveTextContent('Preset must be pulled')) + expect(crowdsecApi.writeCrowdsecFile).not.toHaveBeenCalled() }) it('records backup info on apply failure and generic errors', async () => { @@ -388,59 +386,36 @@ describe('CrowdSecConfig coverage', () => { expect((screen.getByTestId('apply-preset-btn') as HTMLButtonElement).disabled).toBe(true) }) - it('guards local apply prerequisites and succeeds when content exists', async () => { - vi.mocked(crowdsecApi.listCrowdsecFiles).mockResolvedValueOnce({ files: [] }) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(501, 'not implemented')) + it('shows the server error for 413 and 400 on file save and no success toast', async () => { await renderPage() - await userEvent.click(screen.getByTestId('apply-preset-btn')) - await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Select a configuration file to apply the preset')) + await userEvent.selectOptions(screen.getByTestId('crowdsec-file-select'), 'acquis.yaml') + await waitFor(() => expect(crowdsecApi.readCrowdsecFile).toHaveBeenCalledWith('acquis.yaml')) + await waitFor(() => expect(screen.getAllByRole('textbox').some((el) => el.tagName.toLowerCase() === 'textarea')).toBe(true)) - cleanup() - vi.mocked(toast.error).mockClear() + vi.mocked(crowdsecApi.writeCrowdsecFile).mockRejectedValueOnce(axiosError(413, 'too large', { error: 'content too large' })) + await userEvent.click(screen.getByText('Save')) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('content too large')) - vi.mocked(crowdsecApi.listCrowdsecFiles).mockResolvedValue({ files: ['acquis.yaml'] }) - vi.mocked(presetsApi.listCrowdsecPresets).mockResolvedValue({ - presets: [ - { - slug: 'custom-empty', - title: 'Empty', - summary: 'empty preset', - source: 'hub', - requires_hub: false, - available: true, - cached: false, - cache_key: 'cache-empty', - etag: 'etag-empty', - }, - ], - }) - vi.mocked(presetsApi.pullCrowdsecPreset).mockResolvedValue({ - status: 'pulled', - slug: 'custom-empty', - preview: '', - cache_key: 'cache-empty', - }) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(501, 'not implemented')) - await renderPage() - await userEvent.selectOptions(screen.getByTestId('crowdsec-file-select'), 'acquis.yaml') - await userEvent.click(screen.getByTestId('apply-preset-btn')) - await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Preset preview is unavailable; retry pulling before applying')) + vi.mocked(crowdsecApi.writeCrowdsecFile).mockRejectedValueOnce(axiosError(400, 'bad', { error: 'symlink target refused' })) + await userEvent.click(screen.getByText('Save')) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('symlink target refused')) - cleanup() - vi.mocked(toast.error).mockClear() + vi.mocked(crowdsecApi.writeCrowdsecFile).mockRejectedValueOnce(new Error('network')) + await userEvent.click(screen.getByText('Save')) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Failed to save file')) - vi.mocked(presetsApi.pullCrowdsecPreset).mockResolvedValue({ - status: 'pulled', - slug: presetFromCatalog.slug, - preview: 'content', - cache_key: 'cache-123', - }) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValueOnce(axiosError(501, 'not implemented')) - vi.mocked(crowdsecApi.writeCrowdsecFile).mockResolvedValue({}) + expect(toast.success).not.toHaveBeenCalledWith('File saved') + }) + + it('saves successfully when the write response has an empty backup', async () => { + vi.mocked(crowdsecApi.writeCrowdsecFile).mockResolvedValue({ status: 'written', backup: '' }) await renderPage() await userEvent.selectOptions(screen.getByTestId('crowdsec-file-select'), 'acquis.yaml') - await userEvent.click(screen.getByTestId('apply-preset-btn')) - await waitFor(() => expect(crowdsecApi.writeCrowdsecFile).toHaveBeenCalled()) + await waitFor(() => expect(crowdsecApi.readCrowdsecFile).toHaveBeenCalledWith('acquis.yaml')) + await waitFor(() => expect(screen.getAllByRole('textbox').some((el) => el.tagName.toLowerCase() === 'textarea')).toBe(true)) + await userEvent.click(screen.getByText('Save')) + await waitFor(() => expect(toast.success).toHaveBeenCalledWith('File saved')) + expect(toast.error).not.toHaveBeenCalled() }) it('reads, edits, saves, and closes files', async () => { diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.curatedApply.test.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.curatedApply.test.tsx index faa9bd2f9..72dc0ab86 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.curatedApply.test.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.curatedApply.test.tsx @@ -193,7 +193,12 @@ describe('CrowdSecConfig curated preset apply', () => { expect(crowdsecApi.writeCrowdsecFile).not.toHaveBeenCalled() }) - it('keeps the local fallback for hub presets on 501', async () => { + it.each([ + [501, 'apply unsupported'], + [503, 'hub unavailable upstream'], + [500, 'apply blew up'], + [504, 'upstream timed out'], + ])('surfaces the server message for hub preset %i and performs no file write', async (status, message) => { vi.mocked(presetsApi.listCrowdsecPresets).mockResolvedValue({ presets: [hubPreset] }) vi.mocked(presetsApi.pullCrowdsecPreset).mockResolvedValue({ status: 'pulled', @@ -202,12 +207,16 @@ describe('CrowdSecConfig curated preset apply', () => { cache_key: hubPreset.cache_key, source: 'hub', }) - vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValue(axiosError(501, 'not implemented')) + vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValue(axiosError(status, 'Request failed', { error: message })) await renderPage() await userEvent.selectOptions(screen.getByTestId('crowdsec-file-select'), 'acquis.yaml') await waitFor(() => expect(screen.getByTestId('preset-preview')).toHaveTextContent('base-http-scenarios')) await userEvent.click(screen.getByTestId('apply-preset-btn')) - await waitFor(() => expect(crowdsecApi.writeCrowdsecFile).toHaveBeenCalledWith('acquis.yaml', expect.stringContaining('base-http-scenarios'))) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith(`Apply failed: ${message}`)) + expect(toast.success).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() + expect(crowdsecApi.writeCrowdsecFile).not.toHaveBeenCalled() + expect(backupsApi.createBackup).not.toHaveBeenCalled() }) it('shows a generic error for non-HTTP failures on curated presets', async () => { diff --git a/frontend/src/pages/__tests__/CrowdSecConfig.spec.tsx b/frontend/src/pages/__tests__/CrowdSecConfig.spec.tsx index 843d9f772..d6b40b10d 100644 --- a/frontend/src/pages/__tests__/CrowdSecConfig.spec.tsx +++ b/frontend/src/pages/__tests__/CrowdSecConfig.spec.tsx @@ -11,6 +11,7 @@ import * as featureFlagsApi from '../../api/featureFlags' import * as presetsApi from '../../api/presets' import * as api from '../../api/security' import { HUB_PRESET_FIXTURES } from '../../test-utils/crowdsecPresetFixtures' +import { toast } from '../../utils/toast' import CrowdSecConfig from '../CrowdSecConfig' import type { ConsoleEnrollmentStatus } from '../../api/consoleEnrollment' @@ -21,6 +22,9 @@ vi.mock('../../api/backups') vi.mock('../../api/settings') vi.mock('../../api/presets') vi.mock('../../api/featureFlags') +vi.mock('../../utils/toast', () => ({ + toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() }, +})) vi.mock('../../components/CrowdSecBouncerKeyDisplay', () => ({ CrowdSecBouncerKeyDisplay: () => null, })) @@ -271,34 +275,30 @@ describe('CrowdSecConfig', () => { expect(screen.getByRole('link', { name: /Cerberus/i })).toHaveAttribute('href', '/security') }) - it('renders preset preview and applies with backup when backend apply is unavailable', async () => { + it('shows the server error and does not write files when backend apply is unavailable', async () => { const status = { crowdsec: { enabled: true, mode: 'local' as const, api_url: '' }, cerberus: { enabled: true }, waf: { enabled: false, mode: 'disabled' as const }, rate_limit: { enabled: false }, acl: { enabled: false } } - const presetContent = HUB_PRESET_FIXTURES.find((preset) => preset.slug === 'bot-mitigation-essentials')?.content || '' vi.mocked(api.getSecurityStatus).mockResolvedValue(status) vi.mocked(crowdsecApi.listCrowdsecFiles).mockResolvedValue({ files: ['acquis.yaml'] }) vi.mocked(crowdsecApi.readCrowdsecFile).mockResolvedValue({ content: '' }) - vi.mocked(backupsApi.createBackup).mockResolvedValue({ job_id: 'job-1', type: 'create', status: 'pending' }) - vi.mocked(crowdsecApi.writeCrowdsecFile).mockResolvedValue({ status: 'written' }) const axiosError = new AxiosError('not implemented', undefined, undefined, undefined, { status: 501, statusText: 'Not Implemented', headers: {}, config: { headers: {} }, - data: {}, + data: { error: 'apply unsupported' }, } as AxiosResponse) vi.mocked(presetsApi.applyCrowdsecPreset).mockRejectedValue(axiosError) renderWithProviders(<CrowdSecConfig />) expect(await screen.findByText('CrowdSec Configuration')).toBeInTheDocument() await waitFor(() => expect(screen.getByTestId('preset-preview')).toHaveTextContent('configs:')) - const fileSelect = screen.getByTestId('crowdsec-file-select') - await userEvent.selectOptions(fileSelect, 'acquis.yaml') - const applyBtn = screen.getByTestId('apply-preset-btn') - await userEvent.click(applyBtn) + await userEvent.selectOptions(screen.getByTestId('crowdsec-file-select'), 'acquis.yaml') + await userEvent.click(screen.getByTestId('apply-preset-btn')) await waitFor(() => expect(presetsApi.applyCrowdsecPreset).toHaveBeenCalledWith({ slug: 'bot-mitigation-essentials', cache_key: 'cache-123' })) - await waitFor(() => expect(backupsApi.createBackup).toHaveBeenCalled()) - await waitFor(() => expect(crowdsecApi.writeCrowdsecFile).toHaveBeenCalledWith('acquis.yaml', presetContent)) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Apply failed: apply unsupported')) + expect(backupsApi.createBackup).not.toHaveBeenCalled() + expect(crowdsecApi.writeCrowdsecFile).not.toHaveBeenCalled() }) it('surfaces validation error when slug is invalid', async () => { @@ -388,7 +388,7 @@ describe('CrowdSecConfig', () => { // reloadHint is a boolean and renders as empty/true - just verify the info section exists }) - it('shows improved error message when preset is not cached', async () => { + it('shows the server message when preset is not cached', async () => { const axiosError = { isAxiosError: true, response: { @@ -407,7 +407,7 @@ describe('CrowdSecConfig', () => { const applyBtn = await screen.findByTestId('apply-preset-btn') await userEvent.click(applyBtn) - expect(await screen.findByTestId('preset-validation-error')).toBeInTheDocument() - expect(screen.getByTestId('preset-validation-error')).toHaveTextContent('Preset must be pulled before applying') + await waitFor(() => expect(toast.error).toHaveBeenCalledWith(expect.stringContaining('Apply failed: CrowdSec preset not cached. Pull the preset first'))) + expect(screen.queryByTestId('preset-validation-error')).not.toBeInTheDocument() }) }) diff --git a/scripts/crowdsec_data_persistence_test.sh b/scripts/crowdsec_data_persistence_test.sh new file mode 100755 index 000000000..fb54835e8 --- /dev/null +++ b/scripts/crowdsec_data_persistence_test.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Brief: Integration test for CrowdSec hub data file persistence (container recreation). +# +# CrowdSec hub items live under /etc/crowdsec (symlinked into the /app/data volume) but their +# data files (blocklists, GeoLite2 databases, ...) live in CrowdSec's data_dir. The entrypoint +# redirects data_dir onto the volume and, only when installed items are missing data files, +# runs a bounded `cscli hub upgrade`. +# +# Scenarios (all use a throwaway named volume and no published ports): +# 1. Fresh volume: items really install, data_dir is on the volume, data files exist, +# GeoLite2-City.mmdb exists, `crowdsec -t` reports no data-init errors. +# 2. Recreate container with the same volume: files present, `cscli hub upgrade` NOT run. +# 2b. Recreate with one data file present but empty: `cscli hub upgrade` NOT run (empty is valid). +# 3. Recreate with data files deleted from the volume: `cscli hub upgrade` runs once and the +# files are restored. +# 4. Legacy volume (config.yaml still pointing at /var/lib/crowdsec/data, no data files, no +# cscli.hub_branch): data_dir is migrated, the files are restored and hub_branch is added. +# 5. An operator-chosen cscli.hub_branch is preserved (and never duplicated) across restarts. +# +# The entrypoint also pins cscli.hub_branch to master so hub commands never depend on +# version.crowdsec.net; scenarios 1 and 4 assert it is set exactly once. +# +# Requires network access (CrowdSec hub) and a charon image (default: charon:local). + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$PROJECT_ROOT" + +IMAGE="${CHARON_TEST_IMAGE:-charon:local}" +CONTAINER_NAME="charon-crowdsec-persist-test" +VOLUME_NAME="charon_crowdsec_persist_data" +INIT_TIMEOUT_SECONDS="${INIT_TIMEOUT_SECONDS:-300}" +DATA_DIR="/app/data/crowdsec/data" +CONFIG_YAML="/app/data/crowdsec/config/config.yaml" +INIT_DONE_MARKER="CrowdSec configuration initialized" +UPGRADE_MARKER="Running 'cscli hub upgrade'" + +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_test() { echo -e "${BLUE}[TEST]${NC} $1"; } + +PASSED=0 +FAILED=0 + +pass_test() { + PASSED=$((PASSED + 1)) + echo -e " ${GREEN}✓ PASS${NC}" +} + +fail_test() { + FAILED=$((FAILED + 1)) + echo -e " ${RED}✗ FAIL${NC}: $1" +} + +cleanup() { + log_info "Cleaning up test resources..." + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + docker volume rm "${VOLUME_NAME}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is not available; aborting" >&2 + exit 1 +fi + +if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then + log_info "Building ${IMAGE}..." + docker build -t "${IMAGE}" . +fi + +# container_logs: print the container logs. +container_logs() { + docker logs "${CONTAINER_NAME}" 2>&1 +} + +# logs_contain: succeed if the container logs contain the fixed string. The logs are buffered +# first so `grep -q` closing a pipe early cannot trip pipefail. +logs_contain() { + local out + out="$(container_logs)" + grep -qF -- "$1" <<<"${out}" +} + +# start_container: (re)create the test container on the shared volume and wait for hub init. +start_container() { + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + docker run -d --name "${CONTAINER_NAME}" \ + -e CHARON_ENV=development \ + -v "${VOLUME_NAME}:/app/data" \ + "${IMAGE}" >/dev/null + + local waited=0 + until logs_contain "${INIT_DONE_MARKER}"; do + if [ "${waited}" -ge "${INIT_TIMEOUT_SECONDS}" ]; then + echo "Timed out after ${INIT_TIMEOUT_SECONDS}s waiting for CrowdSec init" >&2 + docker logs "${CONTAINER_NAME}" 2>&1 | tail -40 >&2 + exit 1 + fi + if [ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}")" != "true" ]; then + echo "Container exited during startup" >&2 + docker logs "${CONTAINER_NAME}" 2>&1 | tail -40 >&2 + exit 1 + fi + sleep 3 + waited=$((waited + 3)) + done +} + +# volume_sh: run a shell snippet against the volume without starting Charon. +volume_sh() { + docker run --rm --user 0 --entrypoint sh -v "${VOLUME_NAME}:/app/data" "${IMAGE}" -c "$1" +} + +# declared_data_files: data files declared by installed hub items (one per line). +declared_data_files() { + docker exec "${CONTAINER_NAME}" sh -c \ + "find -L /etc/crowdsec/parsers /etc/crowdsec/scenarios /etc/crowdsec/postoverflows -type f -name '*.yaml' -exec grep -h 'dest_file:' {} + 2>/dev/null | sed -e 's/^.*dest_file:[[:space:]]*//' -e 's/[[:space:]]*\$//' | sort -u" +} + +# missing_data_files: declared data files that are absent from data_dir (empty files are valid). +missing_data_files() { + local f + declared_data_files | while IFS= read -r f; do + [ -n "${f}" ] || continue + docker exec "${CONTAINER_NAME}" test -e "${DATA_DIR}/${f}" || echo "${f}" + done +} + +count_upgrades() { + container_logs | grep -c "${UPGRADE_MARKER}" || true +} + +# crowdsec_data_errors: run the CrowdSec agent briefly (it is not auto-started by the entrypoint) +# and print any data-file / GeoIP initialization errors from its log. +crowdsec_data_errors() { + docker exec "${CONTAINER_NAME}" sh -c \ + 'rm -f /var/log/crowdsec/crowdsec.log; timeout 20 crowdsec -c /etc/crowdsec/config.yaml >/dev/null 2>&1; grep -Ei "unable to init data for file|unable to initialize GeoIP" /var/log/crowdsec/crowdsec.log || true' +} + +# hub_branch_lines: every hub_branch line in the persisted config.yaml. +hub_branch_lines() { + docker exec "${CONTAINER_NAME}" grep -E '^[[:space:]]*hub_branch:' "${CONFIG_YAML}" || true +} + +echo "==============================================" +echo "=== CrowdSec Data Persistence Test ===" +echo "==============================================" + +docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true +docker volume rm "${VOLUME_NAME}" >/dev/null 2>&1 || true + +# ---------------------------------------------------------------------------- +log_info "Scenario 1: fresh volume" +start_container + +log_test "Check 1: install guard really installs items on a fresh volume" +PROBING_JSON="$(docker exec "${CONTAINER_NAME}" cscli scenarios inspect crowdsecurity/http-probing -o json)" +if logs_contain "Installing http-probing" && + grep -q '"installed": *true' <<<"${PROBING_JSON}"; then + pass_test +else + fail_test "http-probing was not installed on a fresh volume" +fi + +log_test "Check 2: data_dir points at the persistent volume" +if docker exec "${CONTAINER_NAME}" grep -q "data_dir: ${DATA_DIR}/" "${CONFIG_YAML}"; then + pass_test +else + fail_test "data_dir is not ${DATA_DIR}/ in config.yaml" +fi + +log_test "Check 3: hub data files and GeoLite2-City.mmdb exist in data_dir" +DECLARED="$(declared_data_files)" +if [ -z "${DECLARED}" ]; then + fail_test "installed hub items declare no data files (expected some)" +elif [ -n "$(missing_data_files)" ]; then + fail_test "missing data files: $(missing_data_files | tr '\n' ' ')" +elif ! docker exec "${CONTAINER_NAME}" test -s "${DATA_DIR}/GeoLite2-City.mmdb"; then + fail_test "GeoLite2-City.mmdb is missing" +else + pass_test +fi + +log_test "Check 4: hub upgrade not run on a fresh volume with intact data files" +if [ "$(count_upgrades)" -eq 0 ]; then + pass_test +else + fail_test "hub upgrade ran on first start" +fi + +log_test "Check 5: crowdsec.log has no data/GeoIP initialization errors" +CS_ERRORS="$(crowdsec_data_errors)" +if [ -n "${CS_ERRORS}" ]; then + fail_test "crowdsec reported data/GeoIP init errors" + echo "${CS_ERRORS}" | head -5 +else + pass_test +fi + +log_test "Check 5b: cscli.hub_branch pinned to master exactly once and cscli reports the pinned version" +if [ "$(hub_branch_lines)" != " hub_branch: master" ]; then + fail_test "expected a single 'hub_branch: master' line, got: $(hub_branch_lines | tr '\n' '|')" +elif ! docker exec "${CONTAINER_NAME}" cscli version 2>&1 | grep -Eq '^version: v[0-9]+\.[0-9]+\.[0-9]+'; then + fail_test "cscli version does not report a version" +else + pass_test +fi + +# ---------------------------------------------------------------------------- +log_info "Scenario 2: recreate with intact data (healthy restart)" +start_container + +log_test "Check 6: data files still present and hub upgrade NOT run" +if [ -n "$(missing_data_files)" ]; then + fail_test "data files missing after recreation: $(missing_data_files | tr '\n' ' ')" +elif [ "$(count_upgrades)" -ne 0 ]; then + fail_test "hub upgrade ran although all data files were present" +else + pass_test +fi + +# ---------------------------------------------------------------------------- +log_info "Scenario 2b: recreate with a present-but-empty data file (legitimately empty list)" +EMPTY_FILE="$(declared_data_files | head -n1)" +docker rm -f "${CONTAINER_NAME}" >/dev/null +volume_sh "truncate -s 0 '${DATA_DIR}/${EMPTY_FILE}'" +start_container + +log_test "Check 6b: an empty-but-present data file does NOT trigger hub upgrade and stays empty" +if [ -z "${EMPTY_FILE}" ]; then + fail_test "no declared data file to empty" +elif [ "$(count_upgrades)" -ne 0 ]; then + fail_test "hub upgrade ran although the only 'problem' was an empty data file" +elif docker exec "${CONTAINER_NAME}" test -s "${DATA_DIR}/${EMPTY_FILE}"; then + fail_test "${EMPTY_FILE} was repopulated unexpectedly" +else + pass_test +fi + +# ---------------------------------------------------------------------------- +log_info "Scenario 3: recreate after data files were lost" +docker rm -f "${CONTAINER_NAME}" >/dev/null +volume_sh "find ${DATA_DIR} -mindepth 1 -maxdepth 1 ! -name 'crowdsec.db*' -exec rm -rf {} +" +start_container + +log_test "Check 7: hub upgrade runs once, restores the data files and CrowdSec loads cleanly" +if [ "$(count_upgrades)" -ne 1 ]; then + fail_test "expected exactly one hub upgrade, got $(count_upgrades)" +elif [ -n "$(missing_data_files)" ]; then + fail_test "data files still missing: $(missing_data_files | tr '\n' ' ')" +elif [ -n "$(crowdsec_data_errors)" ]; then + fail_test "crowdsec reported data/GeoIP init errors after restore" +else + pass_test +fi + +# ---------------------------------------------------------------------------- +log_info "Scenario 4: legacy volume (data_dir still at the old location)" +docker rm -f "${CONTAINER_NAME}" >/dev/null +volume_sh "sed -i -e 's|data_dir: ${DATA_DIR}/|data_dir: /var/lib/crowdsec/data/|' -e '/hub_branch:/d' ${CONFIG_YAML} && find ${DATA_DIR} -mindepth 1 -maxdepth 1 ! -name 'crowdsec.db*' -exec rm -rf {} +" +start_container + +log_test "Check 8: data_dir migrated, data files restored and hub_branch re-added" +if ! docker exec "${CONTAINER_NAME}" grep -q "data_dir: ${DATA_DIR}/" "${CONFIG_YAML}"; then + fail_test "data_dir was not migrated to ${DATA_DIR}/" +elif [ "$(hub_branch_lines)" != " hub_branch: master" ]; then + fail_test "hub_branch was not re-added exactly once: $(hub_branch_lines | tr '\n' '|')" +elif [ -n "$(missing_data_files)" ]; then + fail_test "data files missing after migration: $(missing_data_files | tr '\n' ' ')" +else + pass_test +fi + +# ---------------------------------------------------------------------------- +log_info "Scenario 5: operator-chosen hub_branch is preserved" +docker rm -f "${CONTAINER_NAME}" >/dev/null +volume_sh "sed -i 's|^ hub_branch: master| hub_branch: custom-branch|' ${CONFIG_YAML}" +start_container + +log_test "Check 9: custom hub_branch kept, not duplicated" +if [ "$(hub_branch_lines)" = " hub_branch: custom-branch" ]; then + pass_test +else + fail_test "custom hub_branch not preserved: $(hub_branch_lines | tr '\n' '|')" +fi + +echo "" +echo "==============================================" +echo "Passed: ${PASSED} Failed: ${FAILED}" +echo "==============================================" + +if [ "${FAILED}" -ne 0 ]; then + exit 1 +fi diff --git a/scripts/diagnose-crowdsec.sh b/scripts/diagnose-crowdsec.sh index 57dd6503c..c21333d55 100755 --- a/scripts/diagnose-crowdsec.sh +++ b/scripts/diagnose-crowdsec.sh @@ -6,7 +6,7 @@ set -euo pipefail # Default configuration -DATA_DIR="${CROWDSEC_DATA_DIR:-/var/lib/crowdsec}" +DATA_DIR="${CROWDSEC_DATA_DIR:-/app/data/crowdsec}" JSON_OUTPUT=false LAPI_PORT="${CROWDSEC_LAPI_PORT:-8085}" @@ -37,7 +37,7 @@ while [[ $# -gt 0 ]]; do echo "" echo "Options:" echo " --json Output results as JSON" - echo " --data-dir CrowdSec data directory (default: /var/lib/crowdsec)" + echo " --data-dir CrowdSec data directory (default: /app/data/crowdsec)" echo " --lapi-port LAPI port (default: 8085)" echo " -h, --help Show this help message" exit 0 diff --git a/scripts/toolchain-key.sh b/scripts/toolchain-key.sh index 832785e9f..eff270fa0 100755 --- a/scripts/toolchain-key.sh +++ b/scripts/toolchain-key.sh @@ -70,7 +70,7 @@ for pair in "caddy-inline:$caddy_stage" "crowdsec-inline:$crowdsec_stage"; do done # ARG names whose default values feed the key. Keep in sync with spec §2.2 / §3.4.2. -arg_re='^ARG (GO_VERSION|ALPINE_IMAGE|CROWDSEC_VERSION|EXPR_LANG_VERSION|XNET_VERSION|XCRYPTO_VERSION|KLAUSPOST_COMPRESS_VERSION|GRPC_VERSION|CADDY_VERSION|CADDY_CANDIDATE_VERSION|CADDY_USE_CANDIDATE|CADDY_PATCH_SCENARIO|CADDY_SECURITY_VERSION|CORAZA_CADDY_VERSION|CADDY_GEOIP2_VERSION|CADDY_RATELIMIT_VERSION|CADDY_DNS_CLOUDFLARE_VERSION|CADDY_DNS_ROUTE53_VERSION|CADDY_DNS_DIGITALOCEAN_VERSION|CADDY_DNS_GOOGLECLOUDDNS_VERSION|CADDY_DNS_AZURE_VERSION|CADDY_DNS_NAMECHEAP_VERSION|CADDY_DNS_GODADDY_VERSION|CADDY_DNS_HETZNER_VERSION|CADDY_DNS_VULTR_VERSION|CADDY_DNS_DNSIMPLE_VERSION|CADDY_DNS_OVH_VERSION|CADDY_DNS_GANDI_VERSION|CADDY_DNS_LINODE_VERSION|CADDY_DNS_PORKBUN_VERSION|CADDY_DNS_NETLIFY_VERSION|CADDY_DNS_DESEC_VERSION|CADDY_DNS_SCALEWAY_VERSION|CADDY_DNS_DUCKDNS_VERSION|CADDY_DNS_DNSMADEEASY_VERSION|CADDY_DNS_NAMEDOTCOM_VERSION|CADDY_DNS_NAMESILO_VERSION|CADDY_DNS_RFC2136_VERSION|CADDY_DNS_POWERDNS_VERSION|CADDY_DNS_INWX_VERSION|CADDY_DNS_LOOPIA_VERSION|CADDY_DNS_BUNNY_VERSION|CADDY_DNS_VERCEL_VERSION|LIBDNS_NAMEDOTCOM_VERSION|LIBDNS_VERCEL_VERSION|CADDY_DNS_DNSIMPLE_GO_VERSION)=' +arg_re='^ARG (GO_VERSION|ALPINE_IMAGE|CROWDSEC_VERSION|EXPR_LANG_VERSION|XNET_VERSION|XCRYPTO_VERSION|XMOD_VERSION|KLAUSPOST_COMPRESS_VERSION|GRPC_VERSION|CADDY_VERSION|CADDY_CANDIDATE_VERSION|CADDY_USE_CANDIDATE|CADDY_PATCH_SCENARIO|CADDY_SECURITY_VERSION|CORAZA_CADDY_VERSION|CADDY_GEOIP2_VERSION|CADDY_RATELIMIT_VERSION|CADDY_DNS_CLOUDFLARE_VERSION|CADDY_DNS_ROUTE53_VERSION|CADDY_DNS_DIGITALOCEAN_VERSION|CADDY_DNS_GOOGLECLOUDDNS_VERSION|CADDY_DNS_AZURE_VERSION|CADDY_DNS_NAMECHEAP_VERSION|CADDY_DNS_GODADDY_VERSION|CADDY_DNS_HETZNER_VERSION|CADDY_DNS_VULTR_VERSION|CADDY_DNS_DNSIMPLE_VERSION|CADDY_DNS_OVH_VERSION|CADDY_DNS_GANDI_VERSION|CADDY_DNS_LINODE_VERSION|CADDY_DNS_PORKBUN_VERSION|CADDY_DNS_NETLIFY_VERSION|CADDY_DNS_DESEC_VERSION|CADDY_DNS_SCALEWAY_VERSION|CADDY_DNS_DUCKDNS_VERSION|CADDY_DNS_DNSMADEEASY_VERSION|CADDY_DNS_NAMEDOTCOM_VERSION|CADDY_DNS_NAMESILO_VERSION|CADDY_DNS_RFC2136_VERSION|CADDY_DNS_POWERDNS_VERSION|CADDY_DNS_INWX_VERSION|CADDY_DNS_LOOPIA_VERSION|CADDY_DNS_BUNNY_VERSION|CADDY_DNS_VERCEL_VERSION|LIBDNS_NAMEDOTCOM_VERSION|LIBDNS_VERCEL_VERSION|CADDY_DNS_DNSIMPLE_GO_VERSION)=' key="$( { diff --git a/tests/security/crowdsec-config.spec.ts b/tests/security/crowdsec-config.spec.ts index fc988a489..a65bf523a 100644 --- a/tests/security/crowdsec-config.spec.ts +++ b/tests/security/crowdsec-config.spec.ts @@ -54,103 +54,44 @@ test.describe('CrowdSec Configuration @security', () => { }); test('should display presets section', async ({ page }) => { - // Look for presets, packages, scenarios, or collections section - // This feature may not be fully implemented - const presetsSection = page.getByText(/packages|presets|scenarios|collections|bouncers/i).first(); - const presetsVisible = await presetsSection.isVisible().catch(() => false); - - if (presetsVisible) { - await expect(presetsSection).toBeVisible(); - } else { - test.info().annotations.push({ - type: 'info', - description: 'Presets section not visible - feature may not be implemented' - }); - } + await expect(page.getByRole('heading', { name: 'Presets', level: 3 })).toBeVisible(); + await expect(page.getByRole('button', { name: /apply preset/i })).toBeVisible(); }); }); test.describe('Preset Management', () => { - // Preset management may not be fully implemented - test('should display list of available presets', async ({ page }) => { - await test.step('Verify presets are listed', async () => { - // Wait for presets to load - await page.waitForResponse(resp => - resp.url().includes('/presets') || resp.url().includes('/crowdsec') || resp.url().includes('/hub'), - { timeout: 10000 } - ).catch(() => { - // If no API call, presets might be loaded statically or not implemented - }); - - // Should show preset cards or list items - const presetElements = page.locator('[class*="card"], [class*="preset"], button').filter({ - hasText: /apply|install|owasp|basic|advanced|paranoid/i - }); + // Apply outcomes (success, error, fallback) are covered by crowdsec-preset-apply.spec.ts + // and crowdsec-hub-preset-apply.spec.ts. These cases cover the list, search and preview. + const CURATED_TITLE = 'Honeypot Friendly Defaults'; + const OTHER_TITLE = 'GeoIP Enrichment'; - const count = await presetElements.count(); - - if (count === 0) { - // Presets might not be implemented - check for config file management instead - const configSection = page.getByText(/configuration|file|config/i).first(); - const configVisible = await configSection.isVisible().catch(() => false); - - if (!configVisible) { - test.info().annotations.push({ - type: 'info', - description: 'No presets displayed - feature may not be implemented' - }); - } - } - }); + test('should list the curated presets', async ({ page }) => { + await expect(page.getByRole('button', { name: new RegExp(CURATED_TITLE, 'i') })).toBeVisible(); + await expect(page.getByRole('button', { name: new RegExp(OTHER_TITLE, 'i') })).toBeVisible(); }); - test('should allow searching presets', async ({ page }) => { - const searchInput = page.getByPlaceholder(/search/i); - const searchVisible = await searchInput.isVisible().catch(() => false); - - if (searchVisible) { - await test.step('Search for a preset', async () => { - await searchInput.fill('basic'); - // Results should be filtered - await page.waitForTimeout(500); // Debounce - }); - } - }); + test('should filter presets by the search query', async ({ page }) => { + const search = page.getByRole('textbox', { name: /search presets/i }); - test('should show preset preview when selected', async ({ page }) => { - // Find and click on a preset to preview - const presetButton = page.locator('button').filter({ hasText: /preview|view|select/i }).first(); - const buttonVisible = await presetButton.isVisible().catch(() => false); + await test.step('Search for a matching preset', async () => { + await search.fill('honeypot'); + await expect(page.getByRole('button', { name: new RegExp(CURATED_TITLE, 'i') })).toBeVisible(); + await expect(page.getByRole('button', { name: new RegExp(OTHER_TITLE, 'i') })).toHaveCount(0); + }); - if (buttonVisible) { - await presetButton.click(); - // Should show preview content - await page.waitForTimeout(500); - } + await test.step('Search for a non-matching term', async () => { + await search.fill('no-such-preset-xyz'); + await expect(page.getByRole('button', { name: new RegExp(CURATED_TITLE, 'i') })).toHaveCount(0); + await expect(page.getByText(/no presets/i).first()).toBeVisible(); + }); }); - test('should apply preset with confirmation', async ({ page }) => { - // Find apply button - const applyButton = page.locator('button').filter({ hasText: /apply/i }).first(); - const buttonVisible = await applyButton.isVisible().catch(() => false); - - if (buttonVisible) { - await test.step('Click apply button', async () => { - await applyButton.click(); - }); - - await test.step('Handle confirmation or result', async () => { - // Either a confirmation dialog or success toast should appear - const confirmDialog = page.getByRole('dialog'); - const dialogVisible = await confirmDialog.isVisible().catch(() => false); - - if (dialogVisible) { - // Cancel to not make permanent changes - const cancelButton = page.getByRole('button', { name: /cancel/i }); - await cancelButton.click(); - } - }); - } + test('should show the preset details when a preset is selected', async ({ page }) => { + const preset = page.getByRole('button', { name: new RegExp(CURATED_TITLE, 'i') }); + await preset.click(); + await expect(preset).toHaveAttribute('aria-pressed', 'true'); + await expect(page.getByTestId('preset-warning')).toBeVisible(); + await expect(page.getByTestId('preset-preview')).toBeVisible(); }); }); @@ -195,22 +136,8 @@ test.describe('CrowdSec Configuration @security', () => { }); test('should have import functionality', async ({ page }) => { - // Look for import button or file input - const importButton = page.getByRole('button', { name: /import/i }); - const importInput = page.locator('input[type="file"]'); - - const importVisible = await importButton.isVisible().catch(() => false); - const inputVisible = await importInput.isVisible().catch(() => false); - - // Import functionality may not be implemented - if (importVisible || inputVisible) { - await expect(importButton.or(importInput)).toBeVisible(); - } else { - test.info().annotations.push({ - type: 'info', - description: 'Import functionality not visible - feature may not be implemented' - }); - } + await expect(page.getByTestId('import-file')).toBeVisible(); + await expect(page.getByRole('button', { name: /^import$/i })).toBeVisible(); }); }); diff --git a/tests/security/crowdsec-file-editor.spec.ts b/tests/security/crowdsec-file-editor.spec.ts new file mode 100644 index 000000000..8afa3da89 --- /dev/null +++ b/tests/security/crowdsec-file-editor.spec.ts @@ -0,0 +1,144 @@ +/** + * CrowdSec Configuration File Editor E2E Tests + * + * Verifies the generic file editor on the CrowdSec configuration page: + * - A successful save posts exactly one write for exactly the selected path + * - Server rejections (400 disallowed type / invalid YAML, 413 too large) + * surface the server message and never report success + * + * The file list, read, write and backup endpoints are stubbed so the tests are + * deterministic and never touch the real CrowdSec tree. + * + * @see /projects/Charon/docs/plans/current_spec.md + */ + +import type { Page } from '@playwright/test'; +import { test, expect, loginUser } from '../fixtures/auth-fixtures'; +import { waitForLoadingComplete } from '../utils/wait-helpers'; + +const FILES_ROUTE = '**/api/v1/admin/crowdsec/files'; +// Anchored so it does not also match the sibling /crowdsec/files list endpoint. +const FILE_ROUTE = /\/api\/v1\/admin\/crowdsec\/file(\?.*)?$/; +const BACKUPS_ROUTE = '**/api/v1/backups'; +const SELECTED_PATH = 'config.yaml'; +const OTHER_PATH = 'parsers/s01-parse/e2e.yaml'; + +interface WriteBody { + path: string; + content: string; +} + +/** Stubs list/read/backup endpoints and records every POST to the write endpoint. */ +async function stubEditor(page: Page, writeResponse: { status: number; json: Record<string, unknown> }): Promise<{ writes: WriteBody[] }> { + const writes: WriteBody[] = []; + await page.route(FILES_ROUTE, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { files: [SELECTED_PATH, OTHER_PATH] }, + }); + }); + await page.route(BACKUPS_ROUTE, async (route) => { + if (route.request().method() !== 'POST') { + await route.fallback(); + return; + } + await route.fulfill({ status: 202, contentType: 'application/json', json: { id: 'e2e-backup', status: 'queued' } }); + }); + await page.route(FILE_ROUTE, async (route) => { + const request = route.request(); + if (request.method() === 'POST') { + writes.push(request.postDataJSON() as WriteBody); + await route.fulfill({ status: writeResponse.status, contentType: 'application/json', json: writeResponse.json }); + return; + } + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { content: 'common:\n log_level: info\n' }, + }); + }); + return { writes }; +} + +async function editSelectedFile(page: Page, newContent: string): Promise<void> { + await test.step('Open the configuration page', async () => { + await page.goto('/security/crowdsec'); + await waitForLoadingComplete(page); + }); + await test.step('Select the file and edit its content', async () => { + await page.getByRole('combobox', { name: /select a file/i }).selectOption(SELECTED_PATH); + const editor = page.getByRole('textbox', { name: /file content|content/i }); + await expect(editor).toHaveValue(/log_level: info/); + await editor.fill(newContent); + }); + await test.step('Click Save', async () => { + await page.getByRole('button', { name: /^save$/i }).click(); + }); +} + +test.describe('CrowdSec File Editor @security', () => { + test.beforeEach(async ({ page, adminUser }) => { + await loginUser(page, adminUser); + await waitForLoadingComplete(page); + }); + + test('should post exactly one write for exactly the selected path on save', async ({ page }) => { + const stub = await stubEditor(page, { + status: 200, + json: { status: 'written', backup: '/app/data/crowdsec.filebackup.20260101-000000.000000' }, + }); + const newContent = 'common:\n log_level: debug\n'; + + await editSelectedFile(page, newContent); + + await test.step('Verify success toast', async () => { + await expect(page.getByText(/file saved/i)).toBeVisible(); + }); + await test.step('Verify exactly one write for exactly the selected path', async () => { + expect(stub.writes).toHaveLength(1); + expect(stub.writes[0].path).toBe(SELECTED_PATH); + expect(stub.writes[0].content).toBe(newContent); + }); + }); + + test('should show the server message when the write is rejected with 400 (invalid YAML)', async ({ page }) => { + const message = 'file content is not valid YAML'; + const stub = await stubEditor(page, { status: 400, json: { error: message } }); + + await editSelectedFile(page, 'common: [unclosed\n'); + + await test.step('Verify the server message is surfaced and no success toast is shown', async () => { + await expect(page.getByText(message).first()).toBeVisible(); + await expect(page.getByText(/file saved/i)).toHaveCount(0); + }); + await test.step('Verify a single write attempt for the selected path', async () => { + expect(stub.writes).toHaveLength(1); + expect(stub.writes[0].path).toBe(SELECTED_PATH); + }); + }); + + test('should show the server message when the write is rejected with 400 (disallowed file type)', async ({ page }) => { + const message = 'file type is not allowed'; + await stubEditor(page, { status: 400, json: { error: message } }); + + await editSelectedFile(page, 'anything\n'); + + await test.step('Verify the server message is surfaced and no success toast is shown', async () => { + await expect(page.getByText(message).first()).toBeVisible(); + await expect(page.getByText(/file saved/i)).toHaveCount(0); + }); + }); + + test('should show the server message when the write is rejected with 413 (too large)', async ({ page }) => { + const message = 'file content exceeds the maximum allowed size'; + await stubEditor(page, { status: 413, json: { error: message } }); + + await editSelectedFile(page, 'common:\n log_level: debug\n'); + + await test.step('Verify the server message is surfaced and no success toast is shown', async () => { + await expect(page.getByText(message).first()).toBeVisible(); + await expect(page.getByText(/file saved/i)).toHaveCount(0); + }); + }); +}); diff --git a/tests/security/crowdsec-hub-preset-apply.spec.ts b/tests/security/crowdsec-hub-preset-apply.spec.ts new file mode 100644 index 000000000..a4b029832 --- /dev/null +++ b/tests/security/crowdsec-hub-preset-apply.spec.ts @@ -0,0 +1,195 @@ +/** + * CrowdSec Hub Preset Apply E2E Tests + * + * Verifies that applying a non-curated (hub) CrowdSec preset reports the real + * outcome and never falls back to a client-side file write: + * - Success toast only when the backend returns status 'applied' + * - Errors (500/503/504) are surfaced and never shown as success + * - HTTP 501 shows an error instead of writing the preview locally + * + * The preset list, pull and apply endpoints are stubbed so the tests are + * deterministic and do not require the CrowdSec hub or cscli. + * + * @see /projects/Charon/docs/plans/current_spec.md + */ + +import type { Page } from '@playwright/test'; +import { test, expect, loginUser } from '../fixtures/auth-fixtures'; +import { waitForLoadingComplete } from '../utils/wait-helpers'; + +const PRESETS_ROUTE = '**/api/v1/admin/crowdsec/presets'; +const PULL_ROUTE = '**/api/v1/admin/crowdsec/presets/pull'; +const APPLY_ROUTE = '**/api/v1/admin/crowdsec/presets/apply'; +const FILE_WRITE_ROUTE = '**/api/v1/admin/crowdsec/file'; +const HUB_PRESET_SLUG = 'crowdsecurity/e2e-hub-preset'; +const HUB_PRESET_TITLE = 'E2E Hub Preset'; +const CACHE_KEY = 'e2e-hub-cache-key'; + +/** Records POSTs to the generic file-write endpoint (the local fallback path). */ +function trackFileWrites(page: Page): { count: () => number } { + let writes = 0; + void page.route(FILE_WRITE_ROUTE, async (route) => { + if (route.request().method() === 'POST') { + writes += 1; + } + await route.continue(); + }); + return { count: () => writes }; +} + +/** Stubs the preset list and pull endpoints so a single hub preset is available. */ +async function stubHubPreset(page: Page): Promise<void> { + await page.route(PRESETS_ROUTE, async (route) => { + if (route.request().method() !== 'GET') { + await route.fallback(); + return; + } + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { + presets: [ + { + slug: HUB_PRESET_SLUG, + title: HUB_PRESET_TITLE, + summary: 'Deterministic hub preset for E2E', + source: 'hub', + requires_hub: true, + available: true, + cached: true, + cache_key: CACHE_KEY, + }, + ], + }, + }); + }); + await page.route(PULL_ROUTE, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { + status: 'pulled', + slug: HUB_PRESET_SLUG, + preview: 'name: e2e-hub-preset\n', + cache_key: CACHE_KEY, + source: 'hub', + }, + }); + }); +} + +async function openPageAndApplyHubPreset(page: Page): Promise<void> { + await test.step('Reload so the stubbed preset list is used', async () => { + await page.goto('/security/crowdsec'); + await waitForLoadingComplete(page); + }); + await test.step('Select the hub preset', async () => { + await page.getByRole('button', { name: new RegExp(HUB_PRESET_TITLE, 'i') }).click(); + }); + await test.step('Click Apply Preset', async () => { + await page.getByRole('button', { name: /apply preset/i }).click(); + }); +} + +test.describe('CrowdSec Hub Preset Apply @security', () => { + test.beforeEach(async ({ page, adminUser }) => { + await loginUser(page, adminUser); + await waitForLoadingComplete(page); + await stubHubPreset(page); + }); + + test('should show success toast with reload note when backend reports applied', async ({ page }) => { + const fileWrites = trackFileWrites(page); + await page.route(APPLY_ROUTE, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { + status: 'applied', + used_cscli: true, + backup: '/app/data/crowdsec.backup.20260101-000000.000000', + reload_hint: true, + cache_key: CACHE_KEY, + }, + }); + }); + + await openPageAndApplyHubPreset(page); + + await test.step('Verify success toast mentions reload', async () => { + await expect(page.getByText(/preset applied.*reload required/i)).toBeVisible(); + }); + await test.step('Verify no local file write happened', async () => { + expect(fileWrites.count()).toBe(0); + }); + }); + + test('should not show success toast when a 200 response has a non-applied status', async ({ page }) => { + const fileWrites = trackFileWrites(page); + await page.route(APPLY_ROUTE, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { status: 'failed', used_cscli: false, reload_hint: false }, + }); + }); + + await openPageAndApplyHubPreset(page); + + await test.step('Verify the incomplete status is surfaced and no success toast is shown', async () => { + await expect(page.getByText(/preset apply did not complete \(status: failed\)/i).first()).toBeVisible(); + await expect(page.getByText(/preset applied/i)).toHaveCount(0); + }); + await test.step('Verify no local fallback write', async () => { + expect(fileWrites.count()).toBe(0); + }); + }); + + for (const status of [500, 503, 504]) { + test(`should show an error and no success toast when hub apply returns ${status}`, async ({ page }) => { + const fileWrites = trackFileWrites(page); + const message = 'hub preset could not be applied'; + await page.route(APPLY_ROUTE, async (route) => { + await route.fulfill({ + status, + contentType: 'application/json', + json: { error: message }, + }); + }); + + await openPageAndApplyHubPreset(page); + + await test.step('Verify an error is surfaced', async () => { + await expect(page.getByText(message).first()).toBeVisible(); + }); + await test.step('Verify no success toast is shown', async () => { + await expect(page.getByText(/preset applied/i)).toHaveCount(0); + }); + await test.step('Verify zero local fallback writes', async () => { + expect(fileWrites.count()).toBe(0); + }); + }); + } + + test('should show an error and perform no write when hub apply returns 501', async ({ page }) => { + const fileWrites = trackFileWrites(page); + await page.route(APPLY_ROUTE, async (route) => { + await route.fulfill({ + status: 501, + contentType: 'application/json', + json: { error: 'preset apply not implemented' }, + }); + }); + + await openPageAndApplyHubPreset(page); + + await test.step('Verify error shown and no success or local-apply toast', async () => { + await expect(page.getByText(/preset apply not implemented/i).first()).toBeVisible(); + await expect(page.getByText(/preset applied/i)).toHaveCount(0); + await expect(page.getByText(/applying locally/i)).toHaveCount(0); + }); + await test.step('Verify zero local fallback writes', async () => { + expect(fileWrites.count()).toBe(0); + }); + }); +});