diff --git a/.env.example b/.env.example index bb0feb0..7d98b77 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,11 @@ MQTT_HOST=0.0.0.0 # Leave empty to skip audience validation AUTH_EXPECTED_AUDIENCE=mqtt.yourdomain.com +# Where a plain (non-WebSocket) browser request to the broker is 302-redirected. +# This broker only speaks MQTT-over-WS, so a human who opens the URL is sent to +# the front-end. Defaults to https://observers.dutchmeshcore.nl/. +HTTP_REDIRECT_URL=https://observers.dutchmeshcore.nl/ + # Subscribe-Only Users (one per line, format: username:password:role:maxConnections) # Role: 1=admin (full access + can delete retained), 2=full_access (no hidden data), 3=limited (filtered data) # Default role is 3 (limited) if not specified diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6ead13c..2574cff 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,6 +13,7 @@ on: - ".gitignore" - ".github/FUNDING.yml" - "prompts.txt" + - "docker-compose*.yml" pull_request: branches: [main] paths-ignore: @@ -22,6 +23,7 @@ on: - ".gitignore" - ".github/FUNDING.yml" - "prompts.txt" + - "docker-compose*.yml" workflow_dispatch: env: diff --git a/README.md b/README.md index bfe9b3c..e411c4e 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ For setting up with TLS using Cloudflare Tunnels, see [docs/cloudflare-tunnels.m ### Docker (production) -A hardened multi-stage `Dockerfile` and a `docker-compose.prod.yml` are provided. The compose stack runs the broker on a distroless, non-root, read-only-rootfs container alongside a `cloudflared` sidecar that fronts it with TLS via a Cloudflare Tunnel. +A hardened multi-stage `Dockerfile` and a `docker-compose.prod.yml` are provided. The compose stack runs the broker on a distroless, non-root, read-only-rootfs container alongside a `cloudflared` sidecar that fronts it with TLS via a Cloudflare Tunnel. A label-scoped autoheal watcher restarts the broker when Docker marks it unhealthy. ```bash cp .env.example .env diff --git a/docker-compose.yml b/docker-compose.yml index 2c22bf5..2b00e10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,9 @@ services: # dockerfile: Dockerfile container_name: meshcore-mqtt-broker restart: unless-stopped + labels: + autoheal: "true" + autoheal.stop.timeout: "30" init: true # PID 1 reaper; clean SIGTERM forwarding to node. env_file: .env # SUBSCRIBER_*, AUTH_*, ABUSE_*, MQTT_* environment: @@ -54,6 +57,28 @@ services: ports: - "8883:8883" # Optional: expose MQTT port directly (not needed if using cloudflared tunnel) + autoheal: + # Pinned to the multi-platform digest published for willfarrell/autoheal:latest on 2026-07-28. + image: willfarrell/autoheal:latest@sha256:e513881f029803a9214bed90e88a16fbe945dd4c0876bbdf037f29d12b55f8e5 + container_name: meshcore-autoheal + restart: unless-stopped + environment: + AUTOHEAL_CONTAINER_LABEL: autoheal + AUTOHEAL_INTERVAL: "5" + AUTOHEAL_DEFAULT_STOP_TIMEOUT: "30" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + network_mode: none + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + cloudflared: # Pin to a specific release in production. See https://github.com/cloudflare/cloudflared/releases image: cloudflare/cloudflared:latest diff --git a/docs/docker-deployment.md b/docs/docker-deployment.md index 6c9eb00..ca9d7c1 100644 --- a/docs/docker-deployment.md +++ b/docs/docker-deployment.md @@ -126,22 +126,30 @@ The Node one-liner opens a TCP socket to `MQTT_WS_PORT` on `127.0.0.1`, exits 0 - Use the host network and expose 8883 directly. Forces operators to handle TLS themselves at L7 (nginx, caddy) and drops the existing Cloudflare-tunnel deployment story. - Use a volume-based tunnel credentials file instead of `TUNNEL_TOKEN`. Token mode is simpler — operators can rotate via the Cloudflare dashboard without touching the host filesystem. -### D8. Resource limits and log rotation +### D8. Autoheal for unhealthy broker containers -**Choice:** broker capped at 256M memory / 1.0 CPU; cloudflared at 128M / 0.5 CPU. Both use `json-file` driver with `max-size: 10m, max-file: 3`. +**Choice:** only the broker carries the `autoheal=true` label. The autoheal service checks for unhealthy labeled containers every five seconds and restarts the broker when Docker marks it unhealthy. The collector image deliberately remains on `:latest`; the autoheal image is pinned to an immutable multi-platform digest. + +**Why:** Docker health checks only report status and `restart: unless-stopped` only reacts when the main process exits. Autoheal supplies the missing health-status-to-restart behavior without changing the broker application. + +**Security:** the watcher needs read-write access to `/var/run/docker.sock` in order to issue the restart. It is isolated from the network, runs with a read-only root filesystem and all Linux capabilities dropped, and selects only explicitly labeled containers. + +### D9. Resource limits and log rotation + +**Choice:** broker capped at 256M memory / 1.0 CPU; cloudflared at 128M / 0.5 CPU. All services use `json-file` with `max-size: 10m, max-file: 3`. **Why:** - Memory: The broker is mostly in-memory state (rate-limiter Map, abuse-detection working set). 64M reservation, 256M ceiling is comfortable for thousands of clients without permitting runaway leaks. - CPU: The broker is I/O-bound (WebSocket frames, JWT verify). 1 CPU is generous; the limit is there to prevent any one container from saturating the host under attack. - Logs: the app logs heavily on auth and abuse events. Without rotation, a busy broker fills the host disk with `*-json.log` files. 30 MB rotated retention keeps recent events around without being a footgun. -### D9. Secrets via `env_file: .env`, not Docker secrets +### D10. Secrets via `env_file: .env`, not Docker secrets **Choice:** `docker-compose.prod.yml` uses `env_file: .env`. Subscriber credentials (`SUBSCRIBER_N=user:pass:role:max`), `AUTH_EXPECTED_AUDIENCE`, and `TUNNEL_TOKEN` all live in that file. **Why:** Docker Swarm secrets (file-based, mounted at `/run/secrets/`) would be a stronger story — they keep secrets off process env, off `docker inspect` output, and out of `/proc//environ`. But the application reads everything via `process.env` and `dotenv.config()`. Switching to Docker secrets would require code changes (read each secret from a file path, fall back to env). That's beyond the current scope. `.env` with `chmod 600` is the practical choice — match the project's existing configuration model and document it. -### D10. GitHub Actions: multi-arch, attested, built only on push +### D11. GitHub Actions: multi-arch, attested, built only on push **Choice:** `.github/workflows/docker-publish.yml` builds on `push` to `main` and on `v*.*.*` tags, runs on `pull_request` for build verification (no push), publishes to `ghcr.io/${{ github.repository }}` with `linux/amd64` + `linux/arm64`, and attaches SBOM + provenance attestations via `docker/build-push-action@v6` plus `actions/attest-build-provenance@v2`. @@ -168,7 +176,7 @@ The setup was verified end-to-end during implementation: ``` 4. **Healthcheck:** transitions from `starting` to `healthy` after the first probe (verified at t≈25s). 5. **Volume initialization:** the named volume is populated with `/data` owned by `65532:65532 0755`, and the broker creates `abuse-detection.db` (mode `0644`, same owner). Persistence survives container restart (volume is detached from container lifetime). -6. **Compose validation:** `docker compose -f docker-compose.prod.yml config` resolves cleanly with both services, the internal network, and the named volume. +6. **Compose validation:** `docker compose -f docker-compose.prod.yml config` resolves cleanly with all three services, the internal network, and the named volume. ## Operator notes @@ -224,7 +232,7 @@ If the broker hits the 256M memory ceiling under load, expect to tune two things 1. `deploy.resources.limits.memory` in compose. 2. The abuse-detector working set: `ABUSE_TOPIC_HISTORY_SIZE`, `ABUSE_DUPLICATE_WINDOW_SIZE`, and the per-client trust state retention. These all scale with active client count. -The healthcheck and `restart: unless-stopped` together mean the container will be recycled automatically on OOM kill, but data in `/data` is preserved across restarts. +An OOM kill terminates the broker and activates `restart: unless-stopped`. A health-check failure instead activates autoheal once Docker marks the broker unhealthy. Data in `/data` is preserved in both cases. ## Known limitations @@ -240,7 +248,7 @@ The healthcheck and `restart: unless-stopped` together mean the container will b |---|---| | `Dockerfile` | Multi-stage build: glibc builder + distroless runtime, non-root, tsx-at-runtime, TCP healthcheck. | | `.dockerignore` | Keeps `.env`, `dist/`, `node_modules`, secrets, and docs out of the build context. | -| `docker-compose.prod.yml` | Two-service stack: hardened broker + `cloudflared` sidecar on a private bridge. | +| `docker-compose.prod.yml` | Three-service stack: hardened broker, autoheal watcher, and `cloudflared` sidecar. | | `.env.example` | Template for runtime configuration; includes `TUNNEL_TOKEN` for the sidecar. | | `.github/workflows/docker-publish.yml` | Multi-arch build + push to GHCR with SBOM and provenance. | | `docs/cloudflare-tunnels.md` | Existing operator guide for setting up the tunnel side. | diff --git a/src/server.ts b/src/server.ts index b616f88..ace8689 100644 --- a/src/server.ts +++ b/src/server.ts @@ -17,6 +17,10 @@ const subscriberConfig = loadSubscriberConfig(); const WS_PORT = mqttConfig.wsPort; const HOST = mqttConfig.host; const EXPECTED_AUDIENCE = mqttConfig.expectedAudience; +// Where a plain (non-WebSocket) browser request to the broker is redirected. This +// broker only speaks MQTT-over-WS, so a human hitting the URL is sent to the +// front-end. Configurable; defaults to the DMC observers site. +const HTTP_REDIRECT_URL = process.env.HTTP_REDIRECT_URL || 'https://observers.dutchmeshcore.nl/'; // Helper function to validate IATA airport codes function isValidIATACode(code: string): boolean { @@ -977,10 +981,10 @@ aedes.on('clientError', (client, err) => { // Create HTTP server for WebSocket const httpServer = createServer((req, res) => { - // If this is not a WebSocket upgrade request, redirect to analyzer + // If this is not a WebSocket upgrade request, redirect to the front-end. if (!req.headers.upgrade || req.headers.upgrade.toLowerCase() !== 'websocket') { - console.log(`[HTTP] Non-WebSocket request from ${getClientIP(req)}, redirecting to analyzer`); - res.writeHead(301, { 'Location': 'https://analyzer.letsmesh.net/' }); + console.log(`[HTTP] Non-WebSocket request from ${getClientIP(req)}, redirecting to ${HTTP_REDIRECT_URL}`); + res.writeHead(302, { 'Location': HTTP_REDIRECT_URL }); res.end(); return; }