diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 0000000..314e4bc --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,166 @@ +name: Kubernetes + +# Proves the manifests in k8s/ actually work, rather than asserting it in a +# README. Two jobs, deliberately separated by cost: +# +# validate — schema-checks every manifest. Seconds. Catches typos and wrong +# apiVersions without starting anything. +# e2e — builds the image, creates a real cluster with kind, applies the +# CI overlay and drives it until the three runtimes are ready. +# +# The e2e job is what closes the gap declared in PR #25: until this exists, the +# manifests were valid YAML that had never been applied to a cluster. + +on: + push: + branches: + - main + paths: + - "k8s/**" + - "Dockerfile" + - ".github/workflows/k8s.yml" + pull_request: + paths: + - "k8s/**" + - "Dockerfile" + - ".github/workflows/k8s.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + name: Manifest schemas + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install kustomize and kubeconform + run: | + curl -sSfL https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash + sudo mv kustomize /usr/local/bin/ + curl -sSfL -o kubeconform.tar.gz \ + https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + tar xzf kubeconform.tar.gz kubeconform + sudo mv kubeconform /usr/local/bin/ + + - name: Build overlays + run: | + kustomize build k8s/base > /tmp/base.yaml + kustomize build k8s/overlays/ci > /tmp/ci.yaml + echo "--- base ---" && grep -c '^kind:' /tmp/base.yaml + echo "--- ci ---" && grep -c '^kind:' /tmp/ci.yaml + + - name: Validate against Kubernetes schemas + run: | + kubeconform -strict -summary \ + -kubernetes-version 1.31.0 \ + /tmp/base.yaml /tmp/ci.yaml + + # KEDA and Prometheus Operator objects are CRDs, so their schemas are not + # in the upstream Kubernetes catalogue. They are checked against the + # community CRD schema mirror instead; -ignore-missing-schemas would make + # this step pass by simply skipping them, which is worse than useless. + - name: Validate CRD-backed manifests + run: | + kubeconform -strict -summary \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + k8s/keda k8s/monitoring + + e2e: + name: kind end-to-end + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: feedpulse + node_image: kindest/node:v1.31.0 + + - name: Build image + run: docker build -t feedpulse:ci . + + # kind nodes have their own container runtime, so an image built on the + # runner is invisible to them until it is side-loaded. The manifests use + # imagePullPolicy: IfNotPresent, which is what makes this work without a + # registry. + - name: Load image into kind + run: kind load docker-image feedpulse:ci --name feedpulse + + - name: Apply manifests + run: kubectl apply -k k8s/overlays/ci + + - name: Wait for dependencies + run: | + kubectl -n feedpulse rollout status deploy/postgres --timeout=180s + kubectl -n feedpulse rollout status deploy/redis --timeout=120s + + # Migrations must land before the API can pass readiness: /ready resolves + # the base schema, so it fails by design until this Job completes. + - name: Run migrations + run: | + kubectl -n feedpulse wait --for=condition=complete job/feedpulse-migrate --timeout=240s \ + || { kubectl -n feedpulse logs job/feedpulse-migrate --tail=100; exit 1; } + + - name: Wait for the three runtimes + run: | + kubectl -n feedpulse rollout status deploy/feedpulse-api --timeout=240s + kubectl -n feedpulse rollout status deploy/feedpulse-worker --timeout=240s + kubectl -n feedpulse rollout status deploy/feedpulse-scheduler --timeout=240s + + # Asserts the probes mean what k8s/README.md claims they mean. A green + # rollout only proves readiness passed; this proves both endpoints answer + # and are distinct. + - name: Verify liveness and readiness + run: | + kubectl -n feedpulse port-forward svc/feedpulse-api 8080:80 & + sleep 5 + curl -fsS http://127.0.0.1:8080/health && echo " <- /health OK" + curl -fsS http://127.0.0.1:8080/ready && echo " <- /ready OK" + + # The rollout is where graceful shutdown is exercised: every pod receives + # SIGTERM and must drain within terminationGracePeriodSeconds. If + # SHUTDOWN_TIMEOUT_MS ever exceeds the grace period, this step is what + # catches it. + - name: Restart the worker and confirm it drains and returns + run: | + kubectl -n feedpulse rollout restart deploy/feedpulse-worker + kubectl -n feedpulse rollout status deploy/feedpulse-worker --timeout=180s + + - name: Install KEDA and apply the ScaledObject + run: | + helm repo add kedacore https://kedacore.github.io/charts + helm repo update + helm install keda kedacore/keda --namespace keda --create-namespace --wait --timeout 5m + kubectl -n feedpulse apply -f k8s/keda/ + + # A ScaledObject that the admission webhook accepts can still be inert. + # READY=True means KEDA resolved the Redis trigger and created the + # underlying HPA, which is the actual claim being made in k8s/README.md. + - name: Confirm KEDA took ownership of the worker + run: | + kubectl -n feedpulse wait --for=condition=Ready scaledobject/feedpulse-worker --timeout=120s \ + || { kubectl -n feedpulse describe scaledobject feedpulse-worker; exit 1; } + kubectl -n feedpulse get hpa + + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl -n feedpulse get pods -o wide + kubectl -n feedpulse get events --sort-by=.lastTimestamp | tail -40 + kubectl -n feedpulse describe pods | tail -120 + for d in feedpulse-api feedpulse-worker feedpulse-scheduler; do + echo "===== $d =====" + kubectl -n feedpulse logs deploy/$d --tail=80 --all-containers || true + kubectl -n feedpulse logs deploy/$d --tail=80 --previous || true + done diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..5d0977d --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,110 @@ +# Kubernetes deployment + +Manifests for running FeedPulse's three runtimes on Kubernetes. Tracking issue: +[#15](https://github.com/FullFran/feedpulse/issues/15). + +## Why Kubernetes, honestly + +At the volume this project currently runs, Kubernetes is **not** the cheapest or +simplest option. The existing `compose.dokploy.yml` is. Anyone claiming +otherwise is selling something. + +What Kubernetes buys here is specific and measurable: + +| Capability | Why Compose cannot do it | +| --------------------------------------- | ------------------------------------------------------------ | +| Scale `worker` on **queue depth** | Compose has no autoscaler, and no way to read BullMQ backlog | +| Independent scaling per runtime | `api`, `scheduler` and `worker` have unrelated load profiles | +| Rolling updates with automatic rollback | Compose replaces containers; it does not gate on health | +| Declarative recovery | A node dying is rescheduling, not an incident | + +The crossover point — the feed count above which this stops being overkill — is +measured in [#24](https://github.com/FullFran/feedpulse/issues/24) using +`npm run benchmark:stages:mvp`. The number belongs in the README, including the +range where Kubernetes loses. + +## Layout + +``` +k8s/base/ Deployments, Service, ConfigMap, Ingress, migration Job +k8s/keda/ ScaledObject: worker autoscaling on BullMQ queue depth +k8s/monitoring/ ServiceMonitor / PodMonitor for kube-prometheus-stack +``` + +## Apply + +```bash +kubectl create namespace feedpulse + +# Secrets come from a secret manager, never from the repository. +# k8s/base/secret.example.yaml documents the required keys. + +kubectl apply -k k8s/base +kubectl apply -f k8s/keda # requires KEDA installed +kubectl apply -f k8s/monitoring # requires kube-prometheus-stack installed +``` + +## The decisions worth knowing + +**`scheduler` is a Deployment, not a CronJob.** It owns its own timing loop +(`SCHEDULER_TICK_MS`, default 15s) and holds warm database and Redis +connections. A CronJob would pay full process startup every tick for a 15-second +interval. It runs with `strategy: Recreate` so two schedulers never tick at +once; enqueue is deduplicated by job id (`feed-`) so a brief overlap is +survivable, but a visible gap is easier to reason about. + +**Liveness never touches a dependency.** `/health` is process-only; `/ready` +resolves the base schema. Wiring a dependency check into liveness converts a +Postgres blip into a cluster-wide CrashLoopBackOff, because every restarted pod +comes back to the same unreachable database. + +**`terminationGracePeriodSeconds` must exceed `SHUTDOWN_TIMEOUT_MS`.** On +SIGTERM the runtime stops accepting jobs and drains the in-flight one. If the +kubelet SIGKILLs first, that job dies mid-flight and BullMQ only recovers it +once the lock expires. + +**The worker Deployment declares no `replicas`.** KEDA owns that field. Setting +it here would fight the ScaledObject on every apply. + +**Worker probes are TCP, and that is a known compromise.** The worker has no +HTTP API, only its metrics server. A TCP check proves the process is up and its +event loop accepts connections — it cannot prove the BullMQ connection is alive. +Closing that gap needs a readiness endpoint reporting consumer state. + +**Migrations must be backward compatible.** During a RollingUpdate both versions +serve at once. Expand-and-contract; never a destructive change in the release +that also ships the code depending on it. + +## How this is verified + +`.github/workflows/k8s.yml` runs on every change under `k8s/`: + +| Job | What it proves | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Manifest schemas** | `kubeconform -strict` over both overlays, plus the KEDA and Prometheus CRDs against the community catalogue. No `-ignore-missing-schemas`: a skipped schema is worse than no check | +| **kind end-to-end** | Builds the image, side-loads it into a real cluster, applies the CI overlay, runs migrations, waits for all three runtimes, curls `/health` and `/ready`, restarts the worker to exercise graceful shutdown, installs KEDA and waits for the ScaledObject to report `Ready` | + +The ScaledObject check is the one that matters: an object the admission webhook +accepts can still be inert. `Ready=True` means KEDA resolved the Redis trigger +and created the underlying HPA. + +### Two bugs this caught that reasoning did not + +**The KEDA trigger address was wrong twice.** It first used +`addressFromEnv: REDIS_ADDRESS`, but the worker exposes `REDIS_URL` — a full +URL, not a `host:port` pair. Then it used the short name `redis:6379`, which +failed with `server misbehaving`: the connection is opened by the **KEDA +operator**, which runs in the `keda` namespace, and short service names only +resolve within the caller's own namespace. It needs the FQDN. + +**The CI overlay disabled `ENABLE_AUTH`.** `env.schema.ts` refuses to boot with +auth disabled under `NODE_ENV=production`, because that resolves every request +to the shared legacy tenant. The guard was right and the overlay was wrong; CI +now runs with the same guarantee production has. + +## Not done yet + +- Terraform for the cluster itself ([#21](https://github.com/FullFran/feedpulse/issues/21)) +- CI/CD deploy with automatic rollback ([#22](https://github.com/FullFran/feedpulse/issues/22)) +- Grafana dashboards ([#20](https://github.com/FullFran/feedpulse/issues/20)) +- Worker readiness reflecting consumer state diff --git a/k8s/base/api.yaml b/k8s/base/api.yaml new file mode 100644 index 0000000..12934d6 --- /dev/null +++ b/k8s/base/api.yaml @@ -0,0 +1,122 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: feedpulse-api + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api + spec: + # The image runs as uid 1000 and owns nothing it can rewrite; see the + # comments in Dockerfile. These settings assert that at admission time + # instead of trusting the image to have got it right. + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 40 + containers: + - name: api + image: ghcr.io/fullfran/feedpulse:latest + imagePullPolicy: IfNotPresent + args: ["node", "dist/main/api.js"] + ports: + - name: http + containerPort: 3000 + envFrom: + - configMapRef: + name: feedpulse-config + - secretRef: + name: feedpulse-secrets + # Liveness asks one question only: is this process wedged? It must not + # touch Postgres or Redis. A dependency outage that fails liveness + # restarts every pod against the same broken dependency, which turns a + # database blip into a CrashLoopBackOff across the whole Deployment. + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + # Readiness asks the opposite question: can this replica serve right + # now? /ready resolves the base schema through search_path, so it fails + # while Postgres is unreachable and the pod is pulled out of the + # Service endpoints without being killed. + readinessProbe: + httpGet: + path: /ready + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + # Slow first boot must not be mistaken for a wedged process. Until the + # startup probe passes, liveness is not evaluated at all. + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 3 + failureThreshold: 30 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: feedpulse-api + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: feedpulse-api +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api diff --git a/k8s/base/configmap.yaml b/k8s/base/configmap.yaml new file mode 100644 index 0000000..56c2c2a --- /dev/null +++ b/k8s/base/configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: feedpulse-config +data: + NODE_ENV: "production" + PORT: "3000" + LOG_LEVEL: "info" + WORKER_METRICS_PORT: "3001" + WORKER_METRICS_BIND: "0.0.0.0" + # Must stay below the pod's terminationGracePeriodSeconds, or the kubelet + # SIGKILLs the process while it is still draining. + SHUTDOWN_TIMEOUT_MS: "30000" + SCHEDULER_TICK_MS: "15000" + SCHEDULER_BATCH_SIZE: "50" + ENABLE_AUTH: "true" + ENABLE_SWAGGER: "false" + ALLOW_PRIVATE_FEED_HOSTS: "false" diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml new file mode 100644 index 0000000..f9abe54 --- /dev/null +++ b/k8s/base/ingress.yaml @@ -0,0 +1,28 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: feedpulse-api + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + # The app also rate limits via THROTTLER (RATE_LIMIT_*). This is the outer + # layer: it sheds load before it reaches a pod, which the in-process + # limiter by definition cannot do. + nginx.ingress.kubernetes.io/limit-rps: "20" +spec: + ingressClassName: nginx + tls: + - hosts: + - feedpulse.example.com + secretName: feedpulse-tls + rules: + - host: feedpulse.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: feedpulse-api + port: + name: http diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..75c8369 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: feedpulse +commonLabels: + app.kubernetes.io/part-of: feedpulse +resources: + - configmap.yaml + - migrate-job.yaml + - api.yaml + - worker.yaml + - scheduler.yaml + - ingress.yaml +# secret.example.yaml is intentionally NOT listed: it is documentation, not a +# resource. Real secrets come from the cluster's secret manager. +images: + - name: ghcr.io/fullfran/feedpulse + newTag: latest diff --git a/k8s/base/migrate-job.yaml b/k8s/base/migrate-job.yaml new file mode 100644 index 0000000..14b6b54 --- /dev/null +++ b/k8s/base/migrate-job.yaml @@ -0,0 +1,42 @@ +# Runs db/migrations before a new version serves traffic. +# +# Ordering is the part that matters: this Job must complete before the api +# rollout starts, and migrations must be backward compatible with the version +# still running, because during a RollingUpdate both versions are live at once. +# Expand-and-contract, never a destructive change in the same release. +apiVersion: batch/v1 +kind: Job +metadata: + name: feedpulse-migrate +spec: + backoffLimit: 2 + ttlSecondsAfterFinished: 3600 + template: + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: migrate + image: ghcr.io/fullfran/feedpulse:latest + args: ["node", "dist/scripts/migrate.js"] + envFrom: + - configMapRef: + name: feedpulse-config + - secretRef: + name: feedpulse-secrets + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] diff --git a/k8s/base/scheduler.yaml b/k8s/base/scheduler.yaml new file mode 100644 index 0000000..4a7d4f9 --- /dev/null +++ b/k8s/base/scheduler.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: feedpulse-scheduler + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: scheduler +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: scheduler + # Recreate, not RollingUpdate. Two schedulers alive at once would both tick. + # Enqueue is deduplicated by job id (`feed-`, see FetchFeedJobData), so a + # brief overlap is survivable rather than corrupting -- but a visible gap is + # cheaper to reason about than a silent double tick. + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: scheduler + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 40 + containers: + - name: scheduler + image: ghcr.io/fullfran/feedpulse:latest + imagePullPolicy: IfNotPresent + args: ["node", "dist/main/scheduler.js"] + envFrom: + - configMapRef: + name: feedpulse-config + - secretRef: + name: feedpulse-secrets + resources: + requests: + cpu: 50m + memory: 192Mi + limits: + memory: 384Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/k8s/base/secret.example.yaml b/k8s/base/secret.example.yaml new file mode 100644 index 0000000..b11b5b6 --- /dev/null +++ b/k8s/base/secret.example.yaml @@ -0,0 +1,18 @@ +# Example only. Do NOT apply this file and do NOT commit real values. +# +# In a cluster these come from a secret manager (External Secrets Operator or +# the provider's CSI driver), never from a file in the repository. This exists +# so the required keys are discoverable without reading the env schema. +apiVersion: v1 +kind: Secret +metadata: + name: feedpulse-secrets +type: Opaque +stringData: + DATABASE_URL: "postgresql://user:password@postgres:5432/feedpulse" + REDIS_URL: "redis://redis:6379" + TENANT_SECRETS_MASTER_KEY: "replace-me" + METRICS_AUTH_TOKEN: "replace-me" + # KEDA's TriggerAuthentication reads this key. Empty string when Redis has + # no password, but the key must exist or the ScaledObject fails to activate. + REDIS_PASSWORD: "" diff --git a/k8s/base/worker.yaml b/k8s/base/worker.yaml new file mode 100644 index 0000000..3558201 --- /dev/null +++ b/k8s/base/worker.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: feedpulse-worker + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: worker +spec: + # Replica count is deliberately absent: KEDA owns it (see k8s/keda). Setting + # it here would fight the ScaledObject on every apply. + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: worker + template: + metadata: + labels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: worker + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + # Must exceed SHUTDOWN_TIMEOUT_MS. On SIGTERM the runtime stops accepting + # jobs and drains the in-flight one; if the kubelet SIGKILLs first, that + # job dies mid-flight and BullMQ only recovers it when its lock expires. + terminationGracePeriodSeconds: 60 + containers: + - name: worker + image: ghcr.io/fullfran/feedpulse:latest + imagePullPolicy: IfNotPresent + args: ["node", "dist/main/worker.js"] + ports: + - name: metrics + containerPort: 3001 + envFrom: + - configMapRef: + name: feedpulse-config + - secretRef: + name: feedpulse-secrets + # The worker has no HTTP API, only the metrics server started by + # src/main/worker-metrics-server.ts. A TCP check on that port is an + # honest liveness signal: the process is up and its event loop is + # accepting connections. + # + # Known limitation: this cannot distinguish "running" from "consuming". + # A worker whose BullMQ connection dropped while the metrics server + # stayed up would pass. Closing that gap needs a readiness endpoint + # that reports worker connection state -- tracked separately. + livenessProbe: + tcpSocket: + port: metrics + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 3 + startupProbe: + tcpSocket: + port: metrics + periodSeconds: 3 + failureThreshold: 30 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/k8s/keda/scaledobject.yaml b/k8s/keda/scaledobject.yaml new file mode 100644 index 0000000..26ffdba --- /dev/null +++ b/k8s/keda/scaledobject.yaml @@ -0,0 +1,81 @@ +# The reason Kubernetes earns its place in this project. +# +# Worker load is a function of queue depth, not of HTTP traffic. A CPU-based +# HPA is the WRONG signal here: the worker spends most of its time blocked on +# network I/O against slow feeds, so CPU stays low exactly while the backlog +# grows. Scaling on CPU would leave the queue draining at one replica. +# +# BullMQ stores waiting jobs in a Redis list keyed `bull::wait`, which is +# what the redis scaler below measures. +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: feedpulse-worker +spec: + scaleTargetRef: + name: feedpulse-worker + # Scale to zero is deliberately NOT used: minReplicaCount 1 keeps one consumer + # warm so a single feed arriving does not wait a full polling interval plus a + # cold start. Set it to 0 only after measuring that the added latency is + # acceptable -- and record the number. + minReplicaCount: 1 + maxReplicaCount: 10 + pollingInterval: 15 + # Must exceed the longest job. Scaling in mid-job relies on graceful shutdown + # to drain, and draining costs the pod's terminationGracePeriodSeconds. + cooldownPeriod: 120 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + # Feed bursts are spiky. Without this, replicas oscillate: scale up on + # a burst, scale straight back down into the next one. + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + triggers: + - type: redis + metadata: + # Two traps in one line, both found by CI rather than by reasoning: + # + # 1. host:port, NOT a URL. `addressFromEnv` would read from the scale + # target's pod env, and the worker exposes REDIS_URL (a full URL), + # not a host:port pair, so it would never connect. + # 2. Fully qualified, NOT the short name. The connection is opened by + # the KEDA operator, which runs in the `keda` namespace -- short + # names only resolve within the caller's own namespace. `redis:6379` + # fails with "server misbehaving" from there. + address: redis.feedpulse.svc.cluster.local:6379 + listName: bull:fetch-feed:wait + # Target backlog per replica. 20 waiting jobs -> 1 replica; 200 -> 10. + listLength: "20" + authenticationRef: + name: feedpulse-redis-auth + - type: redis + metadata: + # Two traps in one line, both found by CI rather than by reasoning: + # + # 1. host:port, NOT a URL. `addressFromEnv` would read from the scale + # target's pod env, and the worker exposes REDIS_URL (a full URL), + # not a host:port pair, so it would never connect. + # 2. Fully qualified, NOT the short name. The connection is opened by + # the KEDA operator, which runs in the `keda` namespace -- short + # names only resolve within the caller's own namespace. `redis:6379` + # fails with "server misbehaving" from there. + address: redis.feedpulse.svc.cluster.local:6379 + listName: bull:alert-delivery:wait + listLength: "20" + authenticationRef: + name: feedpulse-redis-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: feedpulse-redis-auth +spec: + secretTargetRef: + - parameter: password + name: feedpulse-secrets + key: REDIS_PASSWORD diff --git a/k8s/monitoring/servicemonitor.yaml b/k8s/monitoring/servicemonitor.yaml new file mode 100644 index 0000000..4190070 --- /dev/null +++ b/k8s/monitoring/servicemonitor.yaml @@ -0,0 +1,44 @@ +# prom-client is already a dependency and every runtime exposes metrics: the api +# on /metrics (which also aggregates the worker's), and the worker on its own +# port via src/main/worker-metrics-server.ts. +# +# /metrics is bearer-protected when METRICS_AUTH_TOKEN is set, so the scrape +# config has to carry it. Leaving it unset to make scraping "just work" would +# publish per-tenant operational data to anything that can reach the pod. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: feedpulse-api + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: api + endpoints: + - port: http + path: /metrics + interval: 30s + authorization: + type: Bearer + credentials: + name: feedpulse-secrets + key: METRICS_AUTH_TOKEN +--- +# The worker has no Service, so it is scraped by pod rather than by service. +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: feedpulse-worker + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: feedpulse + app.kubernetes.io/component: worker + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 30s diff --git a/k8s/overlays/ci/deps.yaml b/k8s/overlays/ci/deps.yaml new file mode 100644 index 0000000..7008edf --- /dev/null +++ b/k8s/overlays/ci/deps.yaml @@ -0,0 +1,84 @@ +# Postgres and Redis for CI only. +# +# These mirror the images and credentials in docker-compose.yml so a failure +# here means the manifests are wrong, not that the dependencies drifted. +# +# Deliberately NOT how production should run: no persistence, no backups, no +# StatefulSet. Production uses managed Postgres and Redis. Running a database +# in-cluster is a full-time job (volumes, failover, upgrades) and is explicitly +# called out as a trap in k8s/README.md. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: { app: postgres } + template: + metadata: + labels: { app: postgres } + spec: + containers: + - name: postgres + image: postgres:16-alpine + env: + - { name: POSTGRES_DB, value: rss_monitor } + - { name: POSTGRES_USER, value: postgres } + - { name: POSTGRES_PASSWORD, value: postgres } + - { name: PGDATA, value: /var/lib/postgresql/data/pgdata } + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "rss_monitor"] + periodSeconds: 3 + failureThreshold: 20 + volumeMounts: + - { name: data, mountPath: /var/lib/postgresql/data } + volumes: + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + selector: { app: postgres } + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: { app: redis } + template: + metadata: + labels: { app: redis } + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + periodSeconds: 3 + failureThreshold: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + selector: { app: redis } + ports: + - port: 6379 + targetPort: 6379 diff --git a/k8s/overlays/ci/kustomization.yaml b/k8s/overlays/ci/kustomization.yaml new file mode 100644 index 0000000..e99123a --- /dev/null +++ b/k8s/overlays/ci/kustomization.yaml @@ -0,0 +1,63 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: feedpulse + +resources: + - namespace.yaml + - ../../base + - deps.yaml + - secret.yaml + +# Built locally and side-loaded into kind, so it must never be pulled. +images: + - name: ghcr.io/fullfran/feedpulse + newName: feedpulse + newTag: ci + +patches: + # No ingress controller is installed in the CI cluster, so an Ingress object + # would apply cleanly and then do nothing. Removing it keeps the run honest + # about what is actually being exercised: #18 is not covered here. + - target: + kind: Ingress + name: feedpulse-api + patch: | + $patch: delete + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: feedpulse-api + + # One api replica is enough to prove the manifests. Two would only slow the + # runner down. The PodDisruptionBudget is relaxed to match, otherwise + # minAvailable would equal the replica count and block eviction. + - target: + kind: Deployment + name: feedpulse-api + patch: | + - op: replace + path: /spec/replicas + value: 1 + - target: + kind: PodDisruptionBudget + name: feedpulse-api + patch: | + - op: replace + path: /spec/minAvailable + value: 0 + + # AUTH_PROVIDER defaults to `clerk_api_key`, which would need real Clerk + # credentials. `api_key` verifies against the local api_keys table instead, + # which the migration seeds from BOOTSTRAP_API_KEY. + # + # ENABLE_AUTH is deliberately NOT patched to false: env.schema.ts refuses to + # boot with auth disabled under NODE_ENV=production, and that guard exists for + # a good reason -- disabling auth resolves every request to the shared legacy + # tenant. CI runs with the same guarantee production has. + - target: + kind: ConfigMap + name: feedpulse-config + patch: | + - op: add + path: /data/AUTH_PROVIDER + value: 'api_key' diff --git a/k8s/overlays/ci/namespace.yaml b/k8s/overlays/ci/namespace.yaml new file mode 100644 index 0000000..0e04420 --- /dev/null +++ b/k8s/overlays/ci/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: feedpulse diff --git a/k8s/overlays/ci/secret.yaml b/k8s/overlays/ci/secret.yaml new file mode 100644 index 0000000..6f789c3 --- /dev/null +++ b/k8s/overlays/ci/secret.yaml @@ -0,0 +1,20 @@ +# CI-only values. Safe to commit precisely because they are worthless: this +# namespace is destroyed with the kind cluster at the end of every run and is +# never reachable from outside the runner. +# +# Real deployments source these from a secret manager. See +# k8s/base/secret.example.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: feedpulse-secrets +type: Opaque +stringData: + DATABASE_URL: "postgres://postgres:postgres@postgres:5432/rss_monitor" + REDIS_URL: "redis://redis:6379" + REDIS_PASSWORD: "" + TENANT_SECRETS_MASTER_KEY: "ci-master-key-not-a-real-secret-000000000000" + METRICS_AUTH_TOKEN: "ci-metrics-token" + # Seeded into the api_keys table by the migration so the stack boots with a + # usable credential under AUTH_PROVIDER=api_key. + BOOTSTRAP_API_KEY: "ci-bootstrap-api-key-not-a-real-secret"