diff --git a/infra/environments/prod/backend.tf b/infra/environments/prod/backend.tf index ecf80e7..edc92c4 100644 --- a/infra/environments/prod/backend.tf +++ b/infra/environments/prod/backend.tf @@ -28,6 +28,10 @@ terraform { source = "hashicorp/tls" version = "~> 4.0" } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } } } diff --git a/infra/environments/prod/foundry.tf b/infra/environments/prod/foundry.tf new file mode 100644 index 0000000..d8f6115 --- /dev/null +++ b/infra/environments/prod/foundry.tf @@ -0,0 +1,100 @@ +# Foundry (on-prem k3s) cutover resources — the shared Cloudflare Tunnel +# that connects the on-prem cluster to Cloudflare. Public DNS records for +# individual hostnames are managed by external-dns on the foundry cluster +# (see kubernetes/apps-foundry/external-dns.yaml), not here. +# +# Design: the tunnel has a single catch-all ingress rule that forwards +# ALL traffic to the Traefik service. Traefik does Host-based routing +# in-cluster to the right namespace. Adding a new public hostname is +# just an Ingress resource in that service's manifest set — external-dns +# writes the CNAME automatically, no terraform edit needed. Same shape +# works for a second cluster (gr-other-cluster with its own tunnel + own +# external-dns using txtOwnerId=gr-other-cluster). +# +# Compute (EKS + ALB + ACM origin cert) still lives in main.tf. Once +# every hostname is migrated off EKS and the on-prem stack has soaked, +# delete module.eks / module.argocd / module.origin_cert from main.tf +# and remove kubernetes/apps/ from the repo. +# +# Off-cluster data services (gr-postgres, gr-mqtt, gr-clickhouse) stay +# in AWS — the on-prem cluster reaches them over the public internet via +# their existing Cloudflare A records. +# +# Cutover sequence for the first slice (sentinel): +# +# 1. terraform apply this file. Creates the tunnel; no DNS records +# yet, so no conflict with EKS external-dns. +# 2. Populate cloudflared-secrets + external-dns-config on foundry: +# kubectl -n cloudflared create secret generic cloudflared-secrets \ +# --from-literal=TUNNEL_TOKEN="$(terraform output -raw foundry_tunnel_token)" +# kubectl -n external-dns create configmap foundry-tunnel-target \ +# --from-literal=target="$(terraform output -raw foundry_tunnel_id).cfargotunnel.com" +# kubectl -n external-dns create secret generic cloudflare-api-token \ +# --from-literal=api-token="$CLOUDFLARE_API_TOKEN" +# 3. Apply kubernetes/bootstrap/root-foundry.yaml on the foundry +# ArgoCD, populate sentinel-secrets, wait for pods Healthy. +# 4. Cutover for one hostname: +# a. Delete the existing sentinel-v5.gauchoracing.com record via CF +# dashboard (or scale EKS external-dns to 0 first — it'll take +# its four records with it). +# b. Foundry external-dns notices the sentinel Ingress with no +# record, writes CNAME → .cfargotunnel.com within +# its --interval (1m default). +# c. Traffic starts landing on foundry within CF TTL. +# 5. Bake. Add mapache/vault/argocd in follow-up PRs — each is a copy +# of manifests// into manifests-foundry// with two file +# changes (ingress.yaml → Traefik + external-dns annotation, +# postgres.yaml → public hostname). No terraform edit. + +# Account ID for the tunnel resources. The CF API token used by the +# provider is scoped to a single account — return that account's ID. +data "cloudflare_accounts" "current" {} + +locals { + cloudflare_account_id = data.cloudflare_accounts.current.result[0].id +} + +# 32-byte tunnel secret, stored in TF state. +resource "random_bytes" "foundry_tunnel_secret" { + length = 32 +} + +# Named tunnel. config_src = "cloudflare" means CF hosts the ingress rule +# table (managed by cloudflare_zero_trust_tunnel_cloudflared_config +# below); the on-prem cloudflared pod boots with just a token and pulls +# the ruleset from CF at runtime. +resource "cloudflare_zero_trust_tunnel_cloudflared" "foundry" { + account_id = local.cloudflare_account_id + name = "gr-foundry" + tunnel_secret = random_bytes.foundry_tunnel_secret.base64 + config_src = "cloudflare" +} + +# Catch-all tunnel config. Every request that arrives via a DNS record +# CNAMEd to .cfargotunnel.com goes to the foundry Traefik +# service regardless of Host header; Traefik does the Host-based routing +# to the right in-cluster Service. +# +# Adding a hostname to the tunnel is NOT done here — it's implicit: any +# DNS record (written by external-dns for a matching Ingress) that +# points at this tunnel gets served. +resource "cloudflare_zero_trust_tunnel_cloudflared_config" "foundry" { + account_id = local.cloudflare_account_id + tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.foundry.id + + config = { + ingress = [ + { + service = "http://traefik.kube-system.svc.cluster.local:80" + }, + ] + } +} + +# Data source for the tunnel token; consumed by the cloudflared pods on +# the on-prem cluster via TUNNEL_TOKEN. Read with: +# terraform output -raw foundry_tunnel_token +data "cloudflare_zero_trust_tunnel_cloudflared_token" "foundry" { + account_id = local.cloudflare_account_id + tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.foundry.id +} diff --git a/infra/environments/prod/outputs.tf b/infra/environments/prod/outputs.tf index 8e102ff..4d16cab 100644 --- a/infra/environments/prod/outputs.tf +++ b/infra/environments/prod/outputs.tf @@ -86,3 +86,14 @@ output "clickhouse_admin_password" { value = module.clickhouse.admin_password sensitive = true } + +output "foundry_tunnel_id" { + description = "UUID of the gr-foundry Cloudflare tunnel. DNS CNAMEs point at .cfargotunnel.com." + value = cloudflare_zero_trust_tunnel_cloudflared.foundry.id +} + +output "foundry_tunnel_token" { + description = "Tunnel connector token for the gr-foundry tunnel. Consumed by the cloudflared pods on the on-prem cluster via TUNNEL_TOKEN. Read with `terraform output -raw foundry_tunnel_token`." + value = data.cloudflare_zero_trust_tunnel_cloudflared_token.foundry.token + sensitive = true +} diff --git a/kubernetes/apps-foundry/cloudflared.yaml b/kubernetes/apps-foundry/cloudflared.yaml new file mode 100644 index 0000000..82c8fa6 --- /dev/null +++ b/kubernetes/apps-foundry/cloudflared.yaml @@ -0,0 +1,33 @@ +# cloudflared: outbound tunnel connector. Fronts every public +# gauchoracing.com hostname on this cluster — the tunnel and its route +# table live in Terraform (foundry.tf), the pod that terminates the +# tunnel here lives in this Application. +# +# Manual one-time Secret bootstrap after this first syncs: +# +# kubectl -n cloudflared create secret generic cloudflared-secrets \ +# --from-literal=TUNNEL_TOKEN="$(cd ../../infra/environments/prod && terraform output -raw foundry_tunnel_token)" + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cloudflared + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/Gaucho-Racing/infrastructure.git + targetRevision: main + path: kubernetes/manifests-foundry/cloudflared + destination: + server: https://kubernetes.default.svc + namespace: cloudflared + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/kubernetes/apps-foundry/external-dns.yaml b/kubernetes/apps-foundry/external-dns.yaml new file mode 100644 index 0000000..03d7ed2 --- /dev/null +++ b/kubernetes/apps-foundry/external-dns.yaml @@ -0,0 +1,99 @@ +# external-dns for the foundry k3s cluster. +# +# Watches Ingress resources in this cluster and reconciles matching +# Cloudflare DNS records. Because the foundry Traefik service is +# behind a Cloudflare Tunnel (see infra/environments/prod/foundry.tf), +# every record it writes needs to CNAME to .cfargotunnel.com +# rather than the LB status field's node IPs — that's what --default-targets +# does below. +# +# Multi-cluster: this instance uses txtOwnerId=gr-foundry so it only +# manages records it wrote itself. A second on-prem cluster later gets +# its own external-dns Application with a different txtOwnerId and its +# own tunnel target; the two never touch each other's records. +# +# Manual one-time bootstrap after this Application first syncs: +# +# kubectl -n external-dns create secret generic cloudflare-api-token \ +# --from-literal=api-token="$CLOUDFLARE_API_TOKEN" +# +# kubectl -n external-dns create configmap foundry-tunnel-target \ +# --from-literal=target="$(cd ../../infra/environments/prod && terraform output -raw foundry_tunnel_id).cfargotunnel.com" + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: external-dns + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://kubernetes-sigs.github.io/external-dns + chart: external-dns + targetRevision: 1.18.0 + helm: + releaseName: external-dns + values: | + provider: + name: cloudflare + + domainFilters: + - gauchoracing.com + + # sync = create + update + delete based on cluster state. Same + # policy as the EKS instance so behavior stays consistent while + # both clusters exist. + policy: sync + + # TXT registry with a foundry-specific owner ID keeps the foundry + # external-dns from stepping on records the EKS instance owns + # (gr-prod) and vice-versa. Prefix distinguishes the ownership + # TXT record name too, so a `dig` on any record shows which + # cluster owns it. + registry: txt + txtOwnerId: gr-foundry + txtPrefix: fdry- + + # Watch Ingress only — no Service source, so a LoadBalancer + # Service can't accidentally publish itself. + sources: + - ingress + + env: + - name: CF_API_TOKEN + valueFrom: + secretKeyRef: + name: cloudflare-api-token + key: api-token + # FOUNDRY_TUNNEL_TARGET is populated from a ConfigMap created + # by the manual bootstrap step above. Kubernetes substitutes + # $(VAR) in container args at pod start, so the tunnel UUID + # flows into --default-targets without needing to be in git. + - name: FOUNDRY_TUNNEL_TARGET + valueFrom: + configMapKeyRef: + name: foundry-tunnel-target + key: target + + extraArgs: + # Force every record this instance writes to CNAME the tunnel, + # regardless of what the Ingress's status.loadBalancer field + # says. Traefik on k3s advertises node IPs there via klipper-lb; + # those aren't publicly routable and are irrelevant since + # traffic arrives through the tunnel anyway. + - "--default-targets=$(FOUNDRY_TUNNEL_TARGET)" + + logLevel: info + logFormat: json + destination: + server: https://kubernetes.default.svc + namespace: external-dns + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/kubernetes/apps-foundry/sentinel.yaml b/kubernetes/apps-foundry/sentinel.yaml new file mode 100644 index 0000000..9bb91a8 --- /dev/null +++ b/kubernetes/apps-foundry/sentinel.yaml @@ -0,0 +1,48 @@ +# Sentinel on the foundry k3s cluster. Same manifest set as the EKS tree +# (kubernetes/manifests/sentinel/), diverging only in ingress.yaml +# (Traefik instead of ALB) and postgres.yaml (public Cloudflare hostname +# for the gr-postgres EC2 instead of the internal EC2 hostname). +# +# Manual Secret needed once after this Application first syncs. Copy the +# external-facing values from the EKS cluster verbatim so identity state +# (Discord, INTERNAL_BOOTSTRAP_SECRET) stays coherent across environments +# during cutover: +# +# kubectl --context eks -n sentinel get secret sentinel-secrets -o yaml \ +# | grep -v '^\s*resourceVersion:\|^\s*uid:\|^\s*creationTimestamp:\|^\s*namespace:' \ +# | kubectl --context foundry apply -f - +# +# Or create from scratch: +# +# kubectl -n sentinel create secret generic sentinel-secrets \ +# --from-literal=POSTGRES_PASSWORD="$(cd ../../infra/environments/prod && terraform output -raw postgres_password)" \ +# --from-literal=INTERNAL_BOOTSTRAP_SECRET=... \ +# --from-literal=KERBECS_PASSWORD=... \ +# --from-literal=DISCORD_TOKEN=... \ +# --from-literal=DISCORD_CLIENT_ID=... \ +# --from-literal=DISCORD_CLIENT_SECRET=... \ +# --from-literal=GOOGLE_SERVICE_ACCOUNT=... + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sentinel + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/Gaucho-Racing/infrastructure.git + targetRevision: main + path: kubernetes/manifests-foundry/sentinel + destination: + server: https://kubernetes.default.svc + namespace: sentinel + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/kubernetes/bootstrap/root-foundry.yaml b/kubernetes/bootstrap/root-foundry.yaml new file mode 100644 index 0000000..4792bdb --- /dev/null +++ b/kubernetes/bootstrap/root-foundry.yaml @@ -0,0 +1,43 @@ +# Root ArgoCD Application for the foundry k3s cluster. +# +# The foundry cluster runs its own ArgoCD (installed via foundry-infra). +# This is a SECOND root app-of-apps that lives alongside foundry-infra's +# root — one ArgoCD instance, two independently-reconciled trees. Apply +# once manually on the foundry cluster after the initial cutover prep: +# +# kubectl --context foundry apply -f kubernetes/bootstrap/root-foundry.yaml +# +# From then on, ArgoCD reads kubernetes/apps-foundry/ from this repo and +# reconciles every Application resource it finds there. +# +# EKS is still managed by kubernetes/bootstrap/root.yaml → kubernetes/apps/. +# The two roots do NOT share a cluster; each root's ArgoCD is scoped to +# its own cluster via server: https://kubernetes.default.svc. +# +# See kubernetes/apps-foundry/*.yaml for the per-stack manual bootstrap +# steps (secret creation, DB creation, etc.). + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: root-foundry + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/Gaucho-Racing/infrastructure.git + targetRevision: main + path: kubernetes/apps-foundry + directory: + recurse: true + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/kubernetes/manifests-foundry/cloudflared/deployment.yaml b/kubernetes/manifests-foundry/cloudflared/deployment.yaml new file mode 100644 index 0000000..edb2cd4 --- /dev/null +++ b/kubernetes/manifests-foundry/cloudflared/deployment.yaml @@ -0,0 +1,81 @@ +# cloudflared: outbound Cloudflare Tunnel connector. +# +# One tunnel (gr-foundry) fronts every gauchoracing.com hostname routed +# to this cluster. Tunnel + per-hostname routes + DNS records are all +# terraform-managed (see infra/environments/prod/foundry.tf) — this +# Deployment just runs the connector. +# +# Manual bootstrap after this Application first syncs — the Secret's +# TUNNEL_TOKEN comes from `terraform output -raw foundry_tunnel_token`: +# +# kubectl -n cloudflared create secret generic cloudflared-secrets \ +# --from-literal=TUNNEL_TOKEN="$(cd ../../infra/environments/prod && terraform output -raw foundry_tunnel_token)" +# +# Two replicas so a pod restart doesn't drop the tunnel; each opens +# its own set of connections to the Cloudflare edge and CF load-balances +# across them automatically. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudflared + namespace: cloudflared +spec: + replicas: 2 + selector: + matchLabels: + app: cloudflared + template: + metadata: + labels: + app: cloudflared + spec: + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: cloudflared + containers: + - name: cloudflared + # Pin to a specific tag; bump manually when reviewing releases. + # https://github.com/cloudflare/cloudflared/releases + image: cloudflare/cloudflared:2025.11.1 + args: + - tunnel + - --no-autoupdate + - --metrics + - 0.0.0.0:2000 + - run + env: + - name: TUNNEL_TOKEN + valueFrom: + secretKeyRef: + name: cloudflared-secrets + key: TUNNEL_TOKEN + ports: + - containerPort: 2000 + name: metrics + livenessProbe: + httpGet: + path: /ready + port: 2000 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 2000 + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/kubernetes/manifests-foundry/cloudflared/namespace.yaml b/kubernetes/manifests-foundry/cloudflared/namespace.yaml new file mode 100644 index 0000000..620b8ae --- /dev/null +++ b/kubernetes/manifests-foundry/cloudflared/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cloudflared diff --git a/kubernetes/manifests-foundry/sentinel/configmap-kerbecs.yaml b/kubernetes/manifests-foundry/sentinel/configmap-kerbecs.yaml new file mode 100644 index 0000000..3525624 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/configmap-kerbecs.yaml @@ -0,0 +1,194 @@ +# Kerbecs gateway routing rules. Mounted into the kerbecs pod at +# /etc/kerbecs/kerbecs.yaml — KERBECS_CONFIG points the binary at it. +# +# Upstreams reference the in-cluster Services by name (cross-namespace +# resolution not needed since everything lives in `sentinel`). The web +# upstream points at port 80 (the nginx-served prod build), unlike the +# docker-compose dev setup where vite serves on 5173. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: kerbecs-config + namespace: sentinel +data: + kerbecs.yaml: | + gateway: + name: sentinel-gateway + version: 0.1.0 + env: ${ENV:PROD} + timeouts: + dial: 5s + headers: 30s + idle: 50s + overall: 0 + + listeners: + gateway: + port: "10310" + cors: + enabled: true + allowed_origins: ["*"] + allow_credentials: false + admin: + port: "10300" + auth: + type: basic + username: ${KERBECS_USER:admin} + password: ${KERBECS_PASSWORD:admin} + + providers: + static: + # Hot-reload the config on ConfigMap updates (kerbecs >= 3.1.0). The + # watcher reacts to the kubelet ..data symlink swap, so route changes + # apply without restarting the pod. + watch: true + + upstreams: + core: + name: sentinel-core + version: 0.1.0 + instances: + - http://core.sentinel.svc.cluster.local:9999 + + oauth: + name: sentinel-oauth + version: 0.1.0 + instances: + - http://oauth.sentinel.svc.cluster.local:9997 + + discord: + name: sentinel-discord + version: 0.1.0 + instances: + - http://discord.sentinel.svc.cluster.local:9998 + + saml: + name: sentinel-saml + version: 0.1.0 + instances: + - http://saml.sentinel.svc.cluster.local:9996 + + google: + name: sentinel-google + version: 0.1.0 + instances: + - http://google.sentinel.svc.cluster.local:9995 + + web: + name: sentinel-web + version: 0.1.0 + instances: + - http://web.sentinel.svc.cluster.local:80 + + routes: + - name: core-internal + match: + path: /api/core/* + upstream: core + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: core-users + match: + path: /api/users/* + upstream: core + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: core-applications + match: + path: /api/applications/* + upstream: core + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: core-groups + match: + path: /api/groups/* + upstream: core + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: core-entities + match: + path: /api/entities/* + upstream: core + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: oauth + match: + path: /api/oauth/* + upstream: oauth + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: auth + match: + path: /api/auth/* + upstream: oauth + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: discord + match: + path: /api/discord/* + upstream: discord + rewrite: + strip_prefix: /api + envelope: passthrough + + - name: google + match: + path: /api/google/* + upstream: google + rewrite: + strip_prefix: /api + envelope: passthrough + + # SAML consent endpoints — the SPA calls these through the /api prefix. + - name: saml + match: + path: /api/saml/* + upstream: saml + rewrite: + strip_prefix: /api + envelope: passthrough + + # OIDC discovery must live at the issuer root (no /api prefix), so route + # it straight to oauth without stripping anything. + - name: well-known + match: + path: /.well-known/* + upstream: oauth + envelope: passthrough + + # SAML IdP metadata and SSO endpoint live at the issuer root (no /api + # prefix) so the URLs published in metadata are clean and stable. The + # SPA-served /saml/authorize consent page is NOT listed here, so it falls + # through to the web frontend below. + - name: saml-metadata + match: + path: /saml/metadata + upstream: saml + envelope: passthrough + + - name: saml-sso + match: + path: /saml/sso + upstream: saml + envelope: passthrough + + - name: web-frontend + match: + path: /* + upstream: web + envelope: passthrough diff --git a/kubernetes/manifests-foundry/sentinel/core.yaml b/kubernetes/manifests-foundry/sentinel/core.yaml new file mode 100644 index 0000000..805afa1 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/core.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: core + namespace: sentinel +spec: + replicas: 1 + selector: + matchLabels: + app: core + template: + metadata: + labels: + app: core + spec: + # Spread across at least 2 nodes (minDomains) but tolerate skew + # up to 1 (so 3 replicas → [2,1], not forced [1,1,1]). minDomains + # makes the scheduler treat the missing-domain count as 0 when + # only one node currently has matching pods, so the second + # replica blocks on schedule and Karpenter provisions a node + # (requires k8s ≥ 1.30 — we're on 1.35). + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: core + containers: + - name: core + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-core:latest + ports: + - containerPort: 9999 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: ENV + value: PROD + - name: PORT + value: "9999" + - name: DATABASE_HOST + value: postgres.sentinel.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USER + value: postgres + - name: DATABASE_NAME + value: sentinel + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: POSTGRES_PASSWORD + # OIDC issuer — the public base URL. MUST match oauth's ISSUER + # exactly; it's the `iss` claim stamped into signed tokens and the + # `issuer` advertised by the OAuth discovery document. + - name: ISSUER + value: https://sentinel-v5.gauchoracing.com + # Shared secret each non-core service exchanges at startup + # for its pre-seeded bearer JWT. Same value as on every + # other sentinel-* deployment. Rotating it requires a + # rolling restart of every service so each can re-fetch. + - name: INTERNAL_BOOTSTRAP_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: INTERNAL_BOOTSTRAP_SECRET +--- +apiVersion: v1 +kind: Service +metadata: + name: core + namespace: sentinel +spec: + selector: + app: core + ports: + - port: 9999 + targetPort: 9999 diff --git a/kubernetes/manifests-foundry/sentinel/discord.yaml b/kubernetes/manifests-foundry/sentinel/discord.yaml new file mode 100644 index 0000000..2078a57 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/discord.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discord + namespace: sentinel +spec: + # Stays at 1 — multiple replicas would each open their own gateway + # connection and double-process every Discord event. + replicas: 1 + selector: + matchLabels: + app: discord + template: + metadata: + labels: + app: discord + spec: + containers: + - name: discord + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-discord:latest + ports: + - containerPort: 9998 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: ENV + value: PROD + - name: PORT + value: "9998" + - name: DATABASE_HOST + value: postgres.sentinel.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USER + value: postgres + - name: DATABASE_NAME + value: sentinel + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: POSTGRES_PASSWORD + # Resolve inter-service calls through the kerbecs gateway's admin + # API (replaces the rincon registry). Same admin creds the kerbecs + # deployment uses. + - name: KERBECS_ENDPOINT + value: http://kerbecs.sentinel.svc.cluster.local:10300 + - name: KERBECS_USER + value: admin + - name: KERBECS_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: KERBECS_PASSWORD + - name: DISCORD_GUILD + value: "756738476887638107" + - name: DISCORD_PREFIX + value: "!" + - name: WEB_BASE_URL + value: https://sentinel-v5.gauchoracing.com + - name: DISCORD_TOKEN + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: DISCORD_TOKEN + # Same value as on core; used at startup to exchange for + # this service's pre-seeded bearer JWT. + - name: INTERNAL_BOOTSTRAP_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: INTERNAL_BOOTSTRAP_SECRET +--- +apiVersion: v1 +kind: Service +metadata: + name: discord + namespace: sentinel +spec: + selector: + app: discord + ports: + - port: 9998 + targetPort: 9998 diff --git a/kubernetes/manifests-foundry/sentinel/google.yaml b/kubernetes/manifests-foundry/sentinel/google.yaml new file mode 100644 index 0000000..845f125 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/google.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: google + namespace: sentinel +spec: + # Single replica: the service runs a reconcile cron that mirrors group + # membership out to Google Groups. With >1 replica each pod runs its own + # cron, so two sweeps could write the same Google Group concurrently. The + # writes are idempotent, but the cancel-and-restart coalescing and the + # GOOGLE_SYNC_MAX_REMOVALS guard are per-process — keep one cron owner. This + # is low-volume periodic work, so one replica is plenty. + replicas: 1 + selector: + matchLabels: + app: google + template: + metadata: + labels: + app: google + spec: + # Topology spread — see comment in core.yaml. Inert at replicas: 1. + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: google + containers: + - name: google + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-google:latest + ports: + - containerPort: 9995 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: ENV + value: PROD + - name: PORT + value: "9995" + - name: DATABASE_HOST + value: postgres.sentinel.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USER + value: postgres + - name: DATABASE_NAME + value: sentinel + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: POSTGRES_PASSWORD + # Resolve inter-service calls through the kerbecs gateway's admin + # API. Same admin creds the kerbecs deployment uses. + - name: KERBECS_ENDPOINT + value: http://kerbecs.sentinel.svc.cluster.local:10300 + - name: KERBECS_USER + value: admin + - name: KERBECS_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: KERBECS_PASSWORD + # Shared secret exchanged at startup for this service's pre-seeded + # bearer JWT. Same value as every other sentinel-* deployment. + - name: INTERNAL_BOOTSTRAP_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: INTERNAL_BOOTSTRAP_SECRET + # Service-account JSON key for domain-wide delegation (scope + # admin.directory.group.member). The only real secret here — marked + # optional so the pod still boots before it's populated; sync just + # no-ops until it's set (then restart the pod to pick it up). + - name: GOOGLE_SERVICE_ACCOUNT + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: GOOGLE_SERVICE_ACCOUNT + optional: true + # The super-admin the service account impersonates. Just an email, + # not a credential, so it's a plain value rather than a secret. + - name: GOOGLE_ADMIN_SUBJECT + value: admin@gauchoracing.com +--- +apiVersion: v1 +kind: Service +metadata: + name: google + namespace: sentinel +spec: + selector: + app: google + ports: + - port: 9995 + targetPort: 9995 diff --git a/kubernetes/manifests-foundry/sentinel/ingress.yaml b/kubernetes/manifests-foundry/sentinel/ingress.yaml new file mode 100644 index 0000000..4ac88bc --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/ingress.yaml @@ -0,0 +1,31 @@ +# Public entrypoint on the foundry k3s cluster. Traefik receives plain +# HTTP forwarded by the in-cluster cloudflared tunnel; TLS terminates at +# the Cloudflare edge. Host-based routing to the kerbecs gateway, which +# then routes to core/oauth/discord/saml/google/web internally. +# +# DNS is managed by the foundry external-dns instance: +# - hostname is derived from spec.rules[].host +# - cloudflare-proxied annotation flips the CF record to orange-cloud +# - the record's CNAME target comes from external-dns's +# --default-targets flag (points at the gr-foundry tunnel) + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: sentinel + namespace: sentinel + annotations: + external-dns.alpha.kubernetes.io/cloudflare-proxied: "true" +spec: + ingressClassName: traefik + rules: + - host: sentinel-v5.gauchoracing.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kerbecs + port: + number: 10310 diff --git a/kubernetes/manifests-foundry/sentinel/kerbecs.yaml b/kubernetes/manifests-foundry/sentinel/kerbecs.yaml new file mode 100644 index 0000000..a8ec737 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/kerbecs.yaml @@ -0,0 +1,79 @@ +# Gateway. Public traffic enters through the ALB (see ingress.yaml), +# kerbecs routes to the right upstream Service based on its config in +# the kerbecs-config ConfigMap. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kerbecs + namespace: sentinel +spec: + replicas: 1 + selector: + matchLabels: + app: kerbecs + template: + metadata: + labels: + app: kerbecs + spec: + # Topology spread — see comment in core.yaml. + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: kerbecs + containers: + - name: kerbecs + image: bk1031/kerbecs:latest + ports: + - containerPort: 10310 + name: gateway + - containerPort: 10300 + name: admin + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: KERBECS_CONFIG + value: /etc/kerbecs/kerbecs.yaml + - name: ENV + value: PROD + - name: KERBECS_USER + value: admin + - name: KERBECS_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: KERBECS_PASSWORD + volumeMounts: + - name: kerbecs-config + mountPath: /etc/kerbecs + readOnly: true + volumes: + - name: kerbecs-config + configMap: + name: kerbecs-config +--- +apiVersion: v1 +kind: Service +metadata: + name: kerbecs + namespace: sentinel +spec: + selector: + app: kerbecs + ports: + - name: gateway + port: 10310 + targetPort: 10310 + - name: admin + port: 10300 + targetPort: 10300 diff --git a/kubernetes/manifests-foundry/sentinel/kustomization.yaml b/kubernetes/manifests-foundry/sentinel/kustomization.yaml new file mode 100644 index 0000000..224aa4f --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/kustomization.yaml @@ -0,0 +1,47 @@ +# Kustomize entrypoint — ArgoCD auto-detects this file and runs +# `kustomize build` instead of treating the dir as a flat manifest set. +# +# The `images` block below is the single source of truth for sentinel +# service versions. Bumping a release looks like: +# +# kustomize edit set image \ +# ghcr.io/gaucho-racing/sentinel-core=ghcr.io/gaucho-racing/sentinel-core:5.2.0 +# +# (Run from this directory; CI can wrap it across all four images on +# tag push.) That edit only touches this file — the Deployment YAMLs +# stay unchanged, so the diff in a release PR is one block. +# +# rincon and kerbecs are on independent version streams from the +# sentinel services — bump them separately when there's a reason to. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap-kerbecs.yaml + - postgres.yaml + - kerbecs.yaml + - core.yaml + - oauth.yaml + - discord.yaml + - saml.yaml + - google.yaml + - web.yaml + - ingress.yaml + +images: + - name: ghcr.io/gaucho-racing/sentinel-core + newTag: 5.8.1 + - name: ghcr.io/gaucho-racing/sentinel-oauth + newTag: 5.8.1 + - name: ghcr.io/gaucho-racing/sentinel-discord + newTag: 5.8.1 + - name: ghcr.io/gaucho-racing/sentinel-saml + newTag: 5.8.1 + - name: ghcr.io/gaucho-racing/sentinel-google + newTag: 5.8.1 + - name: ghcr.io/gaucho-racing/sentinel-web + newTag: 5.8.1 + - name: bk1031/kerbecs + newTag: 3.2.0 diff --git a/kubernetes/manifests-foundry/sentinel/namespace.yaml b/kubernetes/manifests-foundry/sentinel/namespace.yaml new file mode 100644 index 0000000..a9464d0 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sentinel diff --git a/kubernetes/manifests-foundry/sentinel/oauth.yaml b/kubernetes/manifests-foundry/sentinel/oauth.yaml new file mode 100644 index 0000000..e2b1969 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/oauth.yaml @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth + namespace: sentinel +spec: + replicas: 1 + selector: + matchLabels: + app: oauth + template: + metadata: + labels: + app: oauth + spec: + # Topology spread — see comment in core.yaml. + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: oauth + containers: + - name: oauth + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-oauth:latest + ports: + - containerPort: 9997 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: ENV + value: PROD + - name: PORT + value: "9997" + - name: DATABASE_HOST + value: postgres.sentinel.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USER + value: postgres + - name: DATABASE_NAME + value: sentinel + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: POSTGRES_PASSWORD + # Resolve inter-service calls through the kerbecs gateway's admin + # API (replaces the rincon registry). Same admin creds the kerbecs + # deployment uses. + - name: KERBECS_ENDPOINT + value: http://kerbecs.sentinel.svc.cluster.local:10300 + - name: KERBECS_USER + value: admin + - name: KERBECS_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: KERBECS_PASSWORD + # OIDC issuer — the public base URL. MUST match core's ISSUER + # exactly; the discovery document's `issuer` and the `iss` claim in + # ID tokens have to be byte-identical for relying parties. + - name: ISSUER + value: https://sentinel-v5.gauchoracing.com + # Discord OAuth client config for the "Continue with Discord" + # login button. Client id/secret are the Discord application's + # credentials; the redirect URI must byte-match what the web + # client sends in its authorize call (and what's registered + # on the Discord application's OAuth2 settings). + - name: DISCORD_CLIENT_ID + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: DISCORD_CLIENT_ID + - name: DISCORD_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: DISCORD_CLIENT_SECRET + - name: DISCORD_REDIRECT_URI + value: https://sentinel-v5.gauchoracing.com/auth/login/discord + # Shared secret exchanged at startup for this service's + # pre-seeded bearer JWT. Same value as every other + # sentinel-* deployment. + - name: INTERNAL_BOOTSTRAP_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: INTERNAL_BOOTSTRAP_SECRET + # client_id of the shared Google Workspace account application + # (team@gauchoracing.com). The oauth service overrides identity + # claims for this client so authorized users sign in as the shared + # account. Not a secret — the client_secret is what's sensitive. + - name: TEAM_GOOGLE_CLIENT_ID + value: uozOEnbeoJ2t +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth + namespace: sentinel +spec: + selector: + app: oauth + ports: + - port: 9997 + targetPort: 9997 diff --git a/kubernetes/manifests-foundry/sentinel/postgres.yaml b/kubernetes/manifests-foundry/sentinel/postgres.yaml new file mode 100644 index 0000000..a207d06 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/postgres.yaml @@ -0,0 +1,17 @@ +# In-cluster handle for the gr-postgres EC2 (still in AWS after the +# compute migration — same instance backing mapache + vault). +# +# From on-prem, this resolves via the Cloudflare A record +# gr-postgres.gauchoracing.com (gray-cloud, unproxied) to the EIP +# 52.32.150.44. Traffic goes over the public internet on 5432; the +# 32-char scram-sha-256 password is the only gate. The SG on the EC2 +# allows 0.0.0.0/0 on 5432 (admin_cidr_blocks in modules/postgres-ec2). + +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: sentinel +spec: + type: ExternalName + externalName: gr-postgres.gauchoracing.com diff --git a/kubernetes/manifests-foundry/sentinel/saml.yaml b/kubernetes/manifests-foundry/sentinel/saml.yaml new file mode 100644 index 0000000..7709345 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/saml.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: saml + namespace: sentinel +spec: + # Single replica for now: on the very first boot the service generates and + # persists its SAML signing key + self-signed cert (saml_signing_key table). + # With >1 replica racing an empty table, two pods could each insert an active + # key and serve different certs, breaking SP signature verification. Once the + # row exists every replica loads the same key, so this can safely go to 2 + # after the first successful deploy. The IdP is interactive and low-volume, so + # one replica is fine in the meantime. + replicas: 1 + selector: + matchLabels: + app: saml + template: + metadata: + labels: + app: saml + spec: + # Topology spread — see comment in core.yaml. Inert at replicas: 1, + # kept so scaling up later spreads pods without an edit here. + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: saml + containers: + - name: saml + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-saml:latest + ports: + - containerPort: 9996 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: ENV + value: PROD + - name: PORT + value: "9996" + - name: DATABASE_HOST + value: postgres.sentinel.svc.cluster.local + - name: DATABASE_PORT + value: "5432" + - name: DATABASE_USER + value: postgres + - name: DATABASE_NAME + value: sentinel + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: POSTGRES_PASSWORD + # Resolve inter-service calls through the kerbecs gateway's admin + # API (replaces the rincon registry). Same admin creds the kerbecs + # deployment uses. + - name: KERBECS_ENDPOINT + value: http://kerbecs.sentinel.svc.cluster.local:10300 + - name: KERBECS_USER + value: admin + - name: KERBECS_PASSWORD + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: KERBECS_PASSWORD + # Public base URL. Doubles as the SAML IdP entityID (the metadata + # URL is derived from it), so it MUST match core's ISSUER exactly + # and stay byte-stable — changing it invalidates trust with every + # registered service provider. + - name: ISSUER + value: https://sentinel-v5.gauchoracing.com + # Shared secret exchanged at startup for this service's + # pre-seeded bearer JWT. Same value as every other + # sentinel-* deployment. + - name: INTERNAL_BOOTSTRAP_SECRET + valueFrom: + secretKeyRef: + name: sentinel-secrets + key: INTERNAL_BOOTSTRAP_SECRET +--- +apiVersion: v1 +kind: Service +metadata: + name: saml + namespace: sentinel +spec: + selector: + app: saml + ports: + - port: 9996 + targetPort: 9996 diff --git a/kubernetes/manifests-foundry/sentinel/web.yaml b/kubernetes/manifests-foundry/sentinel/web.yaml new file mode 100644 index 0000000..f6a30a0 --- /dev/null +++ b/kubernetes/manifests-foundry/sentinel/web.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: sentinel +spec: + replicas: 1 + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + # Topology spread — see comment in core.yaml. + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: web + containers: + - name: web + # Tag is set by kustomization.yaml's images block. + image: ghcr.io/gaucho-racing/sentinel-web:latest + ports: + - containerPort: 80 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: web + namespace: sentinel +spec: + selector: + app: web + ports: + - port: 80 + targetPort: 80