feat(foundry): add sentinel to on-prem k3s cluster via Cloudflare Tunnel

infra/environments/prod/backend.tf

@@ -28,6 +28,10 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.6"
+    }
   }
 }
 

infra/environments/prod/foundry.tf

@@ -0,0 +1,100 @@
+# Foundry (on-prem k3s) cutover resources — the shared Cloudflare Tunnel
+# that connects the on-prem cluster to Cloudflare. Public DNS records for
+# individual hostnames are managed by external-dns on the foundry cluster
+# (see kubernetes/apps-foundry/external-dns.yaml), not here.
+#
+# Design: the tunnel has a single catch-all ingress rule that forwards
+# ALL traffic to the Traefik service. Traefik does Host-based routing
+# in-cluster to the right namespace. Adding a new public hostname is
+# just an Ingress resource in that service's manifest set — external-dns
+# writes the CNAME automatically, no terraform edit needed. Same shape
+# works for a second cluster (gr-other-cluster with its own tunnel + own
+# external-dns using txtOwnerId=gr-other-cluster).
+#
+# Compute (EKS + ALB + ACM origin cert) still lives in main.tf. Once
+# every hostname is migrated off EKS and the on-prem stack has soaked,
+# delete module.eks / module.argocd / module.origin_cert from main.tf
+# and remove kubernetes/apps/ from the repo.
+#
+# Off-cluster data services (gr-postgres, gr-mqtt, gr-clickhouse) stay
+# in AWS — the on-prem cluster reaches them over the public internet via
+# their existing Cloudflare A records.
+#
+# Cutover sequence for the first slice (sentinel):
+#
+#   1. terraform apply this file. Creates the tunnel; no DNS records
+#      yet, so no conflict with EKS external-dns.
+#   2. Populate cloudflared-secrets + external-dns-config on foundry:
+#        kubectl -n cloudflared create secret generic cloudflared-secrets \
+#          --from-literal=TUNNEL_TOKEN="$(terraform output -raw foundry_tunnel_token)"
+#        kubectl -n external-dns create configmap foundry-tunnel-target \
+#          --from-literal=target="$(terraform output -raw foundry_tunnel_id).cfargotunnel.com"
+#        kubectl -n external-dns create secret generic cloudflare-api-token \
+#          --from-literal=api-token="$CLOUDFLARE_API_TOKEN"
+#   3. Apply kubernetes/bootstrap/root-foundry.yaml on the foundry
+#      ArgoCD, populate sentinel-secrets, wait for pods Healthy.
+#   4. Cutover for one hostname:
+#      a. Delete the existing sentinel-v5.gauchoracing.com record via CF
+#         dashboard (or scale EKS external-dns to 0 first — it'll take
+#         its four records with it).
+#      b. Foundry external-dns notices the sentinel Ingress with no
+#         record, writes CNAME → <tunnel-id>.cfargotunnel.com within
+#         its --interval (1m default).
+#      c. Traffic starts landing on foundry within CF TTL.
+#   5. Bake. Add mapache/vault/argocd in follow-up PRs — each is a copy
+#      of manifests/<svc>/ into manifests-foundry/<svc>/ with two file
+#      changes (ingress.yaml → Traefik + external-dns annotation,
+#      postgres.yaml → public hostname). No terraform edit.
+
+# Account ID for the tunnel resources. The CF API token used by the
+# provider is scoped to a single account — return that account's ID.
+data "cloudflare_accounts" "current" {}
+
+locals {
+  cloudflare_account_id = data.cloudflare_accounts.current.result[0].id
+}
+
+# 32-byte tunnel secret, stored in TF state.
+resource "random_bytes" "foundry_tunnel_secret" {
+  length = 32
+}
+
+# Named tunnel. config_src = "cloudflare" means CF hosts the ingress rule
+# table (managed by cloudflare_zero_trust_tunnel_cloudflared_config
+# below); the on-prem cloudflared pod boots with just a token and pulls
+# the ruleset from CF at runtime.
+resource "cloudflare_zero_trust_tunnel_cloudflared" "foundry" {
+  account_id    = local.cloudflare_account_id
+  name          = "gr-foundry"
+  tunnel_secret = random_bytes.foundry_tunnel_secret.base64
+  config_src    = "cloudflare"
+}
+
+# Catch-all tunnel config. Every request that arrives via a DNS record
+# CNAMEd to <tunnel-id>.cfargotunnel.com goes to the foundry Traefik
+# service regardless of Host header; Traefik does the Host-based routing
+# to the right in-cluster Service.
+#
+# Adding a hostname to the tunnel is NOT done here — it's implicit: any
+# DNS record (written by external-dns for a matching Ingress) that
+# points at this tunnel gets served.
+resource "cloudflare_zero_trust_tunnel_cloudflared_config" "foundry" {
+  account_id = local.cloudflare_account_id
+  tunnel_id  = cloudflare_zero_trust_tunnel_cloudflared.foundry.id
+
+  config = {
+    ingress = [
+      {
+        service = "http://traefik.kube-system.svc.cluster.local:80"
+      },
+    ]
+  }
+}
+
+# Data source for the tunnel token; consumed by the cloudflared pods on
+# the on-prem cluster via TUNNEL_TOKEN. Read with:
+#   terraform output -raw foundry_tunnel_token
+data "cloudflare_zero_trust_tunnel_cloudflared_token" "foundry" {
+  account_id = local.cloudflare_account_id
+  tunnel_id  = cloudflare_zero_trust_tunnel_cloudflared.foundry.id
+}

infra/environments/prod/outputs.tf

@@ -86,3 +86,14 @@ output "clickhouse_admin_password" {
   value       = module.clickhouse.admin_password
   sensitive   = true
 }
+
+output "foundry_tunnel_id" {
+  description = "UUID of the gr-foundry Cloudflare tunnel. DNS CNAMEs point at <id>.cfargotunnel.com."
+  value       = cloudflare_zero_trust_tunnel_cloudflared.foundry.id
+}
+
+output "foundry_tunnel_token" {
+  description = "Tunnel connector token for the gr-foundry tunnel. Consumed by the cloudflared pods on the on-prem cluster via TUNNEL_TOKEN. Read with `terraform output -raw foundry_tunnel_token`."
+  value       = data.cloudflare_zero_trust_tunnel_cloudflared_token.foundry.token
+  sensitive   = true
+}

kubernetes/apps-foundry/cloudflared.yaml

@@ -0,0 +1,33 @@
+# cloudflared: outbound tunnel connector. Fronts every public
+# gauchoracing.com hostname on this cluster — the tunnel and its route
+# table live in Terraform (foundry.tf), the pod that terminates the
+# tunnel here lives in this Application.
+#
+# Manual one-time Secret bootstrap after this first syncs:
+#
+#   kubectl -n cloudflared create secret generic cloudflared-secrets \
+#     --from-literal=TUNNEL_TOKEN="$(cd ../../infra/environments/prod && terraform output -raw foundry_tunnel_token)"
+
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cloudflared
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Gaucho-Racing/infrastructure.git
+    targetRevision: main
+    path: kubernetes/manifests-foundry/cloudflared
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cloudflared
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

kubernetes/apps-foundry/external-dns.yaml

@@ -0,0 +1,99 @@
+# external-dns for the foundry k3s cluster.
+#
+# Watches Ingress resources in this cluster and reconciles matching
+# Cloudflare DNS records. Because the foundry Traefik service is
+# behind a Cloudflare Tunnel (see infra/environments/prod/foundry.tf),
+# every record it writes needs to CNAME to <tunnel-id>.cfargotunnel.com
+# rather than the LB status field's node IPs — that's what --default-targets
+# does below.
+#
+# Multi-cluster: this instance uses txtOwnerId=gr-foundry so it only
+# manages records it wrote itself. A second on-prem cluster later gets
+# its own external-dns Application with a different txtOwnerId and its
+# own tunnel target; the two never touch each other's records.
+#
+# Manual one-time bootstrap after this Application first syncs:
+#
+#   kubectl -n external-dns create secret generic cloudflare-api-token \
+#     --from-literal=api-token="$CLOUDFLARE_API_TOKEN"
+#
+#   kubectl -n external-dns create configmap foundry-tunnel-target \
+#     --from-literal=target="$(cd ../../infra/environments/prod && terraform output -raw foundry_tunnel_id).cfargotunnel.com"
+
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: external-dns
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://kubernetes-sigs.github.io/external-dns
+    chart: external-dns
+    targetRevision: 1.18.0
+    helm:
+      releaseName: external-dns
+      values: |
+        provider:
+          name: cloudflare
+
+        domainFilters:
+          - gauchoracing.com
+
+        # sync = create + update + delete based on cluster state. Same
+        # policy as the EKS instance so behavior stays consistent while
+        # both clusters exist.
+        policy: sync
+
+        # TXT registry with a foundry-specific owner ID keeps the foundry
+        # external-dns from stepping on records the EKS instance owns
+        # (gr-prod) and vice-versa. Prefix distinguishes the ownership
+        # TXT record name too, so a `dig` on any record shows which
+        # cluster owns it.
+        registry: txt
+        txtOwnerId: gr-foundry
+        txtPrefix: fdry-
+
+        # Watch Ingress only — no Service source, so a LoadBalancer
+        # Service can't accidentally publish itself.
+        sources:
+          - ingress
+
+        env:
+          - name: CF_API_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: cloudflare-api-token
+                key: api-token
+          # FOUNDRY_TUNNEL_TARGET is populated from a ConfigMap created
+          # by the manual bootstrap step above. Kubernetes substitutes
+          # $(VAR) in container args at pod start, so the tunnel UUID
+          # flows into --default-targets without needing to be in git.
+          - name: FOUNDRY_TUNNEL_TARGET
+            valueFrom:
+              configMapKeyRef:
+                name: foundry-tunnel-target
+                key: target
+
+        extraArgs:
+          # Force every record this instance writes to CNAME the tunnel,
+          # regardless of what the Ingress's status.loadBalancer field
+          # says. Traefik on k3s advertises node IPs there via klipper-lb;
+          # those aren't publicly routable and are irrelevant since
+          # traffic arrives through the tunnel anyway.
+          - "--default-targets=$(FOUNDRY_TUNNEL_TARGET)"
+
+        logLevel: info
+        logFormat: json
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: external-dns
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

kubernetes/apps-foundry/sentinel.yaml

@@ -0,0 +1,48 @@
+# Sentinel on the foundry k3s cluster. Same manifest set as the EKS tree
+# (kubernetes/manifests/sentinel/), diverging only in ingress.yaml
+# (Traefik instead of ALB) and postgres.yaml (public Cloudflare hostname
+# for the gr-postgres EC2 instead of the internal EC2 hostname).
+#
+# Manual Secret needed once after this Application first syncs. Copy the
+# external-facing values from the EKS cluster verbatim so identity state
+# (Discord, INTERNAL_BOOTSTRAP_SECRET) stays coherent across environments
+# during cutover:
+#
+#   kubectl --context eks -n sentinel get secret sentinel-secrets -o yaml \
+#     | grep -v '^\s*resourceVersion:\|^\s*uid:\|^\s*creationTimestamp:\|^\s*namespace:' \
+#     | kubectl --context foundry apply -f -
+#
+# Or create from scratch:
+#
+#   kubectl -n sentinel create secret generic sentinel-secrets \
+#     --from-literal=POSTGRES_PASSWORD="$(cd ../../infra/environments/prod && terraform output -raw postgres_password)" \
+#     --from-literal=INTERNAL_BOOTSTRAP_SECRET=... \
+#     --from-literal=KERBECS_PASSWORD=... \
+#     --from-literal=DISCORD_TOKEN=... \
+#     --from-literal=DISCORD_CLIENT_ID=... \
+#     --from-literal=DISCORD_CLIENT_SECRET=... \
+#     --from-literal=GOOGLE_SERVICE_ACCOUNT=...
+
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: sentinel
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Gaucho-Racing/infrastructure.git
+    targetRevision: main
+    path: kubernetes/manifests-foundry/sentinel
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sentinel
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

kubernetes/bootstrap/root-foundry.yaml

@@ -0,0 +1,43 @@
+# Root ArgoCD Application for the foundry k3s cluster.
+#
+# The foundry cluster runs its own ArgoCD (installed via foundry-infra).
+# This is a SECOND root app-of-apps that lives alongside foundry-infra's
+# root — one ArgoCD instance, two independently-reconciled trees. Apply
+# once manually on the foundry cluster after the initial cutover prep:
+#
+#   kubectl --context foundry apply -f kubernetes/bootstrap/root-foundry.yaml
+#
+# From then on, ArgoCD reads kubernetes/apps-foundry/ from this repo and
+# reconciles every Application resource it finds there.
+#
+# EKS is still managed by kubernetes/bootstrap/root.yaml → kubernetes/apps/.
+# The two roots do NOT share a cluster; each root's ArgoCD is scoped to
+# its own cluster via server: https://kubernetes.default.svc.
+#
+# See kubernetes/apps-foundry/*.yaml for the per-stack manual bootstrap
+# steps (secret creation, DB creation, etc.).
+
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: root-foundry
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Gaucho-Racing/infrastructure.git
+    targetRevision: main
+    path: kubernetes/apps-foundry
+    directory:
+      recurse: true
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true