diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b55ee0e..39d1963 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ env: # Values that are marked Required in the chart. Without them the chart fails # fast with a clear `required` error, so CI must provide dummies. REQUIRED_VALUES: >- - --set services.icc.database_url=postgres://user:pass@db:5432/icc + --set services.icc.database_url=postgres://user:pass@db:5432 --set services.icc.valkey.apps_url=redis://valkey:6379/0 --set services.icc.valkey.icc_url=redis://valkey:6379/1 --set services.icc.prometheus.url=http://prometheus:9090 @@ -49,6 +49,9 @@ jobs: fi echo "OK: chart fails fast when required values are missing" + - name: 'Database URL compatibility' + run: bash test/database-urls.sh + - name: 'Install kubeconform' run: | curl -sSL -o /tmp/kubeconform.tar.gz \ diff --git a/MANUAL.md b/MANUAL.md index 67202dd..1259ce0 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -64,6 +64,13 @@ be accessible to the user in `database_url`. ICC applies its schema migrations automatically when it starts. A `workflow` database is only needed if you deploy the optional workflow service. +For an existing installation with different database names or credentials, set +the complete URLs under `services.icc.database_urls` instead. Supported keys +are `activities`, `cluster_manager`, `cold_storage`, `compliance`, +`control_plane`, `cron`, `scaler`, `trafficante`, `traffic_inspector`, and +`user_manager`. Exact URLs take precedence over `database_url`. See +`MIGRATING-v3-to-v4.md` for the v3 mapping. + For Enterprise installs, also add the registry credentials and the Enterprise image references. Without the `image` overrides the chart pulls the public Docker Hub images and the pull secret has no effect. Use the repositories and @@ -112,7 +119,7 @@ EOF ```sh helm install platformatic oci://ghcr.io/platformatic/helm \ - --version "^4.1.0" \ + --version "^4.2.0" \ --namespace platformatic --create-namespace \ -f my-values.yaml -f my-secrets.yaml ``` @@ -154,7 +161,7 @@ kubectl port-forward -n platformatic svc/icc 8080:80 ```sh helm upgrade platformatic oci://ghcr.io/platformatic/helm \ - --version "^4.1.0" -n platformatic \ + --version "^4.2.0" -n platformatic \ -f my-values.yaml -f my-secrets.yaml \ --wait --timeout 10m @@ -168,8 +175,9 @@ helm uninstall platformatic -n platformatic ## Troubleshooting -- `services.icc.database_url is required` (or valkey / prometheus): a required - value is missing from `my-values.yaml`. See section 2. +- `services.icc.database_urls. or services.icc.database_url is required` + (or valkey / prometheus): a required value is missing from `my-values.yaml`. + See section 2. - `chart requires kubeVersion: >= 1.30.0-0`: your cluster (or Helm's default capabilities) is below 1.30. Upgrade the cluster. - `no matches for kind "PodMonitor"` (or `"ServiceMonitor"`): the Prometheus diff --git a/MIGRATING-v3-to-v4.md b/MIGRATING-v3-to-v4.md index 1267fc3..48760e5 100644 --- a/MIGRATING-v3-to-v4.md +++ b/MIGRATING-v3-to-v4.md @@ -76,8 +76,19 @@ services: deploy: true public_url: https://icc.example.com - # Base PostgreSQL URL without a database name. - database_url: postgres://USER:PASSWORD@HOST:5432 + # Preserve the complete URLs from the v3 ICC secrets. Database names and + # credentials do not need to change. See the PostgreSQL mapping below. + database_urls: + activities: postgres://ACTIVITIES_USER:PASSWORD@HOST:5432/ACTIVITIES_DB + cluster_manager: postgres://CLUSTER_MANAGER_USER:PASSWORD@HOST:5432/CLUSTER_MANAGER_DB + cold_storage: postgres://COLD_STORAGE_USER:PASSWORD@HOST:5432/COLD_STORAGE_DB + compliance: postgres://COMPLIANCE_USER:PASSWORD@HOST:5432/COMPLIANCE_DB + control_plane: postgres://CONTROL_PLANE_USER:PASSWORD@HOST:5432/CONTROL_PLANE_DB + cron: postgres://CRON_USER:PASSWORD@HOST:5432/CRON_DB + scaler: postgres://SCALER_USER:PASSWORD@HOST:5432/SCALER_DB + trafficante: postgres://TRAFFICANTE_USER:PASSWORD@HOST:5432/TRAFFICANTE_DB + traffic_inspector: postgres://TRAFFIC_INSPECTOR_USER:PASSWORD@HOST:5432/TRAFFIC_INSPECTOR_DB + user_manager: postgres://USER_MANAGER_USER:PASSWORD@HOST:5432/USER_MANAGER_DB valkey: apps_url: redis://VALKEY_HOST:6379/0 @@ -185,18 +196,37 @@ one login method must be enabled for users to sign in. ### PostgreSQL -v3 accepted a complete URL for each ICC database. v4 accepts one URL prefix and -appends the database name for each ICC service. For example: +Preserve the complete v3 database URLs under `database_urls`. This keeps the +existing database names, users, passwords, and permissions: + +| v3 `services.icc.secrets` key | v4 `services.icc.database_urls` key | +| --- | --- | +| `PLT_ACTIVITIES_DATABASE_URL` | `activities` | +| `PLT_CLUSTER_MANAGER_DATABASE_URL` | `cluster_manager` | +| `PLT_COLD_STORAGE_DATABASE_URL` | `cold_storage` | +| `PLT_COMPLIANCE_DATABASE_URL` | `compliance` | +| `PLT_CONTROL_PLANE_DATABASE_URL` | `control_plane` | +| `PLT_CRON_DATABASE_URL` | `cron` | +| `PLT_SCALER_DATABASE_URL` | `scaler` | +| `PLT_TRAFFICANTE_DATABASE_URL` | `trafficante` | +| `PLT_TRAFFIC_INSPECTOR_DATABASE_URL` | `traffic_inspector` | +| `PLT_USER_MANAGER_DATABASE_URL` | `user_manager` | + +If Traffic Inspector and Trafficante used the same URL in v3, omit +`traffic_inspector`; it falls back to the `trafficante` URL. + +Installations where every database already uses one role and the standard v4 +database names may use a single URL prefix instead: ```yaml database_url: postgres://icc:secret@postgres.example.com:5432 ``` -The base URL must not contain a database name or a trailing slash. The required -databases are `activities`, `risk_cold_storage`, `control_plane`, `cron`, -`scaler`, `trafficante`, `user_manager`, `cluster_manager`, and `compliance`. -They must exist and be accessible to the configured role. ICC applies its schema -migrations automatically when it starts. The `workflow` database is not used +The base URL must not contain a database name or trailing slash. The chart +appends `activities`, `risk_cold_storage`, `control_plane`, `cron`, `scaler`, +`trafficante`, `user_manager`, `cluster_manager`, and `compliance`. Exact URLs +take precedence over the base URL, so both forms can be combined. ICC applies +schema migrations automatically when it starts. No Workflow URL is required while `services.workflow.deploy` is `false`. ### Valkey and login @@ -276,7 +306,7 @@ Use the same Helm release name and release namespace as v3. Pin the exact v4 chart version that you tested. ```sh -CHART_VERSION=4.1.0 +CHART_VERSION=4.2.0 helm upgrade "$RELEASE" oci://ghcr.io/platformatic/helm \ --version "$CHART_VERSION" \ diff --git a/README-ENTERPRISE.md b/README-ENTERPRISE.md index e1c2450..daadf58 100644 --- a/README-ENTERPRISE.md +++ b/README-ENTERPRISE.md @@ -93,7 +93,8 @@ production-ready set of values except for the `secrets` portion. | `services.icc.image.pullPolicy` | When to pull an image update | IfNotPresent | No | | `services.icc.log_level` | The level to log ICC services | warn | No | | `services.icc.public_url` | The URL to access Intelligent Command Center (Note: ingress and domain must be configured by the user | "" | Yes | -| `services.icc.database_url` | The database connection string | "" | Yes | +| `services.icc.database_url` | Base database connection string used when an exact URL is not set | "" | Conditional | +| `services.icc.database_urls` | Exact connection strings keyed by ICC database service | {} | Conditional | | `services.icc.valkey.apps_url` | Valkey connection string | "" | Yes | | `services.icc.valkey.icc_url` | Valkey connection string | "" | Yes | | `services.icc.prometheus.url` | Prometheus API URL | "" | Yes | diff --git a/README.md b/README.md index e49fee1..e9f3f7d 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,8 @@ production-ready set of values except for the `secrets` portion. | `services.icc.image.pullPolicy` | When to pull an image update | IfNotPresent | No | | `services.icc.log_level` | The level to log ICC services | warn | No | | `services.icc.public_url` | The URL to access Intelligent Command Center (Note: ingress and domain must be configured by the user | "" | Yes | -| `services.icc.database_url` | The database connection string | "" | Yes | +| `services.icc.database_url` | Base database connection string used when an exact URL is not set | "" | Conditional | +| `services.icc.database_urls` | Exact connection strings keyed by ICC database service | {} | Conditional | | `services.icc.valkey.apps_url` | Valkey connection string | "" | Yes | | `services.icc.valkey.icc_url` | Valkey connection string | "" | Yes | | `services.icc.prometheus.url` | Prometheus API URL | "" | Yes | diff --git a/chart/Chart.yaml b/chart/Chart.yaml index c3dfb69..ffbc747 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: helm -version: 4.1.0 +version: 4.2.0 kubeVersion: ">= 1.30.0-0" description: Platformatic microservices type: application diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 6b7ab90..ba5f246 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -96,8 +96,38 @@ platformatic NodePort {{- end }} -{{/* ICC databases */}} -{{/* Using trafficante database name so that we an safely upgrade existing users */}} +{{/* ICC database connection names */}} {{- define "service.icc.databases" -}} -activities risk_cold_storage control_plane cron scaler trafficante user_manager cluster_manager compliance workflow +activities cluster_manager cold_storage compliance control_plane cron scaler trafficante traffic_inspector user_manager +{{- end }} + +{{/* Database name appended to the base URL when no exact URL is configured */}} +{{- define "service.icc.databaseName" -}} +{{- $database := index . 0 -}} +{{- if eq $database "cold_storage" -}} +risk_cold_storage +{{- else if eq $database "traffic_inspector" -}} +trafficante +{{- else -}} +{{- $database -}} +{{- end -}} +{{- end }} + +{{/* Resolve an exact database URL, falling back to the shared base URL */}} +{{- define "service.icc.databaseUrl" -}} +{{- $root := index . 0 -}} +{{- $database := index . 1 -}} +{{- $urls := $root.Values.services.icc.database_urls | default (dict) -}} +{{- $url := get $urls $database -}} +{{- if and (not $url) (eq $database "traffic_inspector") -}} +{{- $url = get $urls "trafficante" -}} +{{- end -}} +{{- if $url -}} +{{- $url -}} +{{- else -}} +{{- $message := printf "services.icc.database_urls.%s or services.icc.database_url is required" $database -}} +{{- $base := required $message $root.Values.services.icc.database_url -}} +{{- $name := include "service.icc.databaseName" (list $database) -}} +{{- printf "%s/%s" $base $name -}} +{{- end -}} {{- end }} diff --git a/chart/templates/deployment/_icc.yaml b/chart/templates/deployment/_icc.yaml index 8369cfc..db2b3fe 100644 --- a/chart/templates/deployment/_icc.yaml +++ b/chart/templates/deployment/_icc.yaml @@ -265,17 +265,6 @@ spec: name: icc-valkey key: apps - - name: PLT_COLD_STORAGE_DATABASE_URL - valueFrom: - secretKeyRef: - name: icc-databases - key: "risk_cold_storage" - - name: PLT_TRAFFIC_INSPECTOR_DATABASE_URL - valueFrom: - secretKeyRef: - name: icc-databases - key: trafficante - {{- range (include "service.icc.databases" . | trim | split " ") }} - name: {{ printf "PLT_%s_DATABASE_URL" (upper .) }} valueFrom: diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml index 0737c07..2933c73 100644 --- a/chart/templates/secrets.yaml +++ b/chart/templates/secrets.yaml @@ -81,7 +81,13 @@ metadata: {{- include "application.labels" $ | nindent 4 }} data: {{- range (include "service.icc.databases" . | trim | split " ") }} - "{{ . }}": {{ printf "%s/%s" (required "services.icc.database_url is required" $.Values.services.icc.database_url) . | b64enc }} + "{{ . }}": {{ include "service.icc.databaseUrl" (list $ .) | b64enc }} + {{- end }} + # Compatibility alias used by chart 4.1 deployments. + "risk_cold_storage": {{ include "service.icc.databaseUrl" (list $ "cold_storage") | b64enc }} + {{- $databaseUrls := .Values.services.icc.database_urls | default (dict) }} + {{- if or (and .Values.services.workflow .Values.services.workflow.deploy) .Values.services.icc.database_url (get $databaseUrls "workflow") }} + "workflow": {{ include "service.icc.databaseUrl" (list $ "workflow") | b64enc }} {{- end }} {{/* Setup valkey */}} diff --git a/chart/values.yaml b/chart/values.yaml index 0d36f6e..33106c2 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -137,11 +137,27 @@ services: # https://example.com/icc #public_url: "" - # The URL to access the ICC database - # Makes sure to include any required credentials and/or port number - # Required: must be set at install time + # Base URL used to derive all ICC database URLs. Do not include a database + # name or trailing slash. Required unless all required exact URLs are set. database_url: "" + # Exact URLs for installations with separate database credentials or names. + # An exact URL takes precedence over database_url. traffic_inspector falls + # back to the trafficante exact URL when it is not set. + #database_urls: + # activities: "" + # cluster_manager: "" + # cold_storage: "" + # compliance: "" + # control_plane: "" + # cron: "" + # scaler: "" + # trafficante: "" + # traffic_inspector: "" + # user_manager: "" + # workflow: "" # Required only when services.workflow.deploy is true + database_urls: {} + # URLs to valkey for ICC caching systems # Required: both must be set at install time valkey: diff --git a/test/database-urls.sh b/test/database-urls.sh new file mode 100644 index 0000000..825a719 --- /dev/null +++ b/test/database-urls.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CHART_DIR="$ROOT_DIR/chart" +FIXTURE="$ROOT_DIR/test/fixtures/database-urls.yaml" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +secret_value() { + local manifest=$1 + local key=$2 + local encoded + + encoded=$(awk -v key="\"$key\":" ' + $0 == " name: icc-databases" { in_secret = 1; next } + in_secret && $0 == "---" { in_secret = 0 } + in_secret && $1 == key { print $2; exit } + ' <<< "$manifest") + + [[ -n "$encoded" ]] || return 1 + printf '%s' "$encoded" | base64 --decode +} + +assert_secret_value() { + local manifest=$1 + local key=$2 + local expected=$3 + local actual + + actual=$(secret_value "$manifest" "$key") || fail "missing icc-databases key $key" + [[ "$actual" == "$expected" ]] || fail "$key is $actual, expected $expected" +} + +assert_env_secret_key() { + local manifest=$1 + local env_name=$2 + local expected=$3 + local actual + + actual=$(awk -v env_name="$env_name" ' + $1 == "-" && $2 == "name:" && $3 == env_name { in_env = 1; next } + in_env && $1 == "key:" { gsub(/"/, "", $2); print $2; exit } + ' <<< "$manifest") + + [[ "$actual" == "$expected" ]] || \ + fail "$env_name references $actual, expected $expected" +} + +common_values=( + --set-string services.icc.valkey.apps_url=redis://valkey:6379/0 + --set-string services.icc.valkey.icc_url=redis://valkey:6379/1 + --set-string services.icc.prometheus.url=http://prometheus:9090 +) + +base_manifest=$(helm template database-base "$CHART_DIR" \ + --kube-version 1.30.0 \ + "${common_values[@]}" \ + --set-string services.icc.database_url=postgresql://shared:secret@db:5432) + +assert_secret_value "$base_manifest" activities \ + postgresql://shared:secret@db:5432/activities +assert_secret_value "$base_manifest" cold_storage \ + postgresql://shared:secret@db:5432/risk_cold_storage +assert_secret_value "$base_manifest" traffic_inspector \ + postgresql://shared:secret@db:5432/trafficante +assert_secret_value "$base_manifest" risk_cold_storage \ + postgresql://shared:secret@db:5432/risk_cold_storage +assert_secret_value "$base_manifest" workflow \ + postgresql://shared:secret@db:5432/workflow + +mixed_manifest=$(helm template database-mixed "$CHART_DIR" \ + --kube-version 1.30.0 \ + "${common_values[@]}" \ + --set-string services.icc.database_url=postgresql://shared:secret@db:5432 \ + --set-string services.icc.database_urls.activities=postgresql://activities_user:secret@db:5432/plt_activities) + +assert_secret_value "$mixed_manifest" activities \ + postgresql://activities_user:secret@db:5432/plt_activities +assert_secret_value "$mixed_manifest" cold_storage \ + postgresql://shared:secret@db:5432/risk_cold_storage + +migration_manifest=$(helm template database-migration "$CHART_DIR" \ + --kube-version 1.30.0 \ + -f "$FIXTURE") + +helm lint "$CHART_DIR" -f "$FIXTURE" >/dev/null + +for database in \ + activities cluster_manager cold_storage compliance control_plane cron scaler \ + trafficante traffic_inspector user_manager; do + expected=$(awk -v key="$database:" '$1 == key { print $2; exit }' "$FIXTURE") + assert_secret_value "$migration_manifest" "$database" "$expected" + env_name="PLT_$(printf '%s' "$database" | tr '[:lower:]' '[:upper:]')_DATABASE_URL" + assert_env_secret_key "$migration_manifest" "$env_name" "$database" +done + +if secret_value "$migration_manifest" workflow >/dev/null; then + fail 'workflow database rendered without an exact or base URL' +fi + +trafficante_fallback_manifest=$(helm template trafficante-fallback "$CHART_DIR" \ + --kube-version 1.30.0 \ + -f "$FIXTURE" \ + --set-string services.icc.database_urls.traffic_inspector=) + +assert_secret_value "$trafficante_fallback_manifest" traffic_inspector \ + postgresql://trafficante_user:secret@db:5432/plt_trafficante + +if helm template workflow-missing-database "$CHART_DIR" \ + --kube-version 1.30.0 \ + -f "$FIXTURE" \ + --set services.workflow.deploy=true >/dev/null 2>&1; then + fail 'workflow rendered without a workflow database URL or base URL' +fi + +workflow_manifest=$(helm template workflow-database "$CHART_DIR" \ + --kube-version 1.30.0 \ + -f "$FIXTURE" \ + --set services.workflow.deploy=true \ + --set-string services.icc.database_urls.workflow=postgresql://workflow_user:secret@db:5432/plt_workflow) + +assert_secret_value "$workflow_manifest" workflow \ + postgresql://workflow_user:secret@db:5432/plt_workflow + +echo 'Database URL tests passed' diff --git a/test/fixtures/database-urls.yaml b/test/fixtures/database-urls.yaml new file mode 100644 index 0000000..4158444 --- /dev/null +++ b/test/fixtures/database-urls.yaml @@ -0,0 +1,18 @@ +services: + icc: + database_urls: + activities: postgresql://activities_user:secret@db:5432/plt_activities + cluster_manager: postgresql://cluster_manager_user:secret@db:5432/plt_cluster_manager + cold_storage: postgresql://cold_storage_user:secret@db:5432/plt_cold_storage + compliance: postgresql://compliance_user:secret@db:5432/plt_compliance + control_plane: postgresql://control_plane_user:secret@db:5432/plt_control_plane + cron: postgresql://cron_user:secret@db:5432/plt_cron + scaler: postgresql://scaler_user:secret@db:5432/plt_scaler + trafficante: postgresql://trafficante_user:secret@db:5432/plt_trafficante + traffic_inspector: postgresql://traffic_inspector_user:secret@db:5432/plt_traffic_inspector + user_manager: postgresql://user_manager_user:secret@db:5432/plt_user_manager + valkey: + apps_url: redis://valkey:6379/0 + icc_url: redis://valkey:6379/1 + prometheus: + url: http://prometheus:9090