From d7f1974130b869cef7d68eead86c7c598a4b72ee Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:47:34 -0600 Subject: [PATCH 1/4] Helm chart 1.0.0: run the native Server, keep PHP selectable The chart was PHP-shaped and refused native 3.x tags (#947). It now has a runtime value. Native, the default for new installs, renders one server Deployment for the native image with /api/health liveness, /api/ready readiness, a drain-length grace period and a payload storage volume, and no worker, scheduler, migration Job or Redis. PHP mode renders exactly what chart 0.3.9 rendered and stays available for existing 2.x installs. An empty runtime stays native on upgrade only when the live server Deployment carries the native runtime marker, so helm upgrade never moves a PHP release from chart 0.x to native on its own. A 3.x tag in PHP mode and a 2.x tag in native mode are refused at render time, as are native settings the binary would refuse at startup. Native schema upgrades run only through an opt-in pre-upgrade Job that invokes the image's schema-upgrade command, and the chart refuses to render it while a server pod can still run. The source release record gains helm_chart.app_version so the chart can package native 3.0.1 while server.version stays the frozen PHP 2.11.8. CI covers both runtimes, including a kind smoke of the published native image that exercises the runtime and schema-upgrade guards. Refs #946 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/helm-chart-checks.yml | 61 +++-- .github/workflows/helm-chart-release.yml | 1 + .github/workflows/helm-chart-validation.yml | 2 + README.md | 2 +- docs/helm-validation.md | 31 ++- k8s/helm/README.md | 10 +- k8s/helm/durable-workflow/Chart.yaml | 26 +- k8s/helm/durable-workflow/README.md | 238 +++++++++++----- .../ci/existing-secrets-values.yaml | 5 + .../ci/ingress-and-hpa-values.yaml | 5 + .../ci/inline-secrets-values.yaml | 5 + .../ci/native-ingress-and-hpa-values.yaml | 45 ++++ .../ci/native-inline-secrets-values.yaml | 19 ++ k8s/helm/durable-workflow/docs/UPGRADING.md | 134 ++++++++- k8s/helm/durable-workflow/templates/NOTES.txt | 35 +++ .../durable-workflow/templates/_helpers.tpl | 206 +++++++++++++- .../durable-workflow/templates/configmap.yaml | 12 +- .../templates/migration-job.yaml | 2 +- .../templates/native-schema-upgrade-job.yaml | 84 ++++++ .../templates/native-server-deployment.yaml | 142 ++++++++++ .../templates/scheduler-cronjob.yaml | 2 +- .../templates/secret-redis.yaml | 2 +- .../templates/server-deployment.yaml | 2 +- .../templates/worker-deployment.yaml | 2 +- .../templates/worker-hpa.yaml | 2 +- .../templates/worker-pdb.yaml | 2 +- k8s/helm/durable-workflow/values.schema.json | 26 ++ k8s/helm/durable-workflow/values.yaml | 144 +++++++--- k8s/helm/examples/values-dev.yaml | 4 + .../values-external-secrets-operator.yaml | 4 + k8s/helm/examples/values-native-dev.yaml | 35 +++ .../values-production-existing-secrets.yaml | 4 + resources/release/source-release.json | 3 +- scripts/ci/helm_chart_release.py | 23 +- scripts/ci/sync-source-release.mjs | 55 ++-- scripts/ci/test-source-release.mjs | 21 +- scripts/ci/test_database_tls_chart.py | 30 ++- scripts/ci/test_helm_chart_release.py | 29 +- scripts/ci/test_helm_native_image_guard.py | 50 ---- scripts/ci/test_helm_runtime_modes.py | 219 +++++++++++++++ scripts/helm-chart-kind-smoke.sh | 6 +- scripts/helm-chart-native-kind-smoke.sh | 254 ++++++++++++++++++ ...eleaseImagePublishWorkflowContractTest.php | 8 +- 43 files changed, 1729 insertions(+), 263 deletions(-) create mode 100644 k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml create mode 100644 k8s/helm/durable-workflow/ci/native-inline-secrets-values.yaml create mode 100644 k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml create mode 100644 k8s/helm/durable-workflow/templates/native-server-deployment.yaml create mode 100644 k8s/helm/examples/values-native-dev.yaml delete mode 100644 scripts/ci/test_helm_native_image_guard.py create mode 100644 scripts/ci/test_helm_runtime_modes.py create mode 100755 scripts/helm-chart-native-kind-smoke.sh diff --git a/.github/workflows/helm-chart-checks.yml b/.github/workflows/helm-chart-checks.yml index d72cffa00..2f949bfce 100644 --- a/.github/workflows/helm-chart-checks.yml +++ b/.github/workflows/helm-chart-checks.yml @@ -73,8 +73,8 @@ jobs: python -I -m pip install --only-binary :all: PyYAML==6.0.2 python scripts/ci/test_database_tls_chart.py - - name: Check that native 3.x images are refused - run: python scripts/ci/test_helm_native_image_guard.py + - name: Check what each runtime renders and refuses + run: python scripts/ci/test_helm_runtime_modes.py - name: Render every CI fixture run: | @@ -228,33 +228,42 @@ jobs: fi scripts/helm-chart-kind-smoke.sh + - name: Run native kind smoke script + env: + K8S_HELM_SMOKE_CLUSTER: ${{ env.HELM_CT_CLUSTER_NAME }} + K8S_HELM_SMOKE_REUSE_CLUSTER: "1" + K8S_HELM_NATIVE_SMOKE_ARTIFACT_DIR: ${{ runner.temp }}/helm-native-smoke + run: scripts/helm-chart-native-kind-smoke.sh + - name: Print smoke diagnostics if: failure() run: | - artifact_dir="${{ runner.temp }}/helm-smoke" - if [ ! -d "${artifact_dir}" ]; then - echo "no smoke artifacts collected" - exit 0 - fi - - for file in \ - helm-status.txt \ - events.txt \ - describe.txt \ - resources.txt \ - migration-job.log \ - server.log \ - worker.log \ - mysql.log \ - redis.log \ - port-forward.log; do - path="${artifact_dir}/${file}" - if [ ! -f "${path}" ]; then + for artifact_dir in "${{ runner.temp }}/helm-smoke" "${{ runner.temp }}/helm-native-smoke"; do + if [ ! -d "${artifact_dir}" ]; then + echo "no smoke artifacts collected in ${artifact_dir}" continue fi - echo "::group::${file}" - tail -n 200 "${path}" || cat "${path}" || true - echo "::endgroup::" + + for file in \ + helm-status.txt \ + events.txt \ + describe.txt \ + resources.txt \ + migration-job.log \ + schema-upgrade.log \ + server.log \ + worker.log \ + mysql.log \ + redis.log \ + port-forward.log; do + path="${artifact_dir}/${file}" + if [ ! -f "${path}" ]; then + continue + fi + echo "::group::$(basename "${artifact_dir}")/${file}" + tail -n 200 "${path}" || cat "${path}" || true + echo "::endgroup::" + done done - name: Upload smoke artifacts @@ -263,7 +272,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: helm-chart-kind-smoke - path: ${{ runner.temp }}/helm-smoke + path: | + ${{ runner.temp }}/helm-smoke + ${{ runner.temp }}/helm-native-smoke if-no-files-found: ignore - name: Clean up kind infrastructure diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index e56b8e0a5..e5766d34c 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -6,6 +6,7 @@ on: paths: - "k8s/helm/**" - "scripts/helm-chart-kind-smoke.sh" + - "scripts/helm-chart-native-kind-smoke.sh" - "scripts/helm-chart-kind-fixtures.yaml" - "scripts/ci/helm_chart_release.py" - "scripts/ci/cleanup-kind-cluster.sh" diff --git a/.github/workflows/helm-chart-validation.yml b/.github/workflows/helm-chart-validation.yml index 667c364ba..e3c3b505d 100644 --- a/.github/workflows/helm-chart-validation.yml +++ b/.github/workflows/helm-chart-validation.yml @@ -5,11 +5,13 @@ on: paths: - "k8s/helm/**" - "scripts/helm-chart-kind-smoke.sh" + - "scripts/helm-chart-native-kind-smoke.sh" - "scripts/helm-chart-kind-fixtures.yaml" - "scripts/ci/helm_chart_release.py" - "scripts/ci/cleanup-kind-cluster.sh" - "scripts/ci/test_helm_chart_release.py" - "scripts/ci/test_database_tls_chart.py" + - "scripts/ci/test_helm_runtime_modes.py" - "uv.toml" - ".github/workflows/helm-chart-checks.yml" - ".github/workflows/helm-chart-validation.yml" diff --git a/README.md b/README.md index c950b6d4b..db3c4229d 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ contract, so a workflow in one language can dispatch activities to another. | SQLite containers | Local evaluation with minimal infrastructure | [Official image and SQLite](docs/server-reference.md#official-image--sqlite) | | Compose with MySQL and Redis | Development and single-host production | [Compose deployment](docs/server-reference.md#official-image--compose) | | Small cluster | Multiple API nodes with external persistence | [Small-cluster contract](docs/small-cluster-validation.md) | -| Kubernetes | Independently scalable API and worker pools | [Helm chart](k8s/helm/durable-workflow/README.md) | +| Kubernetes | Native API replicas on Kubernetes, or the PHP 2.x shape for existing installs | [Helm chart](k8s/helm/durable-workflow/README.md) | | Active/passive regions | Operator-controlled regional failover | [Multi-region contract](docs/multi-region-validation.md) | Server supports SQLite for a single-node runtime and MySQL or PostgreSQL for diff --git a/docs/helm-validation.md b/docs/helm-validation.md index 1957f53da..dd028e775 100644 --- a/docs/helm-validation.md +++ b/docs/helm-validation.md @@ -158,17 +158,38 @@ touches `k8s/helm/**` or the chart smoke script. The harness has two jobs: - runs `helm upgrade` in place and re-checks readiness; - runs `helm uninstall` and asserts no chart-owned resources remain. -The CI fixtures intentionally cover four shapes: + The same job then runs `scripts/helm-chart-native-kind-smoke.sh`, which + installs the native Server from the chart's default published image + against a fresh in-cluster MySQL database and: + - waits for the single server Deployment to become ready and runs + `helm test`; + - asserts that no worker, scheduler, migration Job or Redis Secret exists; + - upgrades with an empty `runtime` and confirms the release stays native; + - confirms that the schema-upgrade hook refuses while a Server pod runs, + then scales to zero and runs the hook, which verifies the current schema; + - confirms that an upgrade of a release whose server Deployment lacks the + native runtime marker refuses an empty `runtime`; + - runs `helm uninstall` and asserts no chart-owned resources remain. + +The CI fixtures intentionally cover six shapes: | Fixture | Purpose | | --- | --- | | `ci/inline-secrets-values.yaml` | Smallest renderable chart — exercises secret rendering. | | `ci/existing-secrets-values.yaml` | GitOps / externally-managed-secret path. The chart renders no Secret resources of its own. | | `ci/ingress-and-hpa-values.yaml` | Optional templates: Ingress, HPA, NetworkPolicy, server + worker PDB. | -| `ci/database-tls-values.yaml` | External CA Secret, canonical TLS settings and read-only mounts in every PHP role. | - -A chart change that affects rendered manifests must keep all four -fixtures green, and the kind smoke must end with a clean uninstall. +| `ci/database-tls-values.yaml` | External CA Secret, canonical TLS settings and read-only mounts. Renders native by default. The TLS test also renders it in PHP mode. | +| `ci/native-inline-secrets-values.yaml` | Native Server with inline secrets: one Deployment, no PHP roles. | +| `ci/native-ingress-and-hpa-values.yaml` | Native with Ingress, HPA, NetworkPolicy, PDB, a payload claim and the schema-upgrade hook. | + +The first three fixtures pin `runtime: php`. A chart change that affects +rendered manifests must keep every fixture green, and both kind smokes must +end with a clean uninstall. + +`scripts/ci/test_helm_runtime_modes.py` checks what each runtime renders, that +a tag from the other runtime is refused, that an upgrade with an empty +`runtime` must find a native release, and the native render-time refusals +that mirror native startup refusals. `scripts/ci/test_database_tls_chart.py` also checks both pre-install-hook and ordinary bootstrap ordering, CA path/key consistency, and preservation of diff --git a/k8s/helm/README.md b/k8s/helm/README.md index bc82b51c0..71c33b72c 100644 --- a/k8s/helm/README.md +++ b/k8s/helm/README.md @@ -6,9 +6,10 @@ standalone server, together with the example values files and CI fixtures. * `durable-workflow/` — the chart itself. See its [README](durable-workflow/README.md) and [UPGRADING guide](durable-workflow/docs/UPGRADING.md). -* `examples/` — copy-and-adapt values files for the most common shapes - (production with externally-managed secrets, dev, External Secrets - Operator companion, etc.). +* `examples/` — copy-and-adapt values files. `values-native-dev.yaml` runs + the native Server, the chart default. The others keep an existing PHP 2.x + installation (`runtime: php`): production with externally-managed secrets, + dev, and an External Secrets Operator companion. * `durable-workflow/ci/` — minimum-viable values fixtures used by chart-CI (`helm lint`, `helm template`, `ct lint-and-install`). @@ -16,4 +17,5 @@ The chart is the supported Helm path for the deployment matrix at . The raw manifests in [`../`](..) remain a supported, lower-level alternative; both share the same external-persistence contract and the same singleton-scheduler -invariant. Pick one or the other per environment, not both. +invariant. The raw manifests run the PHP 2.x Server only. Pick one or the +other per environment, not both. diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 7809c162b..2b83cb0b8 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -1,15 +1,15 @@ apiVersion: v2 name: durable-workflow -description: Durable Workflow self-hosted server, worker, scheduler/maintenance runner, and bootstrap job for Kubernetes. +description: Durable Workflow self-hosted Server for Kubernetes. Runs the native 3.x Server by default, or the PHP 2.x Server with its worker, scheduler and bootstrap job for existing installs. type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/UPGRADING.md alongside the chart. -version: 0.3.9 -# The immutable Durable Workflow Server identity this chart release packages. +version: 1.0.0 +# The immutable native Durable Workflow Server this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the -# checked-in source release record. -appVersion: "2.11.8" +# checked-in source release record (helm_chart.app_version). +appVersion: "3.0.1" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,18 +30,16 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.11.8" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:3.0.1" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - - kind: fixed - description: Preserve duplicate bridge update receipts for SDK-worker workflows. - - kind: fixed - description: Include canonical failure storage, retention recovery and rejected-outcome history readers. - kind: changed - description: Package Server 2.11 with capability-gated activity lease maintenance and configurable lease duration. + description: Run the native 3.x Server by default as one Deployment with no worker, scheduler, migration job or Redis. - kind: added - description: Document compatible worker rollout and the distinction between leases and application deadlines. - - kind: fixed - description: Send application logs to container stderr by default and verify actual API, worker and scheduler logging before chart publication. + description: Add runtime=php so existing PHP 2.x installs keep their worker, scheduler and bootstrap job. + - kind: added + description: Add an opt-in pre-upgrade job that runs the native schema-upgrade command while every Server is stopped. + - kind: changed + description: Refuse to move an existing PHP release to native unless runtime is set explicitly. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index d3c1a5c1e..8c04935e2 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -1,17 +1,29 @@ # Durable Workflow Helm Chart -This chart deploys a self-hosted Durable Workflow standalone server, worker -pool, scheduler/maintenance runner, and bootstrap job onto Kubernetes. It is -the self-serve Helm path for the deployment matrix at +This chart deploys a self-hosted Durable Workflow Server onto Kubernetes. It +is the self-serve Helm path for the deployment matrix at . +The chart runs one of two Server runtimes, selected by the `runtime` value: + +* **native** (the default for new installs) runs the native 3.x Server. Each + pod is one process that serves the API, checks its database schema at + startup and runs timers, repair, retention and schedules itself. The chart + renders one server Deployment and no worker, scheduler, migration Job or + Redis. +* **php** keeps an existing PHP 2.x installation running: a server + Deployment, a queue worker Deployment, a singleton scheduler CronJob and a + bootstrap Job, with Redis. PHP Server 2.x is frozen at 2.11.8 and gets no + further releases. Move to native when you can (see + [docs/UPGRADING.md](docs/UPGRADING.md)). + The chart is engine-aware: the templates encode the engine invariants from [workflow/docs/deployment/multi-node-requirements.md](https://github.com/durable-workflow/workflow/blob/main/docs/deployment/multi-node-requirements.md) and the failover guarantees from [server/docs/ha-failover-validation.md](../../docs/ha-failover-validation.md) -(singleton scheduler, readiness on `/api/ready`, externals-first persistence, -no in-cluster databases bundled). Override-with-care knobs that would break -those guarantees are flagged in `values.yaml`. +(readiness on `/api/ready`, externals-first persistence, no in-cluster +databases bundled, and in PHP mode a singleton scheduler). Override-with-care +knobs that would break those guarantees are flagged in `values.yaml`. ## Status @@ -19,10 +31,10 @@ those guarantees are flagged in `values.yaml`. version, which is `appVersion`). * Chart distribution: published as an OCI artifact at `oci://ghcr.io/durable-workflow/charts/durable-workflow`. -* Stability: **beta** — the externals-first contract, singleton scheduler - invariant, secret layout, and resource naming are committed to. Any breaking - change to those will bump the chart's MAJOR version and ship migration steps - in [docs/UPGRADING.md](docs/UPGRADING.md). +* Stability: **beta** — the externals-first contract, the runtime selection, + the singleton scheduler invariant in PHP mode, secret layout, and resource + naming are committed to. Any breaking change to those will bump the chart's + MAJOR version and ship migration steps in [docs/UPGRADING.md](docs/UPGRADING.md). ## Prerequisites @@ -30,14 +42,16 @@ those guarantees are flagged in `values.yaml`. | --- | --- | | Kubernetes ≥ 1.27 | Older versions are not validated; the chart uses `policy/v1` PDB and `autoscaling/v2` HPA. | | Helm ≥ 3.10 | Required for OCI install and the `lookup` builtin. | -| External MySQL/PostgreSQL | The chart does not bundle a database. | -| External Redis (or compatible) | Required for the multi-node correctness contract. | +| External MySQL/MariaDB/PostgreSQL | The chart does not bundle a database. Native creates its schema in an empty database on first start. The database itself must already exist. | +| External Redis (or compatible) | PHP mode only. Required for the PHP multi-node correctness contract. Native does not use Redis. | | Inbound TLS / ingress | Provided by your platform. The chart can render an `Ingress` but does not terminate TLS itself. | ## Install -The chart will refuse to render until you point it at an external database and -Redis — that is intentional, not a bug. +The chart will refuse to render until you point it at an external database +(and, in PHP mode, Redis) — that is intentional, not a bug. Native mode also +refuses to render without a shared `DW_AUTH_TOKEN`, as the native Server +refuses to start without one. ### From the OCI registry (recommended for production) @@ -63,10 +77,9 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.11.8" + tag: "3.0.1" # Pin a digest in production: # digest: "sha256:abc123..." - # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. externalDatabase: connection: pgsql @@ -75,11 +88,6 @@ externalDatabase: database: durable_workflow existingSecret: durable-workflow-db-credentials -externalRedis: - host: workflow-redis.example.internal - port: 6379 - existingSecret: durable-workflow-redis-credentials - auth: existingSecret: durable-workflow-app-credentials @@ -99,44 +107,85 @@ server: - workflow.example.com ``` -The three referenced Secrets must contain: +The two referenced Secrets must contain: * **db credentials**: `DB_USERNAME`, `DB_PASSWORD` -* **Redis credentials** (only when Redis enforces AUTH): `REDIS_USERNAME`, - `REDIS_PASSWORD` -* **app credentials**: `DW_SERVER_KEY`, `DW_WORKER_TOKEN`, - `DW_OPERATOR_TOKEN`, `DW_ADMIN_TOKEN`, optional `DW_PRINCIPAL_TOKENS` for - named bearer-token principals, plus optional `DW_AUTH_TOKEN` when you keep - `config.auth.backwardCompatible: true`. +* **app credentials**: `DW_AUTH_TOKEN` (required, and native accepts it for every + role, so use a strong random value), `DW_WORKER_TOKEN`, `DW_OPERATOR_TOKEN`, + `DW_ADMIN_TOKEN` for narrower role-scoped tokens, and `DW_SERVER_KEY`. Do not + put `DW_PRINCIPAL_TOKENS` in this Secret: the native Server refuses to start + with it. The chart accepts existing Secret names directly, so it composes cleanly with the External Secrets Operator, Vault Secret Operator, Secrets Store CSI Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`. +### Native capacity and storage + +* Each native pod opens at most four database connections. Size the + database's connection limit for four per pod at the highest replica count + (`server.autoscaling.maxReplicas` when autoscaling), plus one for a schema + upgrade. +* Native keeps external payload objects on local disk at `/app/storage`. + Without `native.storage.existingClaim` the chart mounts an `emptyDir`, and + the objects are lost when a pod is replaced. Set a claim before any + namespace enables external payload storage. With more than one replica the + claim must be `ReadWriteMany` so every pod reads the same objects. +* On `SIGTERM` the native Server holds open worker polls for up to + `DW_WORKER_POLL_TIMEOUT` (30 seconds by default) before it exits. + `native.terminationGracePeriodSeconds` (default 45) must stay above it. The + chart refuses a grace period at or below a `DW_WORKER_POLL_TIMEOUT` set in + `config.extraEnv`. +* The chart supports MySQL, MariaDB and PostgreSQL. It does not run the + native Server on SQLite. + +### PHP 2.x installations + +An existing PHP installation selects PHP mode explicitly and pins its 2.x +image. The chart refuses a 3.x tag in PHP mode and a 2.x tag in native mode. + +```yaml +runtime: php +image: + tag: "2.11.8" # or your current 2.x release +externalRedis: + host: workflow-redis.example.internal + existingSecret: durable-workflow-redis-credentials +``` + +PHP mode renders the same resources as chart 0.3.x. Its app credentials Secret +holds `DW_SERVER_KEY`, the role tokens, optional `DW_PRINCIPAL_TOKENS`, and +`DW_AUTH_TOKEN` when you keep `config.auth.backwardCompatible: true`. The +Redis Secret holds `REDIS_USERNAME` and `REDIS_PASSWORD` when Redis enforces +AUTH. See `examples/` for complete PHP values files. + ## What the chart renders -| Resource | Purpose | Disabled when | -| --- | --- | --- | -| `Job` (`*-migrate`) | Runs `server-bootstrap` (migrations, namespace bootstrap) before workloads accept traffic. Helm hook by default; sync-wave annotation for Argo CD. | `bootstrap.enabled: false` | -| `Deployment` (`*-server`) | Stateless API replicas. Liveness on `/api/health`, readiness on `/api/ready`. | `server.enabled: false` | -| `Service` (`*-server`) | ClusterIP in front of the API. | `server.enabled: false` | -| `PodDisruptionBudget` (`*-server`) | Default `minAvailable: 1`. | `server.pdb.enabled: false` | -| `Ingress` (`*-server`) | Optional. | `server.ingress.enabled: false` (default) | -| `HorizontalPodAutoscaler` (`*-server`) | Optional. | `server.autoscaling.enabled: false` (default) | -| `Deployment` (`*-worker`) | Worker pool. Independent scale axis. | `worker.enabled: false` | -| `CronJob` (`*-scheduler`) | Singleton scheduler/maintenance runner, including unscoped repair for expired remote SDK task leases. **`concurrencyPolicy: Forbid`** is enforced — see below. | `scheduler.enabled: false` | -| `ConfigMap` (`*-config`) | All non-secret config (DB host, Redis host, auth driver, metrics knobs). | Always rendered. | -| `Secret` (`*-app-secrets`) | Server signing key, role-scoped tokens, and named principal token map. | `auth.existingSecret` is set. | -| `Secret` (`*-database`) | DB username/password. | `externalDatabase.existingSecret` is set. | -| `Secret` (`*-redis`) | Redis username/password. | `externalRedis.existingSecret` is set or no inline credentials supplied. | -| `ServiceAccount` (`*`) | Namespace-scoped SA for every workload. | `serviceAccount.create: false` | -| `NetworkPolicy` (`*-server`) | Optional ingress allow-list to the API. | `networkPolicy.enabled: false` (default) | +| Resource | Runtime | Purpose | Disabled when | +| --- | --- | --- | --- | +| `Deployment` (`*-server`) | both | API replicas. Liveness on `/api/health`, readiness on `/api/ready`. In native mode the image's own process runs maintenance, and the Deployment carries the `workflows.durable-workflow.dev/runtime: native` annotation. | `server.enabled: false` | +| `Job` (`*-schema-upgrade`) | native | `pre-upgrade` hook that runs the image's `schema-upgrade` command. Opt-in, see [Native schema upgrades](#native-schema-upgrades). | `native.schemaUpgrade.enabled: false` (default) | +| `Job` (`*-migrate`) | php | Runs `server-bootstrap` (migrations, namespace bootstrap) before workloads accept traffic. Helm hook by default; sync-wave annotation for Argo CD. | `bootstrap.enabled: false` | +| `Service` (`*-server`) | both | ClusterIP in front of the API. | `server.enabled: false` | +| `PodDisruptionBudget` (`*-server`) | both | Default `minAvailable: 1`. | `server.pdb.enabled: false` | +| `Ingress` (`*-server`) | both | Optional. | `server.ingress.enabled: false` (default) | +| `HorizontalPodAutoscaler` (`*-server`) | both | Optional. | `server.autoscaling.enabled: false` (default) | +| `Deployment` (`*-worker`) | php | Worker pool. Independent scale axis. | `worker.enabled: false` | +| `CronJob` (`*-scheduler`) | php | Singleton scheduler/maintenance runner, including unscoped repair for expired remote SDK task leases. **`concurrencyPolicy: Forbid`** is enforced — see below. | `scheduler.enabled: false` | +| `ConfigMap` (`*-config`) | both | All non-secret config. Native: database settings, auth driver and `config.extraEnv`. PHP adds Redis, queue, application and metrics settings. | Always rendered. | +| `Secret` (`*-app-secrets`) | both | Server signing key, role-scoped tokens, and named principal token map. | `auth.existingSecret` is set. | +| `Secret` (`*-database`) | both | DB username/password. | `externalDatabase.existingSecret` is set. | +| `Secret` (`*-redis`) | php | Redis username/password. | `externalRedis.existingSecret` is set or no inline credentials supplied. | +| `ServiceAccount` (`*`) | both | Namespace-scoped SA for every workload. | `serviceAccount.create: false` | +| `NetworkPolicy` (`*-server`) | both | Optional ingress allow-list to the API. | `networkPolicy.enabled: false` (default) | ## Application logs -The chart defaults `config.extraEnv.LOG_CHANNEL` to `stderr` for the API, worker, -scheduler and bootstrap job. Use `kubectl logs` or the cluster's log collector. -To choose the minimum severity and emit newline-delimited JSON: +The native Server writes its logs to stderr. In PHP mode the chart defaults +`config.extraEnv.LOG_CHANNEL` to `stderr` for the API, worker, scheduler and +bootstrap job. The native Server ignores that setting. Use `kubectl logs` or +the cluster's log collector. In PHP mode, to choose the minimum severity and +emit newline-delimited JSON: ```yaml config: @@ -167,8 +216,9 @@ externalDatabase: The chart sets `DB_SSLMODE` and mounts that Secret read-only at `/etc/durable-workflow/database-tls/ca.pem`, referenced by `DB_SSLROOTCERT`. -The API server, worker, scheduler and bootstrap/migration Job all receive the -same trust bundle. The Secret must already exist when the pre-install or +Every role receives the same trust bundle: the API server and the native +schema-upgrade Job, or in PHP mode the API server, worker, scheduler and +bootstrap/migration Job. The Secret must already exist when the pre-install or pre-upgrade hook starts; the chart does not create it. The `database-tls` volume name and `/etc/durable-workflow/database-tls` mount path are reserved when this option is enabled. Existing application extra volumes are retained. @@ -180,8 +230,8 @@ also supports `verify-ca`; MySQL/MariaDB refuse that mode and require contract is in [Database TLS](../../../docs/server-reference.md#database-tls). CA files update through the Secret volume rather than a `subPath` mount. -After rotating trust, restart all persistent PHP roles to replace existing -connections, then check `/api/ready` and run a workflow canary. Poolers need +After rotating trust, restart the server pods (and in PHP mode the worker) to +replace existing connections, then check `/api/ready` and run a workflow canary. Poolers need independent TLS configuration on their database-facing leg. ## Engine invariants the chart enforces @@ -190,19 +240,30 @@ These come from the engine, not the chart. Changing them silently breaks the engine's correctness contract, so they are either locked or annotated as override-with-care in `values.yaml`: -* **Singleton scheduler.** The CronJob's `concurrencyPolicy` defaults to +* **No silent runtime switch.** An upgrade with an empty `runtime` stays + native only when the live server Deployment already carries the native + runtime annotation. A release that chart 0.x installed runs PHP, so its + upgrade refuses until `runtime` is set. See [docs/UPGRADING.md](docs/UPGRADING.md). +* **Native never upgrades its schema on start.** A pod on an older schema + refuses with `native_schema_upgrade_required` and stays unready. Schema + upgrades run only through the explicit hook below. +* **Singleton scheduler (PHP mode).** The CronJob's `concurrencyPolicy` defaults to `Forbid`, the chart never renders a parallel scheduler `Deployment`, and the schema rejects any other concurrency value. This matches the contract in [single-region HA failover](https://durable-workflow.github.io/docs/2.0/deployment#single-region-high-availability-and-failover). -* **Readiness probe is `/api/ready`, not `/api/health`.** `/api/ready` checks - the database and Redis are usable. Load-balancer routing decisions on - `/api/health` would route traffic to a node whose database had failed over. -* **Bootstrap job runs before workloads.** Helm hooks default to +* **Readiness probe is `/api/ready`, not `/api/health`.** In PHP mode + `/api/ready` checks the database and Redis are usable. In native mode it + checks the schema and the in-process maintenance loop. Load-balancer routing + decisions on `/api/health` would route traffic to a node whose database had + failed over. Keep `/api/ready` out of the liveness probe: in native mode a + stalled maintenance tick fails readiness on every pod, and a liveness probe + on it would restart them all. +* **Bootstrap job runs before workloads (PHP mode).** Helm hooks default to `pre-install,pre-upgrade`. For Argo CD/Flux users who don't honour Helm hooks, sync-wave / depends-on annotations enforce the same ordering. * **No in-cluster database is bundled.** External persistence is the - contract; the chart will not render until `externalDatabase.host` and - `externalRedis.host` are populated. + contract; the chart will not render until `externalDatabase.host` (and in + PHP mode `externalRedis.host`) is populated. * **Rolling upgrades default to `maxUnavailable: 0`.** Set `server.strategy.type: Recreate` for stop-the-world if your release does not satisfy the [rolling-upgrade contract](https://durable-workflow.github.io/docs/2.0/rolling-upgrades). @@ -212,16 +273,54 @@ override-with-care in `values.yaml`: `image.memoPayloadStorage` capability; follow the versioned procedure in [UPGRADING.md](docs/UPGRADING.md) before migration. +## Native schema upgrades + +Patch releases within a native minor line keep the schema, so changing +`image.tag` is enough. A release whose notes announce a schema change needs +`schema-upgrade`, which requires every native Server stopped and a database +backup first: + +1. Scale the Server to zero and wait for its pods to exit: + + ```bash + helm upgrade durable-workflow -f my-values.yaml \ + --set server.replicaCount=0 --set server.autoscaling.enabled=false --wait + ``` +2. Back up the database with your usual procedure. +3. Upgrade to the new image with the hook enabled and your usual replica + count: + + ```bash + helm upgrade durable-workflow -f my-values.yaml \ + --set image.tag= \ + --set native.schemaUpgrade.enabled=true --wait + ``` + + The `pre-upgrade` hook runs `schema-upgrade` before the new pods start. + On a database that is already current it only verifies the schema. +4. Leave `native.schemaUpgrade.enabled` false for later upgrades. + +The chart refuses to render the hook while the live server Deployment has +replicas or a server pod is still running, because `schema-upgrade` must not +run under a live Server. That check reads the cluster through Helm's `lookup`, +so it does not run under `helm template`, a client-side dry run, or a GitOps +controller that renders with `helm template`. Scale to zero yourself in those +setups. Restore the backup to go back: there is no schema downgrade. + ## GitOps notes The chart was designed to compose with both Argo CD and Flux without leaning on Helm hooks alone: +* GitOps controllers that render with `helm template` cannot see the live + release, so an empty `runtime` always renders native there. Set `runtime` + explicitly in GitOps values, especially for a PHP installation. + * `argocd.useSyncWaves: true` (default) emits sync-wave annotations so the - bootstrap Job runs in an earlier sync wave than the workload Deployments + PHP bootstrap Job runs in an earlier sync wave than the workload Deployments and CronJob. Flip to `false` only if you actively use Helm hooks under Argo CD's `--enable-helm-hooks` flag. -* `flux.useDependsOn: true` (default) emits +* `flux.useDependsOn: true` (default) emits, in PHP mode, `kustomize.toolkit.fluxcd.io/depends-on` annotations referencing the bootstrap Job. The Job name is deterministic, so the annotation is stable across reconciliations. @@ -233,7 +332,10 @@ helm test durable-workflow --namespace durable-workflow ``` The test pod hits `/api/health` and `/api/ready` through the in-cluster -Service. Failure means the bootstrap Job did not produce a usable cluster. +Service. In native mode a failure usually means the Server could not reach, +create or accept its schema (check the server pod log, for example for +`native_schema_upgrade_required`). In PHP mode it means the bootstrap Job did +not produce a usable cluster. For the human verification path, see `templates/NOTES.txt`. @@ -249,15 +351,21 @@ are governed by separate semver streams. helm uninstall durable-workflow --namespace durable-workflow ``` -`helm uninstall` deletes everything the chart created. The external database -and Redis are explicitly out of the chart's lifecycle — back them up before +`helm uninstall` deletes everything the chart created. The external database, +Redis in PHP mode, and any `native.storage.existingClaim` are explicitly out of +the chart's lifecycle — back them up before you uninstall a production release. ## Validation in this repository * `helm lint` and `helm template` run on every PR (see `.github/workflows/helm-chart-validation.yml`). -* `chart-testing` (`ct lint-and-install`) installs the chart into a kind - cluster, runs `helm test`, and asserts `/api/ready` responds. +* `scripts/ci/test_helm_runtime_modes.py` checks what each runtime renders and + every render-time refusal. +* A kind cluster installs the chart in both runtimes: PHP from the checked-out + source image (`scripts/helm-chart-kind-smoke.sh`) and native from the + published image (`scripts/helm-chart-native-kind-smoke.sh`), runs + `helm test`, and asserts `/api/ready` responds. The native smoke also + exercises the runtime and schema-upgrade guards against a live release. * `kubeconform` validates every rendered manifest against published Kubernetes schemas for the kube versions in our support matrix. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index cbf158ce7..49a3faaa9 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,5 +1,10 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. + +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: tag: "2.11.8" diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index b69353d51..af36dc9f1 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,4 +1,9 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. + +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: tag: "2.11.8" diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 8a09a9e07..346437622 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -1,6 +1,11 @@ # CI fixture: smallest renderable values. Keeps secrets inline so the # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. + +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: tag: "2.11.8" diff --git a/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml new file mode 100644 index 000000000..754c1eaa3 --- /dev/null +++ b/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml @@ -0,0 +1,45 @@ +# CI fixture: the native Server with PostgreSQL, existing Secrets, ingress, +# autoscaling, a shared payload claim and the opt-in schema-upgrade hook. +runtime: native + +externalDatabase: + connection: pgsql + host: db.example.internal + port: 5432 + database: durable_workflow + existingSecret: durable-workflow-db-credentials + +auth: + existingSecret: durable-workflow-app-credentials + +native: + storage: + existingClaim: durable-workflow-payloads + schemaUpgrade: + enabled: true + +server: + replicaCount: 3 + ingress: + enabled: true + className: nginx + hosts: + - host: workflow.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: workflow-tls + hosts: + - workflow.example.com + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 8 + +networkPolicy: + enabled: true + ingressFrom: + - namespaceSelector: + matchLabels: + team: platform-ingress diff --git a/k8s/helm/durable-workflow/ci/native-inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/native-inline-secrets-values.yaml new file mode 100644 index 000000000..ed4dcf81e --- /dev/null +++ b/k8s/helm/durable-workflow/ci/native-inline-secrets-values.yaml @@ -0,0 +1,19 @@ +# CI fixture: the native Server with inline secrets and MySQL. Renders one +# server Deployment and no worker, scheduler, migration Job or Redis. +runtime: native + +externalDatabase: + connection: mysql + host: db.example.internal + port: 3306 + database: durable_workflow + auth: + username: workflow + password: changeme-ci-only + +auth: + serverKey: "base64:CHANGE_ME_CI_ONLY" + authToken: "ci-shared-token" + workerToken: "ci-worker-token" + operatorToken: "ci-operator-token" + adminToken: "ci-admin-token" diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 18e6cbbfd..257fb1742 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -30,8 +30,8 @@ chart MINOR bump and an entry below. 1. Read this guide for every chart-version increment you are crossing (e.g. for a `0.1.0 → 0.3.0` upgrade, read `0.2.0` and `0.3.0`). -2. Back up the workflow database and snapshot Redis if your operator - recovery packet requires it (see +2. Back up the workflow database (and in PHP mode snapshot Redis) if your + operator recovery packet requires it (see [Operator Operating Envelope](https://durable-workflow.github.io/docs/2.0/operator-operating-envelope)). 3. Render a diff before applying: @@ -51,9 +51,11 @@ chart MINOR bump and an entry below. --namespace durable-workflow \ --atomic --wait --timeout 10m ``` - The default Helm hook order runs the bootstrap Job before the new - server/worker pods take traffic. With Argo CD or Flux, the chart's - sync-wave / depends-on annotations enforce the same ordering. + In PHP mode the default Helm hook order runs the bootstrap Job before the + new server/worker pods take traffic. With Argo CD or Flux, the chart's + sync-wave / depends-on annotations enforce the same ordering. In native + mode each pod checks its own schema at startup. A release with a schema + change also needs the [native schema upgrade](../README.md#native-schema-upgrades). 5. After the rollout, confirm the engine view matches the operator view: ```bash @@ -67,6 +69,128 @@ chart MINOR bump and an entry below. ## Per-version migration notes +### Chart 1.0.0 / Server 3.0.1 (native) + +The chart now runs the native 3.x Server by default. Native is one process +per pod: it creates its schema in an empty database, checks it on every start +and runs timers, repair, retention and schedules itself. Native mode renders +one server Deployment and no worker Deployment, scheduler CronJob, migration +Job or Redis wiring. PHP Server 2.x is frozen at 2.11.8, and the chart keeps +it available as `runtime: php` for existing installations. + +New `runtime` value: `native`, `php`, or empty. Empty runs native on a new +install. On an upgrade it stays native only when the live server Deployment +carries the `workflows.durable-workflow.dev/runtime: native` annotation that +chart 1.x writes. Every release installed by chart 0.x runs PHP, so its first +upgrade to chart 1.x refuses to render until you choose. This also holds for +`helm upgrade --reuse-values`, which keeps the 0.x defaults. A client-side dry +run cannot read the live Deployment and refuses too. Use `--dry-run=server` +(Helm 3.13 or later) or set `runtime`. + +**New installations.** Supply the database and a shared token +(`auth.authToken` or `DW_AUTH_TOKEN` in `auth.existingSecret`). Redis is not +needed. See the chart [README](../README.md). + +**Existing PHP installations that stay on PHP.** Add these values before +upgrading the chart. Nothing else changes: PHP mode renders the same +resources as chart 0.3.9. + +```yaml +runtime: php +image: + tag: "2.11.8" # or the 2.x release you run +``` + +The chart refuses a 3.x or `latest` tag in PHP mode, and a 2.x tag in native +mode. A digest pin is accepted in either mode, so pin a digest that matches +the runtime. + +**Existing PHP installations that move to native.** Native takes over a PHP +database once, in place, through the backup-first +[native takeover runbook](../../../../docs/rust-server-upgrade-takeover.md). +Takeover ships in 3.0.x and is planned for deprecation in the 3.1.x line, so +move while running a 3.0.x native release. There is no PHP rollback after +the takeover: recovery restores the verified backup into a new database for +PHP. Native pods started on a PHP database refuse it and stay unready, so do +not set `runtime: native` before the takeover completes. + +1. Upgrade to chart 1.x in PHP mode as above, with Server 2.5.14 or later + (2.11.8 recommended), and confirm the release is healthy. +2. Stop every PHP role that can write and keep the chart from restarting + them. Wait until no server, worker, scheduler or migration pod remains. + + ```bash + helm upgrade durable-workflow -f my-values.yaml \ + --set runtime=php --set image.tag=2.11.8 \ + --set server.replicaCount=0 --set server.autoscaling.enabled=false \ + --set worker.replicaCount=0 --set worker.autoscaling.enabled=false \ + --set scheduler.enabled=false --set bootstrap.enabled=false --wait + kubectl -n durable-workflow get pods + ``` +3. Follow the runbook's inspection, fingerprint, backup, restored-backup + fingerprint and `upgrade-takeover` steps with one native image digest. Run + each command as a one-off pod in the release namespace with the same + `DB_*` settings, reading the password from the release's database Secret + instead of a command line. For example: + + ```yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: native-upgrade-inspect + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + securityContext: {runAsNonRoot: true, runAsUser: 1000} + containers: + - name: inspect + image: docker.io/durableworkflow/server@sha256: + args: ["upgrade-inspect"] + envFrom: + - configMapRef: {name: durable-workflow-config} + env: + - name: DB_USERNAME + valueFrom: {secretKeyRef: {name: durable-workflow-database, key: DB_USERNAME}} + - name: DB_PASSWORD + valueFrom: {secretKeyRef: {name: durable-workflow-database, key: DB_PASSWORD}} + ``` + + Use your `externalDatabase.existingSecret` and its keys if you set one, and + mount the database CA Secret when you use database TLS. The queue + inspection and the takeover also need the `DW_RUST_UPGRADE_QUEUE_*` + values the runbook lists, and the takeover needs + `DW_UPGRADE_BACKUP_FINGERPRINT`. +4. Switch the release to native with the same image and restore the replica + count. Set `auth.authToken` (or `DW_AUTH_TOKEN` in your existing Secret) and + remove `auth.principalTokens` and `DW_PRINCIPAL_TOKENS`, which native + refuses. + + ```bash + helm upgrade durable-workflow -f my-values.yaml \ + --set runtime=native --set image.tag=3.0.1 \ + --set server.replicaCount=2 --wait + ``` + + Helm removes the worker Deployment, scheduler CronJob, migration Job hook + and Redis Secret and updates the server Deployment in place. PHP-only + values such as `externalRedis`, `worker`, `scheduler` and `bootstrap` are + ignored in native mode. Check `/api/ready` and run a workflow canary. + +**GitOps.** Argo CD and other controllers that render with `helm template` +cannot see the live release, so an empty `runtime` always renders native for +them. Set `runtime` explicitly in GitOps values before you move to chart 1.x. + +**Native settings.** `native.terminationGracePeriodSeconds` (default 45) +stays above the native worker poll timeout so pods drain open polls on +shutdown. `native.storage.existingClaim` keeps external payload objects +across pod restarts. `native.schemaUpgrade.enabled` runs the image's +`schema-upgrade` command as an opt-in `pre-upgrade` hook, see +[Native schema upgrades](../README.md#native-schema-upgrades). Native ignores +`config.auth.backwardCompatible` and the metrics and application name +settings, and accepts only the token auth driver. + ### Chart 0.3.9 / Server 2.11.8 No change for existing installs: the default Server stays 2.11.8 and values diff --git a/k8s/helm/durable-workflow/templates/NOTES.txt b/k8s/helm/durable-workflow/templates/NOTES.txt index 1c6f156cc..0f847c866 100644 --- a/k8s/helm/durable-workflow/templates/NOTES.txt +++ b/k8s/helm/durable-workflow/templates/NOTES.txt @@ -1,6 +1,38 @@ Durable Workflow {{ .Chart.AppVersion }} (chart {{ .Chart.Version }}) was installed into namespace "{{ .Release.Namespace }}" as release "{{ .Release.Name }}". +{{- if include "durable-workflow.isNative" . }} +What just happened +------------------ +This release runs the native Server (runtime=native). Each pod is one process +that serves the API, checks its database schema at startup and runs timers, +repair, retention and schedules itself. There is no worker Deployment, +scheduler CronJob, migration Job or Redis. +{{- if .Values.server.enabled }} + +The API Deployment "{{ include "durable-workflow.serverDeploymentName" . }}" +{{- if .Values.server.autoscaling.enabled }} scales between {{ .Values.server.autoscaling.minReplicas }} and {{ .Values.server.autoscaling.maxReplicas }} replicas +{{- else }} runs {{ .Values.server.replicaCount }} replicas +{{- end }} behind the Service +"{{ include "durable-workflow.serverDeploymentName" . }}" on port {{ .Values.server.service.port }}. +{{- end }} + +Each pod holds up to four database connections. A newer native release with a +schema change needs the explicit schema upgrade in the chart's UPGRADING.md: +stop every Server, back up the database, then upgrade with +native.schemaUpgrade.enabled=true. +{{- if not (get (default (dict) (get (default (dict) .Values.native) "storage")) "existingClaim") }} + +External payload objects are stored in an emptyDir and are lost when a pod is +replaced. Before a namespace enables external payload storage, set +native.storage.existingClaim (ReadWriteMany when more than one replica runs). +{{- end }} + +Verify the deployment +--------------------- + kubectl -n {{ .Release.Namespace }} get pods,svc,pdb + kubectl -n {{ .Release.Namespace }} rollout status deploy/{{ include "durable-workflow.serverDeploymentName" . }} +{{- else }} What just happened ------------------ {{- if .Values.bootstrap.enabled }} @@ -31,6 +63,7 @@ Verify the deployment kubectl -n {{ .Release.Namespace }} get pods,svc,pdb kubectl -n {{ .Release.Namespace }} rollout status deploy/{{ include "durable-workflow.serverDeploymentName" . }} kubectl -n {{ .Release.Namespace }} rollout status deploy/{{ include "durable-workflow.workerDeploymentName" . }} +{{- end }} Probe the readiness contract (must return 200 once migrations finish): @@ -45,7 +78,9 @@ If you have an admin token, confirm the topology manifest: What you still own ------------------ * External MySQL/PostgreSQL: {{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port }} ({{ .Values.externalDatabase.connection }}) +{{- if include "durable-workflow.isPhp" . }} * External Redis: {{ .Values.externalRedis.host }}:{{ .Values.externalRedis.port }} +{{- end }} * TLS termination, ingress, and request logging — left to your platform. * Backup/restore, recovery-packet rehearsal, and disaster-recovery runbooks per https://durable-workflow.github.io/docs/2.0/operator-operating-envelope diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 76341b5ec..a3c9846c8 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -67,8 +67,65 @@ app.kubernetes.io/component: {{ .component }} app.kubernetes.io/component: {{ .component }} {{- end -}} +{{/* +The Server runtime this release runs: "native" (the single-process 3.x +Server) or "php" (the frozen 2.x Server with its worker, scheduler and +migration Job). An explicit runtime value wins. Left empty, a new install runs +native, and an upgrade stays native only when the live server Deployment +carries the native runtime marker. Any other upgrade, such as a release that +chart 0.x installed with PHP, must choose, so helm upgrade never moves a PHP +installation to native on its own (server#946). +*/}} +{{- define "durable-workflow.runtime" -}} +{{- $runtime := toString (default "" .Values.runtime) -}} +{{- if $runtime -}} +{{- if not (has $runtime (list "native" "php")) -}} +{{- fail (printf "runtime %q is not supported. Use native or php." $runtime) -}} +{{- end -}} +{{- $runtime -}} +{{- else if .Release.IsUpgrade -}} +{{- $server := lookup "apps/v1" "Deployment" .Release.Namespace (include "durable-workflow.serverDeploymentName" .) -}} +{{- $marker := "" -}} +{{- if $server -}} +{{- $marker = default "" (get (default (dict) $server.metadata.annotations) "workflows.durable-workflow.dev/runtime") -}} +{{- end -}} +{{- if eq $marker "native" -}} +native +{{- else if $server -}} +{{- fail (printf "Release %s runs the PHP 2.x Server: its Deployment %s has no native runtime marker. This chart now defaults to the native 3.x Server but never switches an existing PHP release on its own. To stay on PHP, set runtime=php and pin image.tag to your 2.x release (the final one is 2.11.8). To move to native, take over the PHP database with the native image first, then set runtime=native. See docs/UPGRADING.md in the chart (durable-workflow/server#946)." .Release.Name $server.metadata.name) -}} +{{- else -}} +{{- fail (printf "Cannot confirm the runtime of release %s because its server Deployment is not visible (for example under a client-side dry run). Set runtime=native or runtime=php explicitly. See docs/UPGRADING.md in the chart (durable-workflow/server#946)." .Release.Name) -}} +{{- end -}} +{{- else -}} +native +{{- end -}} +{{- end -}} + +{{- define "durable-workflow.isNative" -}} +{{- if eq (include "durable-workflow.runtime" .) "native" -}}true{{- end -}} +{{- end -}} + +{{- define "durable-workflow.isPhp" -}} +{{- if eq (include "durable-workflow.runtime" .) "php" -}}true{{- end -}} +{{- end -}} + +{{/* +The Server line an image tag names: "native" for 3.x and later and for the +rolling latest tag, "php" for 2.x and earlier, empty for a custom tag. +*/}} +{{- define "durable-workflow.imageTagRuntime" -}} +{{- $tag := toString . -}} +{{- if or (eq $tag "latest") (regexMatch "^v?([3-9]|[1-9][0-9]+)(\\.|$)" $tag) -}} +native +{{- else if regexMatch "^v?[0-2](\\.|$)" $tag -}} +php +{{- end -}} +{{- end -}} + {{/* Image reference. Prefers digest over tag; tag falls back to .Chart.AppVersion. +A tag must belong to the selected runtime: the native image has no artisan, +queue worker or scheduler, and the PHP image cannot serve a native schema. */}} {{- define "durable-workflow.image" -}} {{- $registry := .Values.image.registry | default "docker.io" -}} @@ -76,18 +133,31 @@ Image reference. Prefers digest over tag; tag falls back to .Chart.AppVersion. {{- if .Values.image.digest -}} {{- printf "%s/%s@%s" $registry $repo .Values.image.digest -}} {{- else -}} -{{- $tag := .Values.image.tag | default .Chart.AppVersion -}} -{{- /* This chart runs the PHP 2.x Server: a queue worker, a scheduler and a -migration job that call server-entrypoint and artisan. The native 3.x image -has none of them, so a 3.x tag would render workloads that fail at runtime -(Server #946). The rolling tags 3, 3.0 and latest now point at native too. */ -}} -{{- if or (eq (toString $tag) "latest") (regexMatch "^v?([3-9]|[1-9][0-9]+)(\\.|$)" (toString $tag)) -}} -{{- fail (printf "image.tag %s selects the native 3.x Server; this chart runs PHP 2.x only until its native mode lands (durable-workflow/server#946). Use docker-compose.published.yml for native, or pin a 2.x tag." $tag) -}} +{{- $tag := toString (.Values.image.tag | default .Chart.AppVersion) -}} +{{- $runtime := include "durable-workflow.runtime" . -}} +{{- $tagRuntime := include "durable-workflow.imageTagRuntime" $tag -}} +{{- if and (eq $runtime "php") (eq $tagRuntime "native") -}} +{{- fail (printf "image.tag %s selects the native 3.x Server, but runtime is php, which runs PHP 2.x images only. Set runtime=native to run native, or pin a 2.x tag such as 2.11.8 to stay on PHP. See docs/UPGRADING.md in the chart (durable-workflow/server#946)." $tag) -}} +{{- else if and (eq $runtime "native") (eq $tagRuntime "php") -}} +{{- fail (printf "image.tag %s selects the PHP 2.x Server, but runtime is native. Set runtime=php to keep running PHP 2.x, or use a native 3.x tag. See docs/UPGRADING.md in the chart (durable-workflow/server#946)." $tag) -}} {{- end -}} {{- printf "%s/%s:%s" $registry $repo $tag -}} {{- end -}} {{- end -}} +{{/* +APP_VERSION for PHP roles: an explicit config.appVersion, else a 2.x image +tag. Otherwise it is omitted and the PHP image reports its own source +release, because the chart's appVersion names the native Server. +*/}} +{{- define "durable-workflow.phpAppVersion" -}} +{{- if .Values.config.appVersion -}} +{{- .Values.config.appVersion -}} +{{- else if and (not .Values.image.digest) (eq (include "durable-workflow.imageTagRuntime" (toString .Values.image.tag)) "php") -}} +{{- .Values.image.tag -}} +{{- end -}} +{{- end -}} + {{/* Classify memo storage from an immutable official Server tag. Opaque images are resolved by an explicit capability declaration or an existing workload marker. @@ -179,7 +249,7 @@ live execution image and storage-capability marker, never desired state or the chart version label as proof that an old execution has stopped. */}} {{- define "durable-workflow.validateMemoPayloadTransition" -}} -{{- if .Release.IsUpgrade -}} +{{- if and .Release.IsUpgrade (include "durable-workflow.isPhp" .) -}} {{- $targetImage := include "durable-workflow.image" . -}} {{- $targetStorage := include "durable-workflow.targetMemoPayloadStorage" . | trim -}} {{- if or (eq $targetStorage "unknown") (eq $targetStorage "envelope-v1") -}} @@ -389,7 +459,7 @@ default in-cluster database would silently break the externals-first contract. {{- if not .Values.externalDatabase.host -}} {{- fail "externalDatabase.host is required. The chart does not bundle a database; point this at your managed MySQL/PostgreSQL endpoint. See k8s/helm/durable-workflow/README.md." -}} {{- end -}} -{{- if not .Values.externalRedis.host -}} +{{- if and (include "durable-workflow.isPhp" .) (not .Values.externalRedis.host) -}} {{- fail "externalRedis.host is required for the multi-node correctness contract. Point this at your managed Redis (ElastiCache, Memorystore, Sentinel, etc.). See k8s/helm/durable-workflow/README.md." -}} {{- end -}} {{- end -}} @@ -408,7 +478,7 @@ Standard env block consumed by every workload. Wires: name: {{ include "durable-workflow.appSecretName" . }} {{- end -}} -{{- define "durable-workflow.standardEnv" -}} +{{- define "durable-workflow.databaseCredentialEnv" -}} - name: DB_USERNAME valueFrom: secretKeyRef: @@ -419,6 +489,10 @@ Standard env block consumed by every workload. Wires: secretKeyRef: name: {{ include "durable-workflow.databaseSecretName" . }} key: {{ .Values.externalDatabase.existingSecretPasswordKey | default "DB_PASSWORD" }} +{{- end -}} + +{{- define "durable-workflow.standardEnv" -}} +{{ include "durable-workflow.databaseCredentialEnv" . }} - name: REDIS_USERNAME valueFrom: secretKeyRef: @@ -455,8 +529,10 @@ not depend on regular resources Helm or Argo create afterward. {{- if eq (include "durable-workflow.bootstrapRunsBeforeResources" .) "true" }} - name: APP_NAME value: {{ .Values.config.appName | quote }} +{{- with include "durable-workflow.phpAppVersion" . }} - name: APP_VERSION - value: {{ default .Chart.AppVersion .Values.config.appVersion | quote }} + value: {{ . | quote }} +{{- end }} - name: APP_ENV value: {{ .Values.config.appEnv | quote }} - name: APP_DEBUG @@ -615,3 +691,111 @@ argocd.argoproj.io/sync-wave: {{ .Values.argocd.workloadWave | quote }} kustomize.toolkit.fluxcd.io/depends-on: {{ printf "%s/%s" .Release.Namespace (include "durable-workflow.bootstrapJobName" .) | quote }} {{- end }} {{- end -}} + +{{/* +Native settings the binary would refuse at startup, refused here at render +time instead so a broken rollout never starts. Each rule cites the native +behavior it mirrors. +*/}} +{{- define "durable-workflow.validateNative" -}} +{{- $native := default (dict) .Values.native -}} +{{- if ne (toString .Values.config.auth.driver) "token" -}} +{{- fail (printf "config.auth.driver %q is not supported by the native Server, which accepts only the token driver in a release image." (toString .Values.config.auth.driver)) -}} +{{- end -}} +{{- if .Values.auth.principalTokens -}} +{{- fail "auth.principalTokens is not supported by the native Server, which refuses DW_PRINCIPAL_TOKENS at startup. Use auth.authToken with the role-scoped worker, operator and admin tokens." -}} +{{- end -}} +{{- if and (not .Values.auth.existingSecret) (not .Values.auth.authToken) -}} +{{- fail "auth.authToken is required in native mode: the native Server refuses to start without DW_AUTH_TOKEN. Set auth.authToken, or put DW_AUTH_TOKEN in auth.existingSecret." -}} +{{- end -}} +{{- range $volume := default (list) .Values.server.extraVolumes -}} +{{- if eq $volume.name "server-storage" -}} +{{- fail "The server-storage volume name is reserved for native payload storage. Use native.storage.existingClaim." -}} +{{- end -}} +{{- end -}} +{{- range $mount := default (list) .Values.server.extraVolumeMounts -}} +{{- if or (eq $mount.name "server-storage") (eq (trimSuffix "/" $mount.mountPath) "/app/storage") -}} +{{- fail "The server-storage volume and the /app/storage mount path are reserved for native payload storage. Use native.storage.existingClaim." -}} +{{- end -}} +{{- end -}} +{{- $grace := int (default 45 (get $native "terminationGracePeriodSeconds")) -}} +{{- $poll := get (default (dict) .Values.config.extraEnv) "DW_WORKER_POLL_TIMEOUT" -}} +{{- if and $poll (le $grace (int $poll)) -}} +{{- fail (printf "native.terminationGracePeriodSeconds (%d) must exceed DW_WORKER_POLL_TIMEOUT (%v): the native Server holds open worker polls for up to that long while it drains on SIGTERM." $grace $poll) -}} +{{- end -}} +{{- end -}} + +{{/* +Refuse native.schemaUpgrade while any native Server can still write. The +schema-upgrade command requires every native node stopped and a database +backup taken first (rust/README.md, Native schema upgrades). The check reads +the live Deployment and pods, as the memo transition guard does, and never +treats desired state as proof that an old pod has stopped. +*/}} +{{- define "durable-workflow.validateNativeSchemaUpgradeQuiescence" -}} +{{- if .Release.IsUpgrade -}} +{{- $namespace := .Release.Namespace -}} +{{- $server := lookup "apps/v1" "Deployment" $namespace (include "durable-workflow.serverDeploymentName" .) -}} +{{- if $server -}} +{{- $replicas := 1 -}} +{{- if hasKey $server.spec "replicas" -}} +{{- $replicas = int (get $server.spec "replicas") -}} +{{- end -}} +{{- if gt $replicas 0 -}} +{{- fail (printf "native.schemaUpgrade.enabled runs schema-upgrade, which needs every native Server stopped and a database backup first, but Deployment %s has %d replicas. Scale it to zero (server.replicaCount=0 with server.autoscaling.enabled=false), wait for its pods to exit, back up the database, then upgrade again with native.schemaUpgrade.enabled=true. Leave native.schemaUpgrade.enabled=false for upgrades that keep the schema." $server.metadata.name $replicas) -}} +{{- end -}} +{{- end -}} +{{- $pods := lookup "v1" "Pod" $namespace "" -}} +{{- range $pod := default (list) $pods.items -}} +{{- $labels := default (dict) $pod.metadata.labels -}} +{{- $phase := default "Unknown" (get (default (dict) $pod.status) "phase") -}} +{{- if and (eq (default "" (get $labels "app.kubernetes.io/instance")) $.Release.Name) (eq (default "" (get $labels "app.kubernetes.io/component")) "server") (not (has $phase (list "Succeeded" "Failed"))) -}} +{{- fail (printf "native.schemaUpgrade.enabled needs every native Server stopped, but server pod %s is still in phase %s. Wait for it to exit before retrying." $pod.metadata.name $phase) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Database settings for the native schema-upgrade hook. They are inlined, as +the PHP bootstrap hook's are, so the hook connects with this upgrade's values +rather than the previous release's ConfigMap. +*/}} +{{- define "durable-workflow.nativeSchemaUpgradeEnv" -}} +- name: DB_CONNECTION + value: {{ .Values.externalDatabase.connection | quote }} +- name: DB_HOST + value: {{ .Values.externalDatabase.host | quote }} +- name: DB_PORT + value: {{ .Values.externalDatabase.port | quote }} +- name: DB_DATABASE + value: {{ .Values.externalDatabase.database | quote }} +{{- with .Values.externalDatabase.socket }} +- name: DB_SOCKET + value: {{ . | quote }} +{{- end }} +{{- with .Values.externalDatabase.sslMode }} +- name: DB_SSLMODE + value: {{ . | quote }} +{{- end }} +{{- if .Values.externalDatabase.tls.existingSecret }} +- name: DB_SSLROOTCERT + value: {{ include "durable-workflow.databaseTlsCaPath" . | quote }} +{{- end }} +{{- range $k, $v := .Values.config.extraEnv }} +- name: {{ $k }} + value: {{ $v | quote }} +{{- end }} +{{- if .Values.externalDatabase.existingSecret }} +{{ include "durable-workflow.databaseCredentialEnv" . }} +{{- else }} +- name: DB_USERNAME + value: {{ required "externalDatabase.auth.username is required when externalDatabase.existingSecret is empty" .Values.externalDatabase.auth.username | quote }} +- name: DB_PASSWORD + value: {{ required "externalDatabase.auth.password is required when externalDatabase.existingSecret is empty" .Values.externalDatabase.auth.password | quote }} +{{- end }} +{{- end -}} + +{{- define "durable-workflow.nativeSchemaUpgradeJobName" -}} +{{ include "durable-workflow.componentName" (dict "context" . "component" "schema-upgrade") }} +{{- end -}} diff --git a/k8s/helm/durable-workflow/templates/configmap.yaml b/k8s/helm/durable-workflow/templates/configmap.yaml index c3519743a..93d9e2380 100644 --- a/k8s/helm/durable-workflow/templates/configmap.yaml +++ b/k8s/helm/durable-workflow/templates/configmap.yaml @@ -12,10 +12,14 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: + {{- if include "durable-workflow.isPhp" . }} APP_NAME: {{ .Values.config.appName | quote }} - APP_VERSION: {{ default .Chart.AppVersion .Values.config.appVersion | quote }} + {{- with include "durable-workflow.phpAppVersion" . }} + APP_VERSION: {{ . | quote }} + {{- end }} APP_ENV: {{ .Values.config.appEnv | quote }} APP_DEBUG: {{ .Values.config.appDebug | quote }} + {{- end }} DB_CONNECTION: {{ .Values.externalDatabase.connection | quote }} DB_HOST: {{ .Values.externalDatabase.host | quote }} DB_PORT: {{ .Values.externalDatabase.port | quote }} @@ -29,6 +33,7 @@ data: {{- if .Values.externalDatabase.tls.existingSecret }} DB_SSLROOTCERT: {{ include "durable-workflow.databaseTlsCaPath" . | quote }} {{- end }} + {{- if include "durable-workflow.isPhp" . }} REDIS_HOST: {{ .Values.externalRedis.host | quote }} REDIS_PORT: {{ .Values.externalRedis.port | quote }} REDIS_DB: {{ .Values.externalRedis.database | quote }} @@ -40,6 +45,11 @@ data: DW_AUTH_DRIVER: {{ .Values.config.auth.driver | quote }} DW_AUTH_BACKWARD_COMPATIBLE: {{ .Values.config.auth.backwardCompatible | quote }} DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT: {{ .Values.config.metrics.workflowTaskFailureTypeLimit | quote }} + {{- else }} + {{- /* The native Server has no Redis, queue or cache and always accepts + DW_AUTH_TOKEN for every role, so the PHP-only settings are left out. */}} + DW_AUTH_DRIVER: {{ .Values.config.auth.driver | quote }} + {{- end }} {{- range $k, $v := .Values.config.extraEnv }} {{ $k }}: {{ $v | quote }} {{- end }} diff --git a/k8s/helm/durable-workflow/templates/migration-job.yaml b/k8s/helm/durable-workflow/templates/migration-job.yaml index da2dfed09..0832e0c56 100644 --- a/k8s/helm/durable-workflow/templates/migration-job.yaml +++ b/k8s/helm/durable-workflow/templates/migration-job.yaml @@ -1,4 +1,4 @@ -{{- if .Values.bootstrap.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.bootstrap.enabled }} apiVersion: batch/v1 kind: Job metadata: diff --git a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml new file mode 100644 index 000000000..e87f0401f --- /dev/null +++ b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml @@ -0,0 +1,84 @@ +{{- /* +Explicit native schema upgrade. A native Server never upgrades its schema at +startup: on an older schema it refuses with native_schema_upgrade_required. +The image's schema-upgrade command moves a database to the image's schema, +and needs every native Server stopped and a database backup taken first. +This pre-upgrade hook runs it only when native.schemaUpgrade.enabled is set, +and the chart refuses to render it while a native Server pod can still run. +It never runs on install, where the Server bootstraps an empty database. +*/ -}} +{{- $native := default (dict) .Values.native }} +{{- $upgrade := default (dict) (get $native "schemaUpgrade") }} +{{- if and (include "durable-workflow.isNative" .) (get $upgrade "enabled") }} +{{- include "durable-workflow.validateNativeSchemaUpgradeQuiescence" . }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "durable-workflow.nativeSchemaUpgradeJobName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "durable-workflow.componentLabels" (dict "context" . "component" "schema-upgrade") | nindent 4 }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + backoffLimit: {{ default 0 (get $upgrade "backoffLimit") }} + activeDeadlineSeconds: {{ default 1800 (get $upgrade "activeDeadlineSeconds") }} + template: + metadata: + labels: + {{- include "durable-workflow.componentLabels" (dict "context" . "component" "schema-upgrade") | nindent 8 }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: schema-upgrade + image: {{ include "durable-workflow.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["schema-upgrade"] + env: + {{- include "durable-workflow.nativeSchemaUpgradeEnv" . | nindent 12 }} + {{- with .Values.server.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml (default (dict) (get $upgrade "resources")) | nindent 12 }} + {{- with .Values.server.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.externalDatabase.tls.existingSecret }} + volumeMounts: + {{- include "durable-workflow.databaseTlsVolumeMount" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.externalDatabase.tls.existingSecret }} + volumes: + {{- include "durable-workflow.databaseTlsVolume" . | nindent 8 }} + {{- end }} + {{- with .Values.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml new file mode 100644 index 000000000..d7609085c --- /dev/null +++ b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml @@ -0,0 +1,142 @@ +{{- /* +Native 3.x Server. One process per pod serves the API, checks its schema at +startup (bootstrapping an empty database, never upgrading an older one) and +runs maintenance in process: timers, repair, retention and schedules +(DW_RUST_MAINTENANCE_LOOP=1 in the image). There is no worker, scheduler, +migration Job or Redis. The runtime annotation lets a later upgrade with an +empty runtime value confirm that this release already runs native. +*/ -}} +{{- if and (include "durable-workflow.isNative" .) .Values.server.enabled }} +{{- include "durable-workflow.validateNative" . }} +{{- $native := default (dict) .Values.native }} +{{- $storage := default (dict) (get $native "storage") }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "durable-workflow.serverDeploymentName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "durable-workflow.componentLabels" (dict "context" . "component" "server") | nindent 4 }} + annotations: + workflows.durable-workflow.dev/runtime: native + {{- include "durable-workflow.argoWorkloadAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.server.autoscaling.enabled }} + replicas: {{ .Values.server.replicaCount }} + {{- end }} + revisionHistoryLimit: {{ .Values.server.revisionHistoryLimit }} + progressDeadlineSeconds: {{ .Values.server.progressDeadlineSeconds }} + minReadySeconds: {{ .Values.server.minReadySeconds }} + {{- with .Values.server.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server") | nindent 6 }} + template: + metadata: + labels: + {{- include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server") | nindent 8 }} + {{- with .Values.server.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- include "durable-workflow.configChecksumAnnotations" . | nindent 8 }} + {{- with .Values.server.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- /* On SIGTERM native stops accepting work and holds open worker + polls for up to DW_WORKER_POLL_TIMEOUT (30 seconds by default), so the + grace period must outlast it. */}} + terminationGracePeriodSeconds: {{ default 45 (get $native "terminationGracePeriodSeconds") }} + containers: + - name: server + image: {{ include "durable-workflow.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + envFrom: + {{- include "durable-workflow.standardEnvFrom" . | nindent 12 }} + env: + {{- include "durable-workflow.databaseCredentialEnv" . | nindent 12 }} + {{- with .Values.server.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.server.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.server.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.server.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.server.resources | nindent 12 }} + {{- with .Values.server.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: server-storage + mountPath: /app/storage + {{- include "durable-workflow.databaseTlsVolumeMount" . | nindent 12 }} + {{- with .Values.server.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- /* Native keeps external payload objects on local disk under + /app/storage. Without a claim the objects live only as long as the pod. */}} + - name: server-storage + {{- with (get $storage "existingClaim") }} + persistentVolumeClaim: + claimName: {{ . | quote }} + {{- else }} + emptyDir: {} + {{- end }} + {{- include "durable-workflow.databaseTlsVolume" . | nindent 8 }} + {{- with .Values.server.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml b/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml index e1812f405..dcbb1f6d6 100644 --- a/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml +++ b/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml @@ -11,7 +11,7 @@ The chart enforces this two ways: Operators who need duplicate scheduler runners as a steady-state topology must contact support; that path is outside this chart's contract. */ -}} -{{- if .Values.scheduler.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.scheduler.enabled }} apiVersion: batch/v1 kind: CronJob metadata: diff --git a/k8s/helm/durable-workflow/templates/secret-redis.yaml b/k8s/helm/durable-workflow/templates/secret-redis.yaml index 57a69b5a3..1d92cabff 100644 --- a/k8s/helm/durable-workflow/templates/secret-redis.yaml +++ b/k8s/helm/durable-workflow/templates/secret-redis.yaml @@ -4,7 +4,7 @@ present and no existingSecret is configured. The standard env block mounts the keys as `optional: true` so an unauthenticated Redis is permitted — in that case nothing is rendered here. */ -}} -{{- if and (not .Values.externalRedis.existingSecret) (or .Values.externalRedis.auth.username .Values.externalRedis.auth.password) }} +{{- if and (include "durable-workflow.isPhp" .) (not .Values.externalRedis.existingSecret) (or .Values.externalRedis.auth.username .Values.externalRedis.auth.password) }} apiVersion: v1 kind: Secret metadata: diff --git a/k8s/helm/durable-workflow/templates/server-deployment.yaml b/k8s/helm/durable-workflow/templates/server-deployment.yaml index cc0f8a56d..7cbeca02e 100644 --- a/k8s/helm/durable-workflow/templates/server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/server-deployment.yaml @@ -1,5 +1,5 @@ {{- include "durable-workflow.validateMemoPayloadTransition" . }} -{{- if .Values.server.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.server.enabled }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/k8s/helm/durable-workflow/templates/worker-deployment.yaml b/k8s/helm/durable-workflow/templates/worker-deployment.yaml index 94eadd15e..d723d5eee 100644 --- a/k8s/helm/durable-workflow/templates/worker-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/worker-deployment.yaml @@ -1,5 +1,5 @@ {{- include "durable-workflow.validateMemoPayloadTransition" . }} -{{- if .Values.worker.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.worker.enabled }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/k8s/helm/durable-workflow/templates/worker-hpa.yaml b/k8s/helm/durable-workflow/templates/worker-hpa.yaml index 2b4413342..3d60bb075 100644 --- a/k8s/helm/durable-workflow/templates/worker-hpa.yaml +++ b/k8s/helm/durable-workflow/templates/worker-hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.worker.enabled .Values.worker.autoscaling.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.worker.enabled .Values.worker.autoscaling.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/k8s/helm/durable-workflow/templates/worker-pdb.yaml b/k8s/helm/durable-workflow/templates/worker-pdb.yaml index 8320071ad..eb0e02245 100644 --- a/k8s/helm/durable-workflow/templates/worker-pdb.yaml +++ b/k8s/helm/durable-workflow/templates/worker-pdb.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.worker.enabled .Values.worker.pdb.enabled }} +{{- if and (include "durable-workflow.isPhp" .) .Values.worker.enabled .Values.worker.pdb.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/k8s/helm/durable-workflow/values.schema.json b/k8s/helm/durable-workflow/values.schema.json index 344ee8c1d..3689b74a8 100644 --- a/k8s/helm/durable-workflow/values.schema.json +++ b/k8s/helm/durable-workflow/values.schema.json @@ -5,6 +5,32 @@ "additionalProperties": true, "required": ["image", "externalDatabase", "externalRedis"], "properties": { + "runtime": { + "type": "string", + "enum": ["", "native", "php"], + "description": "Server runtime. Empty runs native on a new install and keeps an existing native release native; an existing PHP release must choose explicitly." + }, + "native": { + "type": "object", + "properties": { + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, + "storage": { + "type": "object", + "properties": { + "existingClaim": { "type": "string" } + } + }, + "schemaUpgrade": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "backoffLimit": { "type": "integer", "minimum": 0 }, + "activeDeadlineSeconds": { "type": "integer", "minimum": 1 }, + "resources": { "type": "object" } + } + } + } + }, "image": { "type": "object", "required": ["repository"], diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index c10202e2e..61724b5ae 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -1,17 +1,35 @@ # Default values for the durable-workflow Helm chart. # -# Defaults match the small-cluster shape documented at -# https://durable-workflow.github.io/docs/2.0/deployment : -# * stateless API replicas behind a ClusterIP Service -# * a separately-scaled worker Deployment -# * exactly one scheduler/maintenance runner (CronJob with concurrencyPolicy=Forbid) -# * a bootstrap Job that runs once per release before the new pods take traffic +# The chart runs one of two Server runtimes (see `runtime` below): +# +# native (default for new installs): the 3.x Server. API replicas behind a +# ClusterIP Service. Each pod is one process that checks its schema at +# startup and runs timers, repair, retention and schedules itself. No +# worker, scheduler, migration Job or Redis. +# php (existing 2.x installs): the frozen 2.x Server in the small-cluster +# shape at https://durable-workflow.github.io/docs/2.0/deployment : +# * stateless API replicas behind a ClusterIP Service +# * a separately-scaled worker Deployment +# * exactly one scheduler/maintenance runner (CronJob with concurrencyPolicy=Forbid) +# * a bootstrap Job that runs once per release before the new pods take traffic # # This chart deliberately does NOT bundle MySQL/PostgreSQL or Redis. External -# persistence is a first-class contract: point .externalDatabase and .externalRedis -# at the endpoints you already operate (RDS, Cloud SQL, Aurora, ElastiCache, -# Memorystore, Sentinel, etc.). For local exercise, the kind smoke harness in -# scripts/k8s-helm-kind-smoke.sh provisions disposable in-cluster services. +# persistence is a first-class contract: point .externalDatabase (and, in PHP +# mode, .externalRedis) at the endpoints you already operate (RDS, Cloud SQL, +# Aurora, ElastiCache, Memorystore, Sentinel, etc.). For local exercise, the +# kind smoke harnesses in scripts/ provision disposable in-cluster services. + +# Server runtime: "native", "php", or empty. +# * Empty: a new install runs native. An upgrade stays native only when the +# release already runs native, and otherwise refuses to render. A release +# installed by chart 0.x runs PHP, so its upgrade must set runtime. +# * "native": the 3.x Server. image.tag must be a 3.x tag, a custom tag or +# a digest. +# * "php": the frozen 2.x Server for existing installs. Pin image.tag to +# your 2.x release. The final one is 2.11.8. +# Moving a PHP release to native takes over its database first. Follow +# docs/UPGRADING.md (chart 1.0.0) before setting runtime=native on it. +runtime: "" # --------------------------------------------------------------------------- # Image and global identity @@ -20,16 +38,20 @@ image: registry: docker.io repository: durableworkflow/server - # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.11.8" + # The native Server this chart release packages. Generated by + # scripts/ci/sync-source-release.mjs. Do not edit this default. + # A tag must match the runtime: 3.x (or latest) for native, 2.x for php. + # The chart refuses a mismatch at render time. + tag: "3.0.1" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." - # This chart runs the PHP 2.x Server only. A 3.x or latest tag is refused at - # render time; pin only a PHP 2.x digest here (native 3.x: server#946). + # The chart cannot see which runtime a digest names. Pin a native digest + # for runtime=native and a 2.x digest for runtime=php. digest: "" - # Required when a custom repository, custom tag, or digest prevents the chart - # from deriving this image capability. Leave empty for recognized official - # release tags. Valid identities are raw-json-v1, envelope-v1, and dual-v1. + # PHP mode only. Required when a custom repository, custom tag, or digest + # prevents the chart from deriving this image capability. Leave empty for + # recognized official release tags. Valid identities are raw-json-v1, + # envelope-v1, and dual-v1. memoPayloadStorage: "" pullPolicy: IfNotPresent pullSecrets: [] @@ -59,18 +81,25 @@ commonAnnotations: {} # --------------------------------------------------------------------------- # Application-level configuration baked into the shared ConfigMap. Keys mirror -# environment variables consumed by server/worker/scheduler/migrate containers. +# environment variables consumed by the Server containers. appName, appEnv, +# appDebug, appVersion, auth.backwardCompatible and metrics apply to PHP mode +# only. The native Server does not read them. config: appName: "Durable Workflow Server" appEnv: "production" appDebug: "false" - appVersion: "" # Defaults to .Chart.AppVersion when empty. + # PHP mode. Defaults to a 2.x image.tag when empty. Otherwise the PHP image + # reports its own release. + appVersion: "" auth: + # Native accepts only "token". driver: "token" - # Set false for production to disable the legacy unscoped DW_AUTH_TOKEN - # path. Existing Compose deployments often start with this true; the - # chart defaults to the production posture. + # PHP mode only. Set false for production to disable the legacy unscoped + # DW_AUTH_TOKEN path. Existing Compose deployments often start with this + # true; the chart defaults to the production posture. The native Server + # always requires DW_AUTH_TOKEN and accepts it for every role, so native + # mode leaves this setting out. backwardCompatible: "false" metrics: @@ -117,7 +146,8 @@ externalDatabase: username: "" password: "" -# Redis acceleration layer. Required for multi-node correctness contract +# Redis acceleration layer. PHP mode only; native mode ignores this block. +# Required for the PHP multi-node correctness contract # (see workflow/docs/deployment/multi-node-requirements.md). Configure either # an existingSecret or inline credentials. Anonymous Redis is permitted; in # that case leave existingSecret empty and credentials blank. @@ -149,8 +179,8 @@ auth: # In production, prefer existingSecret with values populated by your secret # manager (Vault, AWS Secrets Manager, GCP Secret Manager, etc.). serverKey: "" # DW_SERVER_KEY (base64:...) - authToken: "" # DW_AUTH_TOKEN — legacy unscoped token - principalTokens: "" # DW_PRINCIPAL_TOKENS — JSON named-principal token map + authToken: "" # DW_AUTH_TOKEN — shared token; required in native mode + principalTokens: "" # DW_PRINCIPAL_TOKENS — JSON named-principal token map (PHP only, native refuses it) workerToken: "" # DW_WORKER_TOKEN operatorToken: "" # DW_OPERATOR_TOKEN adminToken: "" # DW_ADMIN_TOKEN @@ -161,7 +191,48 @@ auth: # DW_PAYLOAD_CODEC_KEY: "base64:..." # --------------------------------------------------------------------------- -# Bootstrap / migration job +# Native Server settings (runtime=native) +# --------------------------------------------------------------------------- + +native: + # On SIGTERM the native Server holds open worker polls for up to + # DW_WORKER_POLL_TIMEOUT (30 seconds unless config.extraEnv changes it). + # Keep this above that timeout so a rollout drains polls instead of cutting + # them off. + terminationGracePeriodSeconds: 45 + + # Native keeps external payload objects on local disk at /app/storage. + # Without a claim the chart mounts an emptyDir, which is lost with the pod. + # Set a PersistentVolumeClaim before any namespace enables external payload + # storage. With more than one replica the claim must be ReadWriteMany so + # every pod reads the same objects. + storage: + existingClaim: "" + + # Explicit schema upgrade between native releases. A native Server never + # upgrades its schema at startup: on an older schema it refuses with + # native_schema_upgrade_required and stays unready. Patch releases within a + # minor line keep the schema. When a release notes a schema change: + # 1. scale the Server to zero (server.replicaCount=0, autoscaling off), + # 2. back up the database, + # 3. upgrade to the new image with schemaUpgrade.enabled=true and your + # usual replica count, + # 4. set schemaUpgrade.enabled=false again for later upgrades. + # The chart refuses to render this hook while a Server pod still runs. + schemaUpgrade: + enabled: false + backoffLimit: 0 + activeDeadlineSeconds: 1800 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +# --------------------------------------------------------------------------- +# Bootstrap / migration job (PHP mode only) # --------------------------------------------------------------------------- bootstrap: @@ -227,9 +298,11 @@ server: sessionAffinity: "" # Probes. Defaults preserve the small-cluster contract: liveness on - # /api/health, readiness on /api/ready (which tests DB+Redis usability). - # Changing readinessProbe.path away from /api/ready breaks the - # single-region HA contract. + # /api/health, readiness on /api/ready. Changing readinessProbe.path away + # from /api/ready breaks the single-region HA contract. In PHP mode + # /api/ready tests DB and Redis usability. In native mode it tests the + # schema and the in-process maintenance loop, so keep it out of liveness: + # a stalled maintenance tick would otherwise restart every pod. livenessProbe: httpGet: path: /api/health @@ -324,7 +397,7 @@ server: extraVolumeMounts: [] # --------------------------------------------------------------------------- -# Worker deployment +# Worker deployment (PHP mode only) # --------------------------------------------------------------------------- worker: @@ -400,7 +473,7 @@ worker: extraVolumeMounts: [] # --------------------------------------------------------------------------- -# Scheduler / maintenance runner +# Scheduler / maintenance runner (PHP mode only; native runs it in process) # --------------------------------------------------------------------------- # CRITICAL: the scheduler is a singleton. Helm enforces this by setting @@ -461,8 +534,9 @@ scheduler: # --------------------------------------------------------------------------- # Argo CD / Flux do not always honour Helm hook annotations. Setting these -# adds sync-wave / depends-on annotations so the bootstrap Job runs before -# the server/worker/scheduler workloads. +# adds sync-wave / depends-on annotations so the PHP bootstrap Job runs before +# the server/worker/scheduler workloads. Native mode has no bootstrap Job and +# renders no depends-on annotation. argocd: useSyncWaves: true bootstrapWave: "-5" @@ -497,6 +571,6 @@ networkPolicy: priorityClassName: "" # Optional sidecar containers injected on every pod (server, worker, -# scheduler, bootstrap). Use sparingly — the engine's contract is that the -# main container talks to DB and Redis directly. +# scheduler, bootstrap, native schema upgrade). Use sparingly — the engine's +# contract is that the main container talks to its database directly. sidecars: [] diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 5339b9c37..c3b32d499 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -2,6 +2,10 @@ # the deployment is short-lived and not internet-facing. DO NOT use this # shape in production. +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: tag: "2.11.8" diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 7b8314c7f..bbaa7ab2d 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -4,6 +4,10 @@ # live outside the chart so the secret-source backend remains the operator's # concern. +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: tag: "2.11.8" diff --git a/k8s/helm/examples/values-native-dev.yaml b/k8s/helm/examples/values-native-dev.yaml new file mode 100644 index 000000000..ab5e4d28f --- /dev/null +++ b/k8s/helm/examples/values-native-dev.yaml @@ -0,0 +1,35 @@ +# Development reference values for the native Server. Inline secrets are +# acceptable here because the deployment is short-lived and not +# internet-facing. DO NOT use this shape in production. +# +# The chart's default image is the native Server it packages. Native needs +# only a database: no Redis, worker, scheduler or migration Job. + +externalDatabase: + connection: mysql + host: mysql.dev.svc.cluster.local + port: 3306 + database: durable_workflow + auth: + username: workflow + password: dev-only-password + +auth: + serverKey: "base64:dev-server-key-change-me" + # Required. Native accepts this shared token for every role. + authToken: "dev-shared-token" + workerToken: "dev-worker-token" + operatorToken: "dev-operator-token" + adminToken: "dev-admin-token" + +server: + replicaCount: 1 + pdb: + enabled: false + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 9507c65e5..8b0473163 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -5,6 +5,10 @@ # Secrets Store CSI Driver, or any other bridge that produces the listed # Kubernetes Secrets in the target namespace. +# PHP 2.x mode. The chart defaults to the native 3.x Server. This file keeps +# an existing PHP installation on the frozen 2.x line. +runtime: php + image: registry: ghcr.io repository: durable-workflow/server diff --git a/resources/release/source-release.json b/resources/release/source-release.json index d7e392f7b..9feeb1b48 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -4,6 +4,7 @@ "version": "2.11.8" }, "helm_chart": { - "version": "0.3.9" + "version": "1.0.0", + "app_version": "3.0.1" } } diff --git a/scripts/ci/helm_chart_release.py b/scripts/ci/helm_chart_release.py index aa8014199..446a1c4e5 100644 --- a/scripts/ci/helm_chart_release.py +++ b/scripts/ci/helm_chart_release.py @@ -135,7 +135,10 @@ def source_release_metadata( schema = record["schema"] server_version = record["server"]["version"] chart_version = record["helm_chart"]["version"] - except (OSError, json.JSONDecodeError, KeyError, TypeError) as error: + # The Server image the chart packages. Chart 1.x packages the native + # Server while server.version stays the frozen PHP release. + chart_app_version = record["helm_chart"].get("app_version", server_version) + except (OSError, json.JSONDecodeError, KeyError, TypeError, AttributeError) as error: raise ReleaseError(f"source release manifest is invalid: {error}") from error if schema != "durable-workflow.server.source-release/v1": raise ReleaseError(f"source release manifest uses unsupported schema {schema}") @@ -143,11 +146,17 @@ def source_release_metadata( raise ReleaseError("source release Server version is not SemVer") if not isinstance(chart_version, str) or SEMVER_PATTERN.fullmatch(chart_version) is None: raise ReleaseError("source release Helm chart version is not SemVer") - return {"server_version": server_version, "chart_version": chart_version} + if not isinstance(chart_app_version, str) or SEMVER_PATTERN.fullmatch(chart_app_version) is None: + raise ReleaseError("source release Helm chart app_version is not SemVer") + return { + "server_version": server_version, + "chart_version": chart_version, + "chart_app_version": chart_app_version, + } def onboarding_server_version() -> str: - return source_release_metadata()["server_version"] + return source_release_metadata()["chart_app_version"] def validate_source(chart_path: Path = DEFAULT_CHART_PATH) -> dict[str, Any]: @@ -178,10 +187,10 @@ def validate_source(chart_path: Path = DEFAULT_CHART_PATH) -> dict[str, Any]: f"got {metadata['version']} and {source_release['chart_version']}. " "Run node scripts/ci/sync-source-release.mjs --write" ) - if metadata["app_version"] != source_release["server_version"]: + if metadata["app_version"] != source_release["chart_app_version"]: raise ReleaseError( "Chart.yaml.appVersion must match the authoritative Server source release; " - f"got {metadata['app_version']} and {source_release['server_version']}. " + f"got {metadata['app_version']} and {source_release['chart_app_version']}. " "Run node scripts/ci/sync-source-release.mjs --write" ) return metadata @@ -263,6 +272,8 @@ def helm_value_arguments() -> list[str]: "--set-string", "auth.serverKey=base64:bm90LWEtc2VjcmV0", "--set-string", + "auth.authToken=not-a-secret", + "--set-string", "auth.workerToken=not-a-secret", "--set-string", "auth.operatorToken=not-a-secret", @@ -286,6 +297,8 @@ def helm_fixture_value_arguments() -> list[str]: "--set-string", "auth.serverKey=base64:bm90LWEtc2VjcmV0", "--set-string", + "auth.authToken=not-a-secret", + "--set-string", "auth.workerToken=not-a-secret", "--set-string", "auth.operatorToken=not-a-secret", diff --git a/scripts/ci/sync-source-release.mjs b/scripts/ci/sync-source-release.mjs index cd684a084..cbd1ab4d6 100644 --- a/scripts/ci/sync-source-release.mjs +++ b/scripts/ci/sync-source-release.mjs @@ -44,17 +44,21 @@ export async function sourceRelease(repositoryRoot = defaultRepositoryRoot) { fail(`source release schema must be ${schema}`); } + const server = exact( + object(record.server, 'source release server').version, + serverVersion, + 'source release server.version', + ); + const chart = object(record.helm_chart, 'source release helm_chart'); return { - serverVersion: exact( - object(record.server, 'source release server').version, - serverVersion, - 'source release server.version', - ), - chartVersion: exact( - object(record.helm_chart, 'source release helm_chart').version, - serverVersion, - 'source release helm_chart.version', - ), + serverVersion: server, + chartVersion: exact(chart.version, serverVersion, 'source release helm_chart.version'), + // The Server image the Helm chart packages. Chart 1.x packages the native + // Server while server.version stays the frozen PHP release, so the two + // differ. Older records without the field package server.version. + chartAppVersion: chart.app_version === undefined + ? server + : exact(chart.app_version, serverVersion, 'source release helm_chart.app_version'), }; } @@ -274,7 +278,7 @@ const consumers = [ rendered = replaceExactly( rendered, new RegExp(`^(appVersion:\\s*["'])${serverVersion}(["']\\s*)$`, 'm'), - (_match, prefix, suffix) => `${prefix}${release.serverVersion}${suffix}`, + (_match, prefix, suffix) => `${prefix}${release.chartAppVersion}${suffix}`, 1, this.path, 'top-level appVersion', @@ -282,29 +286,31 @@ const consumers = [ return replaceExactly( rendered, new RegExp(`^( dev\\.durable-workflow\\.image-reference:\\s*["']docker\\.io/durableworkflow/server:)${serverVersion}(["']\\s*)$`, 'm'), - (_match, prefix, suffix) => `${prefix}${release.serverVersion}${suffix}`, + (_match, prefix, suffix) => `${prefix}${release.chartAppVersion}${suffix}`, 1, this.path, 'release image annotation', ); }, }, + // The chart default and its README name the image the chart packages. + // The PHP-mode examples and CI fixtures pin the frozen PHP release. ...[ - 'k8s/helm/durable-workflow/values.yaml', - 'k8s/helm/durable-workflow/README.md', - 'k8s/helm/examples/values-dev.yaml', - 'k8s/helm/examples/values-external-secrets-operator.yaml', - 'k8s/helm/examples/values-production-existing-secrets.yaml', - 'k8s/helm/durable-workflow/ci/existing-secrets-values.yaml', - 'k8s/helm/durable-workflow/ci/inline-secrets-values.yaml', - 'k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml', - ].map((path) => ({ + ['k8s/helm/durable-workflow/values.yaml', 'chartAppVersion'], + ['k8s/helm/durable-workflow/README.md', 'chartAppVersion'], + ['k8s/helm/examples/values-dev.yaml', 'serverVersion'], + ['k8s/helm/examples/values-external-secrets-operator.yaml', 'serverVersion'], + ['k8s/helm/examples/values-production-existing-secrets.yaml', 'serverVersion'], + ['k8s/helm/durable-workflow/ci/existing-secrets-values.yaml', 'serverVersion'], + ['k8s/helm/durable-workflow/ci/inline-secrets-values.yaml', 'serverVersion'], + ['k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml', 'serverVersion'], + ].map(([path, version]) => ({ path, render(source, release) { return replaceExactly( source, - new RegExp(`^( tag:\\s*["'])${serverVersion}(["']\\s*)$`, 'm'), - (_match, prefix, suffix) => `${prefix}${release.serverVersion}${suffix}`, + new RegExp(`^( tag:\\s*["'])${serverVersion}(["']\\s*)$`, 'gm'), + (_match, prefix, suffix) => `${prefix}${release[version]}${suffix}`, 1, path, 'generated image tag', @@ -381,7 +387,7 @@ async function synchronize(repositoryRoot, write) { function usage() { process.stderr.write( - 'Usage: node scripts/ci/sync-source-release.mjs (--check | --write | --print | --print-legacy-server-version) [--root ]\n', + 'Usage: node scripts/ci/sync-source-release.mjs (--check | --write | --print | --print-legacy-server-version) [--root ]\n', ); } @@ -412,6 +418,7 @@ async function main() { const fields = { 'server-version': release.serverVersion, 'chart-version': release.chartVersion, + 'chart-app-version': release.chartAppVersion, }; if (!Object.hasOwn(fields, args[1])) { usage(); diff --git a/scripts/ci/test-source-release.mjs b/scripts/ci/test-source-release.mjs index efb47a1a4..683468c05 100644 --- a/scripts/ci/test-source-release.mjs +++ b/scripts/ci/test-source-release.mjs @@ -107,12 +107,19 @@ test('a stable server source identity fans out through current release consumers const check = runGenerator(temporaryRoot, ['--check']); assert.equal(check.status, 0, check.stderr); + // The PHP-mode chart fixtures follow the PHP release; the chart's own + // appVersion follows the image the chart packages. + const fixture = await readFile( + join(temporaryRoot, 'k8s/helm/durable-workflow/ci/inline-secrets-values.yaml'), + 'utf8', + ); + assert.match(fixture, /^ tag: "2\.0\.0"$/m); const chart = await readFile( join(temporaryRoot, 'k8s/helm/durable-workflow/Chart.yaml'), 'utf8', ); - assert.match(chart, /^appVersion: "2\.0\.0"$/m); - assert.match(chart, /docker\.io\/durableworkflow\/server:2\.0\.0/); + const chartAppVersion = manifest.helm_chart.app_version ?? manifest.server.version; + assert.match(chart, new RegExp(`^appVersion: "${chartAppVersion.replaceAll('.', '\\.')}"$`, 'm')); } finally { await rm(temporaryRoot, {recursive: true, force: true}); } @@ -128,8 +135,10 @@ test('a simulated next release fans out without rewriting history or retaining s const previousChart = manifest.helm_chart.version; const nextServer = nextServerRelease(previousServer); const nextChart = nextServerRelease(previousChart); + const nextChartApp = nextServerRelease(manifest.helm_chart.app_version ?? previousServer); manifest.server.version = nextServer; manifest.helm_chart.version = nextChart; + manifest.helm_chart.app_version = nextChartApp; await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); const upgradingPath = 'k8s/helm/durable-workflow/docs/UPGRADING.md'; @@ -182,7 +191,13 @@ test('a simulated next release fans out without rewriting history or retaining s 'release generation must not rewrite the Artifact Hub changelog', ); assert.match(chartAfter, new RegExp(`^version: ${nextChart.replaceAll('.', '\\.')}$`, 'm')); - assert.match(chartAfter, new RegExp(`^appVersion: "${nextServer.replaceAll('.', '\\.')}"$`, 'm')); + assert.match(chartAfter, new RegExp(`^appVersion: "${nextChartApp.replaceAll('.', '\\.')}"$`, 'm')); + assert.match( + chartAfter, + new RegExp(`docker\\.io/durableworkflow/server:${nextChartApp.replaceAll('.', '\\.')}"`), + ); + const values = await readFile(join(temporaryRoot, 'k8s/helm/durable-workflow/values.yaml'), 'utf8'); + assert.match(values, new RegExp(`^ tag: "${nextChartApp.replaceAll('.', '\\.')}"$`, 'm')); const stale = []; for (const path of paths) { diff --git a/scripts/ci/test_database_tls_chart.py b/scripts/ci/test_database_tls_chart.py index e759d0a79..e7fac25cc 100644 --- a/scripts/ci/test_database_tls_chart.py +++ b/scripts/ci/test_database_tls_chart.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Check CA delivery to every role, including the pre-install bootstrap hook.""" +"""Check CA delivery to every role in both runtimes, including the bootstrap and schema-upgrade hooks.""" from pathlib import Path import subprocess @@ -11,6 +11,7 @@ ROOT = Path(__file__).resolve().parents[2] CHART = ROOT / "k8s/helm/durable-workflow" CA_PATH = "/etc/durable-workflow/database-tls/ca.pem" +PHP = ["--set-string", "runtime=php", "--set-string", "image.tag=2.11.8"] def render(*options: str) -> list[dict]: @@ -35,7 +36,7 @@ class DatabaseTlsChartTest(unittest.TestCase): def test_ca_reaches_all_four_roles_in_hook_and_regular_bootstrap(self): for hooks in ["true", "false"]: with self.subTest(hooks=hooks): - resources = render("--set", f"bootstrap.useHelmHooks={hooks}") + resources = render(*PHP, "--set", f"bootstrap.useHelmHooks={hooks}") config = next(item for item in resources if item["kind"] == "ConfigMap")["data"] self.assertEqual(config["DB_SSLMODE"], "verify-full") self.assertEqual(config["DB_SSLROOTCERT"], CA_PATH) @@ -59,9 +60,32 @@ def test_ca_reaches_all_four_roles_in_hook_and_regular_bootstrap(self): self.assertEqual(environment["DB_SSLROOTCERT"], CA_PATH) self.assertEqual(roles, {"server", "worker", "migrate", "scheduler"}) + def test_ca_reaches_the_native_server_and_schema_upgrade_hook(self): + resources = render("--set", "native.schemaUpgrade.enabled=true") + config = next(item for item in resources if item["kind"] == "ConfigMap")["data"] + self.assertEqual(config["DB_SSLMODE"], "verify-full") + self.assertEqual(config["DB_SSLROOTCERT"], CA_PATH) + roles = set() + for resource in resources: + pod = pod_spec(resource) + if pod is None: + continue + container = pod["containers"][0] + roles.add(container["name"]) + volume = next(item for item in pod["volumes"] if item["name"] == "database-tls") + self.assertEqual(volume["secret"]["secretName"], "database-ca") + mount = next(item for item in container["volumeMounts"] if item["name"] == "database-tls") + self.assertEqual(mount["mountPath"] + "/ca.pem", CA_PATH) + if container["name"] == "schema-upgrade": + environment = {item["name"]: item.get("value") for item in container["env"]} + self.assertEqual(environment["DB_SSLMODE"], "verify-full") + self.assertEqual(environment["DB_SSLROOTCERT"], CA_PATH) + self.assertEqual(roles, {"server", "schema-upgrade"}) + def test_existing_volumes_and_mounts_survive_with_or_without_ca(self): for ca in ["database-ca", ""]: resources = render( + *PHP, "--set", f"externalDatabase.tls.existingSecret={ca}", "--set-json", 'server.extraVolumes=[{"name":"application-data","emptyDir":{}}]', "--set", "server.extraVolumeMounts[0].name=application-data", @@ -84,7 +108,7 @@ def test_ca_volume_name_and_mount_path_cannot_be_shadowed(self): with self.subTest(setting=setting): result = subprocess.run( ["helm", "template", "tls-check", str(CHART), "--values", - str(CHART / "ci/database-tls-values.yaml"), "--set", setting], + str(CHART / "ci/database-tls-values.yaml"), *PHP, "--set", setting], capture_output=True, text=True, ) self.assertNotEqual(result.returncode, 0) diff --git a/scripts/ci/test_helm_chart_release.py b/scripts/ci/test_helm_chart_release.py index 89d4df8c4..d795c1525 100644 --- a/scripts/ci/test_helm_chart_release.py +++ b/scripts/ci/test_helm_chart_release.py @@ -2,6 +2,7 @@ import ast import importlib.util +import json import os import re import sys @@ -474,7 +475,7 @@ def test_current_source_has_synchronized_public_identity(self) -> None: source_release = RELEASE.source_release_metadata() self.assertGreater(RELEASE.semver_key(metadata["version"]), (0, 1, 0)) self.assertEqual(metadata["version"], source_release["chart_version"]) - self.assertEqual(metadata["app_version"], source_release["server_version"]) + self.assertEqual(metadata["app_version"], source_release["chart_app_version"]) self.assertEqual( metadata["image_reference"], f"docker.io/durableworkflow/server:{RELEASE.onboarding_server_version()}", @@ -703,6 +704,32 @@ def test_public_install_uses_a_cluster_instead_of_client_dry_run(self) -> None: self.assertNotIn("database.example.invalid", arguments) self.assertNotIn("redis.example.invalid", arguments) self.assertNotIn("--dry-run=client", arguments) + # The default runtime is native, which refuses to start without a + # shared token. + self.assertIn("auth.authToken=not-a-secret", arguments) + self.assertIn("auth.authToken=not-a-secret", RELEASE.helm_value_arguments()) + + def test_chart_app_version_is_separate_from_the_frozen_php_release(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + record = Path(temporary) / "source-release.json" + base = { + "schema": "durable-workflow.server.source-release/v1", + "server": {"version": "2.11.8"}, + "helm_chart": {"version": "1.0.0"}, + } + record.write_text(json.dumps(base)) + self.assertEqual( + RELEASE.source_release_metadata(record)["chart_app_version"], "2.11.8" + ) + base["helm_chart"]["app_version"] = "3.0.1" + record.write_text(json.dumps(base)) + metadata = RELEASE.source_release_metadata(record) + self.assertEqual(metadata["server_version"], "2.11.8") + self.assertEqual(metadata["chart_app_version"], "3.0.1") + base["helm_chart"]["app_version"] = "latest" + record.write_text(json.dumps(base)) + with self.assertRaisesRegex(RELEASE.ReleaseError, "app_version is not SemVer"): + RELEASE.source_release_metadata(record) def test_public_install_and_source_smoke_share_the_fixture_manifest(self) -> None: fixture = "scripts/helm-chart-kind-fixtures.yaml" diff --git a/scripts/ci/test_helm_native_image_guard.py b/scripts/ci/test_helm_native_image_guard.py deleted file mode 100644 index cdff2efb7..000000000 --- a/scripts/ci/test_helm_native_image_guard.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -"""Check that the PHP-only chart refuses native 3.x Server images (Server #946).""" - -from pathlib import Path -import subprocess -import unittest - - -ROOT = Path(__file__).resolve().parents[2] -CHART = ROOT / "k8s/helm/durable-workflow" - - -def render(*options: str) -> subprocess.CompletedProcess: - return subprocess.run( - ["helm", "template", "native-guard", str(CHART), "--namespace", "native-guard", - "--values", str(CHART / "ci/inline-secrets-values.yaml"), *options], - capture_output=True, text=True, - ) - - -class NativeImageGuardTest(unittest.TestCase): - def test_native_tags_are_refused_at_render_time(self): - for tag in ["3.0.0", "v3.0.0", "3.0.1", "3.1.0-rc.1", "3.0", "3", "10.0.0", "latest"]: - with self.subTest(tag=tag): - result = render( - "--set-string", f"image.tag={tag}", - "--set-string", "image.memoPayloadStorage=dual-v1", - ) - self.assertNotEqual(result.returncode, 0, result.stdout[:200]) - self.assertIn(f"image.tag {tag} selects the native 3.x Server", result.stderr) - self.assertIn("durable-workflow/server#946", result.stderr) - - def test_php_tags_still_render(self): - for tag in ["2.11.8", "2.0.0-rc.49", "2.11"]: - with self.subTest(tag=tag): - result = render( - "--set-string", f"image.tag={tag}", - "--set-string", "image.memoPayloadStorage=dual-v1", - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(f"durableworkflow/server:{tag}", result.stdout) - - def test_default_values_render_the_php_release(self): - result = render() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("durableworkflow/server:2.11.8", result.stdout) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/ci/test_helm_runtime_modes.py b/scripts/ci/test_helm_runtime_modes.py new file mode 100644 index 000000000..04273a752 --- /dev/null +++ b/scripts/ci/test_helm_runtime_modes.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Check what the chart renders in native and PHP mode, and what it refuses (Server #946).""" + +from pathlib import Path +import subprocess +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +CHART = ROOT / "k8s/helm/durable-workflow" +NATIVE = CHART / "ci/native-inline-secrets-values.yaml" +PHP = CHART / "ci/inline-secrets-values.yaml" +PHP_ONLY_COMPONENTS = {"worker", "scheduler", "migration"} + + +def chart_app_version() -> str: + return yaml.safe_load((CHART / "Chart.yaml").read_text())["appVersion"] + + +def helm(*options: str, values: Path | None = NATIVE) -> subprocess.CompletedProcess: + arguments = ["helm", "template", "runtime-check", str(CHART), "--namespace", "runtime-check"] + if values is not None: + arguments += ["--values", str(values)] + return subprocess.run([*arguments, *options], capture_output=True, text=True) + + +def render(*options: str, values: Path | None = NATIVE) -> list[dict]: + result = helm(*options, values=values) + if result.returncode != 0: + raise AssertionError(result.stderr) + return [item for item in yaml.safe_load_all(result.stdout) if item] + + +def component(resource: dict) -> str: + return resource["metadata"].get("labels", {}).get("app.kubernetes.io/component", "") + + +def named(resources: list[dict], kind: str, suffix: str) -> dict: + return next( + item for item in resources + if item["kind"] == kind and item["metadata"]["name"].endswith(suffix) + ) + + +class NativeModeTest(unittest.TestCase): + def test_default_install_renders_one_native_server_and_nothing_php(self): + # The fixture names no runtime and no image: the chart defaults. + resources = render( + "--set-string", "externalDatabase.host=db.example.internal", + "--set-string", "externalDatabase.auth.username=workflow", + "--set-string", "externalDatabase.auth.password=ci-only", + "--set-string", "auth.authToken=ci-shared-token", + values=None, + ) + workloads = [item for item in resources if item["kind"] in {"Deployment", "Job", "CronJob"}] + self.assertEqual([item["metadata"]["name"] for item in workloads], ["runtime-check-durable-workflow-server"]) + self.assertFalse({component(item) for item in resources} & PHP_ONLY_COMPONENTS) + self.assertNotIn("HorizontalPodAutoscaler", {item["kind"] for item in resources}) + + server = workloads[0] + self.assertEqual(server["metadata"]["annotations"]["workflows.durable-workflow.dev/runtime"], "native") + pod = server["spec"]["template"]["spec"] + container = pod["containers"][0] + self.assertEqual(container["image"], f"docker.io/durableworkflow/server:{chart_app_version()}") + self.assertNotIn("command", container) + self.assertNotIn("args", container) + self.assertEqual(container["ports"], [{"name": "http", "containerPort": 8080, "protocol": "TCP"}]) + self.assertEqual(container["livenessProbe"]["httpGet"]["path"], "/api/health") + self.assertEqual(container["readinessProbe"]["httpGet"]["path"], "/api/ready") + self.assertGreater(pod["terminationGracePeriodSeconds"], 30) + self.assertEqual(pod["securityContext"]["runAsUser"], 1000) + self.assertEqual(pod["securityContext"]["fsGroup"], 1000) + self.assertIn({"name": "server-storage", "mountPath": "/app/storage"}, container["volumeMounts"]) + self.assertIn({"name": "server-storage", "emptyDir": {}}, pod["volumes"]) + self.assertEqual({item["name"] for item in container["env"]}, {"DB_USERNAME", "DB_PASSWORD"}) + + config = named(resources, "ConfigMap", "-config")["data"] + for key in config: + self.assertFalse(key.startswith(("REDIS_", "QUEUE_", "CACHE_", "APP_")), key) + self.assertNotIn("DW_AUTH_BACKWARD_COMPATIBLE", config) + self.assertNotIn("DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT", config) + self.assertEqual(config["DW_AUTH_DRIVER"], "token") + self.assertEqual(config["DB_CONNECTION"], "mysql") + + def test_server_options_shared_with_php_mode_apply(self): + resources = render(values=CHART / "ci/native-ingress-and-hpa-values.yaml") + kinds = {item["kind"] for item in resources} + self.assertTrue({"Ingress", "HorizontalPodAutoscaler", "PodDisruptionBudget", "NetworkPolicy", "Service"} <= kinds) + server = named(resources, "Deployment", "-server") + self.assertNotIn("replicas", server["spec"]) + volume = next(item for item in server["spec"]["template"]["spec"]["volumes"] if item["name"] == "server-storage") + self.assertEqual(volume, {"name": "server-storage", "persistentVolumeClaim": {"claimName": "durable-workflow-payloads"}}) + + def test_schema_upgrade_is_an_explicit_pre_upgrade_hook(self): + self.assertNotIn("Job", {item["kind"] for item in render()}) + + job = named(render("--set", "native.schemaUpgrade.enabled=true"), "Job", "-schema-upgrade") + self.assertEqual(job["metadata"]["annotations"]["helm.sh/hook"], "pre-upgrade") + container = job["spec"]["template"]["spec"]["containers"][0] + self.assertEqual(container["args"], ["schema-upgrade"]) + self.assertNotIn("command", container) + environment = {item["name"]: item for item in container["env"]} + self.assertEqual(environment["DB_CONNECTION"]["value"], "mysql") + self.assertEqual(environment["DB_HOST"]["value"], "db.example.internal") + self.assertIn("DB_PASSWORD", environment) + self.assertNotIn("DW_AUTH_TOKEN", environment) + + def test_render_time_refusals_match_native_startup_refusals(self): + cases = { + "auth.authToken is required in native mode": ["--set-string", "auth.authToken="], + "auth.principalTokens is not supported": ["--set-string", "auth.principalTokens=[]"], + 'config.auth.driver "none" is not supported': ["--set-string", "config.auth.driver=none"], + "must exceed DW_WORKER_POLL_TIMEOUT": [ + "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=60", + ], + "reserved for native payload storage": [ + "--set-json", 'server.extraVolumes=[{"name":"server-storage","emptyDir":{}}]', + ], + } + for message, options in cases.items(): + with self.subTest(message=message): + result = helm(*options) + self.assertNotEqual(result.returncode, 0, result.stdout[:200]) + self.assertIn(message, result.stderr) + + accepted = render( + "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=60", + "--set", "native.terminationGracePeriodSeconds=75", + ) + server = named(accepted, "Deployment", "-server") + self.assertEqual(server["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 75) + + def test_existing_secret_satisfies_the_token_requirement(self): + render("--set-string", "auth.authToken=", "--set-string", "auth.existingSecret=app-credentials") + + +class PhpModeTest(unittest.TestCase): + def test_php_fixture_keeps_the_php_shape(self): + resources = render(values=PHP) + self.assertTrue(PHP_ONLY_COMPONENTS <= {component(item) for item in resources}) + server = named(resources, "Deployment", "-server") + self.assertNotIn("workflows.durable-workflow.dev/runtime", server["metadata"]["annotations"]) + self.assertIn("workflows.durable-workflow.dev/memo-payload-storage", server["metadata"]["annotations"]) + config = named(resources, "ConfigMap", "-config")["data"] + self.assertEqual(config["APP_VERSION"], "2.11.8") + self.assertEqual(config["QUEUE_CONNECTION"], "redis") + self.assertNotIn("Job", {item["kind"] for item in resources if component(item) == "schema-upgrade"}) + + def test_php_app_version_never_reports_the_native_chart_version(self): + resources = render( + "--set-string", "image.tag=php-custom", + "--set-string", "image.memoPayloadStorage=dual-v1", + values=PHP, + ) + self.assertNotIn("APP_VERSION", named(resources, "ConfigMap", "-config")["data"]) + + def test_php_mode_ignores_native_settings(self): + resources = render("--set", "native.schemaUpgrade.enabled=true", values=PHP) + self.assertFalse(any(component(item) == "schema-upgrade" for item in resources)) + + +class ImageRuntimeGuardTest(unittest.TestCase): + def test_native_tags_are_refused_in_php_mode(self): + for tag in ["3.0.0", "v3.0.0", "3.0.1", "3.1.0-rc.1", "3.0", "3", "10.0.0", "latest"]: + with self.subTest(tag=tag): + result = helm( + "--set-string", f"image.tag={tag}", + "--set-string", "image.memoPayloadStorage=dual-v1", + values=PHP, + ) + self.assertNotEqual(result.returncode, 0, result.stdout[:200]) + self.assertIn(f"image.tag {tag} selects the native 3.x Server, but runtime is php", result.stderr) + self.assertIn("durable-workflow/server#946", result.stderr) + + def test_php_tags_are_refused_in_native_mode(self): + for tag in ["2.11.8", "2.0.0-rc.49", "2.11", "2", "v2.11.8"]: + with self.subTest(tag=tag): + result = helm("--set-string", f"image.tag={tag}") + self.assertNotEqual(result.returncode, 0, result.stdout[:200]) + self.assertIn(f"image.tag {tag} selects the PHP 2.x Server, but runtime is native", result.stderr) + + def test_each_mode_accepts_its_own_tags_and_custom_tags(self): + for tag in ["3.0.1", "3", "latest", "helm-smoke-1"]: + with self.subTest(runtime="native", tag=tag): + self.assertIn(f"durableworkflow/server:{tag}", helm("--set-string", f"image.tag={tag}").stdout) + for tag in ["2.11.8", "2.0.0-rc.49", "helm-smoke-1"]: + with self.subTest(runtime="php", tag=tag): + result = helm( + "--set-string", f"image.tag={tag}", + "--set-string", "image.memoPayloadStorage=dual-v1", + values=PHP, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"durableworkflow/server:{tag}", result.stdout) + + def test_unknown_runtime_is_refused(self): + result = helm("--set-string", "runtime=laravel") + self.assertNotEqual(result.returncode, 0) + self.assertIn("runtime", result.stderr) + + +class UpgradeRuntimeSelectionTest(unittest.TestCase): + """helm template cannot see live objects, so an upgrade without a runtime refuses.""" + + def test_an_upgrade_without_a_visible_native_release_must_choose(self): + native_without_runtime = helm("--is-upgrade", "--set-string", "runtime=") + self.assertNotEqual(native_without_runtime.returncode, 0) + self.assertIn("Set runtime=native or runtime=php explicitly", native_without_runtime.stderr) + + for runtime, values in [("native", NATIVE), ("php", PHP)]: + with self.subTest(runtime=runtime): + result = helm("--is-upgrade", "--set-string", f"runtime={runtime}", values=values) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/helm-chart-kind-smoke.sh b/scripts/helm-chart-kind-smoke.sh index ebd1fb6d2..b280f1deb 100755 --- a/scripts/helm-chart-kind-smoke.sh +++ b/scripts/helm-chart-kind-smoke.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Helm-chart kind smoke. Mirrors scripts/k8s-kind-smoke.sh but exercises the -# self-serve chart at k8s/helm/durable-workflow/ end-to-end: +# self-serve chart at k8s/helm/durable-workflow/ end-to-end in PHP mode: # 1. build a server image and load it into kind # 2. provision in-cluster MySQL + Redis (the chart never bundles them) # 3. helm install the chart @@ -229,6 +229,10 @@ apply_dependencies app_key="base64:$(openssl rand -base64 32)" install_values="${artifact_dir}/install-values.yaml" cat >"${install_values}" </dev/null 2>&1 || { echo "Missing required command: ${command}" >&2; exit 127; } +done +mkdir -p "${artifact_dir}" + +collect_artifacts() { + "${helm_bin}" -n "${namespace}" status "${release}" >"${artifact_dir}/helm-status.txt" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" get all,configmap,secret,pdb,pvc -o wide >"${artifact_dir}/resources.txt" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" describe all >"${artifact_dir}/describe.txt" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" get events --sort-by=.lastTimestamp >"${artifact_dir}/events.txt" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" logs -l app.kubernetes.io/component=server --tail=-1 --all-containers=true >"${artifact_dir}/server.log" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" logs -l app.kubernetes.io/component=schema-upgrade --tail=-1 --all-containers=true >"${artifact_dir}/schema-upgrade.log" 2>&1 || true + "${kubectl_bin}" -n "${namespace}" logs deploy/durable-workflow-mysql --all-containers=true >"${artifact_dir}/mysql.log" 2>&1 || true +} + +cleanup() { + local status=$? + if [ -n "${port_forward_pid}" ]; then + kill "${port_forward_pid}" >/dev/null 2>&1 || true + fi + if [ "${status}" -ne 0 ]; then + collect_artifacts + echo "native helm chart kind smoke failed; artifacts in ${artifact_dir}" >&2 + fi + rm -rf "${work_dir}" + if [ "${created_cluster}" = "1" ]; then + "${kind_bin}" delete cluster --name "${cluster}" >/dev/null 2>&1 || true + fi + exit "${status}" +} +trap cleanup EXIT + +expect_refusal() { + local scenario="$1" expected="$2" output + shift 2 + if output="$("$@" 2>&1)"; then + printf 'expected a refusal: %s\n%s\n' "${scenario}" "${output}" >&2 + exit 1 + fi + if ! grep -qF "${expected}" <<<"${output}"; then + printf 'refusal for %s did not explain itself; expected "%s":\n%s\n' "${scenario}" "${expected}" "${output}" >&2 + exit 1 + fi + printf 'PASS refused: %s\n' "${scenario}" +} + +start_port_forward() { + "${kubectl_bin}" -n "${namespace}" port-forward "svc/${release}-durable-workflow-server" "${port}:8080" \ + >"${artifact_dir}/port-forward.log" 2>&1 & + port_forward_pid=$! + for _ in $(seq 1 30); do + if curl -fsS "http://127.0.0.1:${port}/api/health" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "port-forward to the native Service never answered" >&2 + exit 1 +} + +# 1. Cluster and a fresh MySQL database. The shared fixture manifest is reused +# without Redis, which native does not need, in its own namespace so a +# database the PHP smoke bootstrapped is never offered to native. +if [ "${reuse_cluster}" != "1" ]; then + command -v "${kind_bin}" >/dev/null 2>&1 || { echo "Missing required command: ${kind_bin}" >&2; exit 127; } + "${kind_bin}" create cluster --name "${cluster}" --image "${kind_node_image}" --wait 120s + created_cluster=1 +fi + +python3 - "${fixture_manifest}" "${namespace}" >"${work_dir}/mysql.yaml" <<'PY' +import sys +source, namespace = sys.argv[1], sys.argv[2] +documents = open(source).read().split("\n---\n") +kept = [doc for doc in documents if "redis" not in doc] +text = "\n---\n".join(kept) +text = text.replace("namespace: durable-workflow\n", f"namespace: {namespace}\n") +text = text.replace("kind: Namespace\nmetadata:\n name: durable-workflow\n", f"kind: Namespace\nmetadata:\n name: {namespace}\n") +print(text) +PY +"${kubectl_bin}" apply -f "${work_dir}/mysql.yaml" +"${kubectl_bin}" -n "${namespace}" rollout status deploy/durable-workflow-mysql --timeout=180s + +values="${work_dir}/values.yaml" +cat >"${values}" <&2 + exit 1 +fi +if "${kubectl_bin}" -n "${namespace}" get secret "${release}-durable-workflow-redis" >/dev/null 2>&1; then + echo "native mode rendered a Redis Secret" >&2 + exit 1 +fi +marker="$("${kubectl_bin}" -n "${namespace}" get deploy "${release}-durable-workflow-server" \ + -o jsonpath='{.metadata.annotations.workflows\.durable-workflow\.dev/runtime}')" +[ "${marker}" = "native" ] || { echo "server Deployment lacks the native runtime marker" >&2; exit 1; } + +"${helm_bin}" test "${release}" --namespace "${namespace}" +start_port_forward +curl -fsS "http://127.0.0.1:${port}/api/ready" | tee "${artifact_dir}/ready.json" +echo +curl -fsS -H "Authorization: Bearer kind-native-admin-token" -H "X-Namespace: default" \ + "http://127.0.0.1:${port}/api/cluster/info" >"${artifact_dir}/cluster-info.json" +kill "${port_forward_pid}" >/dev/null 2>&1 || true +port_forward_pid="" + +# 3. An upgrade with an empty runtime finds the native marker and stays native. +"${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set-string server.podAnnotations.kindSmoke="upgrade-${RANDOM}" \ + --wait --timeout 5m +echo "PASS allowed: upgrade with an empty runtime stays native" + +# 4. The schema-upgrade hook refuses while a Server pod runs, then runs once +# the Server is scaled to zero. +expect_refusal "schema upgrade while the Server runs" "needs every native Server stopped" \ + "${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set native.schemaUpgrade.enabled=true --dry-run=server + +"${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set server.replicaCount=0 --wait --timeout 5m +"${kubectl_bin}" -n "${namespace}" wait --for=delete pod \ + -l "app.kubernetes.io/instance=${release},app.kubernetes.io/component=server" --timeout=120s + +"${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set native.schemaUpgrade.enabled=true --wait --timeout 5m +"${kubectl_bin}" -n "${namespace}" rollout status "deploy/${release}-durable-workflow-server" --timeout=180s +echo "PASS allowed: schema-upgrade hook with every Server stopped" + +# 5. A release whose server Deployment lacks the native marker, as every PHP +# release from chart 0.x does, must choose a runtime explicitly. +stub_chart="${work_dir}/php-stub" +mkdir -p "${stub_chart}/templates" +cat >"${stub_chart}/Chart.yaml" <<'YAML' +apiVersion: v2 +name: durable-workflow +version: 0.3.9 +appVersion: "2.11.8" +YAML +cat >"${stub_chart}/templates/server.yaml" <<'YAML' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-durable-workflow-server +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: server + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: server + spec: + containers: + - name: server + image: docker.io/durableworkflow/server:2.11.8 +YAML +"${helm_bin}" install "${stub_release}" "${stub_chart}" --namespace "${namespace}" >/dev/null +expect_refusal "upgrade of a PHP release with an empty runtime" "runs the PHP 2.x Server" \ + "${helm_bin}" upgrade "${stub_release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --dry-run=server +"${helm_bin}" upgrade "${stub_release}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set runtime=native --dry-run=server >/dev/null +echo "PASS allowed: an explicit runtime renders against the PHP release" +"${helm_bin}" uninstall "${stub_release}" --namespace "${namespace}" --wait --timeout 2m + +# 6. Uninstall leaves only the externally managed database behind. +"${helm_bin}" uninstall "${release}" --namespace "${namespace}" --wait --timeout 5m +remaining="" +for _ in $(seq 1 30); do + remaining="$("${kubectl_bin}" -n "${namespace}" get all,configmap,secret,pdb \ + -l "app.kubernetes.io/instance=${release}" -o name 2>/dev/null || true)" + [ -z "${remaining}" ] && break + sleep 2 +done +if [ -n "${remaining}" ]; then + printf 'helm uninstall left chart-owned resources behind:\n%s\n' "${remaining}" >&2 + exit 1 +fi + +if [ "${created_cluster}" != "1" ]; then + "${kubectl_bin}" delete namespace "${namespace}" --wait=false >/dev/null +fi +echo "native helm chart kind smoke OK" diff --git a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php index a16ec81f6..4b87a0397 100644 --- a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php +++ b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php @@ -1117,12 +1117,16 @@ public function test_release_surfaces_are_rendered_from_the_source_release_manif $smallCluster['x-server-environment']['APP_VERSION'] ?? null, ); + // The chart packages its own Server image (the native line), which + // may differ from the frozen PHP source release. + $expectedChartAppVersion = $release['helm_chart']['app_version'] ?? $expectedVersion; + $this->assertIsString($expectedChartAppVersion); $chart = Yaml::parse($this->read('k8s/helm/durable-workflow/Chart.yaml')); $this->assertIsArray($chart); $this->assertSame($expectedChartVersion, $chart['version'] ?? null); - $this->assertSame($expectedVersion, $chart['appVersion'] ?? null); + $this->assertSame($expectedChartAppVersion, $chart['appVersion'] ?? null); $this->assertSame( - "docker.io/durableworkflow/server:{$expectedVersion}", + "docker.io/durableworkflow/server:{$expectedChartAppVersion}", $chart['annotations']['dev.durable-workflow.image-reference'] ?? null, ); } From 9f951481320933c2a8d22ad8e06ba04687fc2802 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:57:16 -0600 Subject: [PATCH 2/4] Helm native mode: fix the PHP memo guard scenario and Sonar findings The memo-transition CI scenario took its PHP successor tag from the chart appVersion, which is now the native 3.0.1, so PHP mode correctly refused it before the guard ran. It now reads the 2.x tag the PHP fixture pins. Native pods mount no service account token (the native Server never calls the Kubernetes API) unless native.automountServiceAccountToken is set, and their containers get an ephemeral-storage request and limit from native.ephemeralStorage unless the resources already name one. The payload emptyDir counts toward that limit. Also tidy the new test and smoke code Sonar flagged: String.raw and a quantifier in the source-release tests, assertLessEqual, and [[ ]] in the native kind smoke. Refs #946 Co-Authored-By: Claude Opus 5.5 --- k8s/helm/durable-workflow/README.md | 6 +++++ .../durable-workflow/templates/_helpers.tpl | 22 +++++++++++++++++++ .../templates/native-schema-upgrade-job.yaml | 3 ++- .../templates/native-server-deployment.yaml | 5 ++++- k8s/helm/durable-workflow/values.schema.json | 8 +++++++ k8s/helm/durable-workflow/values.yaml | 11 ++++++++++ scripts/ci/helm-memo-transition-upgrade.sh | 8 ++++--- scripts/ci/sync-source-release.mjs | 2 +- scripts/ci/test-source-release.mjs | 20 +++++++++-------- scripts/ci/test_helm_runtime_modes.py | 21 ++++++++++++++++-- scripts/helm-chart-native-kind-smoke.sh | 20 ++++++++--------- 11 files changed, 99 insertions(+), 27 deletions(-) diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 8c04935e2..686f242d2 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -136,6 +136,12 @@ Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`. `native.terminationGracePeriodSeconds` (default 45) must stay above it. The chart refuses a grace period at or below a `DW_WORKER_POLL_TIMEOUT` set in `config.extraEnv`. +* Native pods set an ephemeral-storage request and limit from + `native.ephemeralStorage` (128Mi and 2Gi) unless `server.resources` names + one. The payload `emptyDir` counts toward that limit. They also mount no + service account token, because the native Server never calls the + Kubernetes API. Set `native.automountServiceAccountToken: true` only for a + sidecar that needs one. * The chart supports MySQL, MariaDB and PostgreSQL. It does not run the native Server on SQLite. diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index a3c9846c8..5968023ab 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -799,3 +799,25 @@ rather than the previous release's ConfigMap. {{- define "durable-workflow.nativeSchemaUpgradeJobName" -}} {{ include "durable-workflow.componentName" (dict "context" . "component" "schema-upgrade") }} {{- end -}} + +{{/* +Container resources for native pods: the configured requests and limits, plus +an ephemeral-storage request and limit from native.ephemeralStorage unless the +resources already name one. The payload emptyDir counts toward the pod's +ephemeral storage, so the limit also bounds it. +*/}} +{{- define "durable-workflow.nativeResources" -}} +{{- $resources := deepCopy (default (dict) .resources) -}} +{{- $ephemeral := default (dict) .ephemeralStorage -}} +{{- range $kind, $key := dict "requests" "request" "limits" "limit" -}} +{{- $value := get $ephemeral $key -}} +{{- if $value -}} +{{- $section := default (dict) (get $resources $kind) -}} +{{- if not (hasKey $section "ephemeral-storage") -}} +{{- $_ := set $section "ephemeral-storage" $value -}} +{{- end -}} +{{- $_ := set $resources $kind $section -}} +{{- end -}} +{{- end -}} +{{- toYaml $resources -}} +{{- end -}} diff --git a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml index e87f0401f..9f9efe833 100644 --- a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml +++ b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml @@ -35,6 +35,7 @@ spec: spec: restartPolicy: Never serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} + automountServiceAccountToken: false {{- with .Values.image.pullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -57,7 +58,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} resources: - {{- toYaml (default (dict) (get $upgrade "resources")) | nindent 12 }} + {{- include "durable-workflow.nativeResources" (dict "resources" (get $upgrade "resources") "ephemeralStorage" (get $native "ephemeralStorage")) | nindent 12 }} {{- with .Values.server.containerSecurityContext }} securityContext: {{- toYaml . | nindent 12 }} diff --git a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml index d7609085c..69711eca6 100644 --- a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml @@ -10,6 +10,7 @@ empty runtime value confirm that this release already runs native. {{- include "durable-workflow.validateNative" . }} {{- $native := default (dict) .Values.native }} {{- $storage := default (dict) (get $native "storage") }} +{{- $resources := include "durable-workflow.nativeResources" (dict "resources" .Values.server.resources "ephemeralStorage" (get $native "ephemeralStorage")) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -51,6 +52,8 @@ spec: {{- end }} spec: serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} + {{- /* The native Server never calls the Kubernetes API. */}} + automountServiceAccountToken: {{ ternary "true" "false" (eq (toString (get $native "automountServiceAccountToken")) "true") }} {{- with .Values.image.pullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -94,7 +97,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} resources: - {{- toYaml .Values.server.resources | nindent 12 }} + {{- $resources | nindent 12 }} {{- with .Values.server.containerSecurityContext }} securityContext: {{- toYaml . | nindent 12 }} diff --git a/k8s/helm/durable-workflow/values.schema.json b/k8s/helm/durable-workflow/values.schema.json index 3689b74a8..d97e4fb96 100644 --- a/k8s/helm/durable-workflow/values.schema.json +++ b/k8s/helm/durable-workflow/values.schema.json @@ -14,6 +14,14 @@ "type": "object", "properties": { "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, + "automountServiceAccountToken": { "type": "boolean" }, + "ephemeralStorage": { + "type": "object", + "properties": { + "request": { "type": ["string", "integer"] }, + "limit": { "type": ["string", "integer"] } + } + }, "storage": { "type": "object", "properties": { diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 61724b5ae..610600403 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -195,6 +195,17 @@ auth: # --------------------------------------------------------------------------- native: + # The native Server never calls the Kubernetes API, so its pods mount no + # service account token. Set true only for a sidecar that needs one. + automountServiceAccountToken: false + + # Ephemeral storage added to the server and schema-upgrade containers unless + # their resources already set it. The payload emptyDir (no storage claim) + # counts toward the server pod's limit. + ephemeralStorage: + request: 128Mi + limit: 2Gi + # On SIGTERM the native Server holds open worker polls for up to # DW_WORKER_POLL_TIMEOUT (30 seconds unless config.extraEnv changes it). # Keep this above that timeout so a rollout drains polls instead of cutting diff --git a/scripts/ci/helm-memo-transition-upgrade.sh b/scripts/ci/helm-memo-transition-upgrade.sh index 7cd403c8a..2b936ebe2 100755 --- a/scripts/ci/helm-memo-transition-upgrade.sh +++ b/scripts/ci/helm-memo-transition-upgrade.sh @@ -9,10 +9,12 @@ fixture="${chart_path}/ci/inline-secrets-values.yaml" storage_annotation="workflows.durable-workflow.dev/memo-payload-storage" temporary_directory="$(mktemp -d)" stub_chart="${temporary_directory}/stub" -successor_tag="$(sed -nE 's/^appVersion:[[:space:]]*"([^"]+)"/\1/p' "${chart_path}/Chart.yaml")" +# The memo transition guard is PHP-only. Chart 1.x packages the native Server, +# so the PHP successor is the 2.x release the PHP fixture pins, not appVersion. +successor_tag="$(sed -nE 's/^ tag:[[:space:]]*"([^"]+)"/\1/p' "${fixture}")" -if [[ -z "${successor_tag}" ]]; then - printf 'could not resolve the successor image tag from %s/Chart.yaml\n' "${chart_path}" >&2 +if [[ ! "${successor_tag}" =~ ^2\. ]]; then + printf 'could not resolve a PHP 2.x successor image tag from %s\n' "${fixture}" >&2 exit 1 fi diff --git a/scripts/ci/sync-source-release.mjs b/scripts/ci/sync-source-release.mjs index cbd1ab4d6..21be9bcfa 100644 --- a/scripts/ci/sync-source-release.mjs +++ b/scripts/ci/sync-source-release.mjs @@ -309,7 +309,7 @@ const consumers = [ render(source, release) { return replaceExactly( source, - new RegExp(`^( tag:\\s*["'])${serverVersion}(["']\\s*)$`, 'gm'), + new RegExp(String.raw`^( {2}tag:\s*["'])${serverVersion}(["']\s*)$`, 'gm'), (_match, prefix, suffix) => `${prefix}${release[version]}${suffix}`, 1, path, diff --git a/scripts/ci/test-source-release.mjs b/scripts/ci/test-source-release.mjs index 683468c05..1de3f4211 100644 --- a/scripts/ci/test-source-release.mjs +++ b/scripts/ci/test-source-release.mjs @@ -19,6 +19,11 @@ function runGenerator(root, mode) { }); } +// A version as a literal regular expression fragment. +function literal(version) { + return version.replaceAll('.', String.raw`\.`); +} + function nextServerRelease(version) { const match = /^(.*-(?:alpha|beta|rc)\.)(\d+)$/.exec(version); if (match) { @@ -113,13 +118,13 @@ test('a stable server source identity fans out through current release consumers join(temporaryRoot, 'k8s/helm/durable-workflow/ci/inline-secrets-values.yaml'), 'utf8', ); - assert.match(fixture, /^ tag: "2\.0\.0"$/m); + assert.match(fixture, /^ {2}tag: "2\.0\.0"$/m); const chart = await readFile( join(temporaryRoot, 'k8s/helm/durable-workflow/Chart.yaml'), 'utf8', ); const chartAppVersion = manifest.helm_chart.app_version ?? manifest.server.version; - assert.match(chart, new RegExp(`^appVersion: "${chartAppVersion.replaceAll('.', '\\.')}"$`, 'm')); + assert.match(chart, new RegExp(`^appVersion: "${literal(chartAppVersion)}"$`, 'm')); } finally { await rm(temporaryRoot, {recursive: true, force: true}); } @@ -190,14 +195,11 @@ test('a simulated next release fans out without rewriting history or retaining s chartHistoryBefore, 'release generation must not rewrite the Artifact Hub changelog', ); - assert.match(chartAfter, new RegExp(`^version: ${nextChart.replaceAll('.', '\\.')}$`, 'm')); - assert.match(chartAfter, new RegExp(`^appVersion: "${nextChartApp.replaceAll('.', '\\.')}"$`, 'm')); - assert.match( - chartAfter, - new RegExp(`docker\\.io/durableworkflow/server:${nextChartApp.replaceAll('.', '\\.')}"`), - ); + assert.match(chartAfter, new RegExp(`^version: ${literal(nextChart)}$`, 'm')); + assert.match(chartAfter, new RegExp(`^appVersion: "${literal(nextChartApp)}"$`, 'm')); + assert.ok(chartAfter.includes(`docker.io/durableworkflow/server:${nextChartApp}"`)); const values = await readFile(join(temporaryRoot, 'k8s/helm/durable-workflow/values.yaml'), 'utf8'); - assert.match(values, new RegExp(`^ tag: "${nextChartApp.replaceAll('.', '\\.')}"$`, 'm')); + assert.match(values, new RegExp(String.raw`^ {2}tag: "${literal(nextChartApp)}"$`, 'm')); const stale = []; for (const path of paths) { diff --git a/scripts/ci/test_helm_runtime_modes.py b/scripts/ci/test_helm_runtime_modes.py index 04273a752..1ac16a76d 100644 --- a/scripts/ci/test_helm_runtime_modes.py +++ b/scripts/ci/test_helm_runtime_modes.py @@ -75,6 +75,12 @@ def test_default_install_renders_one_native_server_and_nothing_php(self): self.assertIn({"name": "server-storage", "mountPath": "/app/storage"}, container["volumeMounts"]) self.assertIn({"name": "server-storage", "emptyDir": {}}, pod["volumes"]) self.assertEqual({item["name"] for item in container["env"]}, {"DB_USERNAME", "DB_PASSWORD"}) + # Native never calls the Kubernetes API, and its ephemeral storage + # (including the payload emptyDir) is bounded. + self.assertIs(pod["automountServiceAccountToken"], False) + self.assertEqual(container["resources"]["limits"]["ephemeral-storage"], "2Gi") + self.assertEqual(container["resources"]["requests"]["ephemeral-storage"], "128Mi") + self.assertEqual(container["resources"]["limits"]["memory"], "512Mi") config = named(resources, "ConfigMap", "-config")["data"] for key in config: @@ -87,7 +93,7 @@ def test_default_install_renders_one_native_server_and_nothing_php(self): def test_server_options_shared_with_php_mode_apply(self): resources = render(values=CHART / "ci/native-ingress-and-hpa-values.yaml") kinds = {item["kind"] for item in resources} - self.assertTrue({"Ingress", "HorizontalPodAutoscaler", "PodDisruptionBudget", "NetworkPolicy", "Service"} <= kinds) + self.assertLessEqual({"Ingress", "HorizontalPodAutoscaler", "PodDisruptionBudget", "NetworkPolicy", "Service"}, kinds) server = named(resources, "Deployment", "-server") self.assertNotIn("replicas", server["spec"]) volume = next(item for item in server["spec"]["template"]["spec"]["volumes"] if item["name"] == "server-storage") @@ -106,6 +112,17 @@ def test_schema_upgrade_is_an_explicit_pre_upgrade_hook(self): self.assertEqual(environment["DB_HOST"]["value"], "db.example.internal") self.assertIn("DB_PASSWORD", environment) self.assertNotIn("DW_AUTH_TOKEN", environment) + self.assertIs(job["spec"]["template"]["spec"]["automountServiceAccountToken"], False) + self.assertIn("ephemeral-storage", container["resources"]["limits"]) + + def test_explicit_ephemeral_storage_and_token_settings_win(self): + server = named(render( + "--set-string", "server.resources.limits.ephemeral-storage=5Gi", + "--set", "native.automountServiceAccountToken=true", + ), "Deployment", "-server") + pod = server["spec"]["template"]["spec"] + self.assertIs(pod["automountServiceAccountToken"], True) + self.assertEqual(pod["containers"][0]["resources"]["limits"]["ephemeral-storage"], "5Gi") def test_render_time_refusals_match_native_startup_refusals(self): cases = { @@ -139,7 +156,7 @@ def test_existing_secret_satisfies_the_token_requirement(self): class PhpModeTest(unittest.TestCase): def test_php_fixture_keeps_the_php_shape(self): resources = render(values=PHP) - self.assertTrue(PHP_ONLY_COMPONENTS <= {component(item) for item in resources}) + self.assertLessEqual(PHP_ONLY_COMPONENTS, {component(item) for item in resources}) server = named(resources, "Deployment", "-server") self.assertNotIn("workflows.durable-workflow.dev/runtime", server["metadata"]["annotations"]) self.assertIn("workflows.durable-workflow.dev/memo-payload-storage", server["metadata"]["annotations"]) diff --git a/scripts/helm-chart-native-kind-smoke.sh b/scripts/helm-chart-native-kind-smoke.sh index 952f58218..501432ce3 100755 --- a/scripts/helm-chart-native-kind-smoke.sh +++ b/scripts/helm-chart-native-kind-smoke.sh @@ -53,15 +53,15 @@ collect_artifacts() { cleanup() { local status=$? - if [ -n "${port_forward_pid}" ]; then + if [[ -n "${port_forward_pid}" ]]; then kill "${port_forward_pid}" >/dev/null 2>&1 || true fi - if [ "${status}" -ne 0 ]; then + if [[ "${status}" -ne 0 ]]; then collect_artifacts echo "native helm chart kind smoke failed; artifacts in ${artifact_dir}" >&2 fi rm -rf "${work_dir}" - if [ "${created_cluster}" = "1" ]; then + if [[ "${created_cluster}" = "1" ]]; then "${kind_bin}" delete cluster --name "${cluster}" >/dev/null 2>&1 || true fi exit "${status}" @@ -99,7 +99,7 @@ start_port_forward() { # 1. Cluster and a fresh MySQL database. The shared fixture manifest is reused # without Redis, which native does not need, in its own namespace so a # database the PHP smoke bootstrapped is never offered to native. -if [ "${reuse_cluster}" != "1" ]; then +if [[ "${reuse_cluster}" != "1" ]]; then command -v "${kind_bin}" >/dev/null 2>&1 || { echo "Missing required command: ${kind_bin}" >&2; exit 127; } "${kind_bin}" create cluster --name "${cluster}" --image "${kind_node_image}" --wait 120s created_cluster=1 @@ -141,7 +141,7 @@ server: enabled: false EOF image_options=() -if [ -n "${K8S_HELM_NATIVE_SMOKE_IMAGE_TAG:-}" ]; then +if [[ -n "${K8S_HELM_NATIVE_SMOKE_IMAGE_TAG:-}" ]]; then image_options=(--set-string "image.tag=${K8S_HELM_NATIVE_SMOKE_IMAGE_TAG}") fi @@ -151,7 +151,7 @@ fi workloads="$("${kubectl_bin}" -n "${namespace}" get deploy,cronjob,job \ -l "app.kubernetes.io/instance=${release}" -o name)" -if [ "${workloads}" != "deployment.apps/${release}-durable-workflow-server" ]; then +if [[ "${workloads}" != "deployment.apps/${release}-durable-workflow-server" ]]; then printf 'native mode must render only the server Deployment, found:\n%s\n' "${workloads}" >&2 exit 1 fi @@ -161,7 +161,7 @@ if "${kubectl_bin}" -n "${namespace}" get secret "${release}-durable-workflow-re fi marker="$("${kubectl_bin}" -n "${namespace}" get deploy "${release}-durable-workflow-server" \ -o jsonpath='{.metadata.annotations.workflows\.durable-workflow\.dev/runtime}')" -[ "${marker}" = "native" ] || { echo "server Deployment lacks the native runtime marker" >&2; exit 1; } +[[ "${marker}" = "native" ]] || { echo "server Deployment lacks the native runtime marker" >&2; exit 1; } "${helm_bin}" test "${release}" --namespace "${namespace}" start_port_forward @@ -240,15 +240,15 @@ remaining="" for _ in $(seq 1 30); do remaining="$("${kubectl_bin}" -n "${namespace}" get all,configmap,secret,pdb \ -l "app.kubernetes.io/instance=${release}" -o name 2>/dev/null || true)" - [ -z "${remaining}" ] && break + [[ -z "${remaining}" ]] && break sleep 2 done -if [ -n "${remaining}" ]; then +if [[ -n "${remaining}" ]]; then printf 'helm uninstall left chart-owned resources behind:\n%s\n' "${remaining}" >&2 exit 1 fi -if [ "${created_cluster}" != "1" ]; then +if [[ "${created_cluster}" != "1" ]]; then "${kubectl_bin}" delete namespace "${namespace}" --wait=false >/dev/null fi echo "native helm chart kind smoke OK" From 8be71fe517a042c303c49b55283fa3c077811d94 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:11:36 -0600 Subject: [PATCH 3/4] Helm native mode: gate the schema-upgrade hook on live proof, apply Server review Security hold: the schema-upgrade Job rendered whenever the option was on, including installs, helm template and GitOps renders, where a direct apply ignores the hook annotation, and the upgrade check passed an invisible or non-native predecessor. The hook now renders only during helm upgrade, and only when lookup proves every Server is stopped: the release's server Deployment is visible, carries the native runtime marker and has exactly zero replicas, autoscaling is off in the values, no HorizontalPodAutoscaler targets it, and no server pod of the release is running. Everything else is refused with a pointer to a documented manual path (scale to zero, back up, run schema-upgrade as your own Job, start). Server review: - Native liveness is a TCP check on the http port. /api/health fails while the database is unreachable, so probing it would restart every pod during a database outage. Readiness stays on /api/ready. - More than one native replica, or autoscaling, needs native.storage.existingClaim (ReadWriteMany): payload objects are pod-local. Native now defaults to one replica (server.replicaCount empty takes the runtime default, still 2 for PHP), and its PDB renders only when more than one pod can run. - The default grace period is 65 seconds, above the 60-second poll timeout a worker may request. - SQLite is refused in both runtimes until the chart supports it. Tests: runtime-mode tests for each refusal, and kind smoke scenarios for the live hook gate (replicas, client-side render, autoscaling values, a live autoscaler, a live pod, unmarked and PHP-marked predecessors, and the authorized render with the database CA). Refs #946 Co-Authored-By: Claude Opus 5.5 --- docs/helm-validation.md | 9 +- k8s/helm/durable-workflow/README.md | 127 ++++++++++++++---- .../ci/native-ingress-and-hpa-values.yaml | 6 +- k8s/helm/durable-workflow/docs/UPGRADING.md | 31 +++-- k8s/helm/durable-workflow/templates/NOTES.txt | 4 +- .../durable-workflow/templates/_helpers.tpl | 75 ++++++++--- .../templates/native-schema-upgrade-job.yaml | 12 +- .../templates/native-server-deployment.yaml | 14 +- .../templates/server-deployment.yaml | 2 +- .../templates/server-pdb.yaml | 6 +- k8s/helm/durable-workflow/values.schema.json | 15 ++- k8s/helm/durable-workflow/values.yaml | 45 +++++-- scripts/ci/test_database_tls_chart.py | 8 +- scripts/ci/test_helm_runtime_modes.py | 80 ++++++++--- scripts/helm-chart-native-kind-smoke.sh | 92 +++++++++++-- 15 files changed, 412 insertions(+), 114 deletions(-) diff --git a/docs/helm-validation.md b/docs/helm-validation.md index dd028e775..5d72a1526 100644 --- a/docs/helm-validation.md +++ b/docs/helm-validation.md @@ -165,8 +165,11 @@ touches `k8s/helm/**` or the chart smoke script. The harness has two jobs: `helm test`; - asserts that no worker, scheduler, migration Job or Redis Secret exists; - upgrades with an empty `runtime` and confirms the release stays native; - - confirms that the schema-upgrade hook refuses while a Server pod runs, - then scales to zero and runs the hook, which verifies the current schema; + - confirms that the schema-upgrade hook refuses unless live state proves + every Server stopped (replicas above zero, a client-side render, + autoscaling in the values, a live autoscaler, a live server pod, an + unmarked or PHP-marked predecessor), then renders and runs it, and the + hook verifies the current schema; - confirms that an upgrade of a release whose server Deployment lacks the native runtime marker refuses an empty `runtime`; - runs `helm uninstall` and asserts no chart-owned resources remain. @@ -180,7 +183,7 @@ The CI fixtures intentionally cover six shapes: | `ci/ingress-and-hpa-values.yaml` | Optional templates: Ingress, HPA, NetworkPolicy, server + worker PDB. | | `ci/database-tls-values.yaml` | External CA Secret, canonical TLS settings and read-only mounts. Renders native by default. The TLS test also renders it in PHP mode. | | `ci/native-inline-secrets-values.yaml` | Native Server with inline secrets: one Deployment, no PHP roles. | -| `ci/native-ingress-and-hpa-values.yaml` | Native with Ingress, HPA, NetworkPolicy, PDB, a payload claim and the schema-upgrade hook. | +| `ci/native-ingress-and-hpa-values.yaml` | Native with Ingress, HPA, NetworkPolicy, PDB and the shared payload claim that more than one replica needs. | The first three fixtures pin `runtime: php`. A chart change that affects rendered manifests must keep every fixture green, and both kind smokes must diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 686f242d2..887041b21 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -91,6 +91,11 @@ externalDatabase: auth: existingSecret: durable-workflow-app-credentials +native: + storage: + # ReadWriteMany. Required for more than one replica. + existingClaim: durable-workflow-payloads + server: replicaCount: 3 ingress: @@ -129,21 +134,34 @@ Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`. * Native keeps external payload objects on local disk at `/app/storage`. Without `native.storage.existingClaim` the chart mounts an `emptyDir`, and the objects are lost when a pod is replaced. Set a claim before any - namespace enables external payload storage. With more than one replica the - claim must be `ReadWriteMany` so every pod reads the same objects. -* On `SIGTERM` the native Server holds open worker polls for up to - `DW_WORKER_POLL_TIMEOUT` (30 seconds by default) before it exits. - `native.terminationGracePeriodSeconds` (default 45) must stay above it. The - chart refuses a grace period at or below a `DW_WORKER_POLL_TIMEOUT` set in - `config.extraEnv`. + namespace enables external payload storage. +* A native install runs one replica by default. One pod cannot read a payload + object another pod accepted, so the chart refuses `server.replicaCount` + above 1 or `server.autoscaling.enabled` unless `native.storage.existingClaim` + is set, and that claim must be `ReadWriteMany`. The PodDisruptionBudget + renders only when more than one pod can run, because `minAvailable: 1` on a + single pod would block every node drain. +* On `SIGTERM` the native Server holds open worker polls until they end. A + poll waits `DW_WORKER_POLL_TIMEOUT` (30 seconds by default) unless it asks + for up to 60. `native.terminationGracePeriodSeconds` defaults to 65 so a + rollout drains every allowed poll. The chart refuses a grace period at or + below a `DW_WORKER_POLL_TIMEOUT` set in `config.extraEnv`. +* Native liveness probes the listener (`native.livenessProbe`, a TCP check on + the `http` port), not `/api/health`. `/api/health` reports database + liveness, so probing it would restart every pod during a database outage, + while the connection pool recovers on its own. Readiness stays on + `/api/ready` and takes a pod out of service when its database, schema or + maintenance loop fails. `server.livenessProbe` applies to PHP mode only. * Native pods set an ephemeral-storage request and limit from `native.ephemeralStorage` (128Mi and 2Gi) unless `server.resources` names one. The payload `emptyDir` counts toward that limit. They also mount no service account token, because the native Server never calls the Kubernetes API. Set `native.automountServiceAccountToken: true` only for a sidecar that needs one. -* The chart supports MySQL, MariaDB and PostgreSQL. It does not run the - native Server on SQLite. +* The chart supports MySQL, MariaDB and PostgreSQL. It refuses SQLite in + either runtime: native SQLite needs exactly one replica on a ReadWriteOnce + volume at `/app/database`, which this chart does not provision yet. Use + `docker-compose.published.yml` for SQLite. ### PHP 2.x installations @@ -169,11 +187,11 @@ AUTH. See `examples/` for complete PHP values files. | Resource | Runtime | Purpose | Disabled when | | --- | --- | --- | --- | -| `Deployment` (`*-server`) | both | API replicas. Liveness on `/api/health`, readiness on `/api/ready`. In native mode the image's own process runs maintenance, and the Deployment carries the `workflows.durable-workflow.dev/runtime: native` annotation. | `server.enabled: false` | -| `Job` (`*-schema-upgrade`) | native | `pre-upgrade` hook that runs the image's `schema-upgrade` command. Opt-in, see [Native schema upgrades](#native-schema-upgrades). | `native.schemaUpgrade.enabled: false` (default) | +| `Deployment` (`*-server`) | both | API replicas, readiness on `/api/ready`. Liveness is a TCP check in native mode and `/api/health` in PHP mode. In native mode the image's own process runs maintenance, and the Deployment carries the `workflows.durable-workflow.dev/runtime: native` annotation. | `server.enabled: false` | +| `Job` (`*-schema-upgrade`) | native | `pre-upgrade` hook that runs the image's `schema-upgrade` command. Opt-in, and rendered only during `helm upgrade` once live state proves every Server is stopped. See [Native schema upgrades](#native-schema-upgrades). | `native.schemaUpgrade.enabled: false` (default) | | `Job` (`*-migrate`) | php | Runs `server-bootstrap` (migrations, namespace bootstrap) before workloads accept traffic. Helm hook by default; sync-wave annotation for Argo CD. | `bootstrap.enabled: false` | | `Service` (`*-server`) | both | ClusterIP in front of the API. | `server.enabled: false` | -| `PodDisruptionBudget` (`*-server`) | both | Default `minAvailable: 1`. | `server.pdb.enabled: false` | +| `PodDisruptionBudget` (`*-server`) | both | Default `minAvailable: 1`. In native mode only when more than one pod can run. | `server.pdb.enabled: false` | | `Ingress` (`*-server`) | both | Optional. | `server.ingress.enabled: false` (default) | | `HorizontalPodAutoscaler` (`*-server`) | both | Optional. | `server.autoscaling.enabled: false` (default) | | `Deployment` (`*-worker`) | php | Worker pool. Independent scale axis. | `worker.enabled: false` | @@ -263,7 +281,8 @@ override-with-care in `values.yaml`: decisions on `/api/health` would route traffic to a node whose database had failed over. Keep `/api/ready` out of the liveness probe: in native mode a stalled maintenance tick fails readiness on every pod, and a liveness probe - on it would restart them all. + on it would restart them all. Native liveness is a TCP check for the same + reason: `/api/health` fails while the database is unreachable. * **Bootstrap job runs before workloads (PHP mode).** Helm hooks default to `pre-install,pre-upgrade`. For Argo CD/Flux users who don't honour Helm hooks, sync-wave / depends-on annotations enforce the same ordering. @@ -283,18 +302,23 @@ override-with-care in `values.yaml`: Patch releases within a native minor line keep the schema, so changing `image.tag` is enough. A release whose notes announce a schema change needs -`schema-upgrade`, which requires every native Server stopped and a database -backup first: +the image's `schema-upgrade` command, which requires every native Server +stopped and a database backup first. Use the in-cluster hook with +`helm upgrade`, or the manual path with GitOps or any other client that +renders without the cluster. + +### With helm upgrade -1. Scale the Server to zero and wait for its pods to exit: +1. Scale the Server to zero with autoscaling off, and wait for its pods to + exit: ```bash helm upgrade durable-workflow -f my-values.yaml \ --set server.replicaCount=0 --set server.autoscaling.enabled=false --wait ``` 2. Back up the database with your usual procedure. -3. Upgrade to the new image with the hook enabled and your usual replica - count: +3. Upgrade to the new image with the hook enabled, your usual replica count + and autoscaling still off: ```bash helm upgrade durable-workflow -f my-values.yaml \ @@ -304,14 +328,65 @@ backup first: The `pre-upgrade` hook runs `schema-upgrade` before the new pods start. On a database that is already current it only verifies the schema. -4. Leave `native.schemaUpgrade.enabled` false for later upgrades. - -The chart refuses to render the hook while the live server Deployment has -replicas or a server pod is still running, because `schema-upgrade` must not -run under a live Server. That check reads the cluster through Helm's `lookup`, -so it does not run under `helm template`, a client-side dry run, or a GitOps -controller that renders with `helm template`. Scale to zero yourself in those -setups. Restore the backup to go back: there is no schema downgrade. +4. Turn `native.schemaUpgrade.enabled` off again, and autoscaling back on if + you use it, in your next upgrade. + +The chart renders the hook only during `helm upgrade` against the cluster, +and only when Helm's `lookup` proves every Server is stopped: the release's +server Deployment is visible and carries the native runtime annotation, its +`replicas` is exactly 0, `server.autoscaling.enabled` is off and no +HorizontalPodAutoscaler targets it, and no server pod of the release is still +running. Otherwise it refuses. An install, `helm template`, a client-side dry +run or a GitOps render with the option enabled is always refused, because a +Job applied without Helm's hook ordering could run beside a live Server. + +### Manual path (GitOps and helm template) + +1. Set `server.replicaCount: 0` with autoscaling off, sync, and wait until no + server pod of the release remains. +2. Back up the database. +3. Run `schema-upgrade` once with the new image, the release's database + settings and its database Secret, then wait for it to complete: + + ```yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: durable-workflow-schema-upgrade + spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, runAsUser: 1000} + containers: + - name: schema-upgrade + image: docker.io/durableworkflow/server: + args: ["schema-upgrade"] + envFrom: + - configMapRef: {name: durable-workflow-config} + env: + - name: DB_USERNAME + valueFrom: {secretKeyRef: {name: durable-workflow-database, key: DB_USERNAME}} + - name: DB_PASSWORD + valueFrom: {secretKeyRef: {name: durable-workflow-database, key: DB_PASSWORD}} + ``` + + ```bash + kubectl -n durable-workflow apply -f schema-upgrade-job.yaml + kubectl -n durable-workflow wait --for=condition=complete \ + job/durable-workflow-schema-upgrade --timeout=30m + kubectl -n durable-workflow logs job/durable-workflow-schema-upgrade + ``` + + Use your `externalDatabase.existingSecret` and its keys if you set one, + and mount the database CA Secret at + `/etc/durable-workflow/database-tls/ca.pem` when you use database TLS. Keep + this Job out of the GitOps application so it never runs again on its own. +4. Set `image.tag` to the new release and restore `server.replicaCount`. + +Restore the backup to go back: there is no schema downgrade. ## GitOps notes diff --git a/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml index 754c1eaa3..0c3d0ca3d 100644 --- a/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/native-ingress-and-hpa-values.yaml @@ -1,5 +1,7 @@ # CI fixture: the native Server with PostgreSQL, existing Secrets, ingress, -# autoscaling, a shared payload claim and the opt-in schema-upgrade hook. +# autoscaling and the shared payload claim that more than one replica needs. +# The schema-upgrade hook renders only during a live helm upgrade, so the kind +# smoke covers it instead of this fixture. runtime: native externalDatabase: @@ -15,8 +17,6 @@ auth: native: storage: existingClaim: durable-workflow-payloads - schemaUpgrade: - enabled: true server: replicaCount: 3 diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 257fb1742..13c54c34f 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -170,9 +170,12 @@ not set `runtime: native` before the takeover completes. ```bash helm upgrade durable-workflow -f my-values.yaml \ --set runtime=native --set image.tag=3.0.1 \ - --set server.replicaCount=2 --wait + --set server.replicaCount=1 --wait ``` + More than one native replica needs `native.storage.existingClaim` on a + ReadWriteMany volume. + Helm removes the worker Deployment, scheduler CronJob, migration Job hook and Redis Secret and updates the server Deployment in place. PHP-only values such as `externalRedis`, `worker`, `scheduler` and `bootstrap` are @@ -182,14 +185,24 @@ not set `runtime: native` before the takeover completes. cannot see the live release, so an empty `runtime` always renders native for them. Set `runtime` explicitly in GitOps values before you move to chart 1.x. -**Native settings.** `native.terminationGracePeriodSeconds` (default 45) -stays above the native worker poll timeout so pods drain open polls on -shutdown. `native.storage.existingClaim` keeps external payload objects -across pod restarts. `native.schemaUpgrade.enabled` runs the image's -`schema-upgrade` command as an opt-in `pre-upgrade` hook, see -[Native schema upgrades](../README.md#native-schema-upgrades). Native ignores -`config.auth.backwardCompatible` and the metrics and application name -settings, and accepts only the token auth driver. +**Native settings.** +- A native install runs one replica by default. More than one replica, or + autoscaling, needs `native.storage.existingClaim` on a ReadWriteMany + volume, because payload objects are pod-local. The chart refuses it without + one. +- `native.terminationGracePeriodSeconds` (default 65) outlasts the longest + worker poll a client may request (60 seconds), so pods drain open polls on + shutdown. +- Liveness is a TCP check on the `http` port (`native.livenessProbe`), because + `/api/health` fails during a database outage. Readiness stays on + `/api/ready`. +- `native.schemaUpgrade.enabled` runs the image's `schema-upgrade` command as + an opt-in `pre-upgrade` hook during `helm upgrade` only, once live state + proves every Server is stopped. GitOps and `helm template` users follow the + manual path. See [Native schema upgrades](../README.md#native-schema-upgrades). +- The chart refuses SQLite in both runtimes. +- Native ignores `config.auth.backwardCompatible` and the metrics and + application name settings, and accepts only the token auth driver. ### Chart 0.3.9 / Server 2.11.8 diff --git a/k8s/helm/durable-workflow/templates/NOTES.txt b/k8s/helm/durable-workflow/templates/NOTES.txt index 0f847c866..7c992caf7 100644 --- a/k8s/helm/durable-workflow/templates/NOTES.txt +++ b/k8s/helm/durable-workflow/templates/NOTES.txt @@ -12,7 +12,7 @@ scheduler CronJob, migration Job or Redis. The API Deployment "{{ include "durable-workflow.serverDeploymentName" . }}" {{- if .Values.server.autoscaling.enabled }} scales between {{ .Values.server.autoscaling.minReplicas }} and {{ .Values.server.autoscaling.maxReplicas }} replicas -{{- else }} runs {{ .Values.server.replicaCount }} replicas +{{- else }} runs {{ include "durable-workflow.serverReplicas" . }} replicas {{- end }} behind the Service "{{ include "durable-workflow.serverDeploymentName" . }}" on port {{ .Values.server.service.port }}. {{- end }} @@ -42,7 +42,7 @@ What just happened {{- end }} {{- if .Values.server.enabled }} 2. The API Deployment "{{ include "durable-workflow.serverDeploymentName" . }}" - came up with {{ .Values.server.replicaCount }} replicas behind the + came up with {{ include "durable-workflow.serverReplicas" . }} replicas behind the ClusterIP Service "{{ include "durable-workflow.serverDeploymentName" . }}" on port {{ .Values.server.service.port }}. {{- end }} diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 5968023ab..cbc61da38 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -456,6 +456,9 @@ The chart deliberately refuses to render if these are missing — bundling a default in-cluster database would silently break the externals-first contract. */}} {{- define "durable-workflow.validateExternalPersistence" -}} +{{- if and (include "durable-workflow.isPhp" .) (eq (toString .Values.externalDatabase.connection) "sqlite") -}} +{{- fail "PHP mode does not run on SQLite. Use MySQL, MariaDB or PostgreSQL." -}} +{{- end -}} {{- if not .Values.externalDatabase.host -}} {{- fail "externalDatabase.host is required. The chart does not bundle a database; point this at your managed MySQL/PostgreSQL endpoint. See k8s/helm/durable-workflow/README.md." -}} {{- end -}} @@ -718,7 +721,15 @@ behavior it mirrors. {{- fail "The server-storage volume and the /app/storage mount path are reserved for native payload storage. Use native.storage.existingClaim." -}} {{- end -}} {{- end -}} -{{- $grace := int (default 45 (get $native "terminationGracePeriodSeconds")) -}} +{{- $connection := toString .Values.externalDatabase.connection -}} +{{- $extra := default (dict) .Values.config.extraEnv -}} +{{- if or (eq $connection "sqlite") (eq (toString (get $extra "DB_CONNECTION")) "sqlite") (hasPrefix "sqlite" (toString (get $extra "DB_URL"))) -}} +{{- fail "The chart does not run the native Server on SQLite yet: that needs exactly one replica on a ReadWriteOnce volume at /app/database, which this chart does not provision. Use MySQL, MariaDB or PostgreSQL, or run SQLite with docker-compose.published.yml." -}} +{{- end -}} +{{- if and (not (get (default (dict) (get $native "storage")) "existingClaim")) (or .Values.server.autoscaling.enabled (gt (int (include "durable-workflow.serverReplicas" .)) 1)) -}} +{{- fail "More than one native Server replica (server.replicaCount above 1 or server.autoscaling.enabled) needs native.storage.existingClaim: native keeps external payload objects on pod-local disk, so one pod cannot read an upload another pod accepted. Set a ReadWriteMany claim, or run one replica." -}} +{{- end -}} +{{- $grace := int (default 65 (get $native "terminationGracePeriodSeconds")) -}} {{- $poll := get (default (dict) .Values.config.extraEnv) "DW_WORKER_POLL_TIMEOUT" -}} {{- if and $poll (le $grace (int $poll)) -}} {{- fail (printf "native.terminationGracePeriodSeconds (%d) must exceed DW_WORKER_POLL_TIMEOUT (%v): the native Server holds open worker polls for up to that long while it drains on SIGTERM." $grace $poll) -}} @@ -726,23 +737,42 @@ behavior it mirrors. {{- end -}} {{/* -Refuse native.schemaUpgrade while any native Server can still write. The -schema-upgrade command requires every native node stopped and a database -backup taken first (rust/README.md, Native schema upgrades). The check reads -the live Deployment and pods, as the memo transition guard does, and never -treats desired state as proof that an old pod has stopped. +Authorize the native schema-upgrade hook. schema-upgrade mutates the database +and needs every native Server stopped and a backup taken first +(rust/README.md, Native schema upgrades). The hook therefore renders only in a +real helm upgrade, and only when live cluster state proves the stop: the +release's server Deployment is visible, carries the native runtime marker, +has exactly zero replicas, has no HorizontalPodAutoscaler that could scale it +back up, and no server pod of the release is still running. Anything lookup +cannot prove, such as an install, helm template, a GitOps render or a +client-side dry run, is refused rather than emitting an unchecked Job. */}} -{{- define "durable-workflow.validateNativeSchemaUpgradeQuiescence" -}} -{{- if .Release.IsUpgrade -}} +{{- define "durable-workflow.validateNativeSchemaUpgradeAuthority" -}} +{{- $manual := "Use the manual schema upgrade in the chart README (Native schema upgrades) instead: scale the Server to zero, back up the database, run schema-upgrade once as your own Job, then start the Server." -}} +{{- if not .Release.IsUpgrade -}} +{{- fail (printf "native.schemaUpgrade.enabled renders the schema-upgrade hook only during helm upgrade of a live native release, where the chart can check that every Server is stopped. This render is an install, helm template or GitOps render. %s" $manual) -}} +{{- end -}} {{- $namespace := .Release.Namespace -}} -{{- $server := lookup "apps/v1" "Deployment" $namespace (include "durable-workflow.serverDeploymentName" .) -}} -{{- if $server -}} -{{- $replicas := 1 -}} -{{- if hasKey $server.spec "replicas" -}} -{{- $replicas = int (get $server.spec "replicas") -}} +{{- $name := include "durable-workflow.serverDeploymentName" . -}} +{{- $server := lookup "apps/v1" "Deployment" $namespace $name -}} +{{- if not $server -}} +{{- fail (printf "native.schemaUpgrade.enabled needs to see the live Deployment %s to prove every Server is stopped, but it is not visible (for example under a client-side dry run). Run helm upgrade against the cluster, or %s" $name $manual) -}} {{- end -}} -{{- if gt $replicas 0 -}} -{{- fail (printf "native.schemaUpgrade.enabled runs schema-upgrade, which needs every native Server stopped and a database backup first, but Deployment %s has %d replicas. Scale it to zero (server.replicaCount=0 with server.autoscaling.enabled=false), wait for its pods to exit, back up the database, then upgrade again with native.schemaUpgrade.enabled=true. Leave native.schemaUpgrade.enabled=false for upgrades that keep the schema." $server.metadata.name $replicas) -}} +{{- $marker := default "" (get (default (dict) $server.metadata.annotations) "workflows.durable-workflow.dev/runtime") -}} +{{- if ne $marker "native" -}} +{{- fail (printf "native.schemaUpgrade.enabled upgrades a native schema, but Deployment %s does not carry the native runtime marker, so this release does not run native. A PHP database moves to native through the takeover runbook, never schema-upgrade." $name) -}} +{{- end -}} +{{- if or (not (hasKey $server.spec "replicas")) (ne (int (get $server.spec "replicas")) 0) -}} +{{- fail (printf "native.schemaUpgrade.enabled needs every native Server stopped and a database backup first, but Deployment %s has %v replicas. Scale it to zero (server.replicaCount=0 with server.autoscaling.enabled=false), wait for its pods to exit, back up the database, then upgrade with native.schemaUpgrade.enabled=true." $name (default "unset" (get $server.spec "replicas"))) -}} +{{- end -}} +{{- if .Values.server.autoscaling.enabled -}} +{{- fail "native.schemaUpgrade.enabled cannot run with server.autoscaling.enabled: the autoscaler could start a Server during the upgrade. Disable autoscaling for the schema upgrade and enable it again afterwards." -}} +{{- end -}} +{{- $autoscalers := lookup "autoscaling/v2" "HorizontalPodAutoscaler" $namespace "" -}} +{{- range $hpa := default (list) $autoscalers.items -}} +{{- $target := default (dict) $hpa.spec.scaleTargetRef -}} +{{- if and (eq (default "" $target.kind) "Deployment") (eq (default "" $target.name) $name) -}} +{{- fail (printf "native.schemaUpgrade.enabled needs every native Server stopped, but HorizontalPodAutoscaler %s targets Deployment %s and could start one. Remove it (server.autoscaling.enabled=false) before the schema upgrade." $hpa.metadata.name $name) -}} {{- end -}} {{- end -}} {{- $pods := lookup "v1" "Pod" $namespace "" -}} @@ -754,7 +784,6 @@ treats desired state as proof that an old pod has stopped. {{- end -}} {{- end -}} {{- end -}} -{{- end -}} {{/* Database settings for the native schema-upgrade hook. They are inlined, as @@ -821,3 +850,17 @@ ephemeral storage, so the limit also bounds it. {{- end -}} {{- toYaml $resources -}} {{- end -}} + +{{/* +The server replica count. Empty takes the runtime's default: one native pod, +because more than one needs a shared payload claim, or two PHP pods as in +chart 0.x. An explicit value, including 0, is used as given. +*/}} +{{- define "durable-workflow.serverReplicas" -}} +{{- $replicas := .Values.server.replicaCount -}} +{{- if or (kindIs "invalid" $replicas) (eq (toString $replicas) "") -}} +{{- if include "durable-workflow.isNative" . -}}1{{- else -}}2{{- end -}} +{{- else -}} +{{- int $replicas -}} +{{- end -}} +{{- end -}} diff --git a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml index 9f9efe833..1cad6a32c 100644 --- a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml +++ b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml @@ -3,14 +3,17 @@ Explicit native schema upgrade. A native Server never upgrades its schema at startup: on an older schema it refuses with native_schema_upgrade_required. The image's schema-upgrade command moves a database to the image's schema, and needs every native Server stopped and a database backup taken first. -This pre-upgrade hook runs it only when native.schemaUpgrade.enabled is set, -and the chart refuses to render it while a native Server pod can still run. -It never runs on install, where the Server bootstraps an empty database. +This pre-upgrade hook renders only during helm upgrade with +native.schemaUpgrade.enabled, and only when live cluster state proves every +native Server is stopped (validateNativeSchemaUpgradeAuthority). Installs, +helm template and GitOps renders are refused instead of emitting an unchecked +Job; they use the manual path in the chart README. */ -}} {{- $native := default (dict) .Values.native }} {{- $upgrade := default (dict) (get $native "schemaUpgrade") }} {{- if and (include "durable-workflow.isNative" .) (get $upgrade "enabled") }} -{{- include "durable-workflow.validateNativeSchemaUpgradeQuiescence" . }} +{{- include "durable-workflow.validateNativeSchemaUpgradeAuthority" . }} +{{- if .Release.IsUpgrade }} apiVersion: batch/v1 kind: Job metadata: @@ -83,3 +86,4 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- end }} +{{- end }} diff --git a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml index 69711eca6..bd69ffc82 100644 --- a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml @@ -26,7 +26,7 @@ metadata: {{- end }} spec: {{- if not .Values.server.autoscaling.enabled }} - replicas: {{ .Values.server.replicaCount }} + replicas: {{ include "durable-workflow.serverReplicas" . }} {{- end }} revisionHistoryLimit: {{ .Values.server.revisionHistoryLimit }} progressDeadlineSeconds: {{ .Values.server.progressDeadlineSeconds }} @@ -66,9 +66,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- /* On SIGTERM native stops accepting work and holds open worker - polls for up to DW_WORKER_POLL_TIMEOUT (30 seconds by default), so the - grace period must outlast it. */}} - terminationGracePeriodSeconds: {{ default 45 (get $native "terminationGracePeriodSeconds") }} + polls until they end. A poll may ask for up to 60 seconds, so the grace + period must outlast that. */}} + terminationGracePeriodSeconds: {{ default 65 (get $native "terminationGracePeriodSeconds") }} containers: - name: server image: {{ include "durable-workflow.image" . | quote }} @@ -84,7 +84,11 @@ spec: {{- with .Values.server.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.server.livenessProbe }} + {{- /* /api/health reports database liveness, so a liveness probe on + it would restart every pod during a database outage. Native mode + probes the listener instead and leaves database checks to + readiness. */}} + {{- with (get $native "livenessProbe") }} livenessProbe: {{- toYaml . | nindent 12 }} {{- end }} diff --git a/k8s/helm/durable-workflow/templates/server-deployment.yaml b/k8s/helm/durable-workflow/templates/server-deployment.yaml index 7cbeca02e..026909c2f 100644 --- a/k8s/helm/durable-workflow/templates/server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/server-deployment.yaml @@ -16,7 +16,7 @@ metadata: {{- end }} spec: {{- if not .Values.server.autoscaling.enabled }} - replicas: {{ .Values.server.replicaCount }} + replicas: {{ include "durable-workflow.serverReplicas" . }} {{- end }} revisionHistoryLimit: {{ .Values.server.revisionHistoryLimit }} progressDeadlineSeconds: {{ .Values.server.progressDeadlineSeconds }} diff --git a/k8s/helm/durable-workflow/templates/server-pdb.yaml b/k8s/helm/durable-workflow/templates/server-pdb.yaml index 23e1b526f..1c1769280 100644 --- a/k8s/helm/durable-workflow/templates/server-pdb.yaml +++ b/k8s/helm/durable-workflow/templates/server-pdb.yaml @@ -1,4 +1,8 @@ -{{- if and .Values.server.enabled .Values.server.pdb.enabled }} +{{- /* In native mode the budget renders only when more than one pod can +run: minAvailable 1 on a single replica would block every voluntary eviction, +including node drains. */}} +{{- $multiple := or .Values.server.autoscaling.enabled (gt (int (include "durable-workflow.serverReplicas" .)) 1) }} +{{- if and .Values.server.enabled .Values.server.pdb.enabled (or (include "durable-workflow.isPhp" .) $multiple) }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/k8s/helm/durable-workflow/values.schema.json b/k8s/helm/durable-workflow/values.schema.json index d97e4fb96..bc4797e8a 100644 --- a/k8s/helm/durable-workflow/values.schema.json +++ b/k8s/helm/durable-workflow/values.schema.json @@ -14,6 +14,7 @@ "type": "object", "properties": { "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, + "livenessProbe": { "type": "object" }, "automountServiceAccountToken": { "type": "boolean" }, "ephemeralStorage": { "type": "object", @@ -59,7 +60,7 @@ "type": "object", "required": ["connection", "database"], "properties": { - "connection": { "type": "string", "enum": ["mysql", "mariadb", "pgsql"] }, + "connection": { "type": "string", "enum": ["mysql", "mariadb", "pgsql", "sqlite"], "description": "sqlite is listed only so the chart can refuse it with an explanation; neither runtime runs on SQLite in this chart." }, "host": { "type": "string" }, "port": { "type": ["integer", "string"] }, "database": { "type": "string", "minLength": 1 }, @@ -116,6 +117,18 @@ "extra": { "type": "object" } } }, + "server": { + "type": "object", + "properties": { + "replicaCount": { + "anyOf": [ + { "type": "integer", "minimum": 0 }, + { "type": "string", "pattern": "^([0-9]+)?$" }, + { "type": "null" } + ] + } + } + }, "scheduler": { "type": "object", "properties": { diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 610600403..1e26acfd2 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -206,30 +206,51 @@ native: request: 128Mi limit: 2Gi - # On SIGTERM the native Server holds open worker polls for up to - # DW_WORKER_POLL_TIMEOUT (30 seconds unless config.extraEnv changes it). - # Keep this above that timeout so a rollout drains polls instead of cutting - # them off. - terminationGracePeriodSeconds: 45 + # On SIGTERM the native Server holds open worker polls until they end. A + # poll waits DW_WORKER_POLL_TIMEOUT (30 seconds) unless it asks for up to 60, + # so keep this above 60 for a rollout to drain polls instead of cutting them + # off. The chart refuses a value at or below a DW_WORKER_POLL_TIMEOUT set in + # config.extraEnv. + terminationGracePeriodSeconds: 65 + + # Liveness for native pods. /api/health reports database liveness, so + # probing it would restart every pod during a database outage while the + # connection pool would have recovered on its own. Native probes the + # listener instead; readiness (server.readinessProbe on /api/ready) still + # takes a pod out of service when its database, schema or maintenance loop + # fails. server.livenessProbe applies to PHP mode only. + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 3 # Native keeps external payload objects on local disk at /app/storage. # Without a claim the chart mounts an emptyDir, which is lost with the pod. # Set a PersistentVolumeClaim before any namespace enables external payload - # storage. With more than one replica the claim must be ReadWriteMany so - # every pod reads the same objects. + # storage. More than one replica (server.replicaCount above 1 or + # autoscaling) requires a claim, and it must be ReadWriteMany so every pod + # reads the same objects. The chart refuses more than one replica without + # one. storage: existingClaim: "" # Explicit schema upgrade between native releases. A native Server never # upgrades its schema at startup: on an older schema it refuses with # native_schema_upgrade_required and stays unready. Patch releases within a - # minor line keep the schema. When a release notes a schema change: + # minor line keep the schema. When a release notes a schema change, with + # helm upgrade against the cluster: # 1. scale the Server to zero (server.replicaCount=0, autoscaling off), # 2. back up the database, # 3. upgrade to the new image with schemaUpgrade.enabled=true and your - # usual replica count, + # usual replica count (autoscaling still off), # 4. set schemaUpgrade.enabled=false again for later upgrades. - # The chart refuses to render this hook while a Server pod still runs. + # The hook renders only in helm upgrade, and only when the live native + # Deployment has zero replicas, no autoscaler and no running pod. An + # install, helm template or GitOps render with this enabled is refused; use + # the manual path in the chart README instead. schemaUpgrade: enabled: false backoffLimit: 0 @@ -280,7 +301,9 @@ bootstrap: server: enabled: true - replicaCount: 2 + # Empty takes the runtime default: 1 native pod (more than one needs + # native.storage.existingClaim) or 2 PHP pods. 0 stops the Server. + replicaCount: "" # Strategy controls. Default RollingUpdate with maxUnavailable=0 enforces # the rolling-upgrade contract (https://durable-workflow.github.io/docs/2.0/rolling-upgrades). diff --git a/scripts/ci/test_database_tls_chart.py b/scripts/ci/test_database_tls_chart.py index e7fac25cc..66dcb5a07 100644 --- a/scripts/ci/test_database_tls_chart.py +++ b/scripts/ci/test_database_tls_chart.py @@ -60,8 +60,10 @@ def test_ca_reaches_all_four_roles_in_hook_and_regular_bootstrap(self): self.assertEqual(environment["DB_SSLROOTCERT"], CA_PATH) self.assertEqual(roles, {"server", "worker", "migrate", "scheduler"}) - def test_ca_reaches_the_native_server_and_schema_upgrade_hook(self): - resources = render("--set", "native.schemaUpgrade.enabled=true") + def test_ca_reaches_the_native_server(self): + # The schema-upgrade hook renders only against a live release; the + # native kind smoke checks its CA wiring. + resources = render() config = next(item for item in resources if item["kind"] == "ConfigMap")["data"] self.assertEqual(config["DB_SSLMODE"], "verify-full") self.assertEqual(config["DB_SSLROOTCERT"], CA_PATH) @@ -80,7 +82,7 @@ def test_ca_reaches_the_native_server_and_schema_upgrade_hook(self): environment = {item["name"]: item.get("value") for item in container["env"]} self.assertEqual(environment["DB_SSLMODE"], "verify-full") self.assertEqual(environment["DB_SSLROOTCERT"], CA_PATH) - self.assertEqual(roles, {"server", "schema-upgrade"}) + self.assertEqual(roles, {"server"}) def test_existing_volumes_and_mounts_survive_with_or_without_ca(self): for ca in ["database-ca", ""]: diff --git a/scripts/ci/test_helm_runtime_modes.py b/scripts/ci/test_helm_runtime_modes.py index 1ac16a76d..fd54e54fd 100644 --- a/scripts/ci/test_helm_runtime_modes.py +++ b/scripts/ci/test_helm_runtime_modes.py @@ -57,7 +57,10 @@ def test_default_install_renders_one_native_server_and_nothing_php(self): workloads = [item for item in resources if item["kind"] in {"Deployment", "Job", "CronJob"}] self.assertEqual([item["metadata"]["name"] for item in workloads], ["runtime-check-durable-workflow-server"]) self.assertFalse({component(item) for item in resources} & PHP_ONLY_COMPONENTS) - self.assertNotIn("HorizontalPodAutoscaler", {item["kind"] for item in resources}) + # One pod by default: more than one needs a shared payload claim. A + # budget of minAvailable 1 on one pod would block every drain. + self.assertEqual(workloads[0]["spec"]["replicas"], 1) + self.assertFalse({"HorizontalPodAutoscaler", "PodDisruptionBudget"} & {item["kind"] for item in resources}) server = workloads[0] self.assertEqual(server["metadata"]["annotations"]["workflows.durable-workflow.dev/runtime"], "native") @@ -67,9 +70,13 @@ def test_default_install_renders_one_native_server_and_nothing_php(self): self.assertNotIn("command", container) self.assertNotIn("args", container) self.assertEqual(container["ports"], [{"name": "http", "containerPort": 8080, "protocol": "TCP"}]) - self.assertEqual(container["livenessProbe"]["httpGet"]["path"], "/api/health") + # /api/health fails during a database outage, so liveness probes the + # listener and only readiness checks the database. + self.assertEqual(container["livenessProbe"]["tcpSocket"], {"port": "http"}) + self.assertNotIn("httpGet", container["livenessProbe"]) self.assertEqual(container["readinessProbe"]["httpGet"]["path"], "/api/ready") - self.assertGreater(pod["terminationGracePeriodSeconds"], 30) + # A worker poll may ask to be held for up to 60 seconds. + self.assertEqual(pod["terminationGracePeriodSeconds"], 65) self.assertEqual(pod["securityContext"]["runAsUser"], 1000) self.assertEqual(pod["securityContext"]["fsGroup"], 1000) self.assertIn({"name": "server-storage", "mountPath": "/app/storage"}, container["volumeMounts"]) @@ -96,24 +103,28 @@ def test_server_options_shared_with_php_mode_apply(self): self.assertLessEqual({"Ingress", "HorizontalPodAutoscaler", "PodDisruptionBudget", "NetworkPolicy", "Service"}, kinds) server = named(resources, "Deployment", "-server") self.assertNotIn("replicas", server["spec"]) + self.assertNotIn("Job", {item["kind"] for item in resources}) volume = next(item for item in server["spec"]["template"]["spec"]["volumes"] if item["name"] == "server-storage") self.assertEqual(volume, {"name": "server-storage", "persistentVolumeClaim": {"claimName": "durable-workflow-payloads"}}) - def test_schema_upgrade_is_an_explicit_pre_upgrade_hook(self): + def test_schema_upgrade_hook_is_never_rendered_unchecked(self): + """The hook needs live proof that every Server is stopped. + + helm template has no cluster, so an install render and an upgrade + render whose predecessor cannot be seen must both refuse. The live + refusals and the positive render are in + scripts/helm-chart-native-kind-smoke.sh. + """ self.assertNotIn("Job", {item["kind"] for item in render()}) - job = named(render("--set", "native.schemaUpgrade.enabled=true"), "Job", "-schema-upgrade") - self.assertEqual(job["metadata"]["annotations"]["helm.sh/hook"], "pre-upgrade") - container = job["spec"]["template"]["spec"]["containers"][0] - self.assertEqual(container["args"], ["schema-upgrade"]) - self.assertNotIn("command", container) - environment = {item["name"]: item for item in container["env"]} - self.assertEqual(environment["DB_CONNECTION"]["value"], "mysql") - self.assertEqual(environment["DB_HOST"]["value"], "db.example.internal") - self.assertIn("DB_PASSWORD", environment) - self.assertNotIn("DW_AUTH_TOKEN", environment) - self.assertIs(job["spec"]["template"]["spec"]["automountServiceAccountToken"], False) - self.assertIn("ephemeral-storage", container["resources"]["limits"]) + install = helm("--set", "native.schemaUpgrade.enabled=true") + self.assertNotEqual(install.returncode, 0, install.stdout[:200]) + self.assertIn("renders the schema-upgrade hook only during helm upgrade", install.stderr) + self.assertIn("manual schema upgrade in the chart README", install.stderr) + + invisible = helm("--is-upgrade", "--set", "native.schemaUpgrade.enabled=true") + self.assertNotEqual(invisible.returncode, 0, invisible.stdout[:200]) + self.assertIn("needs to see the live Deployment runtime-check-durable-workflow-server", invisible.stderr) def test_explicit_ephemeral_storage_and_token_settings_win(self): server = named(render( @@ -130,7 +141,17 @@ def test_render_time_refusals_match_native_startup_refusals(self): "auth.principalTokens is not supported": ["--set-string", "auth.principalTokens=[]"], 'config.auth.driver "none" is not supported': ["--set-string", "config.auth.driver=none"], "must exceed DW_WORKER_POLL_TIMEOUT": [ - "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=60", + "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=70", + ], + "More than one native Server replica": ["--set", "server.replicaCount=2"], + "native.storage.existingClaim: native keeps external payload objects": [ + "--set", "server.autoscaling.enabled=true", + ], + "does not run the native Server on SQLite": [ + "--set-string", "externalDatabase.connection=sqlite", + ], + "does not run the native Server on SQLite yet": [ + "--set-string", "config.extraEnv.DB_URL=sqlite:///app/database/database.sqlite", ], "reserved for native payload storage": [ "--set-json", 'server.extraVolumes=[{"name":"server-storage","emptyDir":{}}]', @@ -143,12 +164,22 @@ def test_render_time_refusals_match_native_startup_refusals(self): self.assertIn(message, result.stderr) accepted = render( - "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=60", + "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=70", "--set", "native.terminationGracePeriodSeconds=75", ) server = named(accepted, "Deployment", "-server") self.assertEqual(server["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 75) + def test_a_shared_payload_claim_allows_more_than_one_replica(self): + resources = render( + "--set", "server.replicaCount=3", + "--set-string", "native.storage.existingClaim=shared-payloads", + ) + self.assertEqual(named(resources, "Deployment", "-server")["spec"]["replicas"], 3) + self.assertIn("PodDisruptionBudget", {item["kind"] for item in resources}) + stopped = render("--set", "server.replicaCount=0") + self.assertEqual(named(stopped, "Deployment", "-server")["spec"]["replicas"], 0) + def test_existing_secret_satisfies_the_token_requirement(self): render("--set-string", "auth.authToken=", "--set-string", "auth.existingSecret=app-credentials") @@ -173,6 +204,19 @@ def test_php_app_version_never_reports_the_native_chart_version(self): ) self.assertNotIn("APP_VERSION", named(resources, "ConfigMap", "-config")["data"]) + def test_php_default_replicas_and_probes_are_unchanged(self): + resources = render(values=PHP) + server = named(resources, "Deployment", "-server") + self.assertEqual(server["spec"]["replicas"], 2) + self.assertEqual(server["spec"]["template"]["spec"]["containers"][0]["livenessProbe"]["httpGet"]["path"], "/api/health") + self.assertNotIn("terminationGracePeriodSeconds", server["spec"]["template"]["spec"]) + self.assertIn("PodDisruptionBudget", {item["kind"] for item in resources}) + + def test_php_mode_refuses_sqlite(self): + result = helm("--set-string", "externalDatabase.connection=sqlite", values=PHP) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PHP mode does not run on SQLite", result.stderr) + def test_php_mode_ignores_native_settings(self): resources = render("--set", "native.schemaUpgrade.enabled=true", values=PHP) self.assertFalse(any(component(item) == "schema-upgrade" for item in resources)) diff --git a/scripts/helm-chart-native-kind-smoke.sh b/scripts/helm-chart-native-kind-smoke.sh index 501432ce3..f0121d6bd 100755 --- a/scripts/helm-chart-native-kind-smoke.sh +++ b/scripts/helm-chart-native-kind-smoke.sh @@ -6,8 +6,9 @@ # 1. install, wait for the single server Deployment, run helm test and an # authenticated API call; assert that no PHP role or Redis Secret exists # 2. upgrade with an empty runtime and confirm the release stays native -# 3. confirm the schema-upgrade hook refuses while a Server pod runs, scale -# to zero, then run the hook (schema-upgrade verifies a current schema) +# 3. confirm the schema-upgrade hook refuses unless live state proves every +# Server stopped (replicas, client-side render, autoscaling values, a live +# autoscaler, a live pod), then run it (it verifies a current schema) # 4. confirm an upgrade of a release without the native runtime marker, # as chart 0.x left every PHP release, refuses an empty runtime # 5. uninstall cleanly @@ -178,21 +179,82 @@ port_forward_pid="" --wait --timeout 5m echo "PASS allowed: upgrade with an empty runtime stays native" -# 4. The schema-upgrade hook refuses while a Server pod runs, then runs once -# the Server is scaled to zero. -expect_refusal "schema upgrade while the Server runs" "needs every native Server stopped" \ - "${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ - -f "${values}" "${image_options[@]}" --set native.schemaUpgrade.enabled=true --dry-run=server +# 4. The schema-upgrade hook renders only when live state proves every native +# Server is stopped: a visible, native-marked Deployment at zero replicas, +# no autoscaler and no running server pod. Each counterfactual is refused. +schema_upgrade() { + local target="$1" + shift + "${helm_bin}" upgrade "${target}" "${chart_dir}" --namespace "${namespace}" \ + -f "${values}" "${image_options[@]}" --set native.schemaUpgrade.enabled=true "$@" +} +server_deployment="${release}-durable-workflow-server" + +expect_refusal "schema upgrade while the Server runs" "has 1 replicas" \ + schema_upgrade "${release}" --dry-run=server "${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ -f "${values}" "${image_options[@]}" --set server.replicaCount=0 --wait --timeout 5m "${kubectl_bin}" -n "${namespace}" wait --for=delete pod \ -l "app.kubernetes.io/instance=${release},app.kubernetes.io/component=server" --timeout=120s -"${helm_bin}" upgrade "${release}" "${chart_dir}" --namespace "${namespace}" \ - -f "${values}" "${image_options[@]}" --set native.schemaUpgrade.enabled=true --wait --timeout 5m -"${kubectl_bin}" -n "${namespace}" rollout status "deploy/${release}-durable-workflow-server" --timeout=180s -echo "PASS allowed: schema-upgrade hook with every Server stopped" +# An explicit runtime keeps the runtime selection, which also needs the live +# Deployment, from refusing first. +expect_refusal "schema upgrade without a cluster view" "needs to see the live Deployment" \ + schema_upgrade "${release}" --dry-run=client --set runtime=native +expect_refusal "schema upgrade with autoscaling in the values" "cannot run with server.autoscaling.enabled" \ + schema_upgrade "${release}" --dry-run=server --set server.autoscaling.enabled=true \ + --set-string native.storage.existingClaim=unused-claim + +cat >"${work_dir}/stray-autoscaler.yaml" </dev/null +expect_refusal "schema upgrade with a live autoscaler" "HorizontalPodAutoscaler stray-autoscaler" \ + schema_upgrade "${release}" --dry-run=server +"${kubectl_bin}" -n "${namespace}" delete hpa stray-autoscaler --wait >/dev/null + +"${kubectl_bin}" -n "${namespace}" run stray-server --image=registry.k8s.io/pause:3.9 --restart=Never \ + --labels="app.kubernetes.io/instance=${release},app.kubernetes.io/component=server" >/dev/null +expect_refusal "schema upgrade with a live server pod" "server pod stray-server is still" \ + schema_upgrade "${release}" --dry-run=server +"${kubectl_bin}" -n "${namespace}" delete pod stray-server --grace-period=0 --force --wait >/dev/null 2>&1 + +# The positive render: the hook Job runs schema-upgrade with the database CA. +schema_upgrade "${release}" --dry-run=server --set-string externalDatabase.sslMode=verify-full \ + --set-string externalDatabase.tls.existingSecret=smoke-database-ca >"${work_dir}/hook.yaml" +for expected in '"helm.sh/hook": pre-upgrade' 'args: ["schema-upgrade"]' 'automountServiceAccountToken: false' \ + 'name: DB_SSLROOTCERT' 'secretName: "smoke-database-ca"'; do + grep -qF "${expected}" "${work_dir}/hook.yaml" || { + printf 'the authorized schema-upgrade render lacks %s\n' "${expected}" >&2 + exit 1 + } +done +echo "PASS allowed: schema-upgrade hook renders once every Server is stopped" + +schema_upgrade "${release}" --wait --timeout 5m +"${kubectl_bin}" -n "${namespace}" rollout status "deploy/${server_deployment}" --timeout=180s +echo "PASS allowed: schema-upgrade hook ran with every Server stopped" # 5. A release whose server Deployment lacks the native marker, as every PHP # release from chart 0.x does, must choose a runtime explicitly. @@ -232,6 +294,14 @@ expect_refusal "upgrade of a PHP release with an empty runtime" "runs the PHP 2. "${helm_bin}" upgrade "${stub_release}" "${chart_dir}" --namespace "${namespace}" \ -f "${values}" "${image_options[@]}" --set runtime=native --dry-run=server >/dev/null echo "PASS allowed: an explicit runtime renders against the PHP release" +# A stopped PHP predecessor is not a native schema: schema-upgrade must never +# run against it, marked or not. +expect_refusal "schema upgrade of an unmarked predecessor" "does not carry the native runtime marker" \ + schema_upgrade "${stub_release}" --set runtime=native --dry-run=server +"${kubectl_bin}" -n "${namespace}" annotate deploy "${stub_release}-durable-workflow-server" \ + workflows.durable-workflow.dev/runtime=php >/dev/null +expect_refusal "schema upgrade of a PHP-marked predecessor" "does not carry the native runtime marker" \ + schema_upgrade "${stub_release}" --set runtime=native --dry-run=server "${helm_bin}" uninstall "${stub_release}" --namespace "${namespace}" --wait --timeout 2m # 6. Uninstall leaves only the externally managed database behind. From 70acff667f242d5a8fe0650d1ee0091a46fe5556 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:53:12 -0600 Subject: [PATCH 4/4] Helm chart: reserve the chart's authority metadata Security found that user label and annotation maps were emitted after the chart's own markers, so a later duplicate key could replace them: commonAnnotations could give a PHP Deployment the native runtime marker that the next empty-runtime upgrade and the schema-upgrade gate trust, override the schema-upgrade Job's helm.sh/hook so it ran after the new Server started, or replace the memo storage marker the PHP transition guard trusts. The chart now refuses any user map (commonAnnotations, commonLabels, role podAnnotations and podLabels, Service, Ingress and ServiceAccount annotations) that sets a key under workflows.durable-workflow.dev/ or helm.sh/hook, helm.sh/hook-weight or helm.sh/hook-delete-policy. As defense in depth, user maps are emitted first, without reserved keys and without any key the chart sets on that object, and the chart's keys follow, so no rendered map carries a key twice and a chart value always wins. Renders with ordinary user metadata are otherwise unchanged. Tests refuse every reserved key in every user map in both runtimes, reject Security's two examples, and parse renders with a loader that fails on duplicate keys. The native kind smoke refuses a hook override against the live gate and checks the authorized Job keeps user annotations and exactly one of each hook key. Refs #946 Co-Authored-By: Claude Opus 5.5 --- k8s/helm/durable-workflow/README.md | 11 ++ k8s/helm/durable-workflow/docs/UPGRADING.md | 6 + .../durable-workflow/templates/_helpers.tpl | 72 +++++++++++- .../durable-workflow/templates/configmap.yaml | 1 + .../templates/migration-job.yaml | 12 +- .../templates/native-schema-upgrade-job.yaml | 4 +- .../templates/native-server-deployment.yaml | 12 +- .../templates/scheduler-cronjob.yaml | 12 +- .../templates/server-deployment.yaml | 12 +- .../templates/worker-deployment.yaml | 12 +- k8s/helm/durable-workflow/values.yaml | 8 +- scripts/ci/test_helm_runtime_modes.py | 106 ++++++++++++++++++ scripts/helm-chart-native-kind-smoke.sh | 18 ++- 13 files changed, 232 insertions(+), 54 deletions(-) diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 887041b21..175f62bf0 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -268,6 +268,17 @@ override-with-care in `values.yaml`: native only when the live server Deployment already carries the native runtime annotation. A release that chart 0.x installed runs PHP, so its upgrade refuses until `runtime` is set. See [docs/UPGRADING.md](docs/UPGRADING.md). +* **Chart-owned metadata is reserved.** The runtime annotation, the memo + storage annotation and the Helm hook keys decide which runtime an upgrade + selects and when a database-mutating Job runs. The chart refuses any label + or annotation map in the values (`commonAnnotations`, `commonLabels`, the + role `podAnnotations` and `podLabels`, and the Service, Ingress and + ServiceAccount annotations) that sets a key under + `workflows.durable-workflow.dev/` or `helm.sh/hook`, `helm.sh/hook-weight` + or `helm.sh/hook-delete-policy`. Other user keys that the chart also sets on + an object, such as Argo CD sync waves or the standard + `app.kubernetes.io/*` labels, are dropped in favor of the chart's value, so + no rendered map carries a key twice. * **Native never upgrades its schema on start.** A pod on an older schema refuses with `native_schema_upgrade_required` and stays unready. Schema upgrades run only through the explicit hook below. diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 13c54c34f..6ddd3ff51 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -201,6 +201,12 @@ them. Set `runtime` explicitly in GitOps values before you move to chart 1.x. proves every Server is stopped. GitOps and `helm template` users follow the manual path. See [Native schema upgrades](../README.md#native-schema-upgrades). - The chart refuses SQLite in both runtimes. +- The chart refuses label and annotation values that set keys under + `workflows.durable-workflow.dev/` or the `helm.sh/hook` keys, which it sets + itself. Remove any such key from `commonAnnotations`, `commonLabels` or a + role's `podAnnotations`/`podLabels` before upgrading. A user key that + collides with another chart-set key is now dropped in favor of the chart's + value instead of rendering twice. - Native ignores `config.auth.backwardCompatible` and the metrics and application name settings, and accepts only the token auth driver. diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index cbc61da38..25083b262 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -28,15 +28,19 @@ Create a default fully qualified app name. Truncated at 63 chars per DNS-1123. {{/* Common labels applied to every resource. */}} -{{- define "durable-workflow.labels" -}} +{{- define "durable-workflow.chartLabels" -}} helm.sh/chart: {{ include "durable-workflow.chart" . }} {{ include "durable-workflow.selectorLabels" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: durable-workflow -{{- with .Values.commonLabels }} -{{ toYaml . }} +{{- end -}} + +{{- define "durable-workflow.labels" -}} +{{- with include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonLabels) "chart" (list (include "durable-workflow.chartLabels" .) "app.kubernetes.io/component: chart")) }} +{{ . }} {{- end }} +{{ include "durable-workflow.chartLabels" . }} {{- end -}} {{/* @@ -864,3 +868,65 @@ chart 0.x. An explicit value, including 0, is used as given. {{- int $replicas -}} {{- end -}} {{- end -}} + +{{/* +Metadata keys only the chart may set: its authority markers, which the runtime +selection, the memo transition guard and the schema-upgrade gate trust, and +Helm's hook keys, which decide when a database-mutating Job runs. +*/}} +{{- define "durable-workflow.reservedMetadataKey" -}} +{{- if or (hasPrefix "workflows.durable-workflow.dev/" .) (has . (list "helm.sh/hook" "helm.sh/hook-weight" "helm.sh/hook-delete-policy")) -}}true{{- end -}} +{{- end -}} + +{{/* +Refuse any user-supplied label or annotation map that sets a reserved key. +*/}} +{{- define "durable-workflow.validateReservedMetadata" -}} +{{- $maps := dict + "commonAnnotations" .Values.commonAnnotations + "commonLabels" .Values.commonLabels + "server.podAnnotations" .Values.server.podAnnotations + "server.podLabels" .Values.server.podLabels + "server.service.annotations" .Values.server.service.annotations + "server.ingress.annotations" .Values.server.ingress.annotations + "worker.podAnnotations" .Values.worker.podAnnotations + "worker.podLabels" .Values.worker.podLabels + "scheduler.podAnnotations" .Values.scheduler.podAnnotations + "scheduler.podLabels" .Values.scheduler.podLabels + "bootstrap.podAnnotations" .Values.bootstrap.podAnnotations + "bootstrap.podLabels" .Values.bootstrap.podLabels + "serviceAccount.annotations" .Values.serviceAccount.annotations -}} +{{- range $path, $map := $maps -}} +{{- range $key, $_ := default (dict) $map -}} +{{- if include "durable-workflow.reservedMetadataKey" $key -}} +{{- fail (printf "%s sets the reserved key %q. Keys under workflows.durable-workflow.dev/ and the helm.sh/hook keys are set only by the chart, which relies on them for the runtime, memo storage and schema-upgrade checks. Remove it." $path $key) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +User labels or annotations for one object: the given maps merged in order, +without reserved keys and without any key the chart itself sets on that object +("chart" holds the chart's own YAML for it). The caller emits the chart's keys +after this output, so the rendered map has no duplicate key and a user value +can never shadow a chart value, even if validateReservedMetadata were skipped. +*/}} +{{- define "durable-workflow.userMetadata" -}} +{{- $merged := dict -}} +{{- range $map := .maps -}} +{{- $merged = mergeOverwrite $merged (deepCopy (default (dict) $map)) -}} +{{- end -}} +{{- $chart := dict -}} +{{- range $yaml := default (list) .chart -}} +{{- $chart = mergeOverwrite $chart (default (dict) (fromYaml $yaml)) -}} +{{- end -}} +{{- range $key := keys $merged -}} +{{- if or (hasKey $chart $key) (include "durable-workflow.reservedMetadataKey" $key) -}} +{{- $_ := unset $merged $key -}} +{{- end -}} +{{- end -}} +{{- if $merged -}} +{{- toYaml $merged -}} +{{- end -}} +{{- end -}} diff --git a/k8s/helm/durable-workflow/templates/configmap.yaml b/k8s/helm/durable-workflow/templates/configmap.yaml index 93d9e2380..d6117b377 100644 --- a/k8s/helm/durable-workflow/templates/configmap.yaml +++ b/k8s/helm/durable-workflow/templates/configmap.yaml @@ -1,5 +1,6 @@ {{- include "durable-workflow.validateExternalPersistence" . }} {{- include "durable-workflow.validateDatabaseTlsVolumes" . }} +{{- include "durable-workflow.validateReservedMetadata" . }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/k8s/helm/durable-workflow/templates/migration-job.yaml b/k8s/helm/durable-workflow/templates/migration-job.yaml index 0832e0c56..125f64378 100644 --- a/k8s/helm/durable-workflow/templates/migration-job.yaml +++ b/k8s/helm/durable-workflow/templates/migration-job.yaml @@ -7,28 +7,24 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "migration") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations) "chart" (list (include "durable-workflow.argoBootstrapAnnotations" .))) | nindent 4 }} {{- if .Values.bootstrap.useHelmHooks }} "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-weight": "-5" "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded {{- end }} {{- include "durable-workflow.argoBootstrapAnnotations" . | nindent 4 }} - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: backoffLimit: {{ .Values.bootstrap.backoffLimit }} activeDeadlineSeconds: {{ .Values.bootstrap.activeDeadlineSeconds }} template: metadata: labels: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.bootstrap.podLabels) "chart" (list (include "durable-workflow.componentLabels" (dict "context" . "component" "migration")))) | nindent 8 }} {{- include "durable-workflow.componentLabels" (dict "context" . "component" "migration") | nindent 8 }} - {{- with .Values.bootstrap.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.bootstrap.podAnnotations }} + {{- with include "durable-workflow.userMetadata" (dict "maps" (list .Values.bootstrap.podAnnotations)) }} annotations: - {{- toYaml . | nindent 8 }} + {{- . | nindent 8 }} {{- end }} spec: restartPolicy: OnFailure diff --git a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml index 1cad6a32c..859962f63 100644 --- a/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml +++ b/k8s/helm/durable-workflow/templates/native-schema-upgrade-job.yaml @@ -22,12 +22,10 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "schema-upgrade") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations)) | nindent 4 }} "helm.sh/hook": pre-upgrade "helm.sh/hook-weight": "-5" "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: backoffLimit: {{ default 0 (get $upgrade "backoffLimit") }} activeDeadlineSeconds: {{ default 1800 (get $upgrade "activeDeadlineSeconds") }} diff --git a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml index bd69ffc82..4e3066f0a 100644 --- a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml @@ -19,11 +19,9 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "server") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations) "chart" (list (include "durable-workflow.argoWorkloadAnnotations" .))) | nindent 4 }} workflows.durable-workflow.dev/runtime: native {{- include "durable-workflow.argoWorkloadAnnotations" . | nindent 4 }} - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: {{- if not .Values.server.autoscaling.enabled }} replicas: {{ include "durable-workflow.serverReplicas" . }} @@ -41,15 +39,11 @@ spec: template: metadata: labels: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.server.podLabels) "chart" (list (include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server")))) | nindent 8 }} {{- include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server") | nindent 8 }} - {{- with .Values.server.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.server.podAnnotations) "chart" (list (include "durable-workflow.configChecksumAnnotations" .))) | nindent 8 }} {{- include "durable-workflow.configChecksumAnnotations" . | nindent 8 }} - {{- with .Values.server.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} spec: serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} {{- /* The native Server never calls the Kubernetes API. */}} diff --git a/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml b/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml index dcbb1f6d6..83cc29f67 100644 --- a/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml +++ b/k8s/helm/durable-workflow/templates/scheduler-cronjob.yaml @@ -20,12 +20,10 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "scheduler") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations) "chart" (list (include "durable-workflow.argoWorkloadAnnotations" .) (include "durable-workflow.fluxDependsOnAnnotation" .))) | nindent 4 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.argoWorkloadAnnotations" . | nindent 4 }} {{- include "durable-workflow.fluxDependsOnAnnotation" . | nindent 4 }} - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: schedule: {{ .Values.scheduler.schedule | quote }} concurrencyPolicy: {{ .Values.scheduler.concurrencyPolicy }} @@ -44,16 +42,12 @@ spec: template: metadata: labels: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.scheduler.podLabels) "chart" (list (include "durable-workflow.componentLabels" (dict "context" . "component" "scheduler")))) | nindent 12 }} {{- include "durable-workflow.componentLabels" (dict "context" . "component" "scheduler") | nindent 12 }} - {{- with .Values.scheduler.podLabels }} - {{- toYaml . | nindent 12 }} - {{- end }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.scheduler.podAnnotations) "chart" (list (include "durable-workflow.configChecksumAnnotations" .))) | nindent 12 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.configChecksumAnnotations" . | nindent 12 }} - {{- with .Values.scheduler.podAnnotations }} - {{- toYaml . | nindent 12 }} - {{- end }} spec: restartPolicy: Never serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} diff --git a/k8s/helm/durable-workflow/templates/server-deployment.yaml b/k8s/helm/durable-workflow/templates/server-deployment.yaml index 026909c2f..d02269513 100644 --- a/k8s/helm/durable-workflow/templates/server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/server-deployment.yaml @@ -8,12 +8,10 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "server") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations) "chart" (list (include "durable-workflow.argoWorkloadAnnotations" .) (include "durable-workflow.fluxDependsOnAnnotation" .))) | nindent 4 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.argoWorkloadAnnotations" . | nindent 4 }} {{- include "durable-workflow.fluxDependsOnAnnotation" . | nindent 4 }} - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: {{- if not .Values.server.autoscaling.enabled }} replicas: {{ include "durable-workflow.serverReplicas" . }} @@ -31,16 +29,12 @@ spec: template: metadata: labels: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.server.podLabels) "chart" (list (include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server")))) | nindent 8 }} {{- include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "server") | nindent 8 }} - {{- with .Values.server.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.server.podAnnotations) "chart" (list (include "durable-workflow.configChecksumAnnotations" .))) | nindent 8 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.configChecksumAnnotations" . | nindent 8 }} - {{- with .Values.server.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} spec: serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} {{- with .Values.image.pullSecrets }} diff --git a/k8s/helm/durable-workflow/templates/worker-deployment.yaml b/k8s/helm/durable-workflow/templates/worker-deployment.yaml index d723d5eee..afd091698 100644 --- a/k8s/helm/durable-workflow/templates/worker-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/worker-deployment.yaml @@ -8,12 +8,10 @@ metadata: labels: {{- include "durable-workflow.componentLabels" (dict "context" . "component" "worker") | nindent 4 }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.commonAnnotations) "chart" (list (include "durable-workflow.argoWorkloadAnnotations" .) (include "durable-workflow.fluxDependsOnAnnotation" .))) | nindent 4 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.argoWorkloadAnnotations" . | nindent 4 }} {{- include "durable-workflow.fluxDependsOnAnnotation" . | nindent 4 }} - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: {{- if not .Values.worker.autoscaling.enabled }} replicas: {{ .Values.worker.replicaCount }} @@ -28,16 +26,12 @@ spec: template: metadata: labels: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.worker.podLabels) "chart" (list (include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "worker")))) | nindent 8 }} {{- include "durable-workflow.componentSelectorLabels" (dict "context" . "component" "worker") | nindent 8 }} - {{- with .Values.worker.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} annotations: + {{- include "durable-workflow.userMetadata" (dict "maps" (list .Values.worker.podAnnotations) "chart" (list (include "durable-workflow.configChecksumAnnotations" .))) | nindent 8 }} workflows.durable-workflow.dev/memo-payload-storage: {{ include "durable-workflow.targetMemoPayloadStorage" . | quote }} {{- include "durable-workflow.configChecksumAnnotations" . | nindent 8 }} - {{- with .Values.worker.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} spec: serviceAccountName: {{ include "durable-workflow.serviceAccountName" . }} {{- with .Values.image.pullSecrets }} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 1e26acfd2..b9585e661 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -72,7 +72,13 @@ serviceAccount: annotations: {} automountServiceAccountToken: true -# Common labels applied to every rendered resource. +# Common labels applied to every rendered resource. Every label and +# annotation map in these values may not set keys under +# workflows.durable-workflow.dev/ or the helm.sh/hook keys: the chart sets +# those itself and relies on them for its runtime, memo storage and +# schema-upgrade checks, so it refuses to render a map that sets one. A user +# key that the chart also sets on an object is dropped in favor of the +# chart's value. commonLabels: {} commonAnnotations: {} diff --git a/scripts/ci/test_helm_runtime_modes.py b/scripts/ci/test_helm_runtime_modes.py index fd54e54fd..b577200e8 100644 --- a/scripts/ci/test_helm_runtime_modes.py +++ b/scripts/ci/test_helm_runtime_modes.py @@ -2,6 +2,7 @@ """Check what the chart renders in native and PHP mode, and what it refuses (Server #946).""" from pathlib import Path +import json import subprocess import unittest @@ -33,6 +34,30 @@ def render(*options: str, values: Path | None = NATIVE) -> list[dict]: return [item for item in yaml.safe_load_all(result.stdout) if item] +class UniqueKeyLoader(yaml.SafeLoader): + """A YAML loader that refuses a mapping with a repeated key. + + Kubernetes and Helm decoding keep the later of two equal keys, so a user + value emitted after a chart value would silently replace it. + """ + + def construct_mapping(self, node, deep=False): + seen = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + if key in seen: + raise AssertionError(f"duplicate key {key!r} at {key_node.start_mark}") + seen.add(key) + return super().construct_mapping(node, deep=deep) + + +def render_unique(*options: str, values: Path | None = NATIVE) -> list[dict]: + result = helm(*options, values=values) + if result.returncode != 0: + raise AssertionError(result.stderr) + return [item for item in yaml.load_all(result.stdout, Loader=UniqueKeyLoader) if item] + + def component(resource: dict) -> str: return resource["metadata"].get("labels", {}).get("app.kubernetes.io/component", "") @@ -262,6 +287,87 @@ def test_unknown_runtime_is_refused(self): self.assertIn("runtime", result.stderr) +class ReservedMetadataTest(unittest.TestCase): + """User label and annotation maps cannot shadow the chart's authority keys.""" + + RESERVED = [ + "workflows.durable-workflow.dev/runtime", + "workflows.durable-workflow.dev/memo-payload-storage", + "workflows.durable-workflow.dev/anything", + "helm.sh/hook", + "helm.sh/hook-weight", + "helm.sh/hook-delete-policy", + ] + MAPS = [ + "commonAnnotations", "commonLabels", + "server.podAnnotations", "server.podLabels", + "server.service.annotations", "server.ingress.annotations", + "worker.podAnnotations", "worker.podLabels", + "scheduler.podAnnotations", "scheduler.podLabels", + "bootstrap.podAnnotations", "bootstrap.podLabels", + "serviceAccount.annotations", + ] + + def test_every_user_map_refuses_every_reserved_key(self): + for values in (NATIVE, PHP): + for path in self.MAPS: + for key in self.RESERVED: + with self.subTest(values=values.name, path=path, key=key): + result = helm("--set-json", f"{path}={json.dumps({key: 'forged'})}", values=values) + self.assertNotEqual(result.returncode, 0, result.stdout[:200]) + self.assertIn(f'{path} sets the reserved key "{key}"', result.stderr) + + def test_security_examples_are_refused(self): + # PHP release forging the native runtime marker, which the next + # empty-runtime upgrade and the schema-upgrade gate trust. + forged_runtime = helm( + "--set-json", 'commonAnnotations={"workflows.durable-workflow.dev/runtime":"native"}', values=PHP, + ) + self.assertNotEqual(forged_runtime.returncode, 0, forged_runtime.stdout[:200]) + self.assertIn("reserved key", forged_runtime.stderr) + # A hook type override that would run schema-upgrade after the new + # Server starts. + forged_hook = helm("--set-json", 'commonAnnotations={"helm.sh/hook":"post-upgrade"}') + self.assertNotEqual(forged_hook.returncode, 0, forged_hook.stdout[:200]) + self.assertIn("reserved key", forged_hook.stderr) + + def test_ordinary_user_metadata_renders_without_duplicate_keys(self): + user = [ + "--set-json", 'commonAnnotations={"team":"payments","argocd.argoproj.io/sync-wave":"7"}', + "--set-json", 'commonLabels={"tier":"core","app.kubernetes.io/name":"forged","app.kubernetes.io/component":"forged"}', + "--set-json", 'server.podAnnotations={"pod":"server","checksum/config":"forged"}', + "--set-json", 'server.podLabels={"lane":"api","app.kubernetes.io/component":"worker"}', + "--set-json", 'worker.podAnnotations={"pod":"worker"}', + "--set-json", 'scheduler.podLabels={"lane":"cron"}', + "--set-json", 'bootstrap.podAnnotations={"pod":"bootstrap"}', + ] + for values in (NATIVE, PHP): + with self.subTest(values=values.name): + resources = render_unique(*user, values=values) + server = named(resources, "Deployment", "-server") + annotations = server["metadata"]["annotations"] + self.assertEqual(annotations["team"], "payments") + # A chart key wins over the same user key. + self.assertEqual(annotations["argocd.argoproj.io/sync-wave"], "0") + labels = server["metadata"]["labels"] + self.assertEqual(labels["tier"], "core") + self.assertEqual(labels["app.kubernetes.io/name"], "durable-workflow") + self.assertEqual(labels["app.kubernetes.io/component"], "server") + template = server["spec"]["template"]["metadata"] + self.assertEqual(template["labels"]["lane"], "api") + self.assertEqual(template["labels"]["app.kubernetes.io/component"], "server") + self.assertEqual(template["annotations"]["pod"], "server") + self.assertNotEqual(template["annotations"]["checksum/config"], "forged") + if values == NATIVE: + self.assertEqual(annotations["workflows.durable-workflow.dev/runtime"], "native") + else: + self.assertNotIn("workflows.durable-workflow.dev/runtime", annotations) + self.assertEqual(annotations["workflows.durable-workflow.dev/memo-payload-storage"], "dual-v1") + migrate = named(resources, "Job", "-migrate") + self.assertEqual(migrate["metadata"]["annotations"]["helm.sh/hook"], "pre-install,pre-upgrade") + self.assertEqual(migrate["spec"]["template"]["metadata"]["annotations"], {"pod": "bootstrap"}) + + class UpgradeRuntimeSelectionTest(unittest.TestCase): """helm template cannot see live objects, so an upgrade without a runtime refuses.""" diff --git a/scripts/helm-chart-native-kind-smoke.sh b/scripts/helm-chart-native-kind-smoke.sh index f0121d6bd..1ccc33636 100755 --- a/scripts/helm-chart-native-kind-smoke.sh +++ b/scripts/helm-chart-native-kind-smoke.sh @@ -240,16 +240,28 @@ expect_refusal "schema upgrade with a live server pod" "server pod stray-server schema_upgrade "${release}" --dry-run=server "${kubectl_bin}" -n "${namespace}" delete pod stray-server --grace-period=0 --force --wait >/dev/null 2>&1 -# The positive render: the hook Job runs schema-upgrade with the database CA. +# A user annotation cannot turn the hook into one that runs after the new +# Server starts. +expect_refusal "schema upgrade with a user hook override" 'commonAnnotations sets the reserved key "helm.sh/hook"' \ + schema_upgrade "${release}" --dry-run=server --set-json 'commonAnnotations={"helm.sh/hook":"post-upgrade"}' + +# The positive render: the hook Job runs schema-upgrade with the database CA, +# keeps ordinary user annotations, and carries each hook key exactly once. schema_upgrade "${release}" --dry-run=server --set-string externalDatabase.sslMode=verify-full \ - --set-string externalDatabase.tls.existingSecret=smoke-database-ca >"${work_dir}/hook.yaml" + --set-string externalDatabase.tls.existingSecret=smoke-database-ca \ + --set-json 'commonAnnotations={"team":"smoke"}' >"${work_dir}/hook.yaml" for expected in '"helm.sh/hook": pre-upgrade' 'args: ["schema-upgrade"]' 'automountServiceAccountToken: false' \ - 'name: DB_SSLROOTCERT' 'secretName: "smoke-database-ca"'; do + 'name: DB_SSLROOTCERT' 'secretName: "smoke-database-ca"' 'team: smoke'; do grep -qF "${expected}" "${work_dir}/hook.yaml" || { printf 'the authorized schema-upgrade render lacks %s\n' "${expected}" >&2 exit 1 } done +hook_keys="$(sed -n '/^kind: Job$/,/^spec:$/p' "${work_dir}/hook.yaml" | grep -c 'helm.sh/hook')" +if [[ "${hook_keys}" != "3" ]]; then + printf 'the schema-upgrade Job must carry each of its three hook keys once, found %s\n' "${hook_keys}" >&2 + exit 1 +fi echo "PASS allowed: schema-upgrade hook renders once every Server is stopped" schema_upgrade "${release}" --wait --timeout 5m