diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 52aeb35da..26b2bd45f 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/UPGRADING.md alongside the chart. -version: 1.1.0 +version: 1.2.0 # The immutable native Durable Workflow Server this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record (helm_chart.app_version). -appVersion: "3.3.2" +appVersion: "3.4.0" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,18 +30,12 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:3.3.2" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:3.4.0" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - - kind: changed - description: Default image changes from native Server 3.0.5 to 3.3.2. No template, values contract or values schema change. - - kind: changed - description: "Upgrading an existing install needs an explicit native schema upgrade: Server 3.3.2 moves MySQL, MariaDB and PostgreSQL from native schema 8 (3.0.x) to 11 and refuses a version 8 database with native_schema_upgrade_required. Scale the Server to zero, back up the database, then run schema-upgrade with native.schemaUpgrade.enabled=true during helm upgrade or with the manual Job in the chart README. On PostgreSQL the version 11 index is built with a plain CREATE INDEX that blocks writes to workflow_tasks until it finishes. There is no downgrade: going back to 3.0.x means restoring the backup. See docs/UPGRADING.md." - - kind: added - description: "Server 3.1.0: every worker answer carries protocol_version and server_capabilities; cluster info serves client_compatibility and supported_sdk_versions; start_child_workflow may set await started (child_workflow_start_turn); JSON doubles keep every digit (server#1001, server#1005, server#966, server#1010)." - kind: added - description: "Server 3.3.0: an open_condition_wait command can opt in to one wall deadline per condition-wait occurrence (condition_wait_occurrence_deadline). sdk-php 3.10.0 does not send it; SDK support ships in a later sdk-php release (sdk-php#269)." + description: "Native pods sleep native.preStopSleepSeconds (default 5, 0 disables) in a preStop hook before SIGTERM, so a rolling update drops no requests (server#608). native.terminationGracePeriodSeconds now defaults to 70 so worker polls still drain after the sleep; the chart refuses a grace period at or below the sleep plus DW_WORKER_POLL_TIMEOUT." - kind: changed - description: "Servers 3.3.0 to 3.3.2 answer empty non-waiting query polls from one read snapshot without the transition lock, read claim candidates in bounded ranges, issue fewer statements per workflow transition, batch local activity records and read a workflow description from one snapshot. Answers are unchanged; no speedup is claimed. https://github.com/durable-workflow/server/releases/tag/3.3.2" + description: "Default image changes from native Server 3.3.2 to 3.4.0, which adds opt-in activity/workflow continuations. No schema upgrade from 3.3.x: MySQL, MariaDB and PostgreSQL stay at native schema 11. https://github.com/durable-workflow/server/releases/tag/3.4.0" diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 250bd1ff2..7354cbeb4 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -77,7 +77,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "3.3.2" + tag: "3.4.0" # Pin a digest in production: # digest: "sha256:abc123..." @@ -143,9 +143,12 @@ Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`. single pod would block every node drain. * On `SIGTERM` the native Server holds open worker polls until they end. A poll waits `DW_WORKER_POLL_TIMEOUT` (30 seconds by default) unless it asks - for up to 60. `native.terminationGracePeriodSeconds` defaults to 65 so a - rollout drains every allowed poll. The chart refuses a grace period at or - below a `DW_WORKER_POLL_TIMEOUT` set in `config.extraEnv`. + for up to 60. A terminating pod first sleeps `native.preStopSleepSeconds` + (5 by default, 0 disables it) in a `preStop` hook, so Services stop routing + to it before the listener closes. `native.terminationGracePeriodSeconds` + defaults to 70 so a rollout drains every allowed poll after that sleep. The + chart refuses a grace period at or below the sleep plus a + `DW_WORKER_POLL_TIMEOUT` set in `config.extraEnv`. * Native liveness probes the listener (`native.livenessProbe`, a TCP check on the `http` port), not `/api/health`. `/api/health` reports database liveness, so probing it would restart every pod during a database outage, diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 904063c7c..52b9385da 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -69,6 +69,26 @@ chart MINOR bump and an entry below. ## Per-version migration notes +### Chart 1.2.0 / Server 3.4.0 (native) + +No schema upgrade from Server 3.3.x: MySQL, MariaDB and PostgreSQL stay at +native schema 11, so a plain `helm upgrade` is enough. From Server 3.2.x or +earlier, follow the Chart 1.1.0 / Server 3.3.2 steps below first (backup and +explicit `schema-upgrade`). + +* The default image is native Server 3.4.0, which adds opt-in + activity/workflow continuations. Existing workers are unaffected. See the + [3.4.0 release notes](https://github.com/durable-workflow/server/releases/tag/3.4.0). + +* Native server pods now run a `preStop` hook that sleeps + `native.preStopSleepSeconds` (default 5) before `SIGTERM`, so Services stop + routing to a terminating pod before its listener closes (server#608). Set + it to `0` to remove the hook. +* `native.terminationGracePeriodSeconds` now defaults to 70 (was 65), so a + 60-second worker poll still drains after the sleep. If you set the grace + period yourself, keep it above `preStopSleepSeconds` plus any + `DW_WORKER_POLL_TIMEOUT`; the chart refuses a smaller value at render time. + ### Chart 1.1.0 / Server 3.3.2 (native) The default image changes from native Server 3.0.5 to 3.3.2 diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 25083b262..bf4ee1aa8 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -733,10 +733,11 @@ behavior it mirrors. {{- if and (not (get (default (dict) (get $native "storage")) "existingClaim")) (or .Values.server.autoscaling.enabled (gt (int (include "durable-workflow.serverReplicas" .)) 1)) -}} {{- fail "More than one native Server replica (server.replicaCount above 1 or server.autoscaling.enabled) needs native.storage.existingClaim: native keeps external payload objects on pod-local disk, so one pod cannot read an upload another pod accepted. Set a ReadWriteMany claim, or run one replica." -}} {{- end -}} -{{- $grace := int (default 65 (get $native "terminationGracePeriodSeconds")) -}} +{{- $grace := int (default 70 (get $native "terminationGracePeriodSeconds")) -}} +{{- $preStop := int (default 0 (get $native "preStopSleepSeconds")) -}} {{- $poll := get (default (dict) .Values.config.extraEnv) "DW_WORKER_POLL_TIMEOUT" -}} -{{- if and $poll (le $grace (int $poll)) -}} -{{- fail (printf "native.terminationGracePeriodSeconds (%d) must exceed DW_WORKER_POLL_TIMEOUT (%v): the native Server holds open worker polls for up to that long while it drains on SIGTERM." $grace $poll) -}} +{{- if and $poll (le $grace (add $preStop (int $poll))) -}} +{{- fail (printf "native.terminationGracePeriodSeconds (%d) must exceed native.preStopSleepSeconds (%d) plus DW_WORKER_POLL_TIMEOUT (%v): the native Server holds open worker polls for up to that long while it drains on SIGTERM." $grace $preStop $poll) -}} {{- end -}} {{- end -}} diff --git a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml index 4e3066f0a..53e4f31a1 100644 --- a/k8s/helm/durable-workflow/templates/native-server-deployment.yaml +++ b/k8s/helm/durable-workflow/templates/native-server-deployment.yaml @@ -62,7 +62,7 @@ spec: {{- /* On SIGTERM native stops accepting work and holds open worker polls until they end. A poll may ask for up to 60 seconds, so the grace period must outlast that. */}} - terminationGracePeriodSeconds: {{ default 65 (get $native "terminationGracePeriodSeconds") }} + terminationGracePeriodSeconds: {{ default 70 (get $native "terminationGracePeriodSeconds") }} containers: - name: server image: {{ include "durable-workflow.image" . | quote }} @@ -94,6 +94,15 @@ spec: startupProbe: {{- toYaml . | nindent 12 }} {{- end }} + {{- /* Endpoints are removed asynchronously; sleeping before SIGTERM + keeps the listener open until routing has moved off this pod. */}} + {{- $preStop := int (default 0 (get $native "preStopSleepSeconds")) }} + {{- if gt $preStop 0 }} + lifecycle: + preStop: + exec: + command: ["/bin/sleep", {{ $preStop | quote }}] + {{- end }} resources: {{- $resources | nindent 12 }} {{- with .Values.server.containerSecurityContext }} diff --git a/k8s/helm/durable-workflow/values.schema.json b/k8s/helm/durable-workflow/values.schema.json index bc4797e8a..473b6e2ad 100644 --- a/k8s/helm/durable-workflow/values.schema.json +++ b/k8s/helm/durable-workflow/values.schema.json @@ -14,6 +14,7 @@ "type": "object", "properties": { "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, + "preStopSleepSeconds": { "type": "integer", "minimum": 0 }, "livenessProbe": { "type": "object" }, "automountServiceAccountToken": { "type": "boolean" }, "ephemeralStorage": { diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 053d35ecd..be43af34b 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -42,7 +42,7 @@ image: # scripts/ci/sync-source-release.mjs. Do not edit this default. # A tag must match the runtime: 3.x (or latest) for native, 2.x for php. # The chart refuses a mismatch at render time. - tag: "3.3.2" + tag: "3.4.0" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." # The chart cannot see which runtime a digest names. Pin a native digest @@ -214,10 +214,15 @@ native: # On SIGTERM the native Server holds open worker polls until they end. A # poll waits DW_WORKER_POLL_TIMEOUT (30 seconds) unless it asks for up to 60, - # so keep this above 60 for a rollout to drain polls instead of cutting them - # off. The chart refuses a value at or below a DW_WORKER_POLL_TIMEOUT set in - # config.extraEnv. - terminationGracePeriodSeconds: 65 + # so keep this above preStopSleepSeconds plus 60 for a rollout to drain polls + # instead of cutting them off. The chart refuses a value at or below + # preStopSleepSeconds plus a DW_WORKER_POLL_TIMEOUT set in config.extraEnv. + terminationGracePeriodSeconds: 70 + + # Seconds a terminating pod keeps serving before it receives SIGTERM, so + # Services and load balancers stop routing to it first and a rolling update + # drops no requests. 0 removes the preStop hook. + preStopSleepSeconds: 5 # Liveness for native pods. /api/health reports database liveness, so # probing it would restart every pod during a database outage while the diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 64fd65829..7f308fb67 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -4,7 +4,7 @@ "version": "2.11.8" }, "helm_chart": { - "version": "1.1.0", - "app_version": "3.3.2" + "version": "1.2.0", + "app_version": "3.4.0" } } diff --git a/scripts/ci/test_helm_runtime_modes.py b/scripts/ci/test_helm_runtime_modes.py index b577200e8..22f6a034f 100644 --- a/scripts/ci/test_helm_runtime_modes.py +++ b/scripts/ci/test_helm_runtime_modes.py @@ -100,8 +100,10 @@ def test_default_install_renders_one_native_server_and_nothing_php(self): self.assertEqual(container["livenessProbe"]["tcpSocket"], {"port": "http"}) self.assertNotIn("httpGet", container["livenessProbe"]) self.assertEqual(container["readinessProbe"]["httpGet"]["path"], "/api/ready") - # A worker poll may ask to be held for up to 60 seconds. - self.assertEqual(pod["terminationGracePeriodSeconds"], 65) + # A worker poll may ask to be held for up to 60 seconds, after a + # preStop sleep that lets routing move off the pod first (Server #608). + self.assertEqual(pod["terminationGracePeriodSeconds"], 70) + self.assertEqual(container["lifecycle"], {"preStop": {"exec": {"command": ["/bin/sleep", "5"]}}}) self.assertEqual(pod["securityContext"]["runAsUser"], 1000) self.assertEqual(pod["securityContext"]["fsGroup"], 1000) self.assertIn({"name": "server-storage", "mountPath": "/app/storage"}, container["volumeMounts"]) @@ -165,8 +167,8 @@ def test_render_time_refusals_match_native_startup_refusals(self): "auth.authToken is required in native mode": ["--set-string", "auth.authToken="], "auth.principalTokens is not supported": ["--set-string", "auth.principalTokens=[]"], 'config.auth.driver "none" is not supported': ["--set-string", "config.auth.driver=none"], - "must exceed DW_WORKER_POLL_TIMEOUT": [ - "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=70", + "must exceed native.preStopSleepSeconds (5) plus DW_WORKER_POLL_TIMEOUT": [ + "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=66", ], "More than one native Server replica": ["--set", "server.replicaCount=2"], "native.storage.existingClaim: native keeps external payload objects": [ @@ -190,10 +192,13 @@ def test_render_time_refusals_match_native_startup_refusals(self): accepted = render( "--set-string", "config.extraEnv.DW_WORKER_POLL_TIMEOUT=70", - "--set", "native.terminationGracePeriodSeconds=75", + "--set", "native.terminationGracePeriodSeconds=76", ) server = named(accepted, "Deployment", "-server") - self.assertEqual(server["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 75) + self.assertEqual(server["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 76) + without_sleep = render("--set", "native.preStopSleepSeconds=0") + container = named(without_sleep, "Deployment", "-server")["spec"]["template"]["spec"]["containers"][0] + self.assertNotIn("lifecycle", container) def test_a_shared_payload_claim_allows_more_than_one_replica(self): resources = render(