From 177772933a61f62446f6e74b34bafdb7187ccae8 Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:28:19 -0400 Subject: [PATCH 1/5] feat(operator): gate readiness on installation completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `helm install --wait` returned as soon as the apl-operator Deployment was available, which the previous readinessProbe reported within ~30s: it ran `pgrep -f apl-operator`, i.e. "the process exists". The helmfile pipeline then installs the platform for another 10-15 minutes. Bootstrap automation therefore had nothing to gate on and either proceeded into a half-installed platform or polled Argo/pod state and guessed. The operator now writes a `/tmp/ready` marker at the single point where the installation phase has resolved to `completed` — fresh install, recovery install, or a restart on an already-installed cluster — and the readinessProbe tests for that marker. `helm install --wait` and `kubectl wait --for=condition=Available deployment/apl-operator` become truthful convergence gates. Design notes: - Readiness LATCHES. It is not cleared while a later apply runs: the reconcile loop applies every ~5 minutes in steady state, and flapping the Deployment's Available condition would make it useless as a gate. Per-apply status stays in the apl-operator-state ConfigMap. - It FAILS CLOSED. A marker that cannot be written, or an installation that keeps retrying, leaves the pod NotReady — `--wait` times out loudly rather than reporting a convergence that did not happen. - progressDeadlineSeconds is raised to 3600, otherwise `kubectl rollout status` reports ProgressDeadlineExceeded at the 600s default while a perfectly healthy first install is still running. Since the gate makes `--wait` block for the real install duration, it is switchable: `operator.readiness.gateOnInstallationComplete=false` restores the previous process-liveness behaviour. The helmfile-managed charts/apl-operator deployment, which had no readinessProbe at all, gets the same gate. EXECUTION_FLOW.md documents the resulting contract, including the already-present apl-installation-status / apl-operator-state ConfigMaps for phase-level introspection. --- chart/apl/templates/NOTES.txt | 10 +++ chart/apl/templates/deployment.yaml | 23 ++++++- chart/apl/values.yaml | 14 ++++ charts/apl-operator/templates/deployment.yaml | 16 +++++ charts/apl-operator/values.yaml | 4 ++ src/operator/EXECUTION_FLOW.md | 65 +++++++++++++++++++ src/operator/k8s.test.ts | 47 +++++++++++++- src/operator/k8s.ts | 34 ++++++++++ src/operator/main.ts | 6 ++ 9 files changed, 217 insertions(+), 2 deletions(-) diff --git a/chart/apl/templates/NOTES.txt b/chart/apl/templates/NOTES.txt index 569365d68c..44e7c1bc79 100644 --- a/chart/apl/templates/NOTES.txt +++ b/chart/apl/templates/NOTES.txt @@ -1,5 +1,15 @@ The App Platform Operator has been successfully deployed on the cluster. Please inspect the output of the apl-operator deployment (apl-operator/{{ include "apl-operator.fullname" . }}) for any feedback or errors. +{{ if ne ((.Values.operator.readiness | default dict).gateOnInstallationComplete) false }} +Installing the platform takes 10-15 minutes. The operator reports Ready only once +installation has completed, so you can wait for it: + + kubectl wait --for=condition=Available deployment/{{ include "apl-operator.fullname" . }} -n apl-operator --timeout=30m + +Progress is readable at any time from: + + kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}' +{{ end }} Also visit https://techdocs.akamai.com/app-platform/ for further instructions and reference documentation. diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 448a484357..b7906a27d3 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -1,6 +1,8 @@ {{- $kms := .Values.kms | default dict }} {{- $version := .Values.otomi.version | default .Chart.AppVersion }} {{- $skipDeployment := .Values.installation.skipOperatorDeployment }} +{{- $readiness := .Values.operator.readiness | default dict }} +{{- $gateOnInstall := ne $readiness.gateOnInstallationComplete false }} {{- if not $skipDeployment }} apiVersion: apps/v1 kind: Deployment @@ -10,6 +12,11 @@ metadata: labels: {{- include "apl-operator.labels" . | nindent 4 }} spec: replicas: 1 + # Installing the platform takes considerably longer than the 600s default, and + # with the readinessProbe gated on convergence the rollout stays Progressing + # for that whole window. Without this, `kubectl rollout status` reports + # ProgressDeadlineExceeded on a perfectly healthy first install. + progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds | default 3600 }} selector: matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }} strategy: @@ -90,11 +97,25 @@ spec: failureThreshold: 3 readinessProbe: exec: + {{- if $gateOnInstall }} + # Convergence signal: /tmp/ready is written by the operator once the + # platform installation reaches the 'completed' state, so the + # Deployment only goes Available when the helmfile pipeline has + # actually converged. This is what makes `helm install --wait` and + # `kubectl wait --for=condition=Available` usable as bootstrap gates. + # NOTE: a first install takes 10-15 minutes — size --timeout accordingly. + command: ["/bin/sh", "-c", "test -f /tmp/ready"] + {{- else }} + # Liveness-equivalent readiness: reports Ready as soon as the operator + # process is up, long before the platform has converged. command: ["/bin/sh", "-c", "pgrep -f 'apl-operator' > /dev/null"] + {{- end }} initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 - failureThreshold: 3 + # Readiness latches once installation completes; a transient probe + # failure should not take the Deployment out of Available. + failureThreshold: {{ $readiness.failureThreshold | default 3 }} volumes: - name: values-secret secret: diff --git a/chart/apl/values.yaml b/chart/apl/values.yaml index 25932348b6..587dfbf89f 100644 --- a/chart/apl/values.yaml +++ b/chart/apl/values.yaml @@ -129,6 +129,20 @@ operator: installRetries: 1000 installMaxTimeoutMs: 10000 + readiness: + # When true (default), the operator only reports Ready once the platform + # installation has completed, making the apl-operator Deployment a truthful + # "platform converged" signal for `helm install --wait` and `kubectl wait`. + # A first install takes 10-15 minutes, so size `helm --timeout` accordingly + # (e.g. `--wait --timeout 30m`). + # Set to false to restore the previous process-liveness readiness, which + # reports Ready within a minute regardless of installation progress. + gateOnInstallationComplete: true + # Rollout budget for the install window. Only relevant when the readiness + # gate is on — `kubectl rollout status` fails once it is exceeded. + progressDeadlineSeconds: 3600 + failureThreshold: 3 + image: repository: "mirror.registry.linodelke.net/docker/linode/apl-core" diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index b9322ffd2e..b3cc54bd8c 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -73,6 +73,22 @@ spec: periodSeconds: 60 failureThreshold: 3 timeoutSeconds: 10 + {{- if ne (.Values.operator.readiness | default dict).gateOnInstallationComplete false }} + # /tmp/ready is written by the operator once the platform installation + # reaches the 'completed' state — the Deployment reports Available only + # after the helmfile pipeline has converged, not merely once the process + # is up. Readiness latches, so steady-state reconciles do not flap it. + readinessProbe: + exec: + command: + - /bin/sh + - -c + - "test -f /tmp/ready" + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: diff --git a/charts/apl-operator/values.yaml b/charts/apl-operator/values.yaml index e21ff63328..7b45580749 100644 --- a/charts/apl-operator/values.yaml +++ b/charts/apl-operator/values.yaml @@ -60,6 +60,10 @@ operator: gitOpTimeoutMs: 10000 installRetries: 1000 installMaxTimeoutMs: 10000 + readiness: + # When true (default), the operator reports Ready only after the platform + # installation has completed. Set to false to drop the readinessProbe. + gateOnInstallationComplete: true nodeSelector: {} diff --git a/src/operator/EXECUTION_FLOW.md b/src/operator/EXECUTION_FLOW.md index 355ed076b1..f4ff66cfb9 100644 --- a/src/operator/EXECUTION_FLOW.md +++ b/src/operator/EXECUTION_FLOW.md @@ -473,6 +473,71 @@ Shared by both loops with trigger-specific variations: 8. Update apply state to 'succeeded' or 'failed' 9. Release lock (`isApplying = false`) +## Readiness and Convergence Contract + +Bootstrap automation needs a machine-checkable answer to "is the platform installed +yet?". The operator exposes it through the readiness of its own Deployment. + +### The gate + +The operator writes `/tmp/ready` (`markInstallationComplete()`) at exactly one point: +after the installation phase resolves to `completed` — whether that came from a fresh +install, a recovery install, or a restart on an already-installed cluster. The +`readinessProbe` on the apl-operator Deployment tests for that file, so: + +```bash +# blocks until the helmfile pipeline has actually converged +kubectl wait --for=condition=Available deployment/apl-operator -n apl-operator --timeout=30m + +# same signal, via helm +helm install apl … --wait --timeout 30m +``` + +Three properties are deliberate: + +- **It latches.** Readiness is never cleared while a later apply runs. The reconcile + loop applies every ~5 minutes in steady state; flipping the Deployment out of + `Available` on each pass would make the condition useless as a gate. Per-apply + status is reported through the `apl-operator-state` ConfigMap instead (below). +- **It fails closed.** If the marker cannot be written, or installation keeps + retrying, the pod stays NotReady. The signal never claims a convergence that did + not happen — `--wait` times out loudly rather than returning early. +- **A first install takes 10-15 minutes.** Size `--timeout` accordingly; the + Deployment's `progressDeadlineSeconds` is raised to 3600 so `kubectl rollout + status` does not report `ProgressDeadlineExceeded` on a healthy install. + +Set `operator.readiness.gateOnInstallationComplete=false` to restore the previous +behaviour, where readiness only reflected that the operator process was running. + +### Introspection + +For phase detail rather than a binary gate, read the ConfigMaps in the table below: + +```bash +# installation phase: pending | in-progress | completed | failed (+ attempt, timestamp) +kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}' + +# last apply: commitHash, status, timestamp, trigger, errorMessage +kubectl get cm apl-operator-state -n apl-operator -o jsonpath='{.data.state}' +``` + +`apl-operator-state.commitHash` is the answer to "did the operator apply *my* commit +yet?" — poll for `status: succeeded` at the revision you pushed. + +### What this is not + +The Deployment gate covers the operator's own pipeline: essential manifests, CRDs, +`stage=prep`, `app=core`, and the ArgoCD Applications for the remaining apps. Apps +that ArgoCD syncs afterwards report health through ArgoCD, not through this gate. + +An end-to-end smoke check that the platform is externally serving is +`https://auth./ready` (oauth2-proxy behind the ingress). It exercises +DNS, ingress-nginx, the TLS certificate and the auth chain, which the in-cluster +gate does not. It is complementary, not a substitute: it needs public DNS and a +trusted certificate, it cannot tell you *which* revision of your values converged, +and a non-200 cannot distinguish "platform not ready" from a DNS or certificate +problem. + ## Kubernetes Resources ### ConfigMaps diff --git a/src/operator/k8s.test.ts b/src/operator/k8s.test.ts index d004691dd2..643be30526 100644 --- a/src/operator/k8s.test.ts +++ b/src/operator/k8s.test.ts @@ -1,5 +1,8 @@ -import { ApplyState, updateApplyState } from './k8s' +import { ApplyState, markInstallationComplete, READINESS_FILE, updateApplyState } from './k8s' import { CoreV1Api, ApiException } from '@kubernetes/client-node' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' jest.mock('@kubernetes/client-node', () => { const mocks = { @@ -38,6 +41,7 @@ jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), error: jest.fn(), + warn: jest.fn(), })), })) @@ -173,3 +177,44 @@ describe('updateApplyState', () => { expect(mockCoreV1Api.createNamespacedConfigMap).not.toHaveBeenCalled() }) }) + +describe('markInstallationComplete', () => { + let workDir: string + + beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), 'apl-readiness-')) + }) + + afterEach(() => { + rmSync(workDir, { recursive: true, force: true }) + }) + + test('defaults to the path the readinessProbe checks', () => { + expect(READINESS_FILE).toBe('/tmp/ready') + }) + + test('writes the readiness marker with a timestamp', () => { + const marker = join(workDir, 'ready') + + markInstallationComplete(marker) + + expect(existsSync(marker)).toBe(true) + expect(Date.parse(readFileSync(marker, 'utf8'))).not.toBeNaN() + }) + + test('is idempotent — a restart of an installed cluster re-marks readiness', () => { + const marker = join(workDir, 'ready') + + markInstallationComplete(marker) + markInstallationComplete(marker) + + expect(existsSync(marker)).toBe(true) + }) + + test('never throws when the marker cannot be written, leaving the pod NotReady', () => { + const unwritable = join(workDir, 'does', 'not', 'exist', 'ready') + + expect(() => markInstallationComplete(unwritable)).not.toThrow() + expect(existsSync(unwritable)).toBe(false) + }) +}) diff --git a/src/operator/k8s.ts b/src/operator/k8s.ts index 8d963e4331..0a568e0d24 100644 --- a/src/operator/k8s.ts +++ b/src/operator/k8s.ts @@ -39,6 +39,40 @@ export function updateHeartbeatFile(): void { writeFileSync('/tmp/heartbeat', '') } +/** + * Marker file that signals platform installation has completed. The operator + * readinessProbe gates on its existence, so the apl-operator Deployment only + * becomes Available once the helmfile pipeline has actually converged — not + * merely once the operator process is up. That makes `helm install --wait` and + * `kubectl wait --for=condition=Available deployment/apl-operator` meaningful + * gates for bootstrap automation. + * + * The marker lives on the pod's /tmp emptyDir, so it is cleared on every + * restart and re-created as soon as the operator re-confirms the installation + * status from the apl-installation-status ConfigMap. + */ +export const READINESS_FILE = '/tmp/ready' + +/** + * Writes the readiness marker. Called once installation has reached the + * 'completed' state — including on restarts of an already-installed cluster. + * Readiness latches: it is intentionally NOT cleared while a subsequent apply + * runs, because steady-state reconcile loops must not flap the Deployment's + * Available condition. Per-apply status lives in the apl-operator-state + * ConfigMap instead. + */ +export function markInstallationComplete(filePath: string = READINESS_FILE): void { + const d = terminal('operator:k8s:markInstallationComplete') + try { + writeFileSync(filePath, new Date().toISOString()) + d.info(`Installation complete, wrote readiness marker ${filePath}`) + } catch (error) { + // Deliberately non-fatal: a missing marker keeps the pod NotReady, which is + // the safe direction — it never reports convergence that did not happen. + d.warn(`Failed to write readiness marker ${filePath}:`, getErrorMessage(error)) + } +} + export async function updateApplyState( state: ApplyState, namespace: string = APL_OPERATOR_NS, diff --git a/src/operator/main.ts b/src/operator/main.ts index f746a4d243..b811a82831 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -10,6 +10,7 @@ import { AplOperations } from './apl-operations' import { AplOperator, AplOperatorConfig } from './apl-operator' import { GitRepository } from './git-repository' import { Installer } from './installer' +import { markInstallationComplete } from './k8s' import { getErrorMessage } from './utils' import { operatorEnv } from './validators' @@ -88,6 +89,11 @@ async function main(): Promise { await installer.reconcileInstall() } + // Every branch above only falls through once the installation has reached + // the 'completed' state, so this is the single point where the platform is + // known to be installed. Signal it to the readinessProbe. + markInstallationComplete() + // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) await installer.setEnvAndCreateSecrets() From 7c18bd1a5888132f6e670852dff5914ab9924cc8 Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:34:53 -0400 Subject: [PATCH 2/5] ci(dyff): don't fail run-compare on pull requests from forks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chart comparison itself succeeds on a fork PR; the job then dies at the last line with `gh: To use GitHub CLI in a GitHub Actions workflow, set the GH_TOKEN environment variable` (exit 4). GitHub does not expose secrets to workflows triggered by `pull_request` from a fork, so `secrets.BOT_TOKEN` is empty and `gh pr comment` cannot authenticate — a contributor outside the org has no way to make this check pass. Publish the comparison to the job summary unconditionally (capped at 900KB, under the 1MB summary limit), and skip the commenting step with an explicit ::notice:: when no token is present. Behaviour for in-repo pull requests is unchanged: BOT_TOKEN is set, so the PR comment is still created and edited in place — they just also get the summary. --- .github/workflows/svcaplbot-run-dyff.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/svcaplbot-run-dyff.yml b/.github/workflows/svcaplbot-run-dyff.yml index 63e7f5dbbc..44b9e983ef 100644 --- a/.github/workflows/svcaplbot-run-dyff.yml +++ b/.github/workflows/svcaplbot-run-dyff.yml @@ -107,6 +107,16 @@ jobs: echo '```diff' >> "$comment_file" cat "$GITHUB_WORKSPACE/pr/tmp/diff-output.txt" >> "$comment_file" echo '```' >> "$comment_file" + + # Always publish the diff to the job summary. For pull requests from a + # fork, secrets — and therefore BOT_TOKEN — are not available, so this + # is the only channel that can carry the comparison. + head -c 900000 "$comment_file" >> "$GITHUB_STEP_SUMMARY" + if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ -z "$GH_TOKEN" ]; then + echo "::notice::No BOT_TOKEN available (pull request from a fork) — comparison published to the job summary instead of a PR comment." + exit 0 + fi gh pr comment ${{ github.event.pull_request.number }} --body-file "$comment_file" --create-if-none --edit-last fi From 7c69fcd1aa7c29105e386b7934bcbbd55e2b28f6 Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:16:15 -0400 Subject: [PATCH 3/5] fix(operator): mark ready after the first apply, and drop the toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback from @CasLubbers: - markOperatorReady() (was markInstallationComplete) now runs after a successful apply run rather than at the end of the helmfile install. That run is what creates the ArgoCD Applications, so past it the platform can heal itself through ArgoCD — which is what 'the operator is ready' should mean. The function is idempotent, so calling it on every apply is fine. - Removed operator.readiness.gateOnInstallationComplete. The marker is always written, so there is nothing to gate on. - Stopped re-defaulting values the chart's values.yaml already sets, and reverted failureThreshold to the 3 it already was. - progressDeadlineSeconds down to 1800. - EXECUTION_FLOW.md no longer claims the platform is fully up when the operator reports Ready — ArgoCD is still working at that point. - Trimmed the comments that restated their own code. --- chart/apl/templates/NOTES.txt | 3 +- chart/apl/templates/deployment.yaml | 26 ++--------- chart/apl/values.yaml | 14 +----- charts/apl-operator/templates/deployment.yaml | 6 --- charts/apl-operator/values.yaml | 4 -- src/operator/EXECUTION_FLOW.md | 43 ++++++++++--------- src/operator/apl-operator.test.ts | 6 ++- src/operator/apl-operator.ts | 7 ++- src/operator/k8s.test.ts | 14 +++--- src/operator/k8s.ts | 31 ++++--------- src/operator/main.ts | 6 --- 11 files changed, 56 insertions(+), 104 deletions(-) diff --git a/chart/apl/templates/NOTES.txt b/chart/apl/templates/NOTES.txt index 44e7c1bc79..8efcf3bbab 100644 --- a/chart/apl/templates/NOTES.txt +++ b/chart/apl/templates/NOTES.txt @@ -1,7 +1,7 @@ The App Platform Operator has been successfully deployed on the cluster. Please inspect the output of the apl-operator deployment (apl-operator/{{ include "apl-operator.fullname" . }}) for any feedback or errors. -{{ if ne ((.Values.operator.readiness | default dict).gateOnInstallationComplete) false }} + Installing the platform takes 10-15 minutes. The operator reports Ready only once installation has completed, so you can wait for it: @@ -10,6 +10,5 @@ installation has completed, so you can wait for it: Progress is readable at any time from: kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}' -{{ end }} Also visit https://techdocs.akamai.com/app-platform/ for further instructions and reference documentation. diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index b7906a27d3..c147a44fc3 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -1,8 +1,7 @@ {{- $kms := .Values.kms | default dict }} {{- $version := .Values.otomi.version | default .Chart.AppVersion }} {{- $skipDeployment := .Values.installation.skipOperatorDeployment }} -{{- $readiness := .Values.operator.readiness | default dict }} -{{- $gateOnInstall := ne $readiness.gateOnInstallationComplete false }} +{{- $readiness := .Values.operator.readiness }} {{- if not $skipDeployment }} apiVersion: apps/v1 kind: Deployment @@ -12,11 +11,8 @@ metadata: labels: {{- include "apl-operator.labels" . | nindent 4 }} spec: replicas: 1 - # Installing the platform takes considerably longer than the 600s default, and - # with the readinessProbe gated on convergence the rollout stays Progressing - # for that whole window. Without this, `kubectl rollout status` reports - # ProgressDeadlineExceeded on a perfectly healthy first install. - progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds | default 3600 }} + # The rollout stays Progressing until the operator is ready, which is longer than the 600s default. + progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds }} selector: matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }} strategy: @@ -97,25 +93,11 @@ spec: failureThreshold: 3 readinessProbe: exec: - {{- if $gateOnInstall }} - # Convergence signal: /tmp/ready is written by the operator once the - # platform installation reaches the 'completed' state, so the - # Deployment only goes Available when the helmfile pipeline has - # actually converged. This is what makes `helm install --wait` and - # `kubectl wait --for=condition=Available` usable as bootstrap gates. - # NOTE: a first install takes 10-15 minutes — size --timeout accordingly. command: ["/bin/sh", "-c", "test -f /tmp/ready"] - {{- else }} - # Liveness-equivalent readiness: reports Ready as soon as the operator - # process is up, long before the platform has converged. - command: ["/bin/sh", "-c", "pgrep -f 'apl-operator' > /dev/null"] - {{- end }} initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 - # Readiness latches once installation completes; a transient probe - # failure should not take the Deployment out of Available. - failureThreshold: {{ $readiness.failureThreshold | default 3 }} + failureThreshold: 3 volumes: - name: values-secret secret: diff --git a/chart/apl/values.yaml b/chart/apl/values.yaml index 587dfbf89f..d59b57975f 100644 --- a/chart/apl/values.yaml +++ b/chart/apl/values.yaml @@ -130,18 +130,8 @@ operator: installMaxTimeoutMs: 10000 readiness: - # When true (default), the operator only reports Ready once the platform - # installation has completed, making the apl-operator Deployment a truthful - # "platform converged" signal for `helm install --wait` and `kubectl wait`. - # A first install takes 10-15 minutes, so size `helm --timeout` accordingly - # (e.g. `--wait --timeout 30m`). - # Set to false to restore the previous process-liveness readiness, which - # reports Ready within a minute regardless of installation progress. - gateOnInstallationComplete: true - # Rollout budget for the install window. Only relevant when the readiness - # gate is on — `kubectl rollout status` fails once it is exceeded. - progressDeadlineSeconds: 3600 - failureThreshold: 3 + # 30 minutes. If the operator has not installed the platform by then, something is off. + progressDeadlineSeconds: 1800 image: repository: "mirror.registry.linodelke.net/docker/linode/apl-core" diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index b3cc54bd8c..7c10f2cc0c 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -73,11 +73,6 @@ spec: periodSeconds: 60 failureThreshold: 3 timeoutSeconds: 10 - {{- if ne (.Values.operator.readiness | default dict).gateOnInstallationComplete false }} - # /tmp/ready is written by the operator once the platform installation - # reaches the 'completed' state — the Deployment reports Available only - # after the helmfile pipeline has converged, not merely once the process - # is up. Readiness latches, so steady-state reconciles do not flap it. readinessProbe: exec: command: @@ -88,7 +83,6 @@ spec: periodSeconds: 10 failureThreshold: 3 timeoutSeconds: 5 - {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: diff --git a/charts/apl-operator/values.yaml b/charts/apl-operator/values.yaml index 7b45580749..e21ff63328 100644 --- a/charts/apl-operator/values.yaml +++ b/charts/apl-operator/values.yaml @@ -60,10 +60,6 @@ operator: gitOpTimeoutMs: 10000 installRetries: 1000 installMaxTimeoutMs: 10000 - readiness: - # When true (default), the operator reports Ready only after the platform - # installation has completed. Set to false to drop the readinessProbe. - gateOnInstallationComplete: true nodeSelector: {} diff --git a/src/operator/EXECUTION_FLOW.md b/src/operator/EXECUTION_FLOW.md index f4ff66cfb9..e6e3469444 100644 --- a/src/operator/EXECUTION_FLOW.md +++ b/src/operator/EXECUTION_FLOW.md @@ -475,40 +475,42 @@ Shared by both loops with trigger-specific variations: ## Readiness and Convergence Contract -Bootstrap automation needs a machine-checkable answer to "is the platform installed -yet?". The operator exposes it through the readiness of its own Deployment. +Bootstrap automation needs a machine-checkable answer to "has the operator finished +its job yet?". The operator exposes it through the readiness of its own Deployment. ### The gate -The operator writes `/tmp/ready` (`markInstallationComplete()`) at exactly one point: -after the installation phase resolves to `completed` — whether that came from a fresh -install, a recovery install, or a restart on an already-installed cluster. The -`readinessProbe` on the apl-operator Deployment tests for that file, so: +The operator writes `/tmp/ready` (`markOperatorReady()`) at exactly one point: after +an apply run completes successfully. That run is what creates the ArgoCD Applications, +so past it the platform can heal itself through ArgoCD. The `readinessProbe` on the +apl-operator Deployment tests for that file, so: ```bash -# blocks until the helmfile pipeline has actually converged +# blocks until the operator has completed an apply run kubectl wait --for=condition=Available deployment/apl-operator -n apl-operator --timeout=30m # same signal, via helm helm install apl … --wait --timeout 30m ``` +This is **not** the same as "the platform is fully up". When the operator reports +Ready, ArgoCD is still working through the Applications it was just handed. The gate +says the operator is finished and its reconcile loop has started — from there, health +belongs to ArgoCD. + Three properties are deliberate: -- **It latches.** Readiness is never cleared while a later apply runs. The reconcile - loop applies every ~5 minutes in steady state; flipping the Deployment out of - `Available` on each pass would make the condition useless as a gate. Per-apply - status is reported through the `apl-operator-state` ConfigMap instead (below). -- **It fails closed.** If the marker cannot be written, or installation keeps - retrying, the pod stays NotReady. The signal never claims a convergence that did - not happen — `--wait` times out loudly rather than returning early. +- **It latches.** Readiness is never cleared by a later apply. The reconcile loop + applies every ~5 minutes in steady state; flipping the Deployment out of `Available` + on each pass would make the condition useless as a gate. Per-apply status is + reported through the `apl-operator-state` ConfigMap instead (below). +- **It fails closed.** If the marker cannot be written, or the apply keeps failing, + the pod stays NotReady. The signal never claims progress that did not happen — + `--wait` times out loudly rather than returning early. - **A first install takes 10-15 minutes.** Size `--timeout` accordingly; the - Deployment's `progressDeadlineSeconds` is raised to 3600 so `kubectl rollout + Deployment's `progressDeadlineSeconds` is raised to 1800 so `kubectl rollout status` does not report `ProgressDeadlineExceeded` on a healthy install. -Set `operator.readiness.gateOnInstallationComplete=false` to restore the previous -behaviour, where readiness only reflected that the operator process was running. - ### Introspection For phase detail rather than a binary gate, read the ConfigMaps in the table below: @@ -527,8 +529,9 @@ yet?" — poll for `status: succeeded` at the revision you pushed. ### What this is not The Deployment gate covers the operator's own pipeline: essential manifests, CRDs, -`stage=prep`, `app=core`, and the ArgoCD Applications for the remaining apps. Apps -that ArgoCD syncs afterwards report health through ArgoCD, not through this gate. +`stage=prep`, `app=core`, and the creation of the ArgoCD Applications for the +remaining apps. Whether those Applications have actually synced and gone Healthy is +ArgoCD's business, not this gate's. An end-to-end smoke check that the platform is externally serving is `https://auth./ready` (oauth2-proxy behind the ingress). It exercises diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index f6a2b0dca2..0281a07f7b 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -3,7 +3,7 @@ import { waitTillGitRepoAvailable } from '../common/gitea' import { AplOperations } from './apl-operations' import { AplOperator, AplOperatorConfig, ApplyTrigger } from './apl-operator' import { GitRepository } from './git-repository' -import { updateApplyState } from './k8s' +import { markOperatorReady, updateApplyState } from './k8s' const mockInfoFn = jest.fn() const mockWarnFn = jest.fn() @@ -61,6 +61,7 @@ jest.mock('../cmd/commit', () => ({ jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), appRevisionMatches: jest.fn().mockResolvedValue(true), + markOperatorReady: jest.fn(), })) jest.mock('./git-repository', () => ({ @@ -210,6 +211,7 @@ describe('AplOperator', () => { }), ) + expect(markOperatorReady).toHaveBeenCalled() expect((aplOperator as any).isApplying).toBe(false) }) @@ -261,6 +263,8 @@ describe('AplOperator', () => { }), ) + // A failed apply leaves the ArgoCD Applications unaccounted for — the pod must stay NotReady. + expect(markOperatorReady).not.toHaveBeenCalled() expect((aplOperator as any).isApplying).toBe(false) expect(mockErrorFn).toHaveBeenCalledWith('[poll] Apply process failed', 'Apply failed') diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 1da973789d..e3653944fa 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -9,7 +9,7 @@ import { ensureManifestDirectories, ensureTeamGitOpsDirectories } from '../commo import { getDefaultValues, writeValues } from '../common/values' import { AplOperations } from './apl-operations' import { GitRepository } from './git-repository' -import { updateApplyState } from './k8s' +import { markOperatorReady, updateApplyState } from './k8s' import { getErrorMessage } from './utils' export interface AplOperatorConfig { @@ -99,6 +99,11 @@ export class AplOperator { this.d.info(`[${trigger}] Apply process completed`) + // The apply run above is what creates the ArgoCD Applications, so from here on the + // platform can heal itself through ArgoCD. That — not the end of the helmfile install + // — is what the operator being 'ready' means. + markOperatorReady() + await updateApplyState({ commitHash, status: 'succeeded', diff --git a/src/operator/k8s.test.ts b/src/operator/k8s.test.ts index 643be30526..9c5b50317c 100644 --- a/src/operator/k8s.test.ts +++ b/src/operator/k8s.test.ts @@ -1,4 +1,4 @@ -import { ApplyState, markInstallationComplete, READINESS_FILE, updateApplyState } from './k8s' +import { ApplyState, markOperatorReady, READINESS_FILE, updateApplyState } from './k8s' import { CoreV1Api, ApiException } from '@kubernetes/client-node' import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs' import { tmpdir } from 'os' @@ -178,7 +178,7 @@ describe('updateApplyState', () => { }) }) -describe('markInstallationComplete', () => { +describe('markOperatorReady', () => { let workDir: string beforeEach(() => { @@ -196,17 +196,17 @@ describe('markInstallationComplete', () => { test('writes the readiness marker with a timestamp', () => { const marker = join(workDir, 'ready') - markInstallationComplete(marker) + markOperatorReady(marker) expect(existsSync(marker)).toBe(true) expect(Date.parse(readFileSync(marker, 'utf8'))).not.toBeNaN() }) - test('is idempotent — a restart of an installed cluster re-marks readiness', () => { + test('is idempotent — every apply run re-marks readiness', () => { const marker = join(workDir, 'ready') - markInstallationComplete(marker) - markInstallationComplete(marker) + markOperatorReady(marker) + markOperatorReady(marker) expect(existsSync(marker)).toBe(true) }) @@ -214,7 +214,7 @@ describe('markInstallationComplete', () => { test('never throws when the marker cannot be written, leaving the pod NotReady', () => { const unwritable = join(workDir, 'does', 'not', 'exist', 'ready') - expect(() => markInstallationComplete(unwritable)).not.toThrow() + expect(() => markOperatorReady(unwritable)).not.toThrow() expect(existsSync(unwritable)).toBe(false) }) }) diff --git a/src/operator/k8s.ts b/src/operator/k8s.ts index 0a568e0d24..b9350b7941 100644 --- a/src/operator/k8s.ts +++ b/src/operator/k8s.ts @@ -39,36 +39,21 @@ export function updateHeartbeatFile(): void { writeFileSync('/tmp/heartbeat', '') } -/** - * Marker file that signals platform installation has completed. The operator - * readinessProbe gates on its existence, so the apl-operator Deployment only - * becomes Available once the helmfile pipeline has actually converged — not - * merely once the operator process is up. That makes `helm install --wait` and - * `kubectl wait --for=condition=Available deployment/apl-operator` meaningful - * gates for bootstrap automation. - * - * The marker lives on the pod's /tmp emptyDir, so it is cleared on every - * restart and re-created as soon as the operator re-confirms the installation - * status from the apl-installation-status ConfigMap. - */ export const READINESS_FILE = '/tmp/ready' /** - * Writes the readiness marker. Called once installation has reached the - * 'completed' state — including on restarts of an already-installed cluster. - * Readiness latches: it is intentionally NOT cleared while a subsequent apply - * runs, because steady-state reconcile loops must not flap the Deployment's - * Available condition. Per-apply status lives in the apl-operator-state - * ConfigMap instead. + * Idempotent, and safe to call on every apply. Readiness latches: the marker is never + * cleared while a later apply runs, because the steady-state reconcile loop would + * otherwise flap the Deployment's Available condition. Per-apply status lives in the + * apl-operator-state ConfigMap. */ -export function markInstallationComplete(filePath: string = READINESS_FILE): void { - const d = terminal('operator:k8s:markInstallationComplete') +export function markOperatorReady(filePath: string = READINESS_FILE): void { + const d = terminal('operator:k8s:markOperatorReady') try { writeFileSync(filePath, new Date().toISOString()) - d.info(`Installation complete, wrote readiness marker ${filePath}`) + d.info(`Wrote readiness marker ${filePath}`) } catch (error) { - // Deliberately non-fatal: a missing marker keeps the pod NotReady, which is - // the safe direction — it never reports convergence that did not happen. + // Non-fatal: a missing marker keeps the pod NotReady, which is the safe direction. d.warn(`Failed to write readiness marker ${filePath}:`, getErrorMessage(error)) } } diff --git a/src/operator/main.ts b/src/operator/main.ts index b811a82831..f746a4d243 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -10,7 +10,6 @@ import { AplOperations } from './apl-operations' import { AplOperator, AplOperatorConfig } from './apl-operator' import { GitRepository } from './git-repository' import { Installer } from './installer' -import { markInstallationComplete } from './k8s' import { getErrorMessage } from './utils' import { operatorEnv } from './validators' @@ -89,11 +88,6 @@ async function main(): Promise { await installer.reconcileInstall() } - // Every branch above only falls through once the installation has reached - // the 'completed' state, so this is the single point where the platform is - // known to be installed. Signal it to the readinessProbe. - markInstallationComplete() - // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) await installer.setEnvAndCreateSecrets() From 85d09a9e79bbe48685805792f5edc8be4b156b3d Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:20:05 -0400 Subject: [PATCH 4/5] fix(ci): close the fence when the job summary is truncated --- .github/workflows/svcaplbot-run-dyff.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/svcaplbot-run-dyff.yml b/.github/workflows/svcaplbot-run-dyff.yml index ea8eb52927..7ac75f6542 100644 --- a/.github/workflows/svcaplbot-run-dyff.yml +++ b/.github/workflows/svcaplbot-run-dyff.yml @@ -111,7 +111,13 @@ jobs: # Always publish the diff to the job summary. For pull requests from a # fork, secrets — and therefore BOT_TOKEN — are not available, so this # is the only channel that can carry the comparison. - head -c 900000 "$comment_file" >> "$GITHUB_STEP_SUMMARY" + # Truncating mid-diff would leave the ```diff fence unclosed and garble the + # rest of the summary, so close it explicitly and say the output was cut. + summary_limit=900000 + head -c "$summary_limit" "$comment_file" >> "$GITHUB_STEP_SUMMARY" + if [ "$(wc -c < "$comment_file")" -gt "$summary_limit" ]; then + printf '\n```\n\n_Output truncated at %s bytes._\n' "$summary_limit" >> "$GITHUB_STEP_SUMMARY" + fi if [ "${{ github.event_name }}" = "pull_request" ]; then if [ -z "$GH_TOKEN" ]; then From 83df2ff124e5198c4828d14e1f1a3dccdb92baaa Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:34:46 -0400 Subject: [PATCH 5/5] fix(operator): give the helmfile-managed deployment the same rollout budget Moving the ready mark to after the first apply means a recreated pod needs a full apply cycle before it reports Ready, which can outrun the 600s default on a large cluster and report ProgressDeadlineExceeded on a healthy operator. Mirrors the 1800 already set on chart/apl. Also states the marker's actual lifetime in EXECUTION_FLOW: it is on the pod's /tmp emptyDir, so it survives a container restart and is cleared only when the pod is recreated. --- charts/apl-operator/templates/deployment.yaml | 2 ++ src/operator/EXECUTION_FLOW.md | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index 7c10f2cc0c..38b80812c8 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -10,6 +10,8 @@ metadata: {{- end }} spec: replicas: {{ .Values.scale.replicas | default 1 }} + # The rollout stays Progressing until the operator is ready, which is longer than the 600s default. + progressDeadlineSeconds: 1800 selector: matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }} diff --git a/src/operator/EXECUTION_FLOW.md b/src/operator/EXECUTION_FLOW.md index e6e3469444..da640efc55 100644 --- a/src/operator/EXECUTION_FLOW.md +++ b/src/operator/EXECUTION_FLOW.md @@ -500,10 +500,13 @@ belongs to ArgoCD. Three properties are deliberate: -- **It latches.** Readiness is never cleared by a later apply. The reconcile loop - applies every ~5 minutes in steady state; flipping the Deployment out of `Available` - on each pass would make the condition useless as a gate. Per-apply status is - reported through the `apl-operator-state` ConfigMap instead (below). +- **It latches, for the life of the pod.** Readiness is never cleared by a later apply. + The reconcile loop applies every ~5 minutes in steady state; flipping the Deployment + out of `Available` on each pass would make the condition useless as a gate. Per-apply + status is reported through the `apl-operator-state` ConfigMap instead (below). The + marker lives on the pod's `/tmp` emptyDir, so it survives a container restart within + the pod and is only cleared when the pod itself is recreated — a rescheduled or + rolled-out pod goes NotReady until it completes an apply of its own. - **It fails closed.** If the marker cannot be written, or the apply keeps failing, the pod stays NotReady. The signal never claims progress that did not happen — `--wait` times out loudly rather than returning early.