feat(operator): gate readiness on installation completion

.github/workflows/svcaplbot-run-dyff.yml

@@ -107,6 +107,16 @@ jobs:
           echo '```diff' >> "$comment_file"
           cat "$GITHUB_WORKSPACE/pr/tmp/diff-output.txt" >> "$comment_file"
           echo '```' >> "$comment_file"
+
+          # Always publish the diff to the job summary. For pull requests from a
+          # fork, secrets — and therefore BOT_TOKEN — are not available, so this
+          # is the only channel that can carry the comparison.
+          head -c 900000 "$comment_file" >> "$GITHUB_STEP_SUMMARY"
+
           if [ "${{ github.event_name }}" = "pull_request" ]; then
+            if [ -z "$GH_TOKEN" ]; then
+              echo "::notice::No BOT_TOKEN available (pull request from a fork) — comparison published to the job summary instead of a PR comment."
+              exit 0
+            fi
             gh pr comment ${{ github.event.pull_request.number }} --body-file "$comment_file" --create-if-none --edit-last
           fi

chart/apl/templates/NOTES.txt

@@ -1,5 +1,15 @@
 The App Platform Operator has been successfully deployed on the cluster.
 
 Please inspect the output of the apl-operator deployment (apl-operator/{{ include "apl-operator.fullname" . }}) for any feedback or errors.
+{{ if ne ((.Values.operator.readiness | default dict).gateOnInstallationComplete) false }}
+Installing the platform takes 10-15 minutes. The operator reports Ready only once
+installation has completed, so you can wait for it:
+
+  kubectl wait --for=condition=Available deployment/{{ include "apl-operator.fullname" . }} -n apl-operator --timeout=30m
+
+Progress is readable at any time from:
+
+  kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}'
+{{ end }}
 
 Also visit https://techdocs.akamai.com/app-platform/ for further instructions and reference documentation.

chart/apl/templates/deployment.yaml

@@ -1,6 +1,8 @@
 {{- $kms := .Values.kms | default dict }}
 {{- $version := .Values.otomi.version | default .Chart.AppVersion }}
 {{- $skipDeployment := .Values.installation.skipOperatorDeployment }}
+{{- $readiness := .Values.operator.readiness | default dict }}
+{{- $gateOnInstall := ne $readiness.gateOnInstallationComplete false }}
 {{- if not $skipDeployment }}
 apiVersion: apps/v1
 kind: Deployment
@@ -10,6 +12,11 @@ metadata:
   labels: {{- include "apl-operator.labels" . | nindent 4 }}
 spec:
   replicas: 1
+  # Installing the platform takes considerably longer than the 600s default, and
+  # with the readinessProbe gated on convergence the rollout stays Progressing
+  # for that whole window. Without this, `kubectl rollout status` reports
+  # ProgressDeadlineExceeded on a perfectly healthy first install.
+  progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds | default 3600 }}
   selector:
     matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }}
   strategy:
@@ -90,11 +97,25 @@ spec:
             failureThreshold: 3
           readinessProbe:
             exec:
+              {{- if $gateOnInstall }}
+              # Convergence signal: /tmp/ready is written by the operator once the
+              # platform installation reaches the 'completed' state, so the
+              # Deployment only goes Available when the helmfile pipeline has
+              # actually converged. This is what makes `helm install --wait` and
+              # `kubectl wait --for=condition=Available` usable as bootstrap gates.
+              # NOTE: a first install takes 10-15 minutes — size --timeout accordingly.
+              command: ["/bin/sh", "-c", "test -f /tmp/ready"]
+              {{- else }}
+              # Liveness-equivalent readiness: reports Ready as soon as the operator
+              # process is up, long before the platform has converged.
               command: ["/bin/sh", "-c", "pgrep -f 'apl-operator' > /dev/null"]
+              {{- end }}
             initialDelaySeconds: 30
             periodSeconds: 10
             timeoutSeconds: 5
-            failureThreshold: 3
+            # Readiness latches once installation completes; a transient probe
+            # failure should not take the Deployment out of Available.
+            failureThreshold: {{ $readiness.failureThreshold | default 3 }}
       volumes:
         - name: values-secret
           secret:

chart/apl/values.yaml

@@ -129,6 +129,20 @@ operator:
   installRetries: 1000
   installMaxTimeoutMs: 10000
 
+  readiness:
+    # When true (default), the operator only reports Ready once the platform
+    # installation has completed, making the apl-operator Deployment a truthful
+    # "platform converged" signal for `helm install --wait` and `kubectl wait`.
+    # A first install takes 10-15 minutes, so size `helm --timeout` accordingly
+    # (e.g. `--wait --timeout 30m`).
+    # Set to false to restore the previous process-liveness readiness, which
+    # reports Ready within a minute regardless of installation progress.
+    gateOnInstallationComplete: true
+    # Rollout budget for the install window. Only relevant when the readiness
+    # gate is on — `kubectl rollout status` fails once it is exceeded.
+    progressDeadlineSeconds: 3600
+    failureThreshold: 3
+
   image:
     repository: "mirror.registry.linodelke.net/docker/linode/apl-core"
 

charts/apl-operator/templates/deployment.yaml

@@ -73,6 +73,22 @@ spec:
             periodSeconds: 60
             failureThreshold: 3
             timeoutSeconds: 10
+          {{- if ne (.Values.operator.readiness | default dict).gateOnInstallationComplete false }}
+          # /tmp/ready is written by the operator once the platform installation
+          # reaches the 'completed' state — the Deployment reports Available only
+          # after the helmfile pipeline has converged, not merely once the process
+          # is up. Readiness latches, so steady-state reconciles do not flap it.
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - "test -f /tmp/ready"
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
+            timeoutSeconds: 5
+          {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

charts/apl-operator/values.yaml

@@ -60,6 +60,10 @@ operator:
   gitOpTimeoutMs: 10000
   installRetries: 1000
   installMaxTimeoutMs: 10000
+  readiness:
+    # When true (default), the operator reports Ready only after the platform
+    # installation has completed. Set to false to drop the readinessProbe.
+    gateOnInstallationComplete: true
 
 nodeSelector: {}
 

src/operator/EXECUTION_FLOW.md

@@ -473,6 +473,71 @@ Shared by both loops with trigger-specific variations:
 8. Update apply state to 'succeeded' or 'failed'
 9. Release lock (`isApplying = false`)
 
+## Readiness and Convergence Contract
+
+Bootstrap automation needs a machine-checkable answer to "is the platform installed
+yet?". The operator exposes it through the readiness of its own Deployment.
+
+### The gate
+
+The operator writes `/tmp/ready` (`markInstallationComplete()`) at exactly one point:
+after the installation phase resolves to `completed` — whether that came from a fresh
+install, a recovery install, or a restart on an already-installed cluster. The
+`readinessProbe` on the apl-operator Deployment tests for that file, so:
+
+```bash
+# blocks until the helmfile pipeline has actually converged
+kubectl wait --for=condition=Available deployment/apl-operator -n apl-operator --timeout=30m
+
+# same signal, via helm
+helm install apl … --wait --timeout 30m
+```
+
+Three properties are deliberate:
+
+- **It latches.** Readiness is never cleared while a later apply runs. The reconcile
+  loop applies every ~5 minutes in steady state; flipping the Deployment out of
+  `Available` on each pass would make the condition useless as a gate. Per-apply
+  status is reported through the `apl-operator-state` ConfigMap instead (below).
+- **It fails closed.** If the marker cannot be written, or installation keeps
+  retrying, the pod stays NotReady. The signal never claims a convergence that did
+  not happen — `--wait` times out loudly rather than returning early.
+- **A first install takes 10-15 minutes.** Size `--timeout` accordingly; the
+  Deployment's `progressDeadlineSeconds` is raised to 3600 so `kubectl rollout
+  status` does not report `ProgressDeadlineExceeded` on a healthy install.
+
+Set `operator.readiness.gateOnInstallationComplete=false` to restore the previous
+behaviour, where readiness only reflected that the operator process was running.
+
+### Introspection
+
+For phase detail rather than a binary gate, read the ConfigMaps in the table below:
+
+```bash
+# installation phase: pending | in-progress | completed | failed (+ attempt, timestamp)
+kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}'
+
+# last apply: commitHash, status, timestamp, trigger, errorMessage
+kubectl get cm apl-operator-state -n apl-operator -o jsonpath='{.data.state}'
+```
+
+`apl-operator-state.commitHash` is the answer to "did the operator apply *my* commit
+yet?" — poll for `status: succeeded` at the revision you pushed.
+
+### What this is not
+
+The Deployment gate covers the operator's own pipeline: essential manifests, CRDs,
+`stage=prep`, `app=core`, and the ArgoCD Applications for the remaining apps. Apps
+that ArgoCD syncs afterwards report health through ArgoCD, not through this gate.
+
+An end-to-end smoke check that the platform is externally serving is
+`https://auth.<domainSuffix>/ready` (oauth2-proxy behind the ingress). It exercises
+DNS, ingress-nginx, the TLS certificate and the auth chain, which the in-cluster
+gate does not. It is complementary, not a substitute: it needs public DNS and a
+trusted certificate, it cannot tell you *which* revision of your values converged,
+and a non-200 cannot distinguish "platform not ready" from a DNS or certificate
+problem.
+
 ## Kubernetes Resources
 
 ### ConfigMaps

src/operator/k8s.test.ts

@@ -1,5 +1,8 @@
-import { ApplyState, updateApplyState } from './k8s'
+import { ApplyState, markInstallationComplete, READINESS_FILE, updateApplyState } from './k8s'
 import { CoreV1Api, ApiException } from '@kubernetes/client-node'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
 
 jest.mock('@kubernetes/client-node', () => {
   const mocks = {
@@ -38,6 +41,7 @@ jest.mock('../common/debug', () => ({
   terminal: jest.fn().mockImplementation(() => ({
     info: jest.fn(),
     error: jest.fn(),
+    warn: jest.fn(),
   })),
 }))
 
@@ -173,3 +177,44 @@ describe('updateApplyState', () => {
     expect(mockCoreV1Api.createNamespacedConfigMap).not.toHaveBeenCalled()
   })
 })
+
+describe('markInstallationComplete', () => {
+  let workDir: string
+
+  beforeEach(() => {
+    workDir = mkdtempSync(join(tmpdir(), 'apl-readiness-'))
+  })
+
+  afterEach(() => {
+    rmSync(workDir, { recursive: true, force: true })
+  })
+
+  test('defaults to the path the readinessProbe checks', () => {
+    expect(READINESS_FILE).toBe('/tmp/ready')
+  })
+
+  test('writes the readiness marker with a timestamp', () => {
+    const marker = join(workDir, 'ready')
+
+    markInstallationComplete(marker)
+
+    expect(existsSync(marker)).toBe(true)
+    expect(Date.parse(readFileSync(marker, 'utf8'))).not.toBeNaN()
+  })
+
+  test('is idempotent — a restart of an installed cluster re-marks readiness', () => {
+    const marker = join(workDir, 'ready')
+
+    markInstallationComplete(marker)
+    markInstallationComplete(marker)
+
+    expect(existsSync(marker)).toBe(true)
+  })
+
+  test('never throws when the marker cannot be written, leaving the pod NotReady', () => {
+    const unwritable = join(workDir, 'does', 'not', 'exist', 'ready')
+
+    expect(() => markInstallationComplete(unwritable)).not.toThrow()
+    expect(existsSync(unwritable)).toBe(false)
+  })
+})

src/operator/k8s.ts

@@ -39,6 +39,40 @@ export function updateHeartbeatFile(): void {
   writeFileSync('/tmp/heartbeat', '')
 }
 
+/**
+ * Marker file that signals platform installation has completed. The operator
+ * readinessProbe gates on its existence, so the apl-operator Deployment only
+ * becomes Available once the helmfile pipeline has actually converged — not
+ * merely once the operator process is up. That makes `helm install --wait` and
+ * `kubectl wait --for=condition=Available deployment/apl-operator` meaningful
+ * gates for bootstrap automation.
+ *
+ * The marker lives on the pod's /tmp emptyDir, so it is cleared on every
+ * restart and re-created as soon as the operator re-confirms the installation
+ * status from the apl-installation-status ConfigMap.
+ */
+export const READINESS_FILE = '/tmp/ready'
+
+/**
+ * Writes the readiness marker. Called once installation has reached the
+ * 'completed' state — including on restarts of an already-installed cluster.
+ * Readiness latches: it is intentionally NOT cleared while a subsequent apply
+ * runs, because steady-state reconcile loops must not flap the Deployment's
+ * Available condition. Per-apply status lives in the apl-operator-state
+ * ConfigMap instead.
+ */
+export function markInstallationComplete(filePath: string = READINESS_FILE): void {
+  const d = terminal('operator:k8s:markInstallationComplete')
+  try {
+    writeFileSync(filePath, new Date().toISOString())
+    d.info(`Installation complete, wrote readiness marker ${filePath}`)
+  } catch (error) {
+    // Deliberately non-fatal: a missing marker keeps the pod NotReady, which is
+    // the safe direction — it never reports convergence that did not happen.
+    d.warn(`Failed to write readiness marker ${filePath}:`, getErrorMessage(error))
+  }
+}
+
 export async function updateApplyState(
   state: ApplyState,
   namespace: string = APL_OPERATOR_NS,

src/operator/main.ts

@@ -10,6 +10,7 @@ import { AplOperations } from './apl-operations'
 import { AplOperator, AplOperatorConfig } from './apl-operator'
 import { GitRepository } from './git-repository'
 import { Installer } from './installer'
+import { markInstallationComplete } from './k8s'
 import { getErrorMessage } from './utils'
 import { operatorEnv } from './validators'
 
@@ -88,6 +89,11 @@ async function main(): Promise<void> {
       await installer.reconcileInstall()
     }
 
+    // Every branch above only falls through once the installation has reached
+    // the 'completed' state, so this is the single point where the platform is
+    // known to be installed. Signal it to the readinessProbe.
+    markInstallationComplete()
+
     // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use)
     await installer.setEnvAndCreateSecrets()