diff --git a/.github/workflows/svcaplbot-run-dyff.yml b/.github/workflows/svcaplbot-run-dyff.yml index 04e2100a07..7ac75f6542 100644 --- a/.github/workflows/svcaplbot-run-dyff.yml +++ b/.github/workflows/svcaplbot-run-dyff.yml @@ -107,6 +107,22 @@ jobs: echo '```diff' >> "$comment_file" cat "$GITHUB_WORKSPACE/pr/tmp/diff-output.txt" >> "$comment_file" echo '```' >> "$comment_file" + + # Always publish the diff to the job summary. For pull requests from a + # fork, secrets — and therefore BOT_TOKEN — are not available, so this + # is the only channel that can carry the comparison. + # Truncating mid-diff would leave the ```diff fence unclosed and garble the + # rest of the summary, so close it explicitly and say the output was cut. + summary_limit=900000 + head -c "$summary_limit" "$comment_file" >> "$GITHUB_STEP_SUMMARY" + if [ "$(wc -c < "$comment_file")" -gt "$summary_limit" ]; then + printf '\n```\n\n_Output truncated at %s bytes._\n' "$summary_limit" >> "$GITHUB_STEP_SUMMARY" + fi + if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ -z "$GH_TOKEN" ]; then + echo "::notice::No BOT_TOKEN available (pull request from a fork) — comparison published to the job summary instead of a PR comment." + exit 0 + fi gh pr comment ${{ github.event.pull_request.number }} --body-file "$comment_file" --create-if-none --edit-last fi diff --git a/chart/apl/templates/NOTES.txt b/chart/apl/templates/NOTES.txt index 569365d68c..8efcf3bbab 100644 --- a/chart/apl/templates/NOTES.txt +++ b/chart/apl/templates/NOTES.txt @@ -2,4 +2,13 @@ The App Platform Operator has been successfully deployed on the cluster. Please inspect the output of the apl-operator deployment (apl-operator/{{ include "apl-operator.fullname" . }}) for any feedback or errors. +Installing the platform takes 10-15 minutes. The operator reports Ready only once +installation has completed, so you can wait for it: + + kubectl wait --for=condition=Available deployment/{{ include "apl-operator.fullname" . }} -n apl-operator --timeout=30m + +Progress is readable at any time from: + + kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}' + Also visit https://techdocs.akamai.com/app-platform/ for further instructions and reference documentation. diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 448a484357..c147a44fc3 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -1,6 +1,7 @@ {{- $kms := .Values.kms | default dict }} {{- $version := .Values.otomi.version | default .Chart.AppVersion }} {{- $skipDeployment := .Values.installation.skipOperatorDeployment }} +{{- $readiness := .Values.operator.readiness }} {{- if not $skipDeployment }} apiVersion: apps/v1 kind: Deployment @@ -10,6 +11,8 @@ metadata: labels: {{- include "apl-operator.labels" . | nindent 4 }} spec: replicas: 1 + # The rollout stays Progressing until the operator is ready, which is longer than the 600s default. + progressDeadlineSeconds: {{ $readiness.progressDeadlineSeconds }} selector: matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }} strategy: @@ -90,7 +93,7 @@ spec: failureThreshold: 3 readinessProbe: exec: - command: ["/bin/sh", "-c", "pgrep -f 'apl-operator' > /dev/null"] + command: ["/bin/sh", "-c", "test -f /tmp/ready"] initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 diff --git a/chart/apl/values.yaml b/chart/apl/values.yaml index 25932348b6..d59b57975f 100644 --- a/chart/apl/values.yaml +++ b/chart/apl/values.yaml @@ -129,6 +129,10 @@ operator: installRetries: 1000 installMaxTimeoutMs: 10000 + readiness: + # 30 minutes. If the operator has not installed the platform by then, something is off. + progressDeadlineSeconds: 1800 + image: repository: "mirror.registry.linodelke.net/docker/linode/apl-core" diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index b9322ffd2e..38b80812c8 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -10,6 +10,8 @@ metadata: {{- end }} spec: replicas: {{ .Values.scale.replicas | default 1 }} + # The rollout stays Progressing until the operator is ready, which is longer than the 600s default. + progressDeadlineSeconds: 1800 selector: matchLabels: {{- include "apl-operator.selectorLabels" . | nindent 6 }} @@ -73,6 +75,16 @@ spec: periodSeconds: 60 failureThreshold: 3 timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - "test -f /tmp/ready" + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: diff --git a/src/operator/EXECUTION_FLOW.md b/src/operator/EXECUTION_FLOW.md index 355ed076b1..da640efc55 100644 --- a/src/operator/EXECUTION_FLOW.md +++ b/src/operator/EXECUTION_FLOW.md @@ -473,6 +473,77 @@ Shared by both loops with trigger-specific variations: 8. Update apply state to 'succeeded' or 'failed' 9. Release lock (`isApplying = false`) +## Readiness and Convergence Contract + +Bootstrap automation needs a machine-checkable answer to "has the operator finished +its job yet?". The operator exposes it through the readiness of its own Deployment. + +### The gate + +The operator writes `/tmp/ready` (`markOperatorReady()`) at exactly one point: after +an apply run completes successfully. That run is what creates the ArgoCD Applications, +so past it the platform can heal itself through ArgoCD. The `readinessProbe` on the +apl-operator Deployment tests for that file, so: + +```bash +# blocks until the operator has completed an apply run +kubectl wait --for=condition=Available deployment/apl-operator -n apl-operator --timeout=30m + +# same signal, via helm +helm install apl … --wait --timeout 30m +``` + +This is **not** the same as "the platform is fully up". When the operator reports +Ready, ArgoCD is still working through the Applications it was just handed. The gate +says the operator is finished and its reconcile loop has started — from there, health +belongs to ArgoCD. + +Three properties are deliberate: + +- **It latches, for the life of the pod.** Readiness is never cleared by a later apply. + The reconcile loop applies every ~5 minutes in steady state; flipping the Deployment + out of `Available` on each pass would make the condition useless as a gate. Per-apply + status is reported through the `apl-operator-state` ConfigMap instead (below). The + marker lives on the pod's `/tmp` emptyDir, so it survives a container restart within + the pod and is only cleared when the pod itself is recreated — a rescheduled or + rolled-out pod goes NotReady until it completes an apply of its own. +- **It fails closed.** If the marker cannot be written, or the apply keeps failing, + the pod stays NotReady. The signal never claims progress that did not happen — + `--wait` times out loudly rather than returning early. +- **A first install takes 10-15 minutes.** Size `--timeout` accordingly; the + Deployment's `progressDeadlineSeconds` is raised to 1800 so `kubectl rollout + status` does not report `ProgressDeadlineExceeded` on a healthy install. + +### Introspection + +For phase detail rather than a binary gate, read the ConfigMaps in the table below: + +```bash +# installation phase: pending | in-progress | completed | failed (+ attempt, timestamp) +kubectl get cm apl-installation-status -n apl-operator -o jsonpath='{.data.status}' + +# last apply: commitHash, status, timestamp, trigger, errorMessage +kubectl get cm apl-operator-state -n apl-operator -o jsonpath='{.data.state}' +``` + +`apl-operator-state.commitHash` is the answer to "did the operator apply *my* commit +yet?" — poll for `status: succeeded` at the revision you pushed. + +### What this is not + +The Deployment gate covers the operator's own pipeline: essential manifests, CRDs, +`stage=prep`, `app=core`, and the creation of the ArgoCD Applications for the +remaining apps. Whether those Applications have actually synced and gone Healthy is +ArgoCD's business, not this gate's. + +An end-to-end smoke check that the platform is externally serving is +`https://auth./ready` (oauth2-proxy behind the ingress). It exercises +DNS, ingress-nginx, the TLS certificate and the auth chain, which the in-cluster +gate does not. It is complementary, not a substitute: it needs public DNS and a +trusted certificate, it cannot tell you *which* revision of your values converged, +and a non-200 cannot distinguish "platform not ready" from a DNS or certificate +problem. + ## Kubernetes Resources ### ConfigMaps diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index f6a2b0dca2..0281a07f7b 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -3,7 +3,7 @@ import { waitTillGitRepoAvailable } from '../common/gitea' import { AplOperations } from './apl-operations' import { AplOperator, AplOperatorConfig, ApplyTrigger } from './apl-operator' import { GitRepository } from './git-repository' -import { updateApplyState } from './k8s' +import { markOperatorReady, updateApplyState } from './k8s' const mockInfoFn = jest.fn() const mockWarnFn = jest.fn() @@ -61,6 +61,7 @@ jest.mock('../cmd/commit', () => ({ jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), appRevisionMatches: jest.fn().mockResolvedValue(true), + markOperatorReady: jest.fn(), })) jest.mock('./git-repository', () => ({ @@ -210,6 +211,7 @@ describe('AplOperator', () => { }), ) + expect(markOperatorReady).toHaveBeenCalled() expect((aplOperator as any).isApplying).toBe(false) }) @@ -261,6 +263,8 @@ describe('AplOperator', () => { }), ) + // A failed apply leaves the ArgoCD Applications unaccounted for — the pod must stay NotReady. + expect(markOperatorReady).not.toHaveBeenCalled() expect((aplOperator as any).isApplying).toBe(false) expect(mockErrorFn).toHaveBeenCalledWith('[poll] Apply process failed', 'Apply failed') diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 1da973789d..e3653944fa 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -9,7 +9,7 @@ import { ensureManifestDirectories, ensureTeamGitOpsDirectories } from '../commo import { getDefaultValues, writeValues } from '../common/values' import { AplOperations } from './apl-operations' import { GitRepository } from './git-repository' -import { updateApplyState } from './k8s' +import { markOperatorReady, updateApplyState } from './k8s' import { getErrorMessage } from './utils' export interface AplOperatorConfig { @@ -99,6 +99,11 @@ export class AplOperator { this.d.info(`[${trigger}] Apply process completed`) + // The apply run above is what creates the ArgoCD Applications, so from here on the + // platform can heal itself through ArgoCD. That — not the end of the helmfile install + // — is what the operator being 'ready' means. + markOperatorReady() + await updateApplyState({ commitHash, status: 'succeeded', diff --git a/src/operator/k8s.test.ts b/src/operator/k8s.test.ts index d004691dd2..9c5b50317c 100644 --- a/src/operator/k8s.test.ts +++ b/src/operator/k8s.test.ts @@ -1,5 +1,8 @@ -import { ApplyState, updateApplyState } from './k8s' +import { ApplyState, markOperatorReady, READINESS_FILE, updateApplyState } from './k8s' import { CoreV1Api, ApiException } from '@kubernetes/client-node' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' jest.mock('@kubernetes/client-node', () => { const mocks = { @@ -38,6 +41,7 @@ jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), error: jest.fn(), + warn: jest.fn(), })), })) @@ -173,3 +177,44 @@ describe('updateApplyState', () => { expect(mockCoreV1Api.createNamespacedConfigMap).not.toHaveBeenCalled() }) }) + +describe('markOperatorReady', () => { + let workDir: string + + beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), 'apl-readiness-')) + }) + + afterEach(() => { + rmSync(workDir, { recursive: true, force: true }) + }) + + test('defaults to the path the readinessProbe checks', () => { + expect(READINESS_FILE).toBe('/tmp/ready') + }) + + test('writes the readiness marker with a timestamp', () => { + const marker = join(workDir, 'ready') + + markOperatorReady(marker) + + expect(existsSync(marker)).toBe(true) + expect(Date.parse(readFileSync(marker, 'utf8'))).not.toBeNaN() + }) + + test('is idempotent — every apply run re-marks readiness', () => { + const marker = join(workDir, 'ready') + + markOperatorReady(marker) + markOperatorReady(marker) + + expect(existsSync(marker)).toBe(true) + }) + + test('never throws when the marker cannot be written, leaving the pod NotReady', () => { + const unwritable = join(workDir, 'does', 'not', 'exist', 'ready') + + expect(() => markOperatorReady(unwritable)).not.toThrow() + expect(existsSync(unwritable)).toBe(false) + }) +}) diff --git a/src/operator/k8s.ts b/src/operator/k8s.ts index 8d963e4331..b9350b7941 100644 --- a/src/operator/k8s.ts +++ b/src/operator/k8s.ts @@ -39,6 +39,25 @@ export function updateHeartbeatFile(): void { writeFileSync('/tmp/heartbeat', '') } +export const READINESS_FILE = '/tmp/ready' + +/** + * Idempotent, and safe to call on every apply. Readiness latches: the marker is never + * cleared while a later apply runs, because the steady-state reconcile loop would + * otherwise flap the Deployment's Available condition. Per-apply status lives in the + * apl-operator-state ConfigMap. + */ +export function markOperatorReady(filePath: string = READINESS_FILE): void { + const d = terminal('operator:k8s:markOperatorReady') + try { + writeFileSync(filePath, new Date().toISOString()) + d.info(`Wrote readiness marker ${filePath}`) + } catch (error) { + // Non-fatal: a missing marker keeps the pod NotReady, which is the safe direction. + d.warn(`Failed to write readiness marker ${filePath}:`, getErrorMessage(error)) + } +} + export async function updateApplyState( state: ApplyState, namespace: string = APL_OPERATOR_NS,