diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..a28b57f --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,176 @@ +// Agentrax — Declarative Jenkins CI/CD Pipeline +// +// Stages: +// 1. Lint — golangci-lint + Helm chart lint (parallel) +// 2. Test — unit & integration tests via envtest +// 3. Docker Build — build image tagged with short Git SHA, push on main +// 4. Integration Test — deploy to kind test namespace, assert reconciliation +// 5. Helm Deploy — manual approval gate then helm upgrade --install +// +// Requirements: +// - Jenkins agent with label 'docker' and Docker-in-Docker socket access +// - Credentials: GHCR_USER (string), GHCR_TOKEN (secret text) +// - Jenkins Slack plugin configured for Slack notifications + +pipeline { + agent { label 'docker' } + + environment { + // Go workspace inside the Jenkins workspace to avoid polluting $HOME + GOPATH = "${WORKSPACE}/.go" + GOMODCACHE = "${WORKSPACE}/.go/pkg/mod" + // Image tag is the short Git SHA for traceability + IMAGE_TAG = "${env.GIT_COMMIT?.take(8) ?: env.BUILD_NUMBER}" + IMAGE = "ghcr.io/gitcommitankit/agentrax:${IMAGE_TAG}" + // Kubernetes namespace used exclusively for integration testing + TEST_NS = "agentrax-jenkins-test" + // Kind cluster and kubectl context for integration testing + KIND_CLUSTER = "${env.KIND_CLUSTER ?: 'agentrax-dev'}" + KUBE_CONTEXT = "${env.KUBE_CONTEXT ?: 'kind-agentrax-dev'}" + } + + options { + // Abort if the full pipeline exceeds 45 minutes + timeout(time: 45, unit: 'MINUTES') + // Keep the last 10 build logs; discard older ones to save disk space + buildDiscarder(logRotator(numToKeepStr: '10')) + // Prevent concurrent builds on the same branch to avoid races on the + // shared kind cluster used in Stage 4 + disableConcurrentBuilds() + ansiColor('xterm') + } + + stages { + + // ----------------------------------------------------------------------- + // Stage 1: Lint + // Runs golangci-lint and Helm chart lint in parallel. + // ----------------------------------------------------------------------- + stage('Lint') { + parallel { + stage('Go Lint') { + steps { + sh 'make golangci-lint' + sh 'make lint' + } + } + stage('Helm Lint') { + steps { + sh 'helm lint charts/agentrax/' + sh 'helm template test charts/agentrax/ --debug > /dev/null' + } + } + } + } + + // ----------------------------------------------------------------------- + // Stage 2: Test + // Runs the full unit + envtest integration test suite. + // ----------------------------------------------------------------------- + stage('Test') { + steps { + sh 'make envtest' + sh 'make test' + } + post { + always { + archiveArtifacts artifacts: 'cover.out', allowEmptyArchive: true + } + } + } + + // ----------------------------------------------------------------------- + // Stage 3: Docker Build + // Builds the operator image. Pushes to GHCR only on the main branch. + // Credentials are bound strictly within the main-branch push path. + // ----------------------------------------------------------------------- + stage('Docker Build') { + steps { + sh "make docker-build IMG=${IMAGE}" + script { + if (env.BRANCH_NAME == 'main') { + withCredentials([ + string(credentialsId: 'GHCR_USER', variable: 'GHCR_USER'), + string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_TOKEN'), + ]) { + sh 'echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin' + sh "make docker-push IMG=${IMAGE}" + } + } else { + echo "Feature branch — image built but not pushed (branch=${env.BRANCH_NAME})." + } + } + } + } + + // ----------------------------------------------------------------------- + // Stage 4: Integration Test (Agentrax-specific stage) + // + // 1. Installs cert-manager, Prometheus Operator CRDs, and Gateway API CRDs + // via `make deploy-deps` (idempotent). + // 2. Loads the locally built image into the kind cluster if kind is present. + // 3. Deploys the operator into the cluster with the newly built image. + // 4. Runs hack/assert-reconciliation.sh — polls until a sample + // AgentDeployment reaches status.phase == Running (60 s timeout). + // + // In post.always, the test namespace is deleted FIRST while the operator + // is still running so finalizers (agentrax.io/mcp-deregister) can execute + // cleanly, followed by `make undeploy`. + // ----------------------------------------------------------------------- + stage('Integration Test') { + steps { + sh "make deploy-deps KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\"" + sh "kind load docker-image ${IMAGE} --name ${KIND_CLUSTER}" + sh "make deploy IMG=${IMAGE} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\"" + sh "TEST_NS=${TEST_NS} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" ./hack/assert-reconciliation.sh" + } + post { + always { + sh "kubectl --context ${KUBE_CONTEXT} delete namespace ${TEST_NS} --ignore-not-found=true" + sh "make undeploy KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" || true" + } + } + } + + // ----------------------------------------------------------------------- + // Stage 5: Helm Deploy + // + // Runs only on the main branch. Requires explicit approval from the + // ops-team before mutating the production cluster. --atomic ensures Helm + // rolls back automatically if any post-install hook fails. + // ----------------------------------------------------------------------- + stage('Helm Deploy') { + when { branch 'main' } + input { + message "Deploy agentrax:${IMAGE_TAG} to production cluster?" + ok 'Approve' + submitter 'ops-team' + } + steps { + sh """ + helm upgrade --install agentrax charts/agentrax/ \ + --namespace agentrax-system \ + --create-namespace \ + --set image.tag=${IMAGE_TAG} \ + --atomic \ + --timeout 5m + """ + } + } + } + + post { + failure { + slackSend( + color: 'danger', + message: "❌ Agentrax build #${BUILD_NUMBER} FAILED on \`${BRANCH_NAME}\`: ${BUILD_URL}", + ) + } + success { + slackSend( + color: 'good', + message: "✅ Agentrax build #${BUILD_NUMBER} passed on \`${BRANCH_NAME}\` (image: \`${IMAGE_TAG}\`)", + ) + } + } +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 839166f..8def929 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -435,6 +435,45 @@ The `.github/workflows/terraform-lint.yml` workflow runs on every PR touching `i --- +### 4.9 Jenkins CI/CD Pipeline + +Agentrax ships a declarative `Jenkinsfile` at the repository root that **complements** (not replaces) the GitHub Actions workflows. The Jenkinsfile is intended for teams running Jenkins on-prem or as a learning vehicle for a multi-stage CI/CD pipeline with a cluster-level integration test gate. + +#### Stage Topology + +``` +Lint (Go + Helm, parallel) + │ + Test (unit + envtest) + │ + Docker Build (push on main only) + │ + Integration Test ← Agentrax-specific + │ make deploy-deps (cert-manager, Prometheus Operator, Gateway API CRDs) + │ make deploy (operator into cluster) + │ hack/assert-reconciliation.sh (poll status.phase == Running) + │ [post.always] make undeploy + │ + Helm Deploy (main branch + manual approval gate) +``` + +#### Agentrax-Specific: Stage 4 — Integration Test + +This stage distinguishes the Jenkins pipeline from the GitHub Actions `ci.yml`. It: + +1. Installs all cluster dependencies via `make deploy-deps` (idempotent). +2. Deploys the operator image built in Stage 3. +3. Applies `hack/testdata/sample-agentdeployment.yaml` and runs `hack/assert-reconciliation.sh`, which polls `status.phase` every 3 seconds until `Running` (60-second timeout). A `RolloutFailed` or `Degraded` terminal phase exits immediately with a non-zero code, failing the stage. +4. In `post { always }`, deletes `TEST_NS` first while the operator is still active so finalizers (`agentrax.io/mcp-deregister`) process cleanly, then runs `make undeploy`. + +#### Safety Properties + +- **`disableConcurrentBuilds()`**: Prevents race conditions on the shared `kind` cluster between simultaneous branch builds. +- **`--atomic` Helm flag**: Helm rolls back automatically if any hook fails during Stage 5. +- **`submitter 'ops-team'`**: Only members of the `ops-team` Jenkins group can approve production deployments. + +--- + ## 5. Architectural Decision Records (ADRs) & Trade-Offs | Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | diff --git a/hack/assert-reconciliation.sh b/hack/assert-reconciliation.sh new file mode 100755 index 0000000..900f326 --- /dev/null +++ b/hack/assert-reconciliation.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# hack/assert-reconciliation.sh +# +# Polls until a sample AgentDeployment in the integration-test namespace +# reaches status.phase == Running, then exits 0. +# Exits non-zero (and fails the Jenkins stage) if the deadline is exceeded. +# +# Environment variables (all optional — defaults shown): +# TEST_NS Namespace to apply the sample manifest into (default: agentrax-jenkins-test) +# TIMEOUT_SEC Maximum seconds to wait for Running phase (default: 60) +# KUBECTL kubectl binary to use (default: kubectl) + +set -euo pipefail + +: "${TEST_NS:=agentrax-jenkins-test}" +: "${TIMEOUT_SEC:=60}" +: "${KUBECTL:=kubectl}" + +SAMPLE_MANIFEST="$(dirname "$0")/testdata/sample-agentdeployment.yaml" +POLL_INTERVAL=3 + +# --------------------------------------------------------------------------- +# 1. Ensure the test namespace exists +# --------------------------------------------------------------------------- +echo "[assert-reconciliation] Ensuring namespace '${TEST_NS}' exists..." +${KUBECTL} create namespace "${TEST_NS}" --dry-run=client -o yaml | ${KUBECTL} apply -f - + +# --------------------------------------------------------------------------- +# 2. Apply the sample AgentDeployment +# --------------------------------------------------------------------------- +if [[ ! -f "${SAMPLE_MANIFEST}" ]]; then + echo "[assert-reconciliation] ERROR: sample manifest not found at ${SAMPLE_MANIFEST}" >&2 + exit 1 +fi + +echo "[assert-reconciliation] Applying sample AgentDeployment from ${SAMPLE_MANIFEST}..." +${KUBECTL} apply -f "${SAMPLE_MANIFEST}" -n "${TEST_NS}" + +# --------------------------------------------------------------------------- +# 3. Poll until status.phase == Running +# --------------------------------------------------------------------------- +AD_NAME=$(${KUBECTL} get agentdeployment -n "${TEST_NS}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +if [[ -z "${AD_NAME}" ]]; then + echo "[assert-reconciliation] ERROR: no AgentDeployment found in namespace '${TEST_NS}'" >&2 + exit 1 +fi + +echo "[assert-reconciliation] Waiting up to ${TIMEOUT_SEC}s for AgentDeployment '${AD_NAME}' to reach Running..." + +elapsed=0 +while true; do + phase=$(${KUBECTL} get agentdeployment "${AD_NAME}" -n "${TEST_NS}" \ + -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + + echo "[assert-reconciliation] t=${elapsed}s status.phase=${phase}" + + if [[ "${phase}" == "Running" ]]; then + echo "[assert-reconciliation] ✅ AgentDeployment '${AD_NAME}' reached Running in ${elapsed}s." + exit 0 + fi + + if [[ "${phase}" == "RolloutFailed" || "${phase}" == "Degraded" ]]; then + echo "[assert-reconciliation] ❌ Terminal failure phase '${phase}' detected." >&2 + ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2 + exit 1 + fi + + if (( elapsed >= TIMEOUT_SEC )); then + echo "[assert-reconciliation] ❌ Timeout after ${TIMEOUT_SEC}s — last phase: '${phase}'" >&2 + echo "[assert-reconciliation] --- describe output ---" >&2 + ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2 + exit 1 + fi + + sleep "${POLL_INTERVAL}" + elapsed=$(( elapsed + POLL_INTERVAL )) +done diff --git a/hack/testdata/sample-agentdeployment.yaml b/hack/testdata/sample-agentdeployment.yaml new file mode 100644 index 0000000..a2a1d59 --- /dev/null +++ b/hack/testdata/sample-agentdeployment.yaml @@ -0,0 +1,21 @@ +apiVersion: agentrax.io/v1alpha1 +kind: AgentDeployment +metadata: + name: jenkins-smoke-test + # Namespace is set by assert-reconciliation.sh at apply time (-n ${TEST_NS}) + labels: + app.kubernetes.io/managed-by: jenkins-ci + agentrax.io/agent: "true" +spec: + # Minimal agent that exercises the full reconcile loop: + # Deployment + Service + ServiceMonitor + HPA creation. + image: "ghcr.io/gitcommitankit/agentrax-agent:latest" + replicas: 1 + tenant: jenkins-test + resources: + limits: + cpu: "250m" + memory: "256Mi" + requests: + cpu: "50m" + memory: "64Mi"