feat(jenkins): add 5-stage Jenkinsfile, assert-reconciliation.sh, and…

Jenkinsfile

@@ -0,0 +1,176 @@
+// Agentrax — Declarative Jenkins CI/CD Pipeline
+//
+// Stages:
+//   1. Lint            — golangci-lint + Helm chart lint (parallel)
+//   2. Test            — unit & integration tests via envtest
+//   3. Docker Build    — build image tagged with short Git SHA, push on main
+//   4. Integration Test — deploy to kind test namespace, assert reconciliation
+//   5. Helm Deploy     — manual approval gate then helm upgrade --install
+//
+// Requirements:
+//   - Jenkins agent with label 'docker' and Docker-in-Docker socket access
+//   - Credentials: GHCR_USER (string), GHCR_TOKEN (secret text)
+//   - Jenkins Slack plugin configured for Slack notifications
+
+pipeline {
+  agent { label 'docker' }
+
+  environment {
+    // Go workspace inside the Jenkins workspace to avoid polluting $HOME
+    GOPATH     = "${WORKSPACE}/.go"
+    GOMODCACHE = "${WORKSPACE}/.go/pkg/mod"
+    // Image tag is the short Git SHA for traceability
+    IMAGE_TAG  = "${env.GIT_COMMIT?.take(8) ?: env.BUILD_NUMBER}"
+    IMAGE      = "ghcr.io/gitcommitankit/agentrax:${IMAGE_TAG}"
+    // Kubernetes namespace used exclusively for integration testing
+    TEST_NS      = "agentrax-jenkins-test"
+    // Kind cluster and kubectl context for integration testing
+    KIND_CLUSTER = "${env.KIND_CLUSTER ?: 'agentrax-dev'}"
+    KUBE_CONTEXT = "${env.KUBE_CONTEXT ?: 'kind-agentrax-dev'}"
+  }
+
+  options {
+    // Abort if the full pipeline exceeds 45 minutes
+    timeout(time: 45, unit: 'MINUTES')
+    // Keep the last 10 build logs; discard older ones to save disk space
+    buildDiscarder(logRotator(numToKeepStr: '10'))
+    // Prevent concurrent builds on the same branch to avoid races on the
+    // shared kind cluster used in Stage 4
+    disableConcurrentBuilds()
+    ansiColor('xterm')
+  }
+
+  stages {
+
+    // -----------------------------------------------------------------------
+    // Stage 1: Lint
+    // Runs golangci-lint and Helm chart lint in parallel.
+    // -----------------------------------------------------------------------
+    stage('Lint') {
+      parallel {
+        stage('Go Lint') {
+          steps {
+            sh 'make golangci-lint'
+            sh 'make lint'
+          }
+        }
+        stage('Helm Lint') {
+          steps {
+            sh 'helm lint charts/agentrax/'
+            sh 'helm template test charts/agentrax/ --debug > /dev/null'
+          }
+        }
+      }
+    }
+
+    // -----------------------------------------------------------------------
+    // Stage 2: Test
+    // Runs the full unit + envtest integration test suite.
+    // -----------------------------------------------------------------------
+    stage('Test') {
+      steps {
+        sh 'make envtest'
+        sh 'make test'
+      }
+      post {
+        always {
+          archiveArtifacts artifacts: 'cover.out', allowEmptyArchive: true
+        }
+      }
+    }
+
+    // -----------------------------------------------------------------------
+    // Stage 3: Docker Build
+    // Builds the operator image. Pushes to GHCR only on the main branch.
+    // Credentials are bound strictly within the main-branch push path.
+    // -----------------------------------------------------------------------
+    stage('Docker Build') {
+      steps {
+        sh "make docker-build IMG=${IMAGE}"
+        script {
+          if (env.BRANCH_NAME == 'main') {
+            withCredentials([
+              string(credentialsId: 'GHCR_USER',  variable: 'GHCR_USER'),
+              string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_TOKEN'),
+            ]) {
+              sh 'echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin'
+              sh "make docker-push IMG=${IMAGE}"
+            }
+          } else {
+            echo "Feature branch — image built but not pushed (branch=${env.BRANCH_NAME})."
+          }
+        }
+      }
+    }
+
+    // -----------------------------------------------------------------------
+    // Stage 4: Integration Test  (Agentrax-specific stage)
+    //
+    // 1. Installs cert-manager, Prometheus Operator CRDs, and Gateway API CRDs
+    //    via `make deploy-deps` (idempotent).
+    // 2. Loads the locally built image into the kind cluster if kind is present.
+    // 3. Deploys the operator into the cluster with the newly built image.
+    // 4. Runs hack/assert-reconciliation.sh — polls until a sample
+    //    AgentDeployment reaches status.phase == Running (60 s timeout).
+    //
+    // In post.always, the test namespace is deleted FIRST while the operator
+    // is still running so finalizers (agentrax.io/mcp-deregister) can execute
+    // cleanly, followed by `make undeploy`.
+    // -----------------------------------------------------------------------
+    stage('Integration Test') {
+      steps {
+        sh "make deploy-deps KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\""
+        sh "kind load docker-image ${IMAGE} --name ${KIND_CLUSTER}"
+        sh "make deploy IMG=${IMAGE} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\""
+        sh "TEST_NS=${TEST_NS} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" ./hack/assert-reconciliation.sh"
+      }
+      post {
+        always {
+          sh "kubectl --context ${KUBE_CONTEXT} delete namespace ${TEST_NS} --ignore-not-found=true"
+          sh "make undeploy KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" || true"
+        }
+      }
+    }
+
+    // -----------------------------------------------------------------------
+    // Stage 5: Helm Deploy
+    //
+    // Runs only on the main branch. Requires explicit approval from the
+    // ops-team before mutating the production cluster. --atomic ensures Helm
+    // rolls back automatically if any post-install hook fails.
+    // -----------------------------------------------------------------------
+    stage('Helm Deploy') {
+      when { branch 'main' }
+      input {
+        message "Deploy agentrax:${IMAGE_TAG} to production cluster?"
+        ok 'Approve'
+        submitter 'ops-team'
+      }
+      steps {
+        sh """
+          helm upgrade --install agentrax charts/agentrax/ \
+            --namespace agentrax-system \
+            --create-namespace \
+            --set image.tag=${IMAGE_TAG} \
+            --atomic \
+            --timeout 5m
+        """
+      }
+    }
+  }
+
+  post {
+    failure {
+      slackSend(
+        color: 'danger',
+        message: "❌ Agentrax build #${BUILD_NUMBER} FAILED on \`${BRANCH_NAME}\`: ${BUILD_URL}",
+      )
+    }
+    success {
+      slackSend(
+        color: 'good',
+        message: "✅ Agentrax build #${BUILD_NUMBER} passed on \`${BRANCH_NAME}\` (image: \`${IMAGE_TAG}\`)",
+      )
+    }
+  }
+}

docs/ARCHITECTURE.md

@@ -435,6 +435,45 @@ The `.github/workflows/terraform-lint.yml` workflow runs on every PR touching `i
 
 ---
 
+### 4.9 Jenkins CI/CD Pipeline
+
+Agentrax ships a declarative `Jenkinsfile` at the repository root that **complements** (not replaces) the GitHub Actions workflows. The Jenkinsfile is intended for teams running Jenkins on-prem or as a learning vehicle for a multi-stage CI/CD pipeline with a cluster-level integration test gate.
+
+#### Stage Topology
+
+```
+Lint (Go + Helm, parallel)
+         │
+       Test  (unit + envtest)
+         │
+   Docker Build  (push on main only)
+         │
+  Integration Test  ← Agentrax-specific
+         │  make deploy-deps  (cert-manager, Prometheus Operator, Gateway API CRDs)
+         │  make deploy       (operator into cluster)
+         │  hack/assert-reconciliation.sh  (poll status.phase == Running)
+         │  [post.always] make undeploy
+         │
+   Helm Deploy  (main branch + manual approval gate)
+```
+
+#### Agentrax-Specific: Stage 4 — Integration Test
+
+This stage distinguishes the Jenkins pipeline from the GitHub Actions `ci.yml`. It:
+
+1. Installs all cluster dependencies via `make deploy-deps` (idempotent).
+2. Deploys the operator image built in Stage 3.
+3. Applies `hack/testdata/sample-agentdeployment.yaml` and runs `hack/assert-reconciliation.sh`, which polls `status.phase` every 3 seconds until `Running` (60-second timeout). A `RolloutFailed` or `Degraded` terminal phase exits immediately with a non-zero code, failing the stage.
+4. In `post { always }`, deletes `TEST_NS` first while the operator is still active so finalizers (`agentrax.io/mcp-deregister`) process cleanly, then runs `make undeploy`.
+
+#### Safety Properties
+
+- **`disableConcurrentBuilds()`**: Prevents race conditions on the shared `kind` cluster between simultaneous branch builds.
+- **`--atomic` Helm flag**: Helm rolls back automatically if any hook fails during Stage 5.
+- **`submitter 'ops-team'`**: Only members of the `ops-team` Jenkins group can approve production deployments.
+
+---
+
 ## 5. Architectural Decision Records (ADRs) & Trade-Offs
 
 | Decision                                  | Alternative Considered                       | Trade-Off & Rationale for Agentrax                                                                                                                                                                                                                                       |

hack/assert-reconciliation.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# hack/assert-reconciliation.sh
+#
+# Polls until a sample AgentDeployment in the integration-test namespace
+# reaches status.phase == Running, then exits 0.
+# Exits non-zero (and fails the Jenkins stage) if the deadline is exceeded.
+#
+# Environment variables (all optional — defaults shown):
+#   TEST_NS      Namespace to apply the sample manifest into (default: agentrax-jenkins-test)
+#   TIMEOUT_SEC  Maximum seconds to wait for Running phase (default: 60)
+#   KUBECTL      kubectl binary to use (default: kubectl)
+
+set -euo pipefail
+
+: "${TEST_NS:=agentrax-jenkins-test}"
+: "${TIMEOUT_SEC:=60}"
+: "${KUBECTL:=kubectl}"
+
+SAMPLE_MANIFEST="$(dirname "$0")/testdata/sample-agentdeployment.yaml"
+POLL_INTERVAL=3
+
+# ---------------------------------------------------------------------------
+# 1. Ensure the test namespace exists
+# ---------------------------------------------------------------------------
+echo "[assert-reconciliation] Ensuring namespace '${TEST_NS}' exists..."
+${KUBECTL} create namespace "${TEST_NS}" --dry-run=client -o yaml | ${KUBECTL} apply -f -
+
+# ---------------------------------------------------------------------------
+# 2. Apply the sample AgentDeployment
+# ---------------------------------------------------------------------------
+if [[ ! -f "${SAMPLE_MANIFEST}" ]]; then
+  echo "[assert-reconciliation] ERROR: sample manifest not found at ${SAMPLE_MANIFEST}" >&2
+  exit 1
+fi
+
+echo "[assert-reconciliation] Applying sample AgentDeployment from ${SAMPLE_MANIFEST}..."
+${KUBECTL} apply -f "${SAMPLE_MANIFEST}" -n "${TEST_NS}"
+
+# ---------------------------------------------------------------------------
+# 3. Poll until status.phase == Running
+# ---------------------------------------------------------------------------
+AD_NAME=$(${KUBECTL} get agentdeployment -n "${TEST_NS}" \
+  -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
+
+if [[ -z "${AD_NAME}" ]]; then
+  echo "[assert-reconciliation] ERROR: no AgentDeployment found in namespace '${TEST_NS}'" >&2
+  exit 1
+fi
+
+echo "[assert-reconciliation] Waiting up to ${TIMEOUT_SEC}s for AgentDeployment '${AD_NAME}' to reach Running..."
+
+elapsed=0
+while true; do
+  phase=$(${KUBECTL} get agentdeployment "${AD_NAME}" -n "${TEST_NS}" \
+    -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown")
+
+  echo "[assert-reconciliation] t=${elapsed}s  status.phase=${phase}"
+
+  if [[ "${phase}" == "Running" ]]; then
+    echo "[assert-reconciliation] ✅ AgentDeployment '${AD_NAME}' reached Running in ${elapsed}s."
+    exit 0
+  fi
+
+  if [[ "${phase}" == "RolloutFailed" || "${phase}" == "Degraded" ]]; then
+    echo "[assert-reconciliation] ❌ Terminal failure phase '${phase}' detected." >&2
+    ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2
+    exit 1
+  fi
+
+  if (( elapsed >= TIMEOUT_SEC )); then
+    echo "[assert-reconciliation] ❌ Timeout after ${TIMEOUT_SEC}s — last phase: '${phase}'" >&2
+    echo "[assert-reconciliation] --- describe output ---" >&2
+    ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2
+    exit 1
+  fi
+
+  sleep "${POLL_INTERVAL}"
+  elapsed=$(( elapsed + POLL_INTERVAL ))
+done

hack/testdata/sample-agentdeployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: agentrax.io/v1alpha1
+kind: AgentDeployment
+metadata:
+  name: jenkins-smoke-test
+  # Namespace is set by assert-reconciliation.sh at apply time (-n ${TEST_NS})
+  labels:
+    app.kubernetes.io/managed-by: jenkins-ci
+    agentrax.io/agent: "true"
+spec:
+  # Minimal agent that exercises the full reconcile loop:
+  # Deployment + Service + ServiceMonitor + HPA creation.
+  image: "ghcr.io/gitcommitankit/agentrax-agent:latest"
+  replicas: 1
+  tenant: jenkins-test
+  resources:
+    limits:
+      cpu: "250m"
+      memory: "256Mi"
+    requests:
+      cpu: "50m"
+      memory: "64Mi"