From e8978b5967ae3decc87ee25fcc6e3b28bbcfd5b6 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Wed, 2 Sep 2026 14:44:08 +0000 Subject: [PATCH 1/4] feat(jenkins): add 5-stage Jenkinsfile, assert-reconciliation.sh, and Jenkins docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the DevOps roadmap: - Jenkinsfile: declarative 5-stage pipeline (Lint → Test → Docker Build → Integration Test → Helm Deploy). Stage 4 is Agentrax-specific: installs cluster deps, deploys operator, polls status.phase via hack/assert-reconciliation.sh, and tears down unconditionally in post.always. Stage 5 (Helm Deploy) is main-only with manual ops-team approval gate and --atomic rollback. disableConcurrentBuilds() prevents kind cluster races. - hack/assert-reconciliation.sh: polling script used by Stage 4. Applies hack/testdata/sample-agentdeployment.yaml, polls status.phase every 3s until Running (60s timeout), fails immediately on RolloutFailed/Degraded terminal phases, prints kubectl describe on timeout for debugging. - hack/testdata/sample-agentdeployment.yaml: minimal smoke-test AgentDeployment that exercises the full reconcile loop (Deployment + Service + ServiceMonitor + HPA) with a small CPU/memory footprint. - docs/jenkins/README.md: local Docker-based Jenkins setup, credential store config, multibranch pipeline creation, stage explanation table, Slack plugin configuration. - docs/ARCHITECTURE.md §4.9: documents Jenkins stage topology, integration test mechanics, safety properties (disableConcurrentBuilds, --atomic, submitter gate), and reference to docs/jenkins/README.md. Signed-off-by: Ankit Kr. Chowdhury --- Jenkinsfile | 168 ++++++++++++++++++++++ docs/ARCHITECTURE.md | 41 ++++++ docs/jenkins/README.md | 131 +++++++++++++++++ hack/assert-reconciliation.sh | 79 ++++++++++ hack/testdata/sample-agentdeployment.yaml | 21 +++ 5 files changed, 440 insertions(+) create mode 100644 Jenkinsfile create mode 100644 docs/jenkins/README.md create mode 100755 hack/assert-reconciliation.sh create mode 100644 hack/testdata/sample-agentdeployment.yaml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..5aeaf61 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,168 @@ +// Agentrax — Declarative Jenkins CI/CD Pipeline +// +// Stages: +// 1. Lint — golangci-lint + Helm chart lint (parallel) +// 2. Test — unit & integration tests via envtest +// 3. Docker Build — build image tagged with short Git SHA, push on main +// 4. Integration Test — deploy to kind test namespace, assert reconciliation +// 5. Helm Deploy — manual approval gate then helm upgrade --install +// +// Requirements: +// - Jenkins agent with label 'docker' and Docker-in-Docker socket access +// - Credentials: GHCR_USER (string), GHCR_TOKEN (secret text) +// - Jenkins Slack plugin configured for Slack notifications + +pipeline { + agent { label 'docker' } + + environment { + // Go workspace inside the Jenkins workspace to avoid polluting $HOME + GOPATH = "${WORKSPACE}/.go" + GOMODCACHE = "${WORKSPACE}/.go/pkg/mod" + // Image tag is the short Git SHA for traceability + IMAGE_TAG = "${env.GIT_COMMIT?.take(8) ?: env.BUILD_NUMBER}" + IMAGE = "ghcr.io/gitcommitankit/agentrax:${IMAGE_TAG}" + // Kubernetes namespace used exclusively for integration testing + TEST_NS = "agentrax-jenkins-test" + } + + options { + // Abort if the full pipeline exceeds 45 minutes + timeout(time: 45, unit: 'MINUTES') + // Keep the last 10 build logs; discard older ones to save disk space + buildDiscarder(logRotator(numToKeepStr: '10')) + // Prevent concurrent builds on the same branch to avoid races on the + // shared kind cluster used in Stage 4 + disableConcurrentBuilds() + ansiColor('xterm') + } + + stages { + + // ----------------------------------------------------------------------- + // Stage 1: Lint + // Runs golangci-lint and Helm chart lint in parallel. + // ----------------------------------------------------------------------- + stage('Lint') { + parallel { + stage('Go Lint') { + steps { + sh 'make golangci-lint' + sh 'make lint' + } + } + stage('Helm Lint') { + steps { + sh 'helm lint charts/agentrax/' + sh 'helm template test charts/agentrax/ --debug > /dev/null' + } + } + } + } + + // ----------------------------------------------------------------------- + // Stage 2: Test + // Runs the full unit + envtest integration test suite. + // ----------------------------------------------------------------------- + stage('Test') { + steps { + sh 'make envtest' + sh 'make test' + } + post { + always { + archiveArtifacts artifacts: 'cover.out', allowEmptyArchive: true + } + } + } + + // ----------------------------------------------------------------------- + // Stage 3: Docker Build + // Builds the operator image. Pushes to GHCR only on the main branch. + // ----------------------------------------------------------------------- + stage('Docker Build') { + steps { + withCredentials([ + string(credentialsId: 'GHCR_USER', variable: 'GHCR_USER'), + string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_TOKEN'), + ]) { + sh 'echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin' + sh "make docker-build IMG=${IMAGE}" + script { + if (env.BRANCH_NAME == 'main') { + sh "make docker-push IMG=${IMAGE}" + } else { + echo "Feature branch — image built but not pushed (branch=${env.BRANCH_NAME})." + } + } + } + } + } + + // ----------------------------------------------------------------------- + // Stage 4: Integration Test (Agentrax-specific stage) + // + // 1. Installs cert-manager, Prometheus Operator CRDs, and Gateway API CRDs + // via `make deploy-deps` (idempotent). + // 2. Deploys the operator into the cluster with the newly built image. + // 3. Runs hack/assert-reconciliation.sh — polls until a sample + // AgentDeployment reaches status.phase == Running (60 s timeout). + // + // The post.always block tears down the test namespace so the cluster stays + // clean for the next build regardless of pass/fail. + // ----------------------------------------------------------------------- + stage('Integration Test') { + steps { + sh 'make deploy-deps' + sh "make deploy IMG=${IMAGE}" + sh "TEST_NS=${TEST_NS} ./hack/assert-reconciliation.sh" + } + post { + always { + sh 'make undeploy || true' + } + } + } + + // ----------------------------------------------------------------------- + // Stage 5: Helm Deploy + // + // Runs only on the main branch. Requires explicit approval from the + // ops-team before mutating the production cluster. --atomic ensures Helm + // rolls back automatically if any post-install hook fails. + // ----------------------------------------------------------------------- + stage('Helm Deploy') { + when { branch 'main' } + input { + message "Deploy agentrax:${IMAGE_TAG} to production cluster?" + ok 'Approve' + submitter 'ops-team' + } + steps { + sh """ + helm upgrade --install agentrax charts/agentrax/ \ + --namespace agentrax-system \ + --create-namespace \ + --set image.tag=${IMAGE_TAG} \ + --atomic \ + --timeout 5m + """ + } + } + } + + post { + failure { + slackSend( + color: 'danger', + message: "❌ Agentrax build #${BUILD_NUMBER} FAILED on \`${BRANCH_NAME}\`: ${BUILD_URL}", + ) + } + success { + slackSend( + color: 'good', + message: "✅ Agentrax build #${BUILD_NUMBER} passed on \`${BRANCH_NAME}\` (image: \`${IMAGE_TAG}\`)", + ) + } + } +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 839166f..4a7c4c5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -435,6 +435,47 @@ The `.github/workflows/terraform-lint.yml` workflow runs on every PR touching `i --- +### 4.9 Jenkins CI/CD Pipeline + +Agentrax ships a declarative `Jenkinsfile` at the repository root that **complements** (not replaces) the GitHub Actions workflows. The Jenkinsfile is intended for teams running Jenkins on-prem or as a learning vehicle for a multi-stage CI/CD pipeline with a cluster-level integration test gate. + +#### Stage Topology + +``` +Lint (Go + Helm, parallel) + │ + Test (unit + envtest) + │ + Docker Build (push on main only) + │ + Integration Test ← Agentrax-specific + │ make deploy-deps (cert-manager, Prometheus Operator, Gateway API CRDs) + │ make deploy (operator into cluster) + │ hack/assert-reconciliation.sh (poll status.phase == Running) + │ [post.always] make undeploy + │ + Helm Deploy (main branch + manual approval gate) +``` + +#### Agentrax-Specific: Stage 4 — Integration Test + +This stage distinguishes the Jenkins pipeline from the GitHub Actions `ci.yml`. It: + +1. Installs all cluster dependencies via `make deploy-deps` (idempotent). +2. Deploys the operator image built in Stage 3. +3. Applies `hack/testdata/sample-agentdeployment.yaml` and runs `hack/assert-reconciliation.sh`, which polls `status.phase` every 3 seconds until `Running` (60-second timeout). A `RolloutFailed` or `Degraded` terminal phase exits immediately with a non-zero code, failing the stage. +4. Tears down the test namespace unconditionally in `post { always }`. + +#### Safety Properties + +- **`disableConcurrentBuilds()`**: Prevents race conditions on the shared `kind` cluster between simultaneous branch builds. +- **`--atomic` Helm flag**: Helm rolls back automatically if any hook fails during Stage 5. +- **`submitter 'ops-team'`**: Only members of the `ops-team` Jenkins group can approve production deployments. + +See [`docs/jenkins/README.md`](jenkins/README.md) for local Docker-based Jenkins setup instructions. + +--- + ## 5. Architectural Decision Records (ADRs) & Trade-Offs | Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | diff --git a/docs/jenkins/README.md b/docs/jenkins/README.md new file mode 100644 index 0000000..0a09851 --- /dev/null +++ b/docs/jenkins/README.md @@ -0,0 +1,131 @@ +# Jenkins CI/CD for Agentrax + +This document explains how to run the Agentrax Jenkins pipeline locally using Docker. + +--- + +## Prerequisites + +- Docker installed and running +- `kubectl` configured with access to a running `kind` cluster +- A GitHub Container Registry (GHCR) token with `write:packages` scope + +--- + +## 1. Run Jenkins in Docker + +```bash +docker run -d --name jenkins \ + -p 8080:8080 \ + -p 50000:50000 \ + -v jenkins_home:/var/jenkins_home \ + -v /var/run/docker.sock:/var/run/docker.sock \ + jenkins/jenkins:lts-jdk17 +``` + +The `-v /var/run/docker.sock` mount gives Jenkins agents access to the host Docker daemon so `make docker-build` works without Docker-in-Docker complexity. + +Retrieve the initial admin password: + +```bash +docker exec jenkins cat /var/jenkins_home/secrets/initialAdminPassword +``` + +Open `http://localhost:8080` and complete the setup wizard. Install the **recommended plugins** plus: + +- **Pipeline** (usually included) +- **AnsiColor** +- **Slack Notification** + +--- + +## 2. Create Jenkins Credentials + +In **Manage Jenkins → Credentials → (global)**, add: + +| ID | Kind | Value | +| ------------ | ----------- | -------------------------------------- | +| `GHCR_USER` | Secret text | Your GitHub username | +| `GHCR_TOKEN` | Secret text | PAT with `write:packages` scope | + +--- + +## 3. Create a Multibranch Pipeline + +1. **New Item → Multibranch Pipeline** — name it `agentrax`. +2. Under **Branch Sources**, add a **GitHub** source: + - Repository URL: `https://github.com/gitcommitankit/agentrax` + - Credentials: add a GitHub PAT credential for private access if needed. +3. Under **Build Configuration**, set: + - **Mode**: `by Jenkinsfile` + - **Script Path**: `Jenkinsfile` (the default) +4. Save and let Jenkins scan branches. It automatically discovers `main` and any feature branches. + +--- + +## 4. Pipeline Stages + +``` +Lint ──────────────────────────┐ + │ (parallel) +Helm Lint ──────────────────── ┘ + │ + ▼ + Test + │ + ▼ + Docker Build (push on main only) + │ + ▼ + Integration Test ← Agentrax-specific + │ make deploy-deps + │ make deploy + │ hack/assert-reconciliation.sh + │ [always] make undeploy + ▼ + Helm Deploy (main only, manual approval) +``` + +| Stage | What runs | Fail condition | +| :--- | :--- | :--- | +| **Lint** | `make lint` + `helm lint charts/agentrax/` | Any lint error | +| **Test** | `make test` (unit + envtest) | Any test failure | +| **Docker Build** | `make docker-build` + push on `main` | Docker build error | +| **Integration Test** | `deploy-deps` → `deploy` → `assert-reconciliation.sh` | Reconciliation timeout or terminal phase | +| **Helm Deploy** | `helm upgrade --install ... --atomic` | Requires ops-team approval; rollback on hook failure | + +--- + +## 5. Integration Test in Detail + +**Stage 4** is the Agentrax-specific addition. It: + +1. Installs `cert-manager`, Prometheus Operator CRDs, and Gateway API CRDs via `make deploy-deps` (idempotent). +2. Deploys the newly built operator image into the cluster via `make deploy`. +3. Runs [`hack/assert-reconciliation.sh`](../../hack/assert-reconciliation.sh), which: + - Creates the `agentrax-jenkins-test` namespace. + - Applies [`hack/testdata/sample-agentdeployment.yaml`](../../hack/testdata/sample-agentdeployment.yaml). + - Polls `status.phase` every 3 seconds until `Running` or 60-second timeout. + - Exits non-zero on `RolloutFailed`, `Degraded`, or timeout — failing the Jenkins stage. + +The `post { always }` block runs `make undeploy` regardless of pass/fail, keeping the cluster clean for subsequent builds. + +--- + +## 6. Slack Notifications + +Configure the Jenkins Slack plugin (**Manage Jenkins → System → Slack**): + +- Workspace: your Slack workspace name +- Credential: add a **Secret text** credential containing the Slack Bot token +- Default channel: `#agentrax-ci` + +The pipeline posts: +- `❌ FAILED` on any stage failure +- `✅ passed` on a successful full pipeline run + +--- + +## 7. Updating the Pipeline + +The `Jenkinsfile` lives at the repository root. Changes are picked up automatically on the next branch scan or build trigger — no Jenkins UI changes required. diff --git a/hack/assert-reconciliation.sh b/hack/assert-reconciliation.sh new file mode 100755 index 0000000..900f326 --- /dev/null +++ b/hack/assert-reconciliation.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# hack/assert-reconciliation.sh +# +# Polls until a sample AgentDeployment in the integration-test namespace +# reaches status.phase == Running, then exits 0. +# Exits non-zero (and fails the Jenkins stage) if the deadline is exceeded. +# +# Environment variables (all optional — defaults shown): +# TEST_NS Namespace to apply the sample manifest into (default: agentrax-jenkins-test) +# TIMEOUT_SEC Maximum seconds to wait for Running phase (default: 60) +# KUBECTL kubectl binary to use (default: kubectl) + +set -euo pipefail + +: "${TEST_NS:=agentrax-jenkins-test}" +: "${TIMEOUT_SEC:=60}" +: "${KUBECTL:=kubectl}" + +SAMPLE_MANIFEST="$(dirname "$0")/testdata/sample-agentdeployment.yaml" +POLL_INTERVAL=3 + +# --------------------------------------------------------------------------- +# 1. Ensure the test namespace exists +# --------------------------------------------------------------------------- +echo "[assert-reconciliation] Ensuring namespace '${TEST_NS}' exists..." +${KUBECTL} create namespace "${TEST_NS}" --dry-run=client -o yaml | ${KUBECTL} apply -f - + +# --------------------------------------------------------------------------- +# 2. Apply the sample AgentDeployment +# --------------------------------------------------------------------------- +if [[ ! -f "${SAMPLE_MANIFEST}" ]]; then + echo "[assert-reconciliation] ERROR: sample manifest not found at ${SAMPLE_MANIFEST}" >&2 + exit 1 +fi + +echo "[assert-reconciliation] Applying sample AgentDeployment from ${SAMPLE_MANIFEST}..." +${KUBECTL} apply -f "${SAMPLE_MANIFEST}" -n "${TEST_NS}" + +# --------------------------------------------------------------------------- +# 3. Poll until status.phase == Running +# --------------------------------------------------------------------------- +AD_NAME=$(${KUBECTL} get agentdeployment -n "${TEST_NS}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +if [[ -z "${AD_NAME}" ]]; then + echo "[assert-reconciliation] ERROR: no AgentDeployment found in namespace '${TEST_NS}'" >&2 + exit 1 +fi + +echo "[assert-reconciliation] Waiting up to ${TIMEOUT_SEC}s for AgentDeployment '${AD_NAME}' to reach Running..." + +elapsed=0 +while true; do + phase=$(${KUBECTL} get agentdeployment "${AD_NAME}" -n "${TEST_NS}" \ + -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + + echo "[assert-reconciliation] t=${elapsed}s status.phase=${phase}" + + if [[ "${phase}" == "Running" ]]; then + echo "[assert-reconciliation] ✅ AgentDeployment '${AD_NAME}' reached Running in ${elapsed}s." + exit 0 + fi + + if [[ "${phase}" == "RolloutFailed" || "${phase}" == "Degraded" ]]; then + echo "[assert-reconciliation] ❌ Terminal failure phase '${phase}' detected." >&2 + ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2 + exit 1 + fi + + if (( elapsed >= TIMEOUT_SEC )); then + echo "[assert-reconciliation] ❌ Timeout after ${TIMEOUT_SEC}s — last phase: '${phase}'" >&2 + echo "[assert-reconciliation] --- describe output ---" >&2 + ${KUBECTL} describe agentdeployment "${AD_NAME}" -n "${TEST_NS}" >&2 + exit 1 + fi + + sleep "${POLL_INTERVAL}" + elapsed=$(( elapsed + POLL_INTERVAL )) +done diff --git a/hack/testdata/sample-agentdeployment.yaml b/hack/testdata/sample-agentdeployment.yaml new file mode 100644 index 0000000..a2a1d59 --- /dev/null +++ b/hack/testdata/sample-agentdeployment.yaml @@ -0,0 +1,21 @@ +apiVersion: agentrax.io/v1alpha1 +kind: AgentDeployment +metadata: + name: jenkins-smoke-test + # Namespace is set by assert-reconciliation.sh at apply time (-n ${TEST_NS}) + labels: + app.kubernetes.io/managed-by: jenkins-ci + agentrax.io/agent: "true" +spec: + # Minimal agent that exercises the full reconcile loop: + # Deployment + Service + ServiceMonitor + HPA creation. + image: "ghcr.io/gitcommitankit/agentrax-agent:latest" + replicas: 1 + tenant: jenkins-test + resources: + limits: + cpu: "250m" + memory: "256Mi" + requests: + cpu: "50m" + memory: "64Mi" From 5a562c7220cbfae4402c7ef023244cd5fcb8d259 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Wed, 2 Sep 2026 15:04:50 +0000 Subject: [PATCH 2/4] refactor: optimize Jenkins integration test teardown and update image loading process while removing local documentation Signed-off-by: Ankit Kr. Chowdhury --- Jenkinsfile | 33 ++++++----- docs/ARCHITECTURE.md | 4 +- docs/jenkins/README.md | 131 ----------------------------------------- 3 files changed, 20 insertions(+), 148 deletions(-) delete mode 100644 docs/jenkins/README.md diff --git a/Jenkinsfile b/Jenkinsfile index 5aeaf61..82438bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -79,21 +79,22 @@ pipeline { // ----------------------------------------------------------------------- // Stage 3: Docker Build // Builds the operator image. Pushes to GHCR only on the main branch. + // Credentials are bound strictly within the main-branch push path. // ----------------------------------------------------------------------- stage('Docker Build') { steps { - withCredentials([ - string(credentialsId: 'GHCR_USER', variable: 'GHCR_USER'), - string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_TOKEN'), - ]) { - sh 'echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin' - sh "make docker-build IMG=${IMAGE}" - script { - if (env.BRANCH_NAME == 'main') { + sh "make docker-build IMG=${IMAGE}" + script { + if (env.BRANCH_NAME == 'main') { + withCredentials([ + string(credentialsId: 'GHCR_USER', variable: 'GHCR_USER'), + string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_TOKEN'), + ]) { + sh 'echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin' sh "make docker-push IMG=${IMAGE}" - } else { - echo "Feature branch — image built but not pushed (branch=${env.BRANCH_NAME})." } + } else { + echo "Feature branch — image built but not pushed (branch=${env.BRANCH_NAME})." } } } @@ -104,21 +105,25 @@ pipeline { // // 1. Installs cert-manager, Prometheus Operator CRDs, and Gateway API CRDs // via `make deploy-deps` (idempotent). - // 2. Deploys the operator into the cluster with the newly built image. - // 3. Runs hack/assert-reconciliation.sh — polls until a sample + // 2. Loads the locally built image into the kind cluster if kind is present. + // 3. Deploys the operator into the cluster with the newly built image. + // 4. Runs hack/assert-reconciliation.sh — polls until a sample // AgentDeployment reaches status.phase == Running (60 s timeout). // - // The post.always block tears down the test namespace so the cluster stays - // clean for the next build regardless of pass/fail. + // In post.always, the test namespace is deleted FIRST while the operator + // is still running so finalizers (agentrax.io/mcp-deregister) can execute + // cleanly, followed by `make undeploy`. // ----------------------------------------------------------------------- stage('Integration Test') { steps { sh 'make deploy-deps' + sh "kind load docker-image ${IMAGE} || true" sh "make deploy IMG=${IMAGE}" sh "TEST_NS=${TEST_NS} ./hack/assert-reconciliation.sh" } post { always { + sh "kubectl delete namespace ${TEST_NS} --ignore-not-found=true || true" sh 'make undeploy || true' } } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4a7c4c5..8def929 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -464,7 +464,7 @@ This stage distinguishes the Jenkins pipeline from the GitHub Actions `ci.yml`. 1. Installs all cluster dependencies via `make deploy-deps` (idempotent). 2. Deploys the operator image built in Stage 3. 3. Applies `hack/testdata/sample-agentdeployment.yaml` and runs `hack/assert-reconciliation.sh`, which polls `status.phase` every 3 seconds until `Running` (60-second timeout). A `RolloutFailed` or `Degraded` terminal phase exits immediately with a non-zero code, failing the stage. -4. Tears down the test namespace unconditionally in `post { always }`. +4. In `post { always }`, deletes `TEST_NS` first while the operator is still active so finalizers (`agentrax.io/mcp-deregister`) process cleanly, then runs `make undeploy`. #### Safety Properties @@ -472,8 +472,6 @@ This stage distinguishes the Jenkins pipeline from the GitHub Actions `ci.yml`. - **`--atomic` Helm flag**: Helm rolls back automatically if any hook fails during Stage 5. - **`submitter 'ops-team'`**: Only members of the `ops-team` Jenkins group can approve production deployments. -See [`docs/jenkins/README.md`](jenkins/README.md) for local Docker-based Jenkins setup instructions. - --- ## 5. Architectural Decision Records (ADRs) & Trade-Offs diff --git a/docs/jenkins/README.md b/docs/jenkins/README.md deleted file mode 100644 index 0a09851..0000000 --- a/docs/jenkins/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# Jenkins CI/CD for Agentrax - -This document explains how to run the Agentrax Jenkins pipeline locally using Docker. - ---- - -## Prerequisites - -- Docker installed and running -- `kubectl` configured with access to a running `kind` cluster -- A GitHub Container Registry (GHCR) token with `write:packages` scope - ---- - -## 1. Run Jenkins in Docker - -```bash -docker run -d --name jenkins \ - -p 8080:8080 \ - -p 50000:50000 \ - -v jenkins_home:/var/jenkins_home \ - -v /var/run/docker.sock:/var/run/docker.sock \ - jenkins/jenkins:lts-jdk17 -``` - -The `-v /var/run/docker.sock` mount gives Jenkins agents access to the host Docker daemon so `make docker-build` works without Docker-in-Docker complexity. - -Retrieve the initial admin password: - -```bash -docker exec jenkins cat /var/jenkins_home/secrets/initialAdminPassword -``` - -Open `http://localhost:8080` and complete the setup wizard. Install the **recommended plugins** plus: - -- **Pipeline** (usually included) -- **AnsiColor** -- **Slack Notification** - ---- - -## 2. Create Jenkins Credentials - -In **Manage Jenkins → Credentials → (global)**, add: - -| ID | Kind | Value | -| ------------ | ----------- | -------------------------------------- | -| `GHCR_USER` | Secret text | Your GitHub username | -| `GHCR_TOKEN` | Secret text | PAT with `write:packages` scope | - ---- - -## 3. Create a Multibranch Pipeline - -1. **New Item → Multibranch Pipeline** — name it `agentrax`. -2. Under **Branch Sources**, add a **GitHub** source: - - Repository URL: `https://github.com/gitcommitankit/agentrax` - - Credentials: add a GitHub PAT credential for private access if needed. -3. Under **Build Configuration**, set: - - **Mode**: `by Jenkinsfile` - - **Script Path**: `Jenkinsfile` (the default) -4. Save and let Jenkins scan branches. It automatically discovers `main` and any feature branches. - ---- - -## 4. Pipeline Stages - -``` -Lint ──────────────────────────┐ - │ (parallel) -Helm Lint ──────────────────── ┘ - │ - ▼ - Test - │ - ▼ - Docker Build (push on main only) - │ - ▼ - Integration Test ← Agentrax-specific - │ make deploy-deps - │ make deploy - │ hack/assert-reconciliation.sh - │ [always] make undeploy - ▼ - Helm Deploy (main only, manual approval) -``` - -| Stage | What runs | Fail condition | -| :--- | :--- | :--- | -| **Lint** | `make lint` + `helm lint charts/agentrax/` | Any lint error | -| **Test** | `make test` (unit + envtest) | Any test failure | -| **Docker Build** | `make docker-build` + push on `main` | Docker build error | -| **Integration Test** | `deploy-deps` → `deploy` → `assert-reconciliation.sh` | Reconciliation timeout or terminal phase | -| **Helm Deploy** | `helm upgrade --install ... --atomic` | Requires ops-team approval; rollback on hook failure | - ---- - -## 5. Integration Test in Detail - -**Stage 4** is the Agentrax-specific addition. It: - -1. Installs `cert-manager`, Prometheus Operator CRDs, and Gateway API CRDs via `make deploy-deps` (idempotent). -2. Deploys the newly built operator image into the cluster via `make deploy`. -3. Runs [`hack/assert-reconciliation.sh`](../../hack/assert-reconciliation.sh), which: - - Creates the `agentrax-jenkins-test` namespace. - - Applies [`hack/testdata/sample-agentdeployment.yaml`](../../hack/testdata/sample-agentdeployment.yaml). - - Polls `status.phase` every 3 seconds until `Running` or 60-second timeout. - - Exits non-zero on `RolloutFailed`, `Degraded`, or timeout — failing the Jenkins stage. - -The `post { always }` block runs `make undeploy` regardless of pass/fail, keeping the cluster clean for subsequent builds. - ---- - -## 6. Slack Notifications - -Configure the Jenkins Slack plugin (**Manage Jenkins → System → Slack**): - -- Workspace: your Slack workspace name -- Credential: add a **Secret text** credential containing the Slack Bot token -- Default channel: `#agentrax-ci` - -The pipeline posts: -- `❌ FAILED` on any stage failure -- `✅ passed` on a successful full pipeline run - ---- - -## 7. Updating the Pipeline - -The `Jenkinsfile` lives at the repository root. Changes are picked up automatically on the next branch scan or build trigger — no Jenkins UI changes required. From c6c21d364ef935f181fe5c668e3830c794ec2483 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Wed, 2 Sep 2026 19:08:12 +0000 Subject: [PATCH 3/4] fix: remove ignore errors for kind image loading and namespace deletion in Jenkinsfile Signed-off-by: Ankit Kr. Chowdhury --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 82438bc..850377b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,13 +117,13 @@ pipeline { stage('Integration Test') { steps { sh 'make deploy-deps' - sh "kind load docker-image ${IMAGE} || true" + sh "kind load docker-image ${IMAGE}" sh "make deploy IMG=${IMAGE}" sh "TEST_NS=${TEST_NS} ./hack/assert-reconciliation.sh" } post { always { - sh "kubectl delete namespace ${TEST_NS} --ignore-not-found=true || true" + sh "kubectl delete namespace ${TEST_NS} --ignore-not-found=true" sh 'make undeploy || true' } } From 8d0cec9e51abe477517525bd1250d357e4e25ef0 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Wed, 2 Sep 2026 19:43:37 +0000 Subject: [PATCH 4/4] chore: parameterize integration tests to support custom Kind clusters and kubectl contexts Signed-off-by: Ankit Kr. Chowdhury --- Jenkinsfile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 850377b..a28b57f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,10 @@ pipeline { IMAGE_TAG = "${env.GIT_COMMIT?.take(8) ?: env.BUILD_NUMBER}" IMAGE = "ghcr.io/gitcommitankit/agentrax:${IMAGE_TAG}" // Kubernetes namespace used exclusively for integration testing - TEST_NS = "agentrax-jenkins-test" + TEST_NS = "agentrax-jenkins-test" + // Kind cluster and kubectl context for integration testing + KIND_CLUSTER = "${env.KIND_CLUSTER ?: 'agentrax-dev'}" + KUBE_CONTEXT = "${env.KUBE_CONTEXT ?: 'kind-agentrax-dev'}" } options { @@ -116,15 +119,15 @@ pipeline { // ----------------------------------------------------------------------- stage('Integration Test') { steps { - sh 'make deploy-deps' - sh "kind load docker-image ${IMAGE}" - sh "make deploy IMG=${IMAGE}" - sh "TEST_NS=${TEST_NS} ./hack/assert-reconciliation.sh" + sh "make deploy-deps KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\"" + sh "kind load docker-image ${IMAGE} --name ${KIND_CLUSTER}" + sh "make deploy IMG=${IMAGE} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\"" + sh "TEST_NS=${TEST_NS} KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" ./hack/assert-reconciliation.sh" } post { always { - sh "kubectl delete namespace ${TEST_NS} --ignore-not-found=true" - sh 'make undeploy || true' + sh "kubectl --context ${KUBE_CONTEXT} delete namespace ${TEST_NS} --ignore-not-found=true" + sh "make undeploy KUBECTL=\"kubectl --context ${KUBE_CONTEXT}\" || true" } } }