From fc8619041a7ba4fdcd612c00aab9d1bb193a1373 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 26 Mar 2026 11:23:51 -0700 Subject: [PATCH 01/35] MLE-14549 Support ARM Docker images (#416) * Update the build pipeline to handle ARM builds. Adding new Docker image types: ubi9-arm and ubi9-rootless-arm. Restructure the pipeline to run tests on external ARM agent. * Increment Docker release version * Update publishing flow and cleanup * Fix image versioning. * add MarkLogic 12 support * add support for MarkLogic 12 ARM builds * Clean up RPM path logic * revert a bug * remove graviton ip handling since jenkins takes care of the agent now --------- Co-authored-by: Vitaly Korolev --- Jenkinsfile | 343 +++++++++++++++--- Makefile | 41 ++- NOTICE.txt | 2 +- dockerFiles/marklogic-deps-ubi9-arm:base | 29 ++ dockerFiles/marklogic-deps-ubi9:base | 4 +- dockerFiles/marklogic-deps-ubi:base | 4 +- .../marklogic-server-ubi-rootless:base | 14 +- dockerFiles/marklogic-server-ubi9-arm:base | 151 ++++++++ test/docker-tests.robot | 4 + test/keywords.resource | 7 +- 10 files changed, 524 insertions(+), 75 deletions(-) create mode 100644 dockerFiles/marklogic-deps-ubi9-arm:base create mode 100644 dockerFiles/marklogic-server-ubi9-arm:base diff --git a/Jenkinsfile b/Jenkinsfile index 1a7e8efc..822c1f5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,19 +21,40 @@ LINT_OUTPUT = '' SCAN_OUTPUT = '' IMAGE_SIZE = 0 RPMversion = '' +GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' +builtImage = '' +publishImage = '' +latestTag = '' +upgradeDockerImage = '' // Define local funtions +/** + * Determines if the current build is for an ARM image type. + * ARM workers (e.g., Graviton3) are only available for MarkLogic 11. + * @return true if dockerImageType contains 'arm', false otherwise. + */ +@NonCPS +def isArmImage() { + return params.dockerImageType.toLowerCase().contains('arm') +} + /** * Performs pre-build checks: * - Initializes parameters as environment variables. * - Extracts Jira ID from branch name or PR title. * - Checks if the PR is a draft or has requested changes (for PR builds). + * - Validates ARM image types are only used with MarkLogic 11 and 12. */ void preBuildCheck() { // Initialize parameters as env variables (workaround for https://issues.jenkins-ci.org/browse/JENKINS-41929) evaluate """${ def script = ''; params.each { k, v -> script += "env.${k} = '''${v}'''\n" }; return script}""" + // Validate ARM images are only supported for MarkLogic 11 and 12 + if (env.dockerImageType.contains('arm') && !(env.marklogicVersion in ['11', '12'])) { + error "ARM images (${env.dockerImageType}) are only supported for MarkLogic 11 and 12. Current version: ${env.marklogicVersion}" + } + JIRA_ID = extractJiraID() echo 'Jira ticket number: ' + JIRA_ID @@ -180,6 +201,10 @@ void resultNotification(status) { * Sets RPM, CONVERTERS, and marklogicVersion global variables. */ void copyRPMs() { + // Determine architecture suffix based on image type + def archSuffix = dockerImageType.contains('arm') ? 'aarch64' : 'x86_64' + def armRhelSuffix = (marklogicVersion == "12") ? 'rhel' : 'rhel9' + if (marklogicVersion == "10") { RPMsuffix = "-nightly" RPMbranch = "b10" @@ -203,25 +228,42 @@ void copyRPMs() { else { error "Invalid value in marklogicVersion parameter." } + sh """ cd src - if [ -z ${env.ML_RPM} ]; then - wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.x86_64.rpm + ARM_DATE=\$(TZ=America/Los_Angeles date +%Y%m%d) + if [ -z "${env.ML_RPM}" ]; then + if [ "${archSuffix}" = "aarch64" ]; then + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-dev-tierpoint/${RPMbranch}/server-arm/MarkLogic-${RPMversion}.\${ARM_DATE}-${armRhelSuffix}.aarch64.rpm + else + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.${archSuffix}.rpm + fi else - wget --no-verbose ${ML_RPM} + wget --no-verbose "${env.ML_RPM}" fi - if [ -z ${env.ML_CONVERTERS}]; then - wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/converters/MarkLogicConverters-${RPMversion}${RPMsuffix}.x86_64.rpm + if [ -n "${env.ML_CONVERTERS}" ]; then + wget --no-verbose "${env.ML_CONVERTERS}" + elif [ "${env.marklogicVersion}" = "11" ] && [ "${archSuffix}" = "aarch64" ]; then + # Temporary exception: remove once the default ML11 ARM converters package is published. + touch MarkLogicConverters-placeholder.rpm else - wget --no-verbose ${ML_CONVERTERS} + if [ "${archSuffix}" = "aarch64" ]; then + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-dev-tierpoint/${RPMbranch}/converters-arm/MarkLogicConverters-${RPMversion}.\${ARM_DATE}-${armRhelSuffix}.aarch64.rpm + else + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/converters/MarkLogicConverters-${RPMversion}${RPMsuffix}.${archSuffix}.rpm + fi fi """ script { - // Get the RPM and Converters file names - RPM = sh(returnStdout: true, script: 'cd src;file MarkLogic-*.rpm | cut -d: -f1').trim() - CONVERTERS = sh(returnStdout: true, script: 'cd src;file MarkLogicConverters-*.rpm | cut -d: -f1').trim() - // Extract MarkLogic version from RPM file name - marklogicVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim() + // Get the RPM and Converters file names for the correct architecture (archSuffix already defined above) + // Use newest files so we don't accidentally pick a stale RPM left from a previous run. + RPM = sh(returnStdout: true, script: "cd src; ls -1t MarkLogic-*.${archSuffix}.rpm 2>/dev/null | head -1").trim() + CONVERTERS = sh(returnStdout: true, script: "cd src; (ls -1t MarkLogicConverters-*.${archSuffix}.rpm 2>/dev/null || ls -1t MarkLogicConverters-*.rpm 2>/dev/null) | head -1").trim() + // Extract MarkLogic version from RPM file name (handle both x86_64 and aarch64) + marklogicVersion = sh(returnStdout: true, script: "echo ${RPM} | awk -F 'MarkLogic-' '{print \$2;}' | awk -F '.x86_64.rpm' '{print \$1;}' | awk -F '.aarch64.rpm' '{print \$1;}' | awk -F '-rhel' '{print \$1;}'").trim() + echo "Selected server RPM: ${RPM}" + echo "Selected converters RPM: ${CONVERTERS}" + echo "Derived MarkLogic version from RPM: ${marklogicVersion}" } } @@ -235,10 +277,12 @@ void buildDockerImage() { publishImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}" mlVerShort=marklogicVersion.split("\\.")[0] latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}" - timeStamp = new Date().format('yyyyMMdd') + // Use Los Angeles time (same as ARM_DATE in copyRPMs) to ensure consistency across UTC/PST boundaries + timeStamp = sh(returnStdout: true, script: "TZ=America/Los_Angeles date +%Y%m%d").trim() timestamptedTag = builtImage.replace('nightly', timeStamp) sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}" currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})" + echo "Built image: ${builtImage}" } /** @@ -251,6 +295,10 @@ void pullUpgradeDockerImage() { sh """ echo 'dockerImageType is set to ubi-rootless, skipping this stage and Docker upgrade test.' """ + } else if (isArmImage()) { + sh """ + echo 'ARM image type detected. Skipping upgrade test (no previous ARM images available for upgrade testing).' + """ } else { if (upgradeDockerImage != "" ) { sh """ @@ -272,8 +320,14 @@ void pullUpgradeDockerImage() { */ void structureTests() { sh """ - #install container-structure-test 1.16.0 binary - curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test + #install container-structure-test 1.16.0 binary (detect architecture) + ARCH=\$(uname -m) + if [ "\$ARCH" = "aarch64" ]; then + PLATFORM="arm64" + else + PLATFORM="amd64" + fi + curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-\${PLATFORM} && chmod +x container-structure-test-linux-\${PLATFORM} && mv container-structure-test-linux-\${PLATFORM} container-structure-test make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true """ } @@ -330,15 +384,17 @@ void vulnerabilityScan() { * Requires Artifactory and Azure ACR credentials. */ void publishToInternalRegistry() { + // Use the discovered image tag if available (handles date/time mismatches across day boundaries) + def imageToPublish = env.IMAGE_TO_PUBLISH ?: builtImage + echo "Publishing image: ${imageToPublish}" + withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { sh """ docker logout ${dockerRegistry} echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry} - docker tag ${builtImage} ${dockerRegistry}/${builtImage} - docker tag ${builtImage} ${dockerRegistry}/${publishImage} - docker tag ${builtImage} ${dockerRegistry}/${latestTag} - docker tag ${builtImage} ${dockerRegistry}/${timestamptedTag} - docker push ${dockerRegistry}/${builtImage} + docker tag ${imageToPublish} ${dockerRegistry}/${publishImage} + docker tag ${imageToPublish} ${dockerRegistry}/${latestTag} + docker tag ${imageToPublish} ${dockerRegistry}/${timestamptedTag} docker push ${dockerRegistry}/${publishImage} docker push ${dockerRegistry}/${latestTag} docker push ${dockerRegistry}/${timestamptedTag} @@ -370,8 +426,8 @@ void publishToInternalRegistry() { withCredentials([usernamePassword(credentialsId: 'PDC_SANDBOX_USER', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { sh """ echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcSbRegistry} - docker tag ${builtImage} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${builtImage} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} + docker tag ${imageToPublish} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} docker push ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} docker push ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} """ @@ -380,8 +436,8 @@ void publishToInternalRegistry() { withCredentials([usernamePassword(credentialsId: 'pdc-azure-cr', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { sh """ echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcDevRegistry} - docker tag ${builtImage} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${builtImage} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} """ @@ -436,11 +492,7 @@ void scapScan() { } pipeline { - agent { - label { - label 'cld-docker' - } - } + agent none options { checkoutToSubdirectory '.' buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '30', numToKeepStr: '') @@ -450,19 +502,26 @@ pipeline { // Trigger nightly builds on the develop branch for every supported version of MarkLogic // and for every supported image type. // Include SCAP scan for rootless images - parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * * % marklogicVersion=10;dockerImageType=ubi - 00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 05 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 30 05 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : '') + parameterizedCron( + env.BRANCH_NAME == 'develop' ? ''' + 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true + 00 04 * * * % marklogicVersion=10;dockerImageType=ubi + 00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 05 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 30 05 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : + env.BRANCH_NAME == 'Docker-ARM-support' ? ''' + 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') @@ -470,57 +529,74 @@ pipeline { parameters { string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) - string(name: 'dockerVersion', defaultValue: '2.2.3', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) - choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9', description: 'Platform type for Docker image. Will be made part of the docker image tag') + string(name: 'dockerVersion', defaultValue: '2.2.4', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) + choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm') string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true) string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true) booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish image to internal registry') booleanParam(name: 'TEST_STRUCTURE', defaultValue: true, description: 'Run container structure tests') - booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests') - string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true) + booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests') + string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true) booleanParam(name: 'SCAP_SCAN', defaultValue: false, description: 'Run Open SCAP scan on the image.') + booleanParam(name: 'GRAVITON3_AGENT', defaultValue: true, description: '[ARM only] Run ARM-only stages on Graviton3 agent') } stages { // Stage: Perform initial checks (PR status, Jira ID) stage('Pre-Build-Check') { + agent { node { label 'cld-docker' } } steps { preBuildCheck() } } - // Stage: Download MarkLogic Server and Converters RPMs + // Stage: Download MarkLogic Server and Converters RPMs (ARM builds on x86) stage('Copy-RPMs') { + agent { node { label 'cld-docker' } } steps { copyRPMs() } } // Stage: Build the Docker image + // Save image archive to workspace and stash for cross-agent stages. stage('Build-Image') { + agent { node { label 'cld-docker' } } steps { buildDockerImage() + script { + // Always save image for cases where agents might differ + sh """ + echo "Saving ${builtImage} to ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image save ${builtImage} -o ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + ls -lh ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + """ + stash name: 'built-image-archive', includes: "${GRAVITON3_IMAGE_ARCHIVE}", allowEmpty: false + } } } // Stage: Pull the base image needed for upgrade testing stage('Pull-Upgrade-Image') { + agent { node { label 'cld-docker' } } steps { pullUpgradeDockerImage() } } - // Stage: Lint Dockerfile and startup scripts + // Stage: Lint Dockerfile and startup scripts (x86 only) stage('Lint') { + agent { node { label 'cld-docker' } } steps { lint() } } - // Stage: Scan the image for vulnerabilities + // Stage: Scan the image for vulnerabilities (x86 only) stage('Scan') { + agent { node { label 'cld-docker' } } steps { echo 'Skipping vulnerability scan due to compatibility issues.' // vulnerabilityScan() @@ -529,43 +605,156 @@ pipeline { // Stage: Run OpenSCAP compliance scan (conditional) stage('SCAP-Scan') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.SCAP_SCAN } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for SCAP scan..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } scapScan() + stash name: 'scap-results', includes: 'scap/**', allowEmpty: true + } + } + + // Stage: Load image from tar archive (ARM builds only) + stage('Load-Image') { + agent { label 'cld-docker-graviton' } + when { + beforeAgent true + expression { return isArmImage() && params.GRAVITON3_AGENT } + } + steps { + script { + unstash 'built-image-archive' + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image load -i ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + else + echo "Image ${builtImage} already loaded (likely by SCAP-Scan stage)" + fi + docker images | head -5 + """ + } } } // Stage: Run container structure tests (conditional) stage('Structure-Tests') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.TEST_STRUCTURE } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for Structure-Tests..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } structureTests() + stash name: 'structure-test-results', includes: 'container-structure-test.xml', allowEmpty: true } } // Stage: Run Docker functional tests (conditional) stage('Docker-Run-Tests') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.DOCKER_TESTS } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for Docker-Run-Tests..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } dockerTests() + stash name: 'docker-test-results', includes: 'test/test_results/**', allowEmpty: true } } // Stage: Publish image to internal registries (conditional) stage('Publish-Image') { + agent { node { label 'cld-docker' } } when { + beforeAgent true anyOf { branch 'develop' expression { return params.PUBLISH_IMAGE } } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Image not found locally, loading from ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image load -i ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + else + echo "Image ${builtImage} already available locally" + fi + """ + + // If builtImage doesn't exist, find the loaded image by repo pattern + def actualImage = sh( + returnStdout: true, + script: """docker images --format 'table {{.Repository}}:{{.Tag}}' | grep "marklogic/marklogic-server-${dockerImageType}:" | head -1""" + ).trim() + + if (!actualImage) { + actualImage = builtImage + echo "Using builtImage tag: ${actualImage}" + } else { + echo "Found loaded image: ${actualImage}" + } + + // Store for use in publishToInternalRegistry + env.IMAGE_TO_PUBLISH = actualImage + } publishToInternalRegistry() // Trigger downstream QA image build job build job: 'KubeNinjas/docker/docker-nightly-builds-qa', wait: false, parameters: [string(name: 'dockerImageType', value: "${dockerImageType}"), string(name: 'marklogicVersion', value: "${RPMversion}")] @@ -574,6 +763,7 @@ pipeline { // Stage: Trigger BlackDuck security scan (conditional) stage('BlackDuck-Scan') { + agent { node { label 'cld-docker' } } when { anyOf { branch 'develop' @@ -585,19 +775,58 @@ pipeline { } } + // Stage: Cleanup ARM agent (ARM builds only) + stage('Cleanup-ARM') { + agent { label 'cld-docker-graviton' } + when { + beforeAgent true + expression { return isArmImage() && params.GRAVITON3_AGENT } + } + steps { + sh ''' + echo "Cleaning up ARM agent..." + # Stop all running containers + docker stop $(docker ps -a -q) || true + # Docker cleanup + docker system prune --force --all --volumes + docker system df + ''' + } + } + } post { always { - // Clean up the workspace and Docker resources - sh ''' - cd src - rm -rf *.rpm NOTICE.txt - docker stop $(docker ps -a -q) || true - docker system prune --force --all --volumes - docker system df - ''' - publishTestResults() + node('cld-docker') { + // Clean up the workspace and Docker resources + sh """ + # Remove any stale test artifacts before unstash + rm -rf test/test_results scap container-structure-test.xml + # Remove ARM image tar archive + rm -f ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + # Remove ARM image if it was built + if [ -n "${builtImage}" ]; then + docker rmi ${builtImage} || true + fi + # Clean up RPMs + if [ -d src ]; then + cd src + rm -rf *.rpm NOTICE.txt + cd .. + fi + # Docker cleanup applies to both agents + docker stop \$(docker ps -a -q) || true + docker system prune --force --all --volumes + docker system df + """ + script { + try { unstash 'structure-test-results' } catch (e) { echo 'No structure test results to unstash.' } + try { unstash 'docker-test-results' } catch (e) { echo 'No docker test results to unstash.' } + try { unstash 'scap-results' } catch (e) { echo 'No SCAP results to unstash.' } + } + publishTestResults() + } } success { resultNotification('✅ Success') diff --git a/Makefile b/Makefile index cc5b405e..47a4a4cd 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. dockerTag?=internal package?=MarkLogic.rpm repo_dir=marklogic -docker_build_options=--compress --platform linux/amd64 +docker_build_options=--compress build_branch?=local docker_image_type?=ubi upgrade_docker_image_type?=ubi @@ -11,6 +11,17 @@ current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag} # Latest release tag can be found here: https://github.com/ComplianceAsCode/content/releases open_scap_version?=0.1.79 +#*************************************************************************** +# set docker platform based on the docker image type +#*************************************************************************** +ifeq ($(findstring arm,$(docker_image_type)),arm) + docker_build_options += --platform linux/arm64 + export DOCKER_PLATFORM=linux/arm64 +else + docker_build_options += --platform linux/amd64 + export DOCKER_PLATFORM=linux/amd64 +endif + #*************************************************************************** # build docker image #*************************************************************************** @@ -18,6 +29,13 @@ build: # NOTICE file need to be in the build context to be included in the built image cp NOTICE.txt src/NOTICE.txt +# Install ARM64 emulation support on Linux (assuming Jenkins environment which is not aarch64) +ifeq ($(findstring arm,$(docker_image_type)),arm) +ifeq ($(shell uname -s),Linux) + docker run --privileged --rm tonistiigi/binfmt --install arm64 +endif +endif + # rootless images use the same dependencies as ubi image so we copy the file ifeq ($(docker_image_type),ubi9) cp dockerFiles/marklogic-server-ubi\:base dockerFiles/marklogic-server-ubi9\:base @@ -27,10 +45,15 @@ ifeq ($(findstring rootless,$(docker_image_type)),rootless) cp dockerFiles/marklogic-deps-ubi9\:base dockerFiles/marklogic-deps-ubi9-rootless\:base cp dockerFiles/marklogic-server-ubi-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base endif +# ubi9-rootless-arm needs deps from ubi9-arm and server template from ubi-rootless +ifeq ($(docker_image_type),ubi9-rootless-arm) + cp dockerFiles/marklogic-deps-ubi9-arm\:base dockerFiles/marklogic-deps-ubi9-rootless-arm\:base + cp dockerFiles/marklogic-server-ubi-rootless\:base dockerFiles/marklogic-server-ubi9-rootless-arm\:base +endif # retrieve and copy open scap hardening script ifeq ($(findstring rootless,$(docker_image_type)),rootless) - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lo scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip #UBI9 needs a different version of the remediation script ifeq ($(findstring ubi9,$(docker_image_type)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/bash/rhel9-script-cis.sh > src/rhel-script-cis.sh @@ -45,7 +68,7 @@ endif cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_DOCKER_VERSION=${dockerVersion} --build-arg ML_VERSION=${marklogicVersion} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} --build-arg ML_DOCKER_TYPE=${docker_image_type} -f ../dockerFiles/marklogic-server-${docker_image_type}:base . # remove temporary files - rm -f dockerFiles/marklogic-deps-ubi-rootless\:base dockerFiles/marklogic-deps-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9\:base src/NOTICE.txt src/rhel-script-cis.sh + rm -f dockerFiles/marklogic-deps-ubi-rootless\:base dockerFiles/marklogic-deps-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9\:base dockerFiles/marklogic-deps-ubi9-rootless-arm\:base dockerFiles/marklogic-server-ubi9-rootless-arm\:base src/NOTICE.txt src/rhel-script-cis.sh #*************************************************************************** # strcture test docker images @@ -133,15 +156,21 @@ endif # security scan docker images #*************************************************************************** scap-scan: + # Clean up any existing scap-scan container from previous runs + docker rm -f scap-scan 2>/dev/null || true mkdir -p scap - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lo scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip #UBI9 needs a different version of the evaluation profile ifeq ($(findstring ubi9,$(current_image)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/ssg-rhel9-ds.xml > scap/ssg-rhel-ds.xml else unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/ssg-rhel8-ds.xml > scap/ssg-rhel-ds.xml endif - docker run -itd --name scap-scan -v $(PWD)/scap:/scap ${current_image} + docker run -itd --name scap-scan --entrypoint /bin/bash -v $(PWD)/scap:/scap ${current_image} -c "sleep infinity" + # Wait a moment for container to be fully up + sleep 2 + # Verify container is running + docker ps | grep scap-scan || (docker logs scap-scan; exit 1) docker exec -u root scap-scan /bin/bash -c "microdnf update -y; microdnf install -y openscap-scanner" # ensure the file is owned by root in order to avoid permission issues docker exec -u root scap-scan /bin/bash -c "chown root:root /scap/ssg-rhel-ds.xml" diff --git a/NOTICE.txt b/NOTICE.txt index 116953d9..ede40c0f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,6 @@ MarkLogic® Docker Container Image v2 -Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. This project is licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base new file mode 100644 index 00000000..fe93ab01 --- /dev/null +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -0,0 +1,29 @@ +############################################################### +# +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# +############################################################### + +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 +LABEL "com.marklogic.maintainer"="docker@marklogic.com" + +############################################################### +# install libnsl rpm package +############################################################### + +RUN microdnf -y update \ + && rpm -i https://download.rockylinux.org/pub/rocky/9/BaseOS/aarch64/os/Packages/l/libnsl-2.34-231.el9_7.10.aarch64.rpm + +############################################################### +# install networking, base deps and tzdata for timezone +############################################################### +# hadolint ignore=DL3006 +RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ + && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux hostname \ + && microdnf clean all + + +############################################################### +# Enable FIPS Mode +############################################################### +RUN update-crypto-policies --set FIPS \ No newline at end of file diff --git a/dockerFiles/marklogic-deps-ubi9:base b/dockerFiles/marklogic-deps-ubi9:base index c0f2394a..5fe0de0a 100644 --- a/dockerFiles/marklogic-deps-ubi9:base +++ b/dockerFiles/marklogic-deps-ubi9:base @@ -1,10 +1,10 @@ ############################################################### # -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1764794109 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 5abff039..fe415096 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -1,10 +1,10 @@ ############################################################### # -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1765178706 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1771947229 LABEL "com.marklogic.maintainer"="docker@marklogic.com" # MarkLogic version passed from build to enable conditional deps diff --git a/dockerFiles/marklogic-server-ubi-rootless:base b/dockerFiles/marklogic-server-ubi-rootless:base index d1cb4131..3892d6df 100644 --- a/dockerFiles/marklogic-server-ubi-rootless:base +++ b/dockerFiles/marklogic-server-ubi-rootless:base @@ -59,9 +59,15 @@ RUN touch /etc/marklogic.conf \ # Add TINI to serve as PID 1 process ############################################################### ENV TINI_VERSION=v0.19.0 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chown ${ML_USER}:users /tini \ - && chmod +x /tini +ARG ML_DOCKER_TYPE=ubi +RUN if [ "${ML_DOCKER_TYPE}" = "ubi9-rootless-arm" ]; then \ + TINI_BIN="tini-arm64"; \ + else \ + TINI_BIN="tini"; \ + fi && \ + curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN} -o /tini && \ + chown ${ML_USER}:users /tini && \ + chmod +x /tini ############################################################### # second stage for flattening layers @@ -149,7 +155,7 @@ RUN touch /.dockerenv \ ############################################################### WORKDIR / COPY ${ML_CONVERTERS} /tmp/converters.rpm -RUN chown ${ML_USER}:users /tmp/converters.rpm +RUN if [ -s /tmp/converters.rpm ]; then chown ${ML_USER}:users /tmp/converters.rpm; else rm -f /tmp/converters.rpm; fi ############################################################### # Remove optional packages that have known vulnerabilities diff --git a/dockerFiles/marklogic-server-ubi9-arm:base b/dockerFiles/marklogic-server-ubi9-arm:base new file mode 100644 index 00000000..ad81e263 --- /dev/null +++ b/dockerFiles/marklogic-server-ubi9-arm:base @@ -0,0 +1,151 @@ +############################################################### +# +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# +############################################################### + +ARG BASE_IMAGE=marklogic-ubi/marklogic-deps-ubi9-arm:11-internal +FROM ${BASE_IMAGE} AS builder + +############################################################### +# set build args +############################################################### + +ARG ML_RPM=marklogic.rpm +ARG ML_USER="marklogic_user" +ARG ML_VERSION=11-internal +ARG ML_CONVERTERS=marklogic.converters +#################################################### +# inject init, start and clustering scripts +############################################################### + +COPY scripts/start-marklogic.sh /usr/local/bin/start-marklogic.sh + +############################################################### +# install MarkLogic server, sudo, and remove mlcmd packages +############################################################### +COPY ${ML_RPM} /tmp/marklogic-server.rpm +RUN rpm -i /tmp/marklogic-server.rpm \ + && rm /tmp/marklogic-server.rpm \ + && microdnf -y install --setopt install_weak_deps=0 sudo \ + && microdnf -y clean all \ + && rm -rf ./opt/MarkLogic/mlcmd/lib/* \ + && rm -rf ./opt/MarkLogic/mlcmd/ext/* + +############################################################### +# Add TINI to serve as PID 1 process +############################################################### +ENV TINI_VERSION=v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 /tini +RUN chmod +x /tini + +############################################################### +# Copy converters package +############################################################### +WORKDIR / +COPY ${ML_CONVERTERS} converters.rpm +############################################################### +# create system user +############################################################### + +RUN adduser --gid users --uid 1000 ${ML_USER} \ + && echo ${ML_USER}" ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +############################################################### +# second stage for flattening layers +############################################################### +FROM ${BASE_IMAGE} + +COPY --from=builder / / + +ARG ML_USER="marklogic_user" +ARG ML_VERSION=11-internal +ARG ML_DOCKER_VERSION=local +ARG BUILD_BRANCH=local +ARG ML_DOCKER_TYPE=ubi +############################################################### +# define docker labels +############################################################### + +LABEL "com.marklogic.maintainer"="docker@marklogic.com" +LABEL "com.marklogic.name"="MarkLogic Server ${ML_VERSION}" +LABEL "com.marklogic.docker-version"="${ML_DOCKER_VERSION}" +LABEL "com.marklogic.release-version"="${ML_VERSION}" +LABEL "com.marklogic.build-branch"="${BUILD_BRANCH}" +LABEL "com.marklogic"="MarkLogic" +LABEL "com.marklogic.release-type"="production" +LABEL "com.marklogic.license"="MarkLogic EULA" +LABEL "com.marklogic.license.description"="By subscribing to this product, you agree to the terms and conditions outlined in MarkLogic's End User License Agreement (EULA) here https://developer.marklogic.com/eula " +LABEL "com.marklogic.license.url"="https://developer.marklogic.com/eula" +LABEL "com.marklogic.description"="MarkLogic is the only Enterprise NoSQL database. It is a new generation database built with a flexible data model to store, manage, and search JSON, XML, RDF, and more - without sacrificing enterprise features such as ACID transactions, certified security, backup, and recovery. With these capabilities, MarkLogic is ideally suited for making heterogeneous data integration simpler and faster, and for delivering dynamic content at massive scale. The current release of the MarkLogic Server Developer Docker image includes all features and is limited to developer use." +LABEL docker.cmd="docker run -it -p 7997-8010:7997-8010 -e MARKLOGIC_INIT=true -e MARKLOGIC_ADMIN_USERNAME= -e MARKLOGIC_ADMIN_PASSWORD= --mount src=MarkLogic,dst=/var/opt/MarkLogic progressofficial/marklogic-db:${ML_VERSION}" + +############################################################### +# copy notice file +############################################################### +COPY --chown=${ML_USER}:users NOTICE.txt /home/${ML_USER}/NOTICE.txt + +############################################################### +# set env vars +############################################################### + +ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic \ + MARKLOGIC_DATA_DIR=/var/opt/MarkLogic \ + MARKLOGIC_USER=${ML_USER} \ + MARKLOGIC_PID_FILE=/var/run/MarkLogic.pid \ + MARKLOGIC_UMASK=022 \ + LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH:/opt/MarkLogic/lib \ + MARKLOGIC_VERSION="${ML_VERSION}" \ + MARKLOGIC_DOCKER_VERSION="${ML_DOCKER_VERSION}" \ + MARKLOGIC_IMAGE_TYPE="$ML_DOCKER_TYPE" \ + MARKLOGIC_BOOTSTRAP_HOST=bootstrap \ + MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_user \ + MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_password_user \ + MARKLOGIC_WALLET_PASSWORD_FILE=mldb_wallet_password \ + BUILD_BRANCH=${BUILD_BRANCH} \ + MARKLOGIC_JOIN_TLS_ENABLED=false \ + OVERWRITE_ML_CONF=true \ + MARKLOGIC_EC2_HOST=0 + +################################################################ +# Set Timezone +################################################################ + +RUN microdnf -y reinstall tzdata + +############################################################### +# Remove optional packages that have known vulnerabilities +############################################################### +RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ + do rpm -e --nodeps $package || true; \ + done; + +############################################################### +# expose MarkLogic server ports +############################################################### + +EXPOSE 25 7997-8010 + +############################################################### +# set system user +############################################################### + +USER ${ML_USER} + +#################################################### +# Set Linux Language Settings +############################################################### + +ENV LANG=en_US.UTF-8 +ENV LC_ALL=C.UTF-8 + +############################################################### +# define volume for persistent MarkLogic server data +############################################################### + +VOLUME /var/opt/MarkLogic + +############################################################### +# set entrypoint +############################################################### +ENTRYPOINT ["/tini", "--", "/usr/local/bin/start-marklogic.sh"] diff --git a/test/docker-tests.robot b/test/docker-tests.robot index 5c97c9fd..8a3f1c40 100644 --- a/test/docker-tests.robot +++ b/test/docker-tests.robot @@ -124,6 +124,7 @@ Initialized MarkLogic container with latency Upgrade MarkLogic container Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -148,6 +149,7 @@ Upgrade MarkLogic container Upgrade MarkLogic container with init parameter Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -172,6 +174,7 @@ Upgrade MarkLogic container with init parameter Upgrade MarkLogic container with init and credential parameters Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -582,6 +585,7 @@ Initialized MarkLogic Server with wallet password and realm [Teardown] Delete container Initialized MarkLogic container with ML converters + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping ML converters test for ARM image (converters not available) Create container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} diff --git a/test/keywords.resource b/test/keywords.resource index f2c71040..cb23823b 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -1,4 +1,4 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. *** Settings *** Library Process Library String @@ -8,7 +8,8 @@ Library Collections Library DateTime *** Variables *** -@{DOCKER DEFAULTS} -it -d -p 8000:8000 -p 8001:8001 -p 8002:8002 -p7997:7997 --platform linux/amd64 +${DOCKER_PLATFORM} %{DOCKER_PLATFORM=linux/amd64} +@{DOCKER DEFAULTS} -it -d -p 8000:8000 -p 8001:8001 -p 8002:8002 -p7997:7997 --platform ${DOCKER_PLATFORM} ${DEFAULT ADMIN USER} test_admin ${DEFAULT ADMIN PASS} test_admin_pass ${SPEC CHARS ADMIN PASS} Admin@2$s%^&*! @@ -50,7 +51,7 @@ Create container with latency ... --name ${container name} ... --cap-add NET_ADMIN --entrypoint /bin/bash ... ${TEST_IMAGE} - ... -c sudo microdnf -y install iproute iptables && sudo curl -s -O https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/Packages/i/iproute-tc-6.2.0-6.el8_10.x86_64.rpm && sudo rpm -i iproute-tc-6.2.0-6.el8_10.x86_64.rpm && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh + ... -c sudo microdnf -y install iproute iptables && (sudo yum install -y iproute-tc || sudo microdnf install -y iproute) && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh ... stderr=test_results/stderr-${container name}.txt ... stdout=test_results/stdout-${container name}.txt ... timeout=15000 From 10896a813966a8bb26739090b4b2ffb7fe49092d Mon Sep 17 00:00:00 2001 From: Vitaly Date: Mon, 30 Mar 2026 13:14:10 -0700 Subject: [PATCH 02/35] add publish flag to ensure ARM image publishing from feature branch and schedule ML12 ARM builds --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 822c1f5d..ebd2d483 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -516,12 +516,14 @@ pipeline { 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 05 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 30 05 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 06 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 06 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency 00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : env.BRANCH_NAME == 'Docker-ARM-support' ? ''' - 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm - 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true''' : '') + 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true + 00 05 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true + 00 05 * * * % marklogicVersion=12;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') From a2b525caeae0877154281d7a72bb16f2e4e88056 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Tue, 31 Mar 2026 09:04:44 +0200 Subject: [PATCH 03/35] add missing agent --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 83a69ac4..49a8e270 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -548,6 +548,7 @@ pipeline { stages { // Stage: Remove stale test results from previous builds stage('Clean-Previous-Results') { + agent { node { label 'cld-docker' } } steps { sh ''' rm -f container-structure-test.xml From 9aacf7ce558ba02c70aa27a98544b3d1539e3cec Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 9 Apr 2026 07:18:55 -0700 Subject: [PATCH 04/35] Update Jenkinsfile Prevent concurrent runs on a single EC2 instance --- Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 49a8e270..2600de60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -504,7 +504,7 @@ pipeline { // Include SCAP scan for rootless images parameterizedCron( env.BRANCH_NAME == 'develop' ? ''' - 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true 00 04 * * * % marklogicVersion=10;dockerImageType=ubi 00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true @@ -516,14 +516,14 @@ pipeline { 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 06 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 06 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : + 00 10 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 10 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : env.BRANCH_NAME == 'Docker-ARM-support' ? ''' - 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true - 00 05 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 05 * * * % marklogicVersion=12;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') + 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true + 00 06 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true + 00 07 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true + 00 08 * * * % marklogicVersion=12;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') From bbf2fe9010971bf7a3320fc68fee247d8e97471c Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 9 Apr 2026 08:32:00 -0700 Subject: [PATCH 05/35] Update Jenkinsfile Add def keyword to top level variables --- Jenkinsfile | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2600de60..47a11567 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,24 +8,24 @@ import groovy.json.JsonSlurperClassic // email list for scheduled builds (includes security vulnerability) -emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' +def emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' // email list for security vulnerabilities only -emailSecList = 'Mahalakshmi.Srinivasan@progress.com' -gitCredID = 'marklogic-builder-github' -dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' -pdcSbRegistry = 'sandboxpdc.azurecr.io' -pdcDevRegistry = 'marklogicclouddev.azurecr.io' -JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/ -JIRA_ID = '' -LINT_OUTPUT = '' -SCAN_OUTPUT = '' -IMAGE_SIZE = 0 -RPMversion = '' -GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' -builtImage = '' -publishImage = '' -latestTag = '' -upgradeDockerImage = '' +def emailSecList = 'Mahalakshmi.Srinivasan@progress.com' +def gitCredID = 'marklogic-builder-github' +def dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' +def pdcSbRegistry = 'sandboxpdc.azurecr.io' +def pdcDevRegistry = 'marklogicclouddev.azurecr.io' +def JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/ +def JIRA_ID = '' +def LINT_OUTPUT = '' +def SCAN_OUTPUT = '' +def IMAGE_SIZE = 0 +def RPMversion = '' +def GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' +def builtImage = '' +def publishImage = '' +def latestTag = '' +def upgradeDockerImage = '' // Define local funtions From 2b349faabe7602af604d475f701657da8eba602a Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 9 Apr 2026 08:55:14 -0700 Subject: [PATCH 06/35] Update Jenkinsfile Revert last change. --- Jenkinsfile | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47a11567..2600de60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,24 +8,24 @@ import groovy.json.JsonSlurperClassic // email list for scheduled builds (includes security vulnerability) -def emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' +emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' // email list for security vulnerabilities only -def emailSecList = 'Mahalakshmi.Srinivasan@progress.com' -def gitCredID = 'marklogic-builder-github' -def dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' -def pdcSbRegistry = 'sandboxpdc.azurecr.io' -def pdcDevRegistry = 'marklogicclouddev.azurecr.io' -def JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/ -def JIRA_ID = '' -def LINT_OUTPUT = '' -def SCAN_OUTPUT = '' -def IMAGE_SIZE = 0 -def RPMversion = '' -def GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' -def builtImage = '' -def publishImage = '' -def latestTag = '' -def upgradeDockerImage = '' +emailSecList = 'Mahalakshmi.Srinivasan@progress.com' +gitCredID = 'marklogic-builder-github' +dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' +pdcSbRegistry = 'sandboxpdc.azurecr.io' +pdcDevRegistry = 'marklogicclouddev.azurecr.io' +JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/ +JIRA_ID = '' +LINT_OUTPUT = '' +SCAN_OUTPUT = '' +IMAGE_SIZE = 0 +RPMversion = '' +GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' +builtImage = '' +publishImage = '' +latestTag = '' +upgradeDockerImage = '' // Define local funtions From 60078403abc9f9cd4de6cec98cf007f0d7ccd25a Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 9 Apr 2026 09:24:11 -0700 Subject: [PATCH 07/35] Update Jenkinsfile Stash RPMs to ensure they're available across copy and builds stages. --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 2600de60..643e667f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -570,6 +570,7 @@ pipeline { agent { node { label 'cld-docker' } } steps { copyRPMs() + stash name: 'rpms', includes: 'src/*.rpm' } } @@ -578,6 +579,7 @@ pipeline { stage('Build-Image') { agent { node { label 'cld-docker' } } steps { + unstash 'rpms' buildDockerImage() script { // Always save image for cases where agents might differ From 45e2f89e6ee633e2c0b362a9809552c24204baaf Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Thu, 9 Apr 2026 14:56:57 -0700 Subject: [PATCH 08/35] Improve handling of scap-security-guide download in Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 47a4a4cd..50653bf4 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ endif # retrieve and copy open scap hardening script ifeq ($(findstring rootless,$(docker_image_type)),rootless) - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + ([ -f scap-security-guide-${open_scap_version}.zip ] && unzip -t scap-security-guide-${open_scap_version}.zip > /dev/null 2>&1) || (rm -f scap-security-guide-${open_scap_version}.zip && curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip) #UBI9 needs a different version of the remediation script ifeq ($(findstring ubi9,$(docker_image_type)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/bash/rhel9-script-cis.sh > src/rhel-script-cis.sh @@ -159,7 +159,7 @@ scap-scan: # Clean up any existing scap-scan container from previous runs docker rm -f scap-scan 2>/dev/null || true mkdir -p scap - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + ([ -f scap-security-guide-${open_scap_version}.zip ] && unzip -t scap-security-guide-${open_scap_version}.zip > /dev/null 2>&1) || (rm -f scap-security-guide-${open_scap_version}.zip && curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip) #UBI9 needs a different version of the evaluation profile ifeq ($(findstring ubi9,$(current_image)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/ssg-rhel9-ds.xml > scap/ssg-rhel-ds.xml From 1f405bdc8df4e91135978d0dfe2a349053df47ae Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Sun, 12 Apr 2026 16:25:20 -0700 Subject: [PATCH 09/35] MLE-28391: Update UBI base images to latest in ARM Dockerfiles (8.10-1775152441, 9.7-1775623882) --- dockerFiles/marklogic-deps-ubi9-arm:base | 2 +- dockerFiles/marklogic-deps-ubi9:base | 2 +- dockerFiles/marklogic-deps-ubi:base | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base index fe93ab01..0d325c13 100644 --- a/dockerFiles/marklogic-deps-ubi9-arm:base +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -4,7 +4,7 @@ # ############################################################### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1775623882 LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### diff --git a/dockerFiles/marklogic-deps-ubi9:base b/dockerFiles/marklogic-deps-ubi9:base index 5fe0de0a..138df2f0 100644 --- a/dockerFiles/marklogic-deps-ubi9:base +++ b/dockerFiles/marklogic-deps-ubi9:base @@ -4,7 +4,7 @@ # ############################################################### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1775623882 LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 9b1c51d6..175be2e5 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -4,7 +4,7 @@ # ############################################################### -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1771947229 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1775152441 LABEL "com.marklogic.maintainer"="docker@marklogic.com" # MarkLogic version passed from build to enable conditional deps From b4863c88f24414a16d027f40aaa6118d2869c691 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Thu, 23 Apr 2026 18:20:59 -0700 Subject: [PATCH 10/35] Update Docker image types for ARM builds in Jenkinsfile Co-authored-by: Copilot --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 643e667f..76bb02fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -521,9 +521,9 @@ pipeline { 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : env.BRANCH_NAME == 'Docker-ARM-support' ? ''' 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 06 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true + 00 06 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;PUBLISH_IMAGE=true 00 07 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 08 * * * % marklogicVersion=12;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') + 00 08 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') From 102a5ca8fc4eaff3b42bdd9b436648759c8467f6 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Wed, 27 May 2026 10:47:55 -0700 Subject: [PATCH 11/35] MLE-29914: update libnsl repo path to AlmaLinux 9.7 --- dockerFiles/marklogic-deps-ubi9:base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerFiles/marklogic-deps-ubi9:base b/dockerFiles/marklogic-deps-ubi9:base index 138df2f0..8e509924 100644 --- a/dockerFiles/marklogic-deps-ubi9:base +++ b/dockerFiles/marklogic-deps-ubi9:base @@ -12,7 +12,7 @@ LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### RUN microdnf -y update \ - && rpm -i https://repo.almalinux.org/almalinux/9/BaseOS/x86_64/os/Packages/libnsl-2.34-231.el9_7.10.x86_64.rpm + && rpm -i https://repo.almalinux.org/almalinux/9.7/BaseOS/x86_64/os/Packages/libnsl-2.34-231.el9_7.10.x86_64.rpm ############################################################### # install networking, base deps and tzdata for timezone From d2a75fbc0b767fe1d6df64eef9448437b0d444c2 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Wed, 27 May 2026 11:42:21 -0700 Subject: [PATCH 12/35] MLE-29914: update UBI8 libnsl repo path to AlmaLinux 8.10, sync version to .34 --- dockerFiles/marklogic-deps-ubi:base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 175be2e5..1d26b748 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -15,7 +15,7 @@ ARG ML_VERSION ############################################################### RUN microdnf -y upgrade glibc \ - && rpm -i https://repo.almalinux.org/almalinux/8/BaseOS/x86_64/os/Packages/libnsl-2.28-251.el8_10.31.x86_64.rpm \ + && rpm -i https://repo.almalinux.org/almalinux/8.10/BaseOS/x86_64/os/Packages/libnsl-2.28-251.el8_10.34.x86_64.rpm \ && microdnf clean all ############################################################### From 8efb9e67abdb4319bb309ae995cedc9e314f8d8d Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Wed, 27 May 2026 11:48:30 -0700 Subject: [PATCH 13/35] MLE-29914: add --nodeps to UBI8 libnsl install to fix glibc compatibility --- dockerFiles/marklogic-deps-ubi:base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 1d26b748..2013703b 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -15,7 +15,7 @@ ARG ML_VERSION ############################################################### RUN microdnf -y upgrade glibc \ - && rpm -i https://repo.almalinux.org/almalinux/8.10/BaseOS/x86_64/os/Packages/libnsl-2.28-251.el8_10.34.x86_64.rpm \ + && rpm -i --nodeps https://repo.almalinux.org/almalinux/8.10/BaseOS/x86_64/os/Packages/libnsl-2.28-251.el8_10.34.x86_64.rpm \ && microdnf clean all ############################################################### From 8b2ab5a1696e677bf7dbbfa0226c2b69e4acd398 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Wed, 27 May 2026 13:40:51 -0700 Subject: [PATCH 14/35] MLE-29914: add --nodeps and pin Rocky Linux path to 9.7 for aarch64 libnsl --- dockerFiles/marklogic-deps-ubi9-arm:base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base index 0d325c13..48264b0f 100644 --- a/dockerFiles/marklogic-deps-ubi9-arm:base +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -12,7 +12,7 @@ LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### RUN microdnf -y update \ - && rpm -i https://download.rockylinux.org/pub/rocky/9/BaseOS/aarch64/os/Packages/l/libnsl-2.34-231.el9_7.10.aarch64.rpm + && rpm -i --nodeps https://download.rockylinux.org/pub/rocky/9.7/BaseOS/aarch64/os/Packages/l/libnsl-2.34-231.el9_7.10.aarch64.rpm ############################################################### # install networking, base deps and tzdata for timezone From 5d22f5eadb193b6876dab87f288f5b6607b296af Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Wed, 10 Jun 2026 07:03:50 -0700 Subject: [PATCH 15/35] MLE-30361: Update aarch64 libnsl URL to Rocky Linux 9.8 to match glibc upgrade --- dockerFiles/marklogic-deps-ubi9-arm:base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base index 48264b0f..14b2a733 100644 --- a/dockerFiles/marklogic-deps-ubi9-arm:base +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -12,7 +12,7 @@ LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### RUN microdnf -y update \ - && rpm -i --nodeps https://download.rockylinux.org/pub/rocky/9.7/BaseOS/aarch64/os/Packages/l/libnsl-2.34-231.el9_7.10.aarch64.rpm + && rpm -i --nodeps https://download.rockylinux.org/pub/rocky/9.8/BaseOS/aarch64/os/Packages/l/libnsl-2.34-270.el9_8.aarch64.rpm ############################################################### # install networking, base deps and tzdata for timezone From 613a500b34fd163ea2f5da77eeaafb9f414187f9 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 11 Jun 2026 12:50:46 -0700 Subject: [PATCH 16/35] increment docker version --- Jenkinsfile | 2 +- test/keywords.resource | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 76bb02fa..b35e9e75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -531,7 +531,7 @@ pipeline { parameters { string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) - string(name: 'dockerVersion', defaultValue: '2.2.4', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) + string(name: 'dockerVersion', defaultValue: '2.2.5', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm') diff --git a/test/keywords.resource b/test/keywords.resource index 379c6ec0..418f0ba6 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -13,7 +13,7 @@ ${DOCKER_PLATFORM} %{DOCKER_PLATFORM=linux/amd64} ${DEFAULT ADMIN USER} test_admin ${DEFAULT ADMIN PASS} test_admin_pass ${SPEC CHARS ADMIN PASS} Admin@2$s%^&*! -${TEST_IMAGE} %{DOCKER_TEST_IMAGE=progressofficial/marklogic-db:11.3.1-ubi-rootless-2.2.4} +${TEST_IMAGE} %{DOCKER_TEST_IMAGE=progressofficial/marklogic-db:11.3.1-ubi-rootless-2.2.5} ${UPGRADE_TEST_IMAGE} progressofficial/marklogic-db:${MARKLOGIC_VERSION}-${IMAGE_TYPE}-${MARKLOGIC_DOCKER_VERSION} ${DOCKER TIMEOUT} 300s ${LICENSE KEY} %{QA_LICENSE_KEY=none} @@ -23,7 +23,7 @@ ${BUILD_BRANCH} release_2.2.1 ${IMAGE_TYPE} ubi-rootless ${VOL_NAME} MarkLogic_vol_1 ${VOL_INFO} src=${VOL_NAME},dst=/var/opt/MarkLogic -${MARKLOGIC_DOCKER_VERSION} 2.2.4 +${MARKLOGIC_DOCKER_VERSION} 2.2.5 ${TEST_RESULTS_DIR} test_results *** Keywords *** From c9f8d403878023e7785b4ce5f899d0fcf39c055e Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Fri, 10 Jul 2026 08:26:59 -0700 Subject: [PATCH 17/35] Increment Docker version and remove ML10 option --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b35e9e75..face87f9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -531,10 +531,10 @@ pipeline { parameters { string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) - string(name: 'dockerVersion', defaultValue: '2.2.5', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) + string(name: 'dockerVersion', defaultValue: '2.2.6', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) - choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm') + choice(name: 'marklogicVersion', choices: '12\n11', description: 'MarkLogic Server Branch. used to pick appropriate rpm') string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true) string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true) booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish image to internal registry') From 49b6a3776a61c998a9909ba5b56d0f31d2b013c3 Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:24:09 -0700 Subject: [PATCH 18/35] MLE-31141: Rebasing Docker-arm-support feature branch with develop (#466) * Update UBI base versions * Remove UBI9 update to resolve libnsl conflict * Fix notice file and increment Docker version for release * MLE-24929 Add internal ECR publishing for K8s tests (#431) * Add internal ECR publishing for K8s tests * fix typo * chore: derive ECR registry URL via STS instead of hardcoded account ID Remove the global kubeNinjasEcrRegistry = '308453789681.dkr.ecr.us-west-1.amazonaws.com' constant. Instead, resolve the AWS account ID at runtime inside the existing KUBE_NINJAS_OPS_AWS_JENKINS withCredentials block using: aws sts get-caller-identity --query Account --output text This eliminates the hardcoded account number and is consistent with the approach used in marklogic-operator-kubernetes Jenkinsfile. --------- Co-authored-by: Vitaly Korolev * Update Jenkinsfile Add explicit region Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * MLE-22732 : BUGFIX stacktrace for root (#423) * add missing python3 lib for root image * fix some bugs and update README * update lbnsl for UBI8 * review of the README * rewording in README * add lightweight test for gdb and python3 * fix issue with tzdata install * sync with develop branch * add microdnf clean all * fix TZ issue * fix gliches * try fix issue with timezone * fix test for gdb * fix structure-tests.yaml * MLE-27788: add tests for dynamic-host api changes * Update test/keywords.resource Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * MLE-28295: restore redhat-lsb-core for UBI8 images PR #423 removed redhat-lsb-core unconditionally from the UBI8 deps image. Both MarkLogic 10 and 11 RPMs require the lsb-core-amd64 package it provides; ML12 does not. Restore redhat-lsb-core to the main microdnf install line so it is present for all ML versions on UBI8. The ML10 conditional block is retained for libstdc++.i686 only. * MLE-27707 : Add stack trace capability for rootless image (#429) * review lib for stack trace and enable gdp for rootless * Update dockerFiles/add mising microdnf clean Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix tzdata install * fix TZ issue * fix gliches * new fix for tzdata * fix test --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * PDP-1182: Remove per-repo pr-workflow.yaml * PDP-1182: Remove per-repo pr-workflow.yaml * MLE-3247: Load pipeline notification emails from Jenkins secret file Replace hardcoded emailList and emailSecList global variables with a loadEmailConfig() helper that reads both values from the KUBE_NINJAS_PIPELINE_EMAILS Jenkins secret file credential at runtime. The emailList pipeline parameter is kept as an optional override at the bottom of the parameter list; when left blank the credential value is used, preserving the existing override behaviour for ad-hoc build runs. * MLE-28391: Update UBI base images to latest (8.10-1775152441, 9.7-1775623882) * MLE-29349: Update libnsl RPM version to match upgraded glibc 2.28-251.el8_10.34 * MLE-29350: Publish MarkLogic 11 nightly Docker images to PDC * MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency - Update UBI8 base image: 8.10-1775152441 -> 8.10-1778735208 - Update UBI9 base image: 9.7-1775623882 -> 9.7-1778562320 - Fix UBI8 libnsl: bump to el8_10.34 (matches updated glibc) - Fix UBI9 libnsl: use --nodeps to bypass glibc el9_8 version pin (UBI CDN serves glibc el9_8 after microdnf update; AlmaLinux 9.8 stable not yet released; libnsl.so.1 ABI compatible across all RHEL 9.x) - Bump dockerVersion 2.2.4 -> 2.2.5 in Jenkinsfile and keywords.resource * MLE-29703 Address PR review: targeted glibc upgrade and cache cleanup in UBI9 deps - Replace broad microdnf -y update with targeted microdnf -y upgrade glibc - Add microdnf clean all in same RUN layer to reduce image layer size - Add comment explaining why --nodeps is required and safe for libnsl install * MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency - Update UBI8 base image: 8.10-1771947229 -> 8.10-1778735208 - Update UBI9 base image: 9.7-1771346502 -> 9.7-1778562320 - Fix UBI8 libnsl: bump to el8_10.34, targeted glibc upgrade, cache cleanup - Fix UBI9 libnsl: use --nodeps to bypass glibc el9_8 version pin, targeted glibc upgrade, cache cleanup; add comment explaining rationale - Bump dockerVersion 2.2.4 -> 2.2.5 in Jenkinsfile and keywords.resource * Revert "MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency" This reverts commit a50402c7c1a8c05518d6b619a339b0d13167ef9b. * MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency - Update UBI8 base image: 8.10-1771947229 -> 8.10-1778735208 - Update UBI9 base image: 9.7-1771346502 -> 9.7-1778562320 - Fix UBI8 libnsl: bump to el8_10.34, targeted glibc upgrade, cache cleanup - Fix UBI9 libnsl: use --nodeps to bypass glibc el9_8 version pin, targeted glibc upgrade, cache cleanup; add comment explaining rationale - Bump dockerVersion 2.2.4 -> 2.2.5 in Jenkinsfile and keywords.resource * Revert "MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency" This reverts commit 15ad66fc53feacc7d756b64ca6fc5522f08822ed. * MLE-29703 MLE-29709 Update UBI base images and fix libnsl dependency - Update UBI8 base image: 8.10-1771947229 -> 8.10-1778735208 - Update UBI9 base image: 9.7-1771346502 -> 9.7-1778562320 - Fix UBI8 libnsl: bump to el8_10.34, targeted glibc upgrade, cache cleanup - Fix UBI9 libnsl: use --nodeps to bypass glibc el9_8 version pin, targeted glibc upgrade, cache cleanup; add comment explaining rationale - Bump dockerVersion 2.2.4 -> 2.2.5 in Jenkinsfile and keywords.resource * MLE-29737: Remove MarkLogic 10 builds from pipeline (#453) Co-authored-by: sumanthravipati * MLE-29914: update libnsl repo path to AlmaLinux 9.7 * MLE-29914: update libnsl repo path to AlmaLinux 9.7 * MLE-29914: update UBI8 libnsl repo path to AlmaLinux 8.10 * MLE-29914: update UBI8 libnsl repo path to AlmaLinux 8.10 * MLE-29914: add --nodeps to UBI8 libnsl install to fix glibc compatibility * MLE-29914: add --nodeps to UBI8 libnsl install to fix glibc compatibility * MLE-29914: upgrade tzdata in UBI8 deps to fix reinstall failure in server image * MLE-30361: Update libnsl URL to AlmaLinux 9.8 to match glibc upgrade * Update libnsl version * Increment Docker version to 2.2.6 * increment marklogic version in tests * implement copilot suggestions (typos) * increment docker version with the release * fix mixed up merge * another missing commit * MLE-4163: robot tests enhancements (#464) * robot tests enhancements * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * recommendation fixes * updated test tags * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * updated compose file names and removed redundant compose yaml. * updated copyright info in compose files * added test numbers --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Vitaly Co-authored-by: Vitaly Co-authored-by: Vitaly Korolev Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Romain Winieski Co-authored-by: tposham Co-authored-by: SameeraPriyathamTadikonda Co-authored-by: sumanthravipati <43222215+sumanthravipati@users.noreply.github.com> Co-authored-by: sumanthravipati --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/copilot-instructions.md | 2 +- .github/workflows/pr-workflow.yaml | 23 -- Jenkinsfile | 107 ++++-- Makefile | 6 + README.md | 128 ++++--- dockerFiles/marklogic-deps-ubi9:base | 15 +- dockerFiles/marklogic-deps-ubi:base | 6 +- .../marklogic-server-ubi-rootless:base | 13 +- dockerFiles/marklogic-server-ubi:base | 5 +- ...=> compose-1node-bootstrap-env-creds.yaml} | 3 +- ...st-8.yaml => compose-1node-self-join.yaml} | 4 +- ...yaml => compose-2node-bootstrap-only.yaml} | 4 +- ...l => compose-2node-cluster-env-creds.yaml} | 4 +- ...compose-2node-invalid-bootstrap-host.yaml} | 4 +- ... => compose-2node-join-enode-secrets.yaml} | 4 +- ...compose-2node-join-https-invalid-tls.yaml} | 3 +- ...pose-2node-join-https-missing-cacert.yaml} | 3 +- ... => compose-2node-join-https-secrets.yaml} | 3 +- ...ml => compose-2node-second-uncoupled.yaml} | 4 +- ...> compose-2node-second-uninitialized.yaml} | 4 +- test/compose-2x2node-clusters-secrets.yaml | 110 ++++++ ...aml => compose-3core-11dynamic-hosts.yaml} | 4 +- test/compose-test-1.yaml | 25 -- test/compose-test-14.yaml | 25 -- test/compose-test-15.yaml | 42 --- test/compose-test-2.yaml | 42 --- test/docker-tests.robot | 334 ++++++++++++------ test/keywords.resource | 293 ++++++++++++++- test/structure-test.yaml | 10 +- 30 files changed, 853 insertions(+), 379 deletions(-) delete mode 100644 .github/workflows/pr-workflow.yaml rename test/{compose-test-12.yaml => compose-1node-bootstrap-env-creds.yaml} (80%) rename test/{compose-test-8.yaml => compose-1node-self-join.yaml} (81%) rename test/{compose-test-6.yaml => compose-2node-bootstrap-only.yaml} (81%) rename test/{compose-test-3.yaml => compose-2node-cluster-env-creds.yaml} (88%) rename test/{compose-test-9.yaml => compose-2node-invalid-bootstrap-host.yaml} (89%) rename test/{compose-test-7.yaml => compose-2node-join-enode-secrets.yaml} (85%) rename test/{compose-test-10.yaml => compose-2node-join-https-invalid-tls.yaml} (89%) rename test/{compose-test-11.yaml => compose-2node-join-https-missing-cacert.yaml} (87%) rename test/{compose-test-13.yaml => compose-2node-join-https-secrets.yaml} (88%) rename test/{compose-test-4.yaml => compose-2node-second-uncoupled.yaml} (88%) rename test/{compose-test-5.yaml => compose-2node-second-uninitialized.yaml} (89%) create mode 100644 test/compose-2x2node-clusters-secrets.yaml rename test/{compose-test-16.yaml => compose-3core-11dynamic-hosts.yaml} (98%) delete mode 100644 test/compose-test-1.yaml delete mode 100644 test/compose-test-14.yaml delete mode 100644 test/compose-test-15.yaml delete mode 100644 test/compose-test-2.yaml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 812d66f8..47042866 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,4 +12,4 @@ - ##### Reviewer: - [ ] Reviewed Tests -- [ ] Added to Release Wiki/Jira +- [ ] Added to Release Wiki/Jira \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 68481009..64bae2c9 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -228,4 +228,4 @@ The pipeline supports: - Always create an issue before starting significant work - Tests must be added/updated for new features - Linting must pass: `hadolint` for Dockerfiles, `shellcheck` for scripts -- Security scan reports reviewed before merging +- Security scan reports reviewed before merging \ No newline at end of file diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml deleted file mode 100644 index 9165227c..00000000 --- a/.github/workflows/pr-workflow.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: PR Workflow - -on: - # Using pull_request_target instead of pull_request to handle PRs from forks - pull_request_target: - types: [opened, edited, reopened, synchronize] - # No branch filtering - will run on all PRs - -jobs: - jira-pr-check: - name: 🏷️ Validate JIRA ticket ID - # Use the reusable workflow from the central repository - uses: marklogic/pr-workflows/.github/workflows/jira-id-check.yml@main - with: - # Pass the PR title from the event context - pr-title: ${{ github.event.pull_request.title }} - copyright-validation: - name: © Validate Copyright Headers - uses: marklogic/pr-workflows/.github/workflows/copyright-check.yml@main - permissions: - contents: read - pull-requests: write - issues: write diff --git a/Jenkinsfile b/Jenkinsfile index face87f9..70f7333c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,10 +7,6 @@ @Library('shared-libraries@1.0-declarative') import groovy.json.JsonSlurperClassic -// email list for scheduled builds (includes security vulnerability) -emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' -// email list for security vulnerabilities only -emailSecList = 'Mahalakshmi.Srinivasan@progress.com' gitCredID = 'marklogic-builder-github' dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' pdcSbRegistry = 'sandboxpdc.azurecr.io' @@ -38,6 +34,23 @@ upgradeDockerImage = '' def isArmImage() { return params.dockerImageType.toLowerCase().contains('arm') } +/** + * Loads email configuration from the KUBE_NINJAS_PIPELINE_EMAILS Jenkins secret file credential. + * The credential file must contain key=value lines for 'emailList' and 'emailSecList'. + * @return A map with keys 'emailList' and 'emailSecList'. + */ +Map loadEmailConfig() { + def result = [emailList: '', emailSecList: ''] + withCredentials([file(credentialsId: 'KUBE_NINJAS_PIPELINE_EMAILS', variable: 'emailConfigFile')]) { + def props = readProperties file: emailConfigFile + result.emailList = (props.emailList ?: '').trim() + result.emailSecList = (props.emailSecList ?: '').trim() + if (!result.emailList || !result.emailSecList) { + error("KUBE_NINJAS_PIPELINE_EMAILS must define non-empty 'emailList' and 'emailSecList' properties") + } + } + return result +} /** * Performs pre-build checks: @@ -147,13 +160,18 @@ def getReviewState() { * @param status The build status string (e.g., 'Success', 'Failure'). */ void resultNotification(status) { + def paramEmailList = params.emailList?.trim() + def needSecList = params.SCAP_SCAN && BRANCH_NAME == 'develop' + def emailConfig = (!paramEmailList || needSecList) ? loadEmailConfig() : null + def baseEmailList = paramEmailList ?: emailConfig.emailList + def emailSecList = emailConfig?.emailSecList ?: '' def author, authorEmail, emailList if (env.CHANGE_AUTHOR) { author = env.CHANGE_AUTHOR.toString().trim().toLowerCase() authorEmail = getEmailFromGITUser author - emailList = params.emailList + ',' + authorEmail + emailList = baseEmailList + ',' + authorEmail } else { - emailList = params.emailList + emailList = baseEmailList } email_body = "Build URL: ${env.BUILD_URL}
" + @@ -336,6 +354,7 @@ void structureTests() { * Runs Docker functional tests using the 'make docker-tests' target. */ void dockerTests() { + sh "make docker-test-ids" sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType} DOCKER_TEST_LIST=\"${params.DOCKER_TEST_LIST}\"" } @@ -372,7 +391,8 @@ void vulnerabilityScan() { SCAN_OUTPUT = sh(returnStdout: true, script: "cat scan/report-${env.dockerImageType}.txt") sh 'echo "SCAN_OUTPUT: ${SCAN_OUTPUT}"' if (SCAN_OUTPUT.size()) { - mail charset: 'UTF-8', mimeType: 'text/html', to: "${emailSecList}", body: "
Jenkins pipeline for ${env.JOB_NAME}
Build Number: ${env.BUILD_NUMBER}
Vulnerabilities:
${SCAN_OUTPUT}
", subject: "Critical or High Security Vulnerabilities Found: ${env.JOB_NAME} #${env.BUILD_NUMBER}" + def emailConfig = loadEmailConfig() + mail charset: 'UTF-8', mimeType: 'text/html', to: "${emailConfig.emailSecList}", body: "
Jenkins pipeline for ${env.JOB_NAME}
Build Number: ${env.BUILD_NUMBER}
Vulnerabilities:
${SCAN_OUTPUT}
", subject: "Critical or High Security Vulnerabilities Found: ${env.JOB_NAME} #${env.BUILD_NUMBER}" } archiveArtifacts artifacts: 'scan/*', onlyIfSuccessful: true } @@ -420,7 +440,7 @@ void publishToInternalRegistry() { // } // } - // Publish to private ACR repositories that are used by PDC. (only ML12) + // Publish to private ACR repositories that are used by PDC. if ( params.marklogicVersion == "12" ) { // Publish to Sandbox PDC registry withCredentials([usernamePassword(credentialsId: 'PDC_SANDBOX_USER', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { @@ -432,6 +452,8 @@ void publishToInternalRegistry() { docker push ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} """ } + } + if ( params.marklogicVersion == "11" || params.marklogicVersion == "12" ) { // Publish to Dev PDC registry withCredentials([usernamePassword(credentialsId: 'pdc-azure-cr', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { sh """ @@ -443,6 +465,27 @@ void publishToInternalRegistry() { """ } } + if ( params.marklogicVersion == "12" ) { + // Publish to Kubernetes ECR for testing on EKS + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'KUBE_NINJAS_OPS_AWS_JENKINS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + // Resolve account ID via STS — no account number is hardcoded in this file. + def awsAccountId = sh(returnStdout: true, + script: 'aws sts get-caller-identity --region us-west-1 --query Account --output text').trim() + def kubeNinjasEcrRegistry = "${awsAccountId}.dkr.ecr.us-west-1.amazonaws.com" + def ecrRepo = "${kubeNinjasEcrRegistry}/jenkins-kube-ninjas/marklogic-server-${dockerImageType}" + sh """ + aws ecr get-login-password --region us-west-1 | \\ + docker login --username AWS --password-stdin ${kubeNinjasEcrRegistry} + docker tag ${builtImage} ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${builtImage} ${ecrRepo}:latest-${mlVerShort} + docker push ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker push ${ecrRepo}:latest-${mlVerShort} + """ + } + } currentBuild.description = "Published" } @@ -502,38 +545,31 @@ pipeline { // Trigger nightly builds on the develop branch for every supported version of MarkLogic // and for every supported image type. // Include SCAP scan for rootless images - parameterizedCron( - env.BRANCH_NAME == 'develop' ? ''' - 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm - 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true - 00 04 * * * % marklogicVersion=10;dockerImageType=ubi - 00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 - 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 10 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 10 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : - env.BRANCH_NAME == 'Docker-ARM-support' ? ''' - 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 06 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;PUBLISH_IMAGE=true - 00 07 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;PUBLISH_IMAGE=true - 00 08 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;PUBLISH_IMAGE=true''' : '') - } + parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 03 * * * % marklogicVersion=11;dockerImageType=ubi + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 30 05 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true + 00 06 * * * % marklogicVersion=12;dockerImageType=ubi9-arm + 30 06 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true + 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : '') + } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') } parameters { - string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) string(name: 'dockerVersion', defaultValue: '2.2.6', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) - choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') - string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) + choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9', description: 'Platform type for Docker image. Will be made part of the docker image tag') + string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rootless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) choice(name: 'marklogicVersion', choices: '12\n11', description: 'MarkLogic Server Branch. used to pick appropriate rpm') string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true) string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true) @@ -543,6 +579,7 @@ pipeline { string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true) booleanParam(name: 'SCAP_SCAN', defaultValue: false, description: 'Run Open SCAP scan on the image.') booleanParam(name: 'GRAVITON3_AGENT', defaultValue: true, description: '[ARM only] Run ARM-only stages on Graviton3 agent') + string(name: 'emailList', defaultValue: '', description: 'Optional override for the build notification email list. If left blank, the list is loaded from the KUBE_NINJAS_PIPELINE_EMAILS Jenkins credential file. Specify a comma-separated list only to send notifications to additional or different recipients for a specific build run.', trim: true) } stages { @@ -856,4 +893,4 @@ pipeline { resultNotification('🚫 Aborted') } } -} +} \ No newline at end of file diff --git a/Makefile b/Makefile index 50653bf4..4045cdcd 100644 --- a/Makefile +++ b/Makefile @@ -90,8 +90,14 @@ endif #*************************************************************************** # docker image tests #*************************************************************************** +.PHONY: docker-test-ids +docker-test-ids: + @echo "Docker test ID catalog (Robot test names):" + @awk 'BEGIN{in_tests=0} /^\*\*\* Test Cases \*\*\*/{in_tests=1;next} /^\*\*\*/{if(in_tests)exit} in_tests && /^[DC][0-9][0-9] /{print " - " $$0}' ./test/docker-tests.robot + docker-tests: cd test; \ + $(MAKE) -s docker-test-ids; \ python3 -m venv python_env; \ source ./python_env/bin/activate; \ pip3 install -r requirements.txt; \ diff --git a/README.md b/README.md index b1463183..8148d324 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ * [MarkLogic](#MarkLogic) * [Using this Image](#Using-this-Image) * [Configuration](#Configuration) + * [Enabling Stack Trace Generation](#enabling-stack-trace-generation) * [Clustering](#Clustering) * [Upgrading to the latest MarkLogic Docker Release](#Upgrading-to-the-latest-MarkLogic-Docker-Release) * [Backing Up and Restoring a Database](#Backing-Up-and-Restoring-a-Database) @@ -79,8 +80,9 @@ For an initialized MarkLogic Server, admin credentials are required to be passed To create an initialized MarkLogic Server, pass in the environment variables MARKLOGIC_ADMIN_USERNAME and MARKLOGIC_ADMIN_PASSWORD, and replace {insert admin username}/{insert admin password} with actual values for admin credentials. Use the optional environment variable MARKLOGIC_WALLET_PASSWORD and REALM to set the wallet password and authentication realm of the admin user. If not provided, the wallet-password will default to the value set for admin-password and realm will be set to public. Optionally, you can pass license information in `{insert license}`/`{insert licensee}` to apply your MarkLogic license. To do this, run this this command: -``` +```bash $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) -e MARKLOGIC_INIT=true \ -e MARKLOGIC_ADMIN_USERNAME={insert admin username} \ -e MARKLOGIC_ADMIN_PASSWORD={insert admin password} \ @@ -90,14 +92,18 @@ $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ -e LICENSEE="{insert licensee}" \ progressofficial/marklogic-db ``` + Example run: -``` + +```bash $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) -e MARKLOGIC_INIT=true \ -e MARKLOGIC_ADMIN_USERNAME='admin' \ -e MARKLOGIC_ADMIN_PASSWORD='Areally!PowerfulPassword1337' \ progressofficial/marklogic-db ``` + Wait about a minute for MarkLogic Server to initialize before checking the ports. To verify the successful installation and initialization, log into the MarkLogic Server Admin Interface using the admin credentials used in the command above. Go to http://localhost:8001. You can also verify the configuration by following the procedures outlined in the MarkLogic Server documentation. See the MarkLogic Installation documentation [here](https://docs.marklogic.com/guide/installation/procedures#id_84772). ## Uninitialized MarkLogic Server @@ -105,8 +111,9 @@ For an uninitialized MarkLogic Server, admin credentials or license information To create an uninitialized MarkLogic Server with [Docker CLI](https://docs.docker.com/engine/reference/commandline/cli/), run this command: -``` +```bash $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) progressofficial/marklogic-db ``` The example output will contain a hash of the image ID: `f484a784d99838a918e384eca5d5c0a35e7a4b0f0545d1389e31a65d57b2573d` @@ -122,7 +129,7 @@ A MarkLogic Docker container stores data in `/var/opt/MarkLogic` which is persis The following command will list previously created volumes: -``` +```bash $ docker volume ls ``` If the instructions in the **Using this Image** section are followed, the previous command should output at least two volume identifiers: @@ -134,8 +141,9 @@ local 1b65575a84be319222a4ff9ba9eecdff06ffb3143edbd03720f4b808be0e6d18 The following command uses a named volume and named container in order to make management easier: -``` +```bash $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) --name MarkLogic_cont_1 \ --mount src=MarkLogic_vol_1,dst=/var/opt/MarkLogic \ -e MARKLOGIC_INIT=true \ @@ -147,7 +155,7 @@ $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ Above command will start a Docker container `MarkLogic_cont_1` running MarkLogic Server and associate the named Docker volume `MarkLogic_vol_1` with it. Run this command to check the volumes: -``` +```bash $ docker volume ls ``` @@ -239,6 +247,18 @@ You can change the number of HugePages available to each MarkLogic container by -e ML_HUGEPAGES_TOTAL=0 ``` +## Enabling Stack Trace Generation + + To enable detailed native stack trace generation in the event of a crash (for example, a segmentation fault), you can grant the container the `SYS_PTRACE` capability. + + Add this flag to your `docker run` invocation when you require enhanced crash diagnostics and your container runtime policy permits it: + +```bash +--cap-add=SYS_PTRACE +``` + +This capability is optional and not required for normal MarkLogic Server operation. In hardened or restricted environments where `SYS_PTRACE` is disallowed, omit this flag; crash diagnostics will be more limited, but the server will still run. + # Clustering MarkLogic Server Docker containers ship with a small set of scripts, making it easy to create clusters. See the [MarkLogic documentation](https://docs.marklogic.com/guide/concepts/clustering) for more about clusters. The following three examples show how to create MarkLogic Server clusters with Docker containers. The first two use Docker compose scripts to create one-node and three-node clusters. See the documentation for [Docker compose](https://docs.docker.com/compose/) for more details. The third example demonstrates a container setup on separate VMs. @@ -252,7 +272,7 @@ Create these files on your host machine: `marklogic-single-node.yaml`, `mldb_adm **marklogic-single-node.yaml** -``` +```yaml #Docker compose file sample to setup single node cluster version: '3.6' services: @@ -261,6 +281,8 @@ services: container_name: bootstrap hostname: bootstrap dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -318,13 +340,13 @@ volumes: Once the files are ready, run this command to start the MarkLogic Server container. -``` +```bash $ docker-compose -f marklogic-single-node.yaml up -d ``` The previous command starts a container running MarkLogic Server named "bootstrap". Run this next command to verify if the container is running: -``` +```bash $ docker ps ``` If the containers are running correctly, this command lists all the Docker containers running on the host. @@ -337,7 +359,7 @@ The following is an example of a three-node MarkLogic server cluster created usi **marklogic-multi-node.yaml** -``` +```yaml #Docker compose file sample to setup a three node cluster version: '3.6' services: @@ -346,6 +368,8 @@ services: container_name: bootstrap_3n hostname: bootstrap_3n dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -366,6 +390,8 @@ services: container_name: node2 hostname: node2 dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -390,6 +416,8 @@ services: container_name: node3 hostname: node3 dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -438,14 +466,14 @@ volumes: Once the files have been created, run the following command to start the MarkLogic Server container: -``` +```bash $ docker-compose -f marklogic-multi-node.yaml up -d ``` This command will start three Docker containers running MarkLogic Server, named "bootstrap_3n", "node2" and, "node3". Run this command to verify if the containers are running: -``` +```bash $ docker ps ``` This command lists all the Docker containers running on the host. @@ -479,13 +507,15 @@ Using Docker secrets, username and password information are secured when transmi $docker secret create mldb_wallet_password_v1 mldb_wallet_password_v1.txt ``` 3. Create marklogic-multi-node.yaml using below: -``` +```yaml version: '3.6' services: bootstrap: image: progressofficial/marklogic-db hostname: bootstrap dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -511,6 +541,8 @@ services: image: progressofficial/marklogic-db hostname: node2 dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -539,6 +571,8 @@ services: image: progressofficial/marklogic-db hostname: node3 dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -599,7 +633,7 @@ Now that the nodes have been initialized, we rotate the secrets files to overwri $docker secret create mldb_wallet_password_v2 mldb_wallet_password_v2.txt ``` 6. Use the following commands to rotate the Docker secrets for all the Docker services created above using Docker stack: -``` +```bash docker service update \ --secret-rm mldb_admin_username_v1 \ --secret-rm mldb_admin_password_v1 \ @@ -609,7 +643,7 @@ docker service update \ --secret-add source=mldb_wallet_password_v2,target=mldb_wallet_password \ mlstack_bootstrap ``` -``` +```bash docker service update \ --secret-rm mldb_admin_username_v1 \ --secret-rm mldb_admin_password_v1 \ @@ -619,7 +653,7 @@ docker service update \ --secret-add source=mldb_wallet_password_v2,target=mldb_wallet_password \ mlstack_node2 ``` -``` +```bash docker service update \ --secret-rm mldb_admin_username_v1 \ --secret-rm mldb_admin_password_v1 \ @@ -643,28 +677,29 @@ Follow these steps to set up the first node ("bootstrap") on VM1. Initialize the Docker Swarm with this command: -``` +```bash $ docker swarm init ``` Copy the output from this step. The other VMs will need this information to connect them to the swarm. The output will be similar to this: `docker swarm join --token xxxxxxxxxxxxx {VM1_IP}:2377`. Use this command to create a new network: -``` +```bash $ docker network create --driver=overlay --attachable ml-cluster-network ``` Use this command to verify the ml-cluster-network has been created: -``` +```bash $ docker network ls ``` The `network ls` command will list all the networks on the host. Run this command to start the Docker container, adding your username and password to the command. It will start the Docker container (named "bootstrap") with MarkLogic Server initialized. -``` +```bash $ docker run -d -it -p 7100:8000 -p 7101:8001 -p 7102:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) --name bootstrap -h bootstrap.marklogic.com \ -e MARKLOGIC_ADMIN_USERNAME={insert admin username} \ -e MARKLOGIC_ADMIN_PASSWORD={insert admin password} \ @@ -682,15 +717,16 @@ Follow the next steps to set up an additional node (for example ml2) on VM#n. Run the Docker `swarm join` command that you got as output when you set up VM#1 previously. -``` +```bash $ docker swarm join --token xxxxxxxxxxxxx {VM1_IP}:2377 ``` This command adds the current node to the swarm initialized earlier. Start the Docker container (ml2.marklogic.com) with MarkLogic Server initialized, and join the container to the same cluster as you started/initialized on VM#1. Be sure to add your admin username and password for the bootstrap host in the Docker start up command that follows. To join this host to a specific MarkLogic Group, use the MARKLOGIC_GROUP environment parameter as below. -``` +```bash $ docker run -d -it -p 7200:8000 -p 7201:8001 -p 7202:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) --name ml2 -h ml2.marklogic.com \ -e MARKLOGIC_ADMIN_USERNAME={insert admin username} \ -e MARKLOGIC_ADMIN_PASSWORD={insert admin password} \ @@ -711,7 +747,7 @@ This example shows how to join a node to a TLS enabled cluster. There are two pr Below example uses docker stack for MarkLogic cluster deployment. It will create a docker stack named mlstack with two services named bootstrap and node2. 1. Create a bootstrap host using the following compose file: -``` +```yaml version: '3.6' services: bootstrap_3n: @@ -719,6 +755,8 @@ services: container_name: bootstrap_3n hostname: bootstrap_3n dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=test_admin @@ -745,7 +783,7 @@ docker stack deploy -c bootstrap-compose.yaml mlstack 4. Obtain the CA certificate for SSL enabled app servers on the bootstrap host and store it in the same directory as the compose file. The CA certificate/certificate chain used to join the cluster will be stored as Docker secret. 5. Create files `mldb_admin_username.txt` and `mldb_admin_password.txt` to set the admin username/password used for joining the bootstrap host. 6. Use the compose file below to create node2. Please note the {MARKLOGIC_JOIN_TLS_ENABLED} parameter is set to true and the {MARKLOGIC_JOIN_CACERT_FILE} is set as a Docker secret with the value set to the CA certificate/certificate chain file path. Please see the [Configuration](#Configuration) section for more details on these two parameters. -``` +```yaml version: '3.6' services: node2: @@ -753,6 +791,8 @@ services: container_name: node2 hostname: node2 dns_search: "" + cap_add: + - SYS_PTRACE environment: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username @@ -764,7 +804,7 @@ services: - TZ=Europe/Prague volumes: - MarkLogic_2n_vol2:/var/opt/MarkLogic - secrets:sta + secrets: - source: mldb_admin_username target: mldb_admin_username - source: mldb_admin_password @@ -806,7 +846,7 @@ Follow below steps to update the certificate: $docker secret create certificate_v2.cer certificate_v2.cer ``` 2. Use the below command to rotate the Docker secret for the mlstack_node2 Docker services created above using Docker stack: -``` +```bash docker service update \ --secret-rm certificate_v1.cer \ --secret-add source=certificate_v2.cer,target=certificate.cer \ @@ -825,19 +865,20 @@ Note: In the below example, we are upgrading an initialized MarkLogic host to th 1. If you are upgrading to a rootless image, you need to update the ownership of all files and directories under /var/opt/MarkLogic in the container. Otherwise skip to step 2. Use the following two commands to stop the MarkLogic server and update the ownership of the files and directories: -``` +```bash $ docker exec -it -u root container_id /etc/init.d/MarkLogic stop $ docker exec -it -u root container_id chown -R 1000:100 /var/opt/MarkLogic ``` 2. Stop the MarkLogic Docker container. Use following command to stop the container: -``` +```bash $ docker stop container_id ``` 3. To upgrade MarkLogic, create a new container with the latest Docker image while using the same volume mounted to the container that was running the older release. To prevent conflicts, you should either remove the old container or assign a distinct name to the new container. The following commands use a unique name for the new container with the existing volume. -``` +```bash $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) --name MarkLogic_cont_2 \ --mount src=MarkLogic_vol_1,dst=/var/opt/MarkLogic \ progressofficial/marklogic-db @@ -851,8 +892,9 @@ $ docker run -d -it -p 8000:8000 -p 8001:8001 -p 8002:8002 \ When creating a backup for a database on a MarkLogic Docker container, verify that the directory used for the backup is mounted to a directory on the Docker host machine or Docker volume. This is so that the database backup persists even after the container is stopped. This command is an example of mounting the directory /space used for backup on a Docker volume, while running the MarkLogic Docker container. -``` +```bash $ docker run -d -it -p 7000:8000 -p 7001:8001 -p 7002:8002 \ + # --cap-add=SYS_PTRACE \ (see "Enabling Stack Trace Generation" section below for details) --mount src=MarkLogic_vol_1,dst=/var/opt/MarkLogic \ --mount src=MarkLogic_vol_1,dst=/space \ -e MARKLOGIC_INIT=true \ @@ -874,7 +916,7 @@ Add the `--platform linux/amd64` flag to the `docker run` command to avoid this ## View MarkLogic Server Startup Status To check the MarkLogic Server startup status, run the below command to tail the MarkLogic log file -``` +```bash $ docker exec -it tail -f /var/opt/MarkLogic/Logs/ErrorLog.txt ``` @@ -885,7 +927,7 @@ The following is a set of steps to run to access a container while it is running 1. Access the machine running the Docker container. This is typically done using SSH or by having physical access to the machine hosting the container. 2. Get the container ID for the MarkLogic container running on the machine. To do this, run the following command: -``` +```bash $ docker container ps --filter ancestor=progressofficial/marklogic-db -q ``` In this example command `progressofficial/marklogic-db` is an image ID. Your container ID may be different for your machine. @@ -898,7 +940,7 @@ f484a784d998 If you don't know the image name, you can search for it without a filter: -``` +```bash $ docker container ps ``` @@ -913,13 +955,13 @@ f484a784d998 progressofficial/marklogic-db "/usr/local/bin/star…" 16 min For this example command, `f484a784d998` is the container ID from the prior step. The one assigned to your container will be different. -``` +```bash $ docker exec -it f484a784d998 /bin/bash ``` 4. To verify that MarkLogic is running, use this command: -``` +```bash $ service MarkLogic status ``` @@ -933,13 +975,13 @@ MarkLogic (pid 34) is running... For example, you can list the 8001 error logs, and view them with a single command: -``` +```bash $ cd /var/opt/MarkLogic/Logs && ls && cat ./8001_ErrorLog.txt ``` 6. To exit the container when you are through debugging, use the exit command: -``` +```bash $ exit ``` @@ -949,13 +991,13 @@ $ exit These are the steps you can use to remove the containers created in the "Using this Image" section of the text. It is important to remove resources after development is complete to free up ports and resources when they are not in use. Use this command to stop a container, replacing `container_name` with the name(s) of the container(s) found when using the command: `docker container ps`. -``` +```bash $ docker stop container_name ``` Use this command to remove a stopped container: -``` +```bash $ docker rm container_name ``` @@ -966,20 +1008,20 @@ This section describes the teardown process for clusters set up on a single VM u Resources such as containers, volumes, and networks that were created with compose command can be removed using this command: -``` +```bash $ docker-compose -f marklogic-single-node.yaml down ``` ### Remove volumes Volumes can be removed in a few ways. Adding the `–rm` option while running a container will remove the volume when the container dies. You can also remove a volume by using `prune`. See the following examples for more information. -``` +```bash $ docker run --rm -v /foo -v awesome:/bar container image ``` To remove all other unused volumes use this command: -``` +```bash $ docker volume prune ``` If the process is successful, the output will list all of the removed volumes. @@ -990,7 +1032,7 @@ Then remove all the volumes with the commands described in the "Remove volumes" Finally, disconnect VMs from the swarm running the following command on each VM: -``` +```bash docker swarm leave --force ``` If the process is successful, a message saying the node has left the swarm will be displayed. diff --git a/dockerFiles/marklogic-deps-ubi9:base b/dockerFiles/marklogic-deps-ubi9:base index 8e509924..db92e2fd 100644 --- a/dockerFiles/marklogic-deps-ubi9:base +++ b/dockerFiles/marklogic-deps-ubi9:base @@ -4,22 +4,27 @@ # ############################################################### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1775623882 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1778562320 LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### # install libnsl rpm package ############################################################### -RUN microdnf -y update \ - && rpm -i https://repo.almalinux.org/almalinux/9.7/BaseOS/x86_64/os/Packages/libnsl-2.34-231.el9_7.10.x86_64.rpm +# microdnf -y upgrade glibc brings the UBI9 glibc to el9_8 (currently 2.34-270.el9_8). +# --nodeps is required because the AlmaLinux libnsl RPM declares a dependency on +# AlmaLinux's glibc packaging; UBI9 provides the same ABI but different package provenance. +# The AlmaLinux 9.8 libnsl version is kept in sync with the glibc version UBI9 pulls. +RUN microdnf -y upgrade glibc \ + && rpm -i --nodeps https://repo.almalinux.org/almalinux/9.8/BaseOS/x86_64/os/Packages/libnsl-2.34-270.el9_8.x86_64.rpm \ + && microdnf clean all ############################################################### -# install networking, base deps and tzdata for timezone +# install gdb and dependencies for stack traces, networking, base deps and tzdata for timezone ############################################################### # hadolint ignore=DL3006 RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ - && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux hostname \ + && microdnf -y install --setopt install_weak_deps=0 gdb python3-rpm nss libcap procps-ng python3 libtool-ltdl cpio initscripts tzdata glibc libstdc++ util-linux hostname \ && microdnf clean all diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 2013703b..4d5f9361 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -4,7 +4,7 @@ # ############################################################### -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1775152441 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1778735208 LABEL "com.marklogic.maintainer"="docker@marklogic.com" # MarkLogic version passed from build to enable conditional deps @@ -19,11 +19,11 @@ RUN microdnf -y upgrade glibc \ && microdnf clean all ############################################################### -# install networking, base deps and tzdata for timezone +# install gdb and dependencies for stack traces, networking, base deps and tzdata for timezone ############################################################### # hadolint ignore=DL3006 RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ - && microdnf -y install --setopt install_weak_deps=0 gdb redhat-lsb-core initscripts tzdata glibc libstdc++ hostname \ + && microdnf -y install --setopt install_weak_deps=0 gdb python3-rpm nss libcap procps-ng python3 libtool-ltdl cpio initscripts tzdata glibc libstdc++ util-linux hostname redhat-lsb-core \ && microdnf -y upgrade tzdata \ && microdnf clean all diff --git a/dockerFiles/marklogic-server-ubi-rootless:base b/dockerFiles/marklogic-server-ubi-rootless:base index 3892d6df..da625cda 100644 --- a/dockerFiles/marklogic-server-ubi-rootless:base +++ b/dockerFiles/marklogic-server-ubi-rootless:base @@ -157,10 +157,21 @@ WORKDIR / COPY ${ML_CONVERTERS} /tmp/converters.rpm RUN if [ -s /tmp/converters.rpm ]; then chown ${ML_USER}:users /tmp/converters.rpm; else rm -f /tmp/converters.rpm; fi +############################################################### +# Configure GDB for debugging and set capabilities for non-root usage +############################################################### +RUN microdnf -y install libcap \ + && setcap cap_sys_ptrace+ep $(readlink -f /usr/bin/gdb) \ + && echo "set auto-load safe-path /" > /home/${ML_USER}/.gdbinit \ + && chown ${ML_USER}:users /home/${ML_USER}/.gdbinit \ + && chmod 644 /home/${ML_USER}/.gdbinit \ + && microdnf clean all + ############################################################### # Remove optional packages that have known vulnerabilities +# (Excluding python/gdb dependencies needed for stack traces) ############################################################### -RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ +RUN for package in vim-minimal cups-client cups-libs tar avahi-libs binutils libarchive binutils-gold; \ do rpm -e --nodeps $package || true; \ done; diff --git a/dockerFiles/marklogic-server-ubi:base b/dockerFiles/marklogic-server-ubi:base index 7fa8ca2a..95a275a0 100644 --- a/dockerFiles/marklogic-server-ubi:base +++ b/dockerFiles/marklogic-server-ubi:base @@ -115,8 +115,9 @@ RUN microdnf -y reinstall tzdata ############################################################### # Remove optional packages that have known vulnerabilities +# (Excluding python/gdb dependencies needed for stack traces) ############################################################### -RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ +RUN for package in vim-minimal cups-client cups-libs tar avahi-libs binutils libarchive binutils-gold; \ do rpm -e --nodeps $package || true; \ done; @@ -148,4 +149,4 @@ VOLUME /var/opt/MarkLogic ############################################################### # set entrypoint ############################################################### -ENTRYPOINT ["/tini", "--", "/usr/local/bin/start-marklogic.sh"] +ENTRYPOINT ["/tini", "--", "/usr/local/bin/start-marklogic.sh"] \ No newline at end of file diff --git a/test/compose-test-12.yaml b/test/compose-1node-bootstrap-env-creds.yaml similarity index 80% rename from test/compose-test-12.yaml rename to test/compose-1node-bootstrap-env-creds.yaml index 8a5a9459..1210f981 100644 --- a/test/compose-test-12.yaml +++ b/test/compose-1node-bootstrap-env-creds.yaml @@ -1,4 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Single bootstrap node with admin credentials from environment variables. version: '3.6' services: bootstrap_3n: diff --git a/test/compose-test-8.yaml b/test/compose-1node-self-join.yaml similarity index 81% rename from test/compose-test-8.yaml rename to test/compose-1node-self-join.yaml index d444747d..95cfd347 100644 --- a/test/compose-test-8.yaml +++ b/test/compose-1node-self-join.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a one node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Single node configured to join itself to validate self-join handling. version: '3.6' services: bootstrap: diff --git a/test/compose-test-6.yaml b/test/compose-2node-bootstrap-only.yaml similarity index 81% rename from test/compose-test-6.yaml rename to test/compose-2node-bootstrap-only.yaml index 08c16eae..61de17f1 100644 --- a/test/compose-test-6.yaml +++ b/test/compose-2node-bootstrap-only.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file to setup the bootstrap node on cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Bootstrap-only node used as base for follow-up join tests. version: '3.6' services: bootstrap_2n: diff --git a/test/compose-test-3.yaml b/test/compose-2node-cluster-env-creds.yaml similarity index 88% rename from test/compose-test-3.yaml rename to test/compose-2node-cluster-env-creds.yaml index 8af814a9..8cb5cde0 100644 --- a/test/compose-test-3.yaml +++ b/test/compose-2node-cluster-env-creds.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a three node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Two-node cluster using admin credentials from environment variables. version: '3.6' services: bootstrap: diff --git a/test/compose-test-9.yaml b/test/compose-2node-invalid-bootstrap-host.yaml similarity index 89% rename from test/compose-test-9.yaml rename to test/compose-2node-invalid-bootstrap-host.yaml index d3563f99..c3bc70e1 100644 --- a/test/compose-test-9.yaml +++ b/test/compose-2node-invalid-bootstrap-host.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a two node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Two-node deployment with an invalid bootstrap host name for node2. version: '3.6' services: bootstrap: diff --git a/test/compose-test-7.yaml b/test/compose-2node-join-enode-secrets.yaml similarity index 85% rename from test/compose-test-7.yaml rename to test/compose-2node-join-enode-secrets.yaml index aaf145ed..465dcd89 100644 --- a/test/compose-test-7.yaml +++ b/test/compose-2node-join-enode-secrets.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file to setup and join node2 on cluster using MARKLOGIC_GROUP param +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Second node joins bootstrap and is assigned to the enode group using secrets. version: '3.6' services: node2: diff --git a/test/compose-test-10.yaml b/test/compose-2node-join-https-invalid-tls.yaml similarity index 89% rename from test/compose-test-10.yaml rename to test/compose-2node-join-https-invalid-tls.yaml index 99bf90aa..c719fd1e 100644 --- a/test/compose-test-10.yaml +++ b/test/compose-2node-join-https-invalid-tls.yaml @@ -1,4 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: HTTPS join with invalid MARKLOGIC_JOIN_TLS_ENABLED value. version: '3.6' services: node2: diff --git a/test/compose-test-11.yaml b/test/compose-2node-join-https-missing-cacert.yaml similarity index 87% rename from test/compose-test-11.yaml rename to test/compose-2node-join-https-missing-cacert.yaml index 46897cfb..d67ee492 100644 --- a/test/compose-test-11.yaml +++ b/test/compose-2node-join-https-missing-cacert.yaml @@ -1,4 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: HTTPS join with missing MARKLOGIC_JOIN_CACERT_FILE value. version: '3.6' services: node3: diff --git a/test/compose-test-13.yaml b/test/compose-2node-join-https-secrets.yaml similarity index 88% rename from test/compose-test-13.yaml rename to test/compose-2node-join-https-secrets.yaml index 493060f4..8daf1e5d 100644 --- a/test/compose-test-13.yaml +++ b/test/compose-2node-join-https-secrets.yaml @@ -1,4 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Second node joins bootstrap over HTTPS using username/password/CA secrets. version: '3.6' services: node2: diff --git a/test/compose-test-4.yaml b/test/compose-2node-second-uncoupled.yaml similarity index 88% rename from test/compose-test-4.yaml rename to test/compose-2node-second-uncoupled.yaml index 0203c9a9..ee71392f 100644 --- a/test/compose-test-4.yaml +++ b/test/compose-2node-second-uncoupled.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a three node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Two-node deployment where second node is initialized but does not join the cluster. version: '3.6' services: testbootstrap: diff --git a/test/compose-test-5.yaml b/test/compose-2node-second-uninitialized.yaml similarity index 89% rename from test/compose-test-5.yaml rename to test/compose-2node-second-uninitialized.yaml index 2df9c725..f1b3df94 100644 --- a/test/compose-test-5.yaml +++ b/test/compose-2node-second-uninitialized.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a three node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Two-node deployment where second node remains uninitialized. version: '3.6' services: testbootstrap: diff --git a/test/compose-2x2node-clusters-secrets.yaml b/test/compose-2x2node-clusters-secrets.yaml new file mode 100644 index 00000000..edf7cc44 --- /dev/null +++ b/test/compose-2x2node-clusters-secrets.yaml @@ -0,0 +1,110 @@ +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Two separate 2-node clusters with secrets for cross-cluster coupling tests. +version: '3.6' +services: + # === CLUSTER 1 === + cluster1_bootstrap: + image: progressofficial/marklogic-db + container_name: cluster1_bootstrap + hostname: cluster1_bootstrap + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_cluster1_vol1:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7100-7110:8000-8010 + - 7197:7997 + networks: + - external_net + + cluster1_node2: + image: progressofficial/marklogic-db + container_name: cluster1_node2 + hostname: cluster1_node2 + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - MARKLOGIC_JOIN_CLUSTER=true + - MARKLOGIC_BOOTSTRAP_HOST=cluster1_bootstrap + - TZ=Europe/Prague + volumes: + - MarkLogic_cluster1_vol2:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7200-7210:8000-8010 + - 7297:7997 + depends_on: + - cluster1_bootstrap + networks: + - external_net + + # === CLUSTER 2 === + cluster2_bootstrap: + image: progressofficial/marklogic-db + container_name: cluster2_bootstrap + hostname: cluster2_bootstrap + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_cluster2_vol1:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7300-7310:8000-8010 + - 7397:7997 + networks: + - external_net + + cluster2_node2: + image: progressofficial/marklogic-db + container_name: cluster2_node2 + hostname: cluster2_node2 + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - MARKLOGIC_JOIN_CLUSTER=true + - MARKLOGIC_BOOTSTRAP_HOST=cluster2_bootstrap + - TZ=Europe/Prague + volumes: + - MarkLogic_cluster2_vol2:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7400-7410:8000-8010 + - 7497:7997 + depends_on: + - cluster2_bootstrap + networks: + - external_net + +secrets: + mldb_admin_password: + file: ./mldb_admin_password.txt + mldb_admin_username: + file: ./mldb_admin_username.txt +networks: + external_net: {} +volumes: + MarkLogic_cluster1_vol1: + MarkLogic_cluster1_vol2: + MarkLogic_cluster2_vol1: + MarkLogic_cluster2_vol2: diff --git a/test/compose-test-16.yaml b/test/compose-3core-11dynamic-hosts.yaml similarity index 98% rename from test/compose-test-16.yaml rename to test/compose-3core-11dynamic-hosts.yaml index 4728689c..5a7f69b4 100644 --- a/test/compose-test-16.yaml +++ b/test/compose-3core-11dynamic-hosts.yaml @@ -1,5 +1,5 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -#Docker compose file sample to setup a three node cluster +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Compose scenario: Dynamic host test topology with 3 core nodes and 11 dynamic nodes. version: '3.6' services: bootstrap_3n: diff --git a/test/compose-test-1.yaml b/test/compose-test-1.yaml deleted file mode 100644 index 8a5a9459..00000000 --- a/test/compose-test-1.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -version: '3.6' -services: - bootstrap_3n: - image: progressofficial/marklogic-db - container_name: bootstrap_3n - hostname: bootstrap_3n - dns_search: "" - environment: - - MARKLOGIC_INIT=true - - MARKLOGIC_ADMIN_USERNAME=test_admin - - MARKLOGIC_ADMIN_PASSWORD=test_admin_pass - - REALM=public - - TZ=Europe/Prague - volumes: - - MarkLogic_3n_vol1:/var/opt/MarkLogic - ports: - - 7100-7110:8000-8010 - - 7197:7997 - networks: - - external_net -networks: - external_net: {} -volumes: - MarkLogic_3n_vol1: \ No newline at end of file diff --git a/test/compose-test-14.yaml b/test/compose-test-14.yaml deleted file mode 100644 index 8a5a9459..00000000 --- a/test/compose-test-14.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -version: '3.6' -services: - bootstrap_3n: - image: progressofficial/marklogic-db - container_name: bootstrap_3n - hostname: bootstrap_3n - dns_search: "" - environment: - - MARKLOGIC_INIT=true - - MARKLOGIC_ADMIN_USERNAME=test_admin - - MARKLOGIC_ADMIN_PASSWORD=test_admin_pass - - REALM=public - - TZ=Europe/Prague - volumes: - - MarkLogic_3n_vol1:/var/opt/MarkLogic - ports: - - 7100-7110:8000-8010 - - 7197:7997 - networks: - - external_net -networks: - external_net: {} -volumes: - MarkLogic_3n_vol1: \ No newline at end of file diff --git a/test/compose-test-15.yaml b/test/compose-test-15.yaml deleted file mode 100644 index 493060f4..00000000 --- a/test/compose-test-15.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -version: '3.6' -services: - node2: - image: progressofficial/marklogic-db - container_name: node2 - hostname: node2 - dns_search: "" - environment: - - MARKLOGIC_INIT=true - - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username - - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password - - MARKLOGIC_JOIN_TLS_ENABLED=true - - MARKLOGIC_JOIN_CACERT_FILE=certificate.cer - - MARKLOGIC_JOIN_CLUSTER=true - - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n - - TZ=Europe/Prague - volumes: - - MarkLogic_3n_vol2:/var/opt/MarkLogic - secrets: - - source: mldb_admin_username - target: mldb_admin_username - - source: mldb_admin_password - target: mldb_admin_password - - source: certificate.cer - target: certificate.cer - ports: - - 7200-7210:8000-8010 - - 7297:7997 - networks: - - external_net -secrets: - mldb_admin_password: - file: ./mldb_admin_password.txt - mldb_admin_username: - file: ./mldb_admin_username.txt - certificate.cer: - file: ./certificate.cer -networks: - external_net: {} -volumes: - MarkLogic_3n_vol2: \ No newline at end of file diff --git a/test/compose-test-2.yaml b/test/compose-test-2.yaml deleted file mode 100644 index 493060f4..00000000 --- a/test/compose-test-2.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -version: '3.6' -services: - node2: - image: progressofficial/marklogic-db - container_name: node2 - hostname: node2 - dns_search: "" - environment: - - MARKLOGIC_INIT=true - - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username - - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password - - MARKLOGIC_JOIN_TLS_ENABLED=true - - MARKLOGIC_JOIN_CACERT_FILE=certificate.cer - - MARKLOGIC_JOIN_CLUSTER=true - - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n - - TZ=Europe/Prague - volumes: - - MarkLogic_3n_vol2:/var/opt/MarkLogic - secrets: - - source: mldb_admin_username - target: mldb_admin_username - - source: mldb_admin_password - target: mldb_admin_password - - source: certificate.cer - target: certificate.cer - ports: - - 7200-7210:8000-8010 - - 7297:7997 - networks: - - external_net -secrets: - mldb_admin_password: - file: ./mldb_admin_password.txt - mldb_admin_username: - file: ./mldb_admin_username.txt - certificate.cer: - file: ./certificate.cer -networks: - external_net: {} -volumes: - MarkLogic_3n_vol2: \ No newline at end of file diff --git a/test/docker-tests.robot b/test/docker-tests.robot index 8a3f1c40..0656d854 100644 --- a/test/docker-tests.robot +++ b/test/docker-tests.robot @@ -1,4 +1,4 @@ -# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. *** Settings *** Resource keywords.resource Documentation Test all initialization options using Docker run and Docker Compose. @@ -8,12 +8,18 @@ Suite Setup Ensure Test Results Directory Exists *** Test Cases *** -Smoke Test +D01 Smoke Test + [Tags] docker-run positive + [Documentation] Detailed scenario: Smoke Test. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with Docker log should contain *MARKLOGIC_INIT is set to false or not defined, not initializing.* [Teardown] Delete container -Uninitialized MarkLogic container +D02 Uninitialized MarkLogic container + [Tags] docker-run positive + [Documentation] Detailed scenario: Uninitialized MarkLogic container. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=false IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* @@ -37,7 +43,10 @@ Uninitialized MarkLogic container Verify response for authenticated request with 8002 *Forbidden* [Teardown] Delete container -Uninitialized MarkLogic container with no parameters +D03 Uninitialized MarkLogic container with no parameters + [Tags] docker-run positive + [Documentation] Detailed scenario: Uninitialized MarkLogic container with no parameters. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* @@ -62,7 +71,10 @@ Uninitialized MarkLogic container with no parameters Verify response for authenticated request with 8002 *Forbidden* [Teardown] Delete container -Initialized MarkLogic container +D04 Initialized MarkLogic container + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -89,9 +101,9 @@ Initialized MarkLogic container Verify response for authenticated request with 8002 *Monitoring Dashboard* [Teardown] Delete container -Initialized MarkLogic container with latency - [Tags] long_running - [Documentation] This test verifies the initialization of the MarkLogic container with high latency. +D05 Initialized MarkLogic container with latency + [Tags] docker-run positive long_running + [Documentation] This test verifies the initialization of the MarkLogic container with high latency. Detailed scenario: Initialized MarkLogic container with latency. ... Setup on a linux host can be done with the following commands: ... sudo dnf install kernel-modules-extra ... sudo modprobe sch_netem @@ -122,9 +134,11 @@ Initialized MarkLogic container with latency Verify response for authenticated request with 8002 *Monitoring Dashboard* [Teardown] Delete container -Upgrade MarkLogic container - Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image - Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image +D06 Upgrade MarkLogic container + [Tags] docker-run positive + [Documentation] Detailed scenario: Upgrade MarkLogic container. + ... Covers setup, execution, and expected outcome validation for this scenario. + Skip If 'rootless' in '${IMAGE_TYPE}' msg=Skipping Upgrade MarkLogic test for rootless image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -147,9 +161,11 @@ Upgrade MarkLogic container [Teardown] Run Keywords Delete container True ... AND Delete Volume -Upgrade MarkLogic container with init parameter - Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image - Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image +D07 Upgrade MarkLogic container with init parameter + [Tags] docker-run positive + [Documentation] Detailed scenario: Upgrade MarkLogic container with init parameter. + ... Covers setup, execution, and expected outcome validation for this scenario. + Skip If 'rootless' in '${IMAGE_TYPE}' msg=Skipping Upgrade MarkLogic test for rootless image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -172,9 +188,11 @@ Upgrade MarkLogic container with init parameter [Teardown] Run Keywords Delete container True ... AND Delete Volume -Upgrade MarkLogic container with init and credential parameters - Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image - Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image +D08 Upgrade MarkLogic container with init and credential parameters + [Tags] docker-run positive + [Documentation] Detailed scenario: Upgrade MarkLogic container with init and credential parameters. + ... Covers setup, execution, and expected outcome validation for this scenario. + Skip If 'rootless' in '${IMAGE_TYPE}' msg=Skipping Upgrade MarkLogic test for rootless image Create test container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -199,7 +217,10 @@ Upgrade MarkLogic container with init and credential parameters [Teardown] Run Keywords Delete container True ... AND Delete Volume -Initialized MarkLogic container with admin password containing special characters +D09 Initialized MarkLogic container with admin password containing special characters + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container with admin password containing special characters. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${SPEC CHARS ADMIN PASS} @@ -214,7 +235,10 @@ Initialized MarkLogic container with admin password containing special character Verify response for authenticated request with 8002 *Monitoring Dashboard* ${SPEC CHARS ADMIN PASS} [Teardown] Delete container -Initialized MarkLogic container with license key installed and MARKLOGIC_INIT set to TRUE +D10 Initialized MarkLogic container with license key installed and MARKLOGIC_INIT set to TRUE + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container with license key installed and MARKLOGIC_INIT set to TRUE. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=TRUE ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -230,14 +254,18 @@ Initialized MarkLogic container with license key installed and MARKLOGIC_INIT se Verify response for authenticated request with 8002 *Monitoring Dashboard* [Teardown] Delete container -Initialized MarkLogic container without credentials - [Tags] negative +D11 Initialized MarkLogic container without credentials + [Tags] docker-run negative + [Documentation] Detailed scenario: Initialized MarkLogic container without credentials. + ... Covers setup, execution, and expected outcome validation for this scenario. Create failing container with -e MARKLOGIC_INIT=true Docker log should contain *MARKLOGIC_ADMIN_USERNAME and MARKLOGIC_ADMIN_PASSWORD must be set.* [Teardown] Delete container -Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER - [Tags] negative +D12 Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER + [Tags] docker-run negative + [Documentation] Detailed scenario: Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER. + ... Covers setup, execution, and expected outcome validation for this scenario. Create failing container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -246,16 +274,20 @@ Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER Docker log should contain *Error: MARKLOGIC_JOIN_CLUSTER must be true or false.* [Teardown] Delete container -Invalid value for INIT - [Tags] negative +D13 Invalid value for INIT + [Tags] docker-run negative + [Documentation] Detailed scenario: Invalid value for INIT. + ... Covers setup, execution, and expected outcome validation for this scenario. Create failing container with -e MARKLOGIC_INIT=invalid ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} Docker log should contain *Error: MARKLOGIC_INIT must be true or false.* [Teardown] Delete container -Invalid value for HOSTNAME - [Tags] negative +D14 Invalid value for HOSTNAME + [Tags] docker-run negative + [Documentation] Detailed scenario: Invalid value for HOSTNAME. + ... Covers setup, execution, and expected outcome validation for this scenario. Create failing container with -e HOSTNAME=invalid_hostname ... -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} @@ -263,7 +295,10 @@ Invalid value for HOSTNAME Docker log should contain *Error: Failed to restart invalid_hostname* [Teardown] Delete container -Initialized MarkLogic container without config overrides +D15 Initialized MarkLogic container without config overrides + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container without config overrides. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e OVERWRITE_ML_CONF=false ... -e TZ=America/Los_Angeles @@ -298,7 +333,10 @@ Initialized MarkLogic container without config overrides Verify container timezone America/Los_Angeles [Teardown] Delete container -Initialized MarkLogic container with config overrides +D16 Initialized MarkLogic container with config overrides + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container with config overrides. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e OVERWRITE_ML_CONF=true ... -e TZ=America/Los_Angeles @@ -323,8 +361,10 @@ Initialized MarkLogic container with config overrides Verify container timezone America/Los_Angeles [Teardown] Delete container -Single node compose example - [Tags] compose +C01 Single node compose example + [Tags] compose positive + [Documentation] Detailed scenario: Single node compose example. + ... Covers setup, execution, and expected outcome validation for this scenario. ${compose test file}= Set Variable ../docker-compose/marklogic-single-node.yaml Start compose from ${compose test file} Verify response for unauthenticated request with 8000 *Unauthorized* @@ -340,8 +380,10 @@ Single node compose example Verify container timezone Europe/Prague [Teardown] Delete compose from ../docker-compose/marklogic-single-node.yaml -Single node compose example with special characters in secrets file - [Tags] compose +C02 Single node compose example with special characters in secrets file + [Tags] compose positive + [Documentation] Detailed scenario: Single node compose example with special characters in secrets file. + ... Covers setup, execution, and expected outcome validation for this scenario. Start compose from ../docker-compose/marklogic-single-node.yaml ${SPEC CHARS ADMIN PASS} Verify response for unauthenticated request with 8000 *Unauthorized* Verify response for unauthenticated request with 8001 *Unauthorized* @@ -351,19 +393,23 @@ Single node compose example with special characters in secrets file Verify response for authenticated request with 8002 *Monitoring Dashboard* ${SPEC CHARS ADMIN PASS} [Teardown] Delete compose from ../docker-compose/marklogic-single-node.yaml -Single node compose with special characters in yaml - [Tags] compose - Start compose from ../test/compose-test-1.yaml ${SPEC CHARS ADMIN PASS} +C03 Single node compose with special characters in yaml + [Tags] compose positive + [Documentation] Detailed scenario: Single node compose with special characters in yaml. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-1node-bootstrap-env-creds.yaml ${SPEC CHARS ADMIN PASS} Verify response for unauthenticated request with 7100 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7102 *Unauthorized* Verify response for authenticated request with 7100 *Query Console* ${SPEC CHARS ADMIN PASS} Verify response for authenticated request with 7101 *No license key has been entered* ${SPEC CHARS ADMIN PASS} Verify response for authenticated request with 7102 *Monitoring Dashboard* ${SPEC CHARS ADMIN PASS} - [Teardown] Delete compose from ../test/compose-test-1.yaml + [Teardown] Delete compose from ./compose-1node-bootstrap-env-creds.yaml -Three node compose example - [Tags] compose +C04 Three node compose example + [Tags] compose positive + [Documentation] Detailed scenario: Three node compose example. + ... Covers setup, execution, and expected outcome validation for this scenario. Start compose from ../docker-compose/marklogic-multi-node.yaml Verify response for unauthenticated request with 7100 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* @@ -388,9 +434,11 @@ Three node compose example Host count on port 7302 should be 3 [Teardown] Delete compose from ../docker-compose/marklogic-multi-node.yaml -Two node compose example with node joining enode group - [Tags] compose - Start compose from ./compose-test-6.yaml +C05 Two node compose example with node joining enode group + [Tags] compose positive + [Documentation] Detailed scenario: Two node compose example with node joining enode group. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-2node-bootstrap-only.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7102 *Unauthorized* @@ -398,30 +446,36 @@ Two node compose example with node joining enode group Verify response for authenticated request with 7101 *No license key has been entered* Verify response for authenticated request with 7102 *Monitoring Dashboard* Add group enode on host on port 7102 - Start compose from ./compose-test-7.yaml readiness=False - Compose logs should contain ./compose-test-7.yaml *Cluster config complete, marking this container as ready.* + Start compose from ./compose-2node-join-enode-secrets.yaml readiness=False + Compose logs should contain ./compose-2node-join-enode-secrets.yaml *Cluster config complete, marking this container as ready.* Host node2 should be part of group enode [Teardown] Run keywords - ... Delete compose from ./compose-test-6.yaml - ... AND Delete compose from ./compose-test-7.yaml + ... Delete compose from ./compose-2node-bootstrap-only.yaml + ... AND Delete compose from ./compose-2node-join-enode-secrets.yaml # Tests for invalid certificate/CA, invalid value for MARKLOGIC_JOIN_TLS_ENABLED -Compose example with node joining cluster using https with invalid parameter values +C06 Compose example with node joining cluster using https with invalid parameter values [Tags] compose negative + [Documentation] Detailed scenario: Compose example with node joining cluster using https with invalid parameter values. + ... Covers setup, execution, and expected outcome validation for this scenario. Create invalid certificate file - Start compose from ./compose-test-10.yaml readiness=False - Compose logs should contain ./compose-test-10.yaml *MARKLOGIC_JOIN_TLS_ENABLED must be set to true or false, please review the configuration. Container shutting down.* - [Teardown] Delete compose from ./compose-test-10.yaml + Start compose from ./compose-2node-join-https-invalid-tls.yaml readiness=False + Compose logs should contain ./compose-2node-join-https-invalid-tls.yaml *MARKLOGIC_JOIN_TLS_ENABLED must be set to true or false, please review the configuration. Container shutting down.* + [Teardown] Delete compose from ./compose-2node-join-https-invalid-tls.yaml -Compose example with node joining cluster using https and missing certificate parameter +C07 Compose example with node joining cluster using https and missing certificate parameter [Tags] compose negative - Start compose from ./compose-test-11.yaml readiness=False - Compose logs should contain ./compose-test-11.yaml *MARKLOGIC_JOIN_CACERT_FILE is not set, please review the configuration. Container shutting down.* - [Teardown] Delete compose from ./compose-test-11.yaml + [Documentation] Detailed scenario: Compose example with node joining cluster using https and missing certificate parameter. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-2node-join-https-missing-cacert.yaml readiness=False + Compose logs should contain ./compose-2node-join-https-missing-cacert.yaml *MARKLOGIC_JOIN_CACERT_FILE is not set, please review the configuration. Container shutting down.* + [Teardown] Delete compose from ./compose-2node-join-https-missing-cacert.yaml -Two node compose example with bootstrap node without SSL enabled and node joining cluster using https +C08 Two node compose example with bootstrap node without SSL enabled and node joining cluster using https [Tags] compose negative - Start compose from ./compose-test-12.yaml + [Documentation] Detailed scenario: Two node compose example with bootstrap node without SSL enabled and node joining cluster using https. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-1node-bootstrap-env-creds.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7102 *Unauthorized* @@ -429,15 +483,17 @@ Two node compose example with bootstrap node without SSL enabled and node joinin Verify response for authenticated request with 7101 *No license key has been entered* Verify response for authenticated request with 7102 *Monitoring Dashboard* Create invalid certificate file - Start compose from ./compose-test-13.yaml readiness=False - Compose logs should contain ./compose-test-13.yaml *TLS is not enabled on bootstrap_host_name host, please verify the configuration. Container shutting down.* + Start compose from ./compose-2node-join-https-secrets.yaml readiness=False + Compose logs should contain ./compose-2node-join-https-secrets.yaml *TLS is not enabled on bootstrap_host_name host, please verify the configuration. Container shutting down.* [Teardown] Run keywords - ... Delete compose from ./compose-test-12.yaml - ... AND Delete compose from ./compose-test-13.yaml + ... Delete compose from ./compose-1node-bootstrap-env-creds.yaml + ... AND Delete compose from ./compose-2node-join-https-secrets.yaml -Two node compose example with node joining cluster using invalid CAcertificate +C09 Two node compose example with node joining cluster using invalid CAcertificate [Tags] compose negative - Start compose from ./compose-test-14.yaml + [Documentation] Detailed scenario: Two node compose example with node joining cluster using invalid CAcertificate. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-1node-bootstrap-env-creds.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for authenticated request with 7101 *No license key has been entered* Add certificate template on bootstrap host ./test_template.json 7102 @@ -445,30 +501,34 @@ Two node compose example with node joining cluster using invalid CAcertificate Apply certificate testTemplate on App Server Admin 7102 Apply certificate testTemplate on App Server Manage 7102 Create invalid certificate file - Start compose from ./compose-test-15.yaml readiness=False - Compose logs should contain ./compose-test-15.yaml *MARKLOGIC_JOIN_CACERT_FILE is not valid, please check above error for details. Node shutting down.* + Start compose from ./compose-2node-join-https-secrets.yaml readiness=False + Compose logs should contain ./compose-2node-join-https-secrets.yaml *MARKLOGIC_JOIN_CACERT_FILE is not valid, please check above error for details. Node shutting down.* [Teardown] Run keywords - ... Delete compose from ./compose-test-14.yaml - ... AND Delete compose from ./compose-test-15.yaml - -Two node compose example with node joining cluster using https - [Tags] compose - Start compose from ./compose-test-1.yaml + ... Delete compose from ./compose-1node-bootstrap-env-creds.yaml + ... AND Delete compose from ./compose-2node-join-https-secrets.yaml + +C10 Two node compose example with node joining cluster using https + [Tags] compose positive + [Documentation] Detailed scenario: Two node compose example with node joining cluster using https. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-1node-bootstrap-env-creds.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for authenticated request with 7101 *No license key has been entered* Add certificate template on bootstrap host ./test_template.json 7102 Get CAcertificate for testTemplate 7100 Apply certificate testTemplate on App Server Admin 7102 Apply certificate testTemplate on App Server Manage 7102 - Start compose from ./compose-test-2.yaml readiness=False - Compose logs should contain ./compose-test-2.yaml *Cluster config complete, marking this container as ready.* + Start compose from ./compose-2node-join-https-secrets.yaml readiness=False + Compose logs should contain ./compose-2node-join-https-secrets.yaml *Cluster config complete, marking this container as ready.* [Teardown] Run keywords - ... Delete compose from ./compose-test-1.yaml - ... AND Delete compose from ./compose-test-2.yaml + ... Delete compose from ./compose-1node-bootstrap-env-creds.yaml + ... AND Delete compose from ./compose-2node-join-https-secrets.yaml -Single node compose example with bootstrap node joining trying to itself +C11 Single node compose example with bootstrap node joining trying to itself [Tags] compose negative - ${compose test file}= Set Variable ./compose-test-8.yaml + [Documentation] Detailed scenario: Single node compose example with bootstrap node joining trying to itself. + ... Covers setup, execution, and expected outcome validation for this scenario. + ${compose test file}= Set Variable ./compose-1node-self-join.yaml Start compose from ${compose test file} Verify response for unauthenticated request with 7100 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* @@ -486,9 +546,11 @@ Single node compose example with bootstrap node joining trying to itself Verify container timezone America/Los_Angeles port=7100 [Teardown] Delete compose from ${compose test file} -Two node compose example with incorrect bootstrap host name +C12 Two node compose example with incorrect bootstrap host name [Tags] compose negative - ${compose test file}= Set Variable ./compose-test-9.yaml + [Documentation] Detailed scenario: Two node compose example with incorrect bootstrap host name. + ... Covers setup, execution, and expected outcome validation for this scenario. + ${compose test file}= Set Variable ./compose-2node-invalid-bootstrap-host.yaml Start compose from ${compose test file} Verify response for unauthenticated request with 7100 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* @@ -512,9 +574,11 @@ Two node compose example with incorrect bootstrap host name [Teardown] Delete compose from ${compose test file} -Two node compose with credentials in env and verify restart logic - [Tags] compose - ${compose test file}= Set Variable ./compose-test-3.yaml +C13 Two node compose with credentials in env and verify restart logic + [Tags] compose positive + [Documentation] Detailed scenario: Two node compose with credentials in env and verify restart logic. + ... Covers setup, execution, and expected outcome validation for this scenario. + ${compose test file}= Set Variable ./compose-2node-cluster-env-creds.yaml Start compose from ${compose test file} Verify response for unauthenticated request with 7100 *Unauthorized* Verify response for unauthenticated request with 7101 *Unauthorized* @@ -550,27 +614,34 @@ Two node compose with credentials in env and verify restart logic Verify container timezone America/Los_Angeles port=7200 [Teardown] Delete compose from ${compose test file} -Two node compose with second node uncoupled - [Tags] compose - Start compose from ./compose-test-4.yaml +C14 Two node compose with second node uncoupled + [Tags] compose positive + [Documentation] Detailed scenario: Two node compose with second node uncoupled. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-2node-second-uncoupled.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7201 *Unauthorized* Host count on port 7102 should be 1 Host count on port 7202 should be 1 - [Teardown] Delete compose from ./compose-test-4.yaml + [Teardown] Delete compose from ./compose-2node-second-uncoupled.yaml -Two node compose with second node uninitialized - [Tags] compose - Start compose from ./compose-test-5.yaml +C15 Two node compose with second node uninitialized + [Tags] compose positive + [Documentation] Detailed scenario: Two node compose with second node uninitialized. + ... Covers setup, execution, and expected outcome validation for this scenario. + Start compose from ./compose-2node-second-uninitialized.yaml Verify response for unauthenticated request with 7101 *Unauthorized* Verify response for unauthenticated request with 7201 *This server must now self-install the initial databases and application servers. Click OK to continue.* Host count on port 7102 should be 1 Verify response for authenticated request with 7200 *Forbidden* Verify response for authenticated request with 7201 *This server must now self-install the initial databases and application servers. Click OK to continue.* Verify response for authenticated request with 7202 *Forbidden* - [Teardown] Delete compose from ./compose-test-5.yaml + [Teardown] Delete compose from ./compose-2node-second-uninitialized.yaml -Initialized MarkLogic Server with wallet password and realm +D17 Initialized MarkLogic Server with wallet password and realm + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic Server with wallet password and realm. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -584,8 +655,10 @@ Initialized MarkLogic Server with wallet password and realm Verify response for authenticated request with 8002 *Monitoring Dashboard* [Teardown] Delete container -Initialized MarkLogic container with ML converters - Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping ML converters test for ARM image (converters not available) +D18 Initialized MarkLogic container with ML converters + [Tags] docker-run positive + [Documentation] Detailed scenario: Initialized MarkLogic container with ML converters. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e MARKLOGIC_INIT=true ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} @@ -596,11 +669,13 @@ Initialized MarkLogic container with ML converters Verify converter package installation [Teardown] Delete container -Dynamic Host Cluster Test - [Tags] dynamic-hosts +C16 Dynamic Host Cluster Test + [Tags] compose positive dynamic-hosts + [Documentation] Detailed scenario: Dynamic Host Cluster Test. + ... Covers setup, execution, and expected outcome validation for this scenario. ${major_version}= Set Variable ${MARKLOGIC_VERSION.split('.')[0]} Skip If '${major_version}' == '' or '${major_version}' == 'None' or int('${major_version}' or '0') < 12 msg=Dynamic Host Concurrency Test requires MarkLogic 12 or higher (current version: ${MARKLOGIC_VERSION}) - Start compose from compose-test-16.yaml + Start compose from ./compose-3core-11dynamic-hosts.yaml # give it some time to prepare the large cluster Sleep 60s ${group}= set Variable dynamic @@ -624,14 +699,61 @@ Dynamic Host Cluster Test Enable dynamic host feature on 7102 for group Default Dynamic Host Join Fails When Token Expires ${group} Dynamic Host Join Fails After Token Revoked ${group} + Delete Token By JTI Succeeds on ${group} + Verify Decoded Tokens Contain Fields on port 7102 + Delete Token By Invalid JTI on port 7102 + Delete Token By Host ID Succeeds on ${group} + Delete Token By Invalid Host ID on port 7102 + Verify Invalid Cluster Name Returns 404 on port 7102 Verify Dynamic Host Can Execute Query Default 7902 - [Teardown] Delete compose from compose-test-16.yaml + [Teardown] Delete compose from ./compose-3core-11dynamic-hosts.yaml -Dynamic Host Cluster Concurrecy Join Test - [Tags] dynamic-hosts +C17 Coupled Clusters Cross-Cluster API Test + [Tags] compose positive dynamic-hosts coupled-clusters + [Documentation] Tests that foreign cluster dynamic host endpoints return expected cross-cluster responses. + ... GET /dynamic-host-token=200(empty), POST /dynamic-host-token=400, and DELETE operations on foreign-cluster resources=404. + ... Detailed scenario: Coupled Clusters Cross-Cluster API Test. + ${major_version}= Set Variable ${MARKLOGIC_VERSION.split('.')[0]} + Skip If '${major_version}' == '' or '${major_version}' == 'None' or int('${major_version}' or '0') < 12 msg=Coupled Clusters Test requires MarkLogic 12 or higher (current version: ${MARKLOGIC_VERSION}) + + # Start two separate clusters + Start compose from ./compose-2x2node-clusters-secrets.yaml + + # Get cluster names + ${cluster1_name}= Get Local Cluster Name on port 7102 + ${cluster2_name}= Get Local Cluster Name on port 7302 + Log Cluster 1 name: ${cluster1_name} + Log Cluster 2 name: ${cluster2_name} + + # Couple the two clusters (bidirectional) + ${foreign_name}= Couple Cluster on port 7102 with Foreign Cluster on port 7302 + Log Cluster 1 coupled with foreign cluster: ${foreign_name} + ${foreign_name}= Couple Cluster on port 7302 with Foreign Cluster on port 7102 + Log Cluster 2 coupled with foreign cluster: ${foreign_name} + + # Enable dynamic host feature and API token auth on both clusters (required for token creation) + Enable dynamic host feature on 7102 for group Default + Enable API token authentication on 7102 for group Default + Enable dynamic host feature on 7302 for group Default + Enable API token authentication on 7302 for group Default + + # Test 1: From Cluster 1, try to access Cluster 2's dynamic host token API - should return 400 + Verify Cross Cluster API Returns Error on port 7102 for cluster ${cluster2_name} + + # Test 2: From Cluster 2, try to access Cluster 1's dynamic host token API - should return 400 + Verify Cross Cluster API Returns Error on port 7302 for cluster ${cluster1_name} + + Log Successfully verified coupled cluster API behaviour: GET /dynamic-host-token=200(empty), POST /dynamic-host-token=400, DELETE /dynamic-host-token/{real-jti}=404, DELETE /dynamic-hosts/{real-host-id}=404 + + [Teardown] Delete compose from ./compose-2x2node-clusters-secrets.yaml + +C18 Dynamic Host Cluster Concurrecy Join Test + [Tags] compose positive dynamic-hosts + [Documentation] Detailed scenario: Dynamic Host Cluster Concurrency Join Test. + ... Covers setup, execution, and expected outcome validation for this scenario. ${major_version}= Set Variable ${MARKLOGIC_VERSION.split('.')[0]} Skip If '${major_version}' == '' or '${major_version}' == 'None' or int('${major_version}' or '0') < 12 msg=Dynamic Host Concurrency Test requires MarkLogic 12 or higher (current version: ${MARKLOGIC_VERSION}) - Start compose from compose-test-16.yaml + Start compose from ./compose-3core-11dynamic-hosts.yaml # give it some time to prepare the large cluster Sleep 60s ${group}= set Variable dynamic @@ -639,9 +761,12 @@ Dynamic Host Cluster Concurrecy Join Test Enable API token authentication on 7202 for group Default Concurrent Dynamic Host Join Test - [Teardown] Delete compose from compose-test-16.yaml + [Teardown] Delete compose from ./compose-3core-11dynamic-hosts.yaml -Verify parameter overrides +D19 Verify parameter overrides + [Tags] docker-run positive + [Documentation] Detailed scenario: Verify parameter overrides. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e OVERWRITE_ML_CONF=true ... -e TZ=America/Los_Angeles ... -e MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test @@ -660,7 +785,10 @@ Verify parameter overrides Verify That marklogic.conf contains TZ=America/Los_Angeles MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test MARKLOGIC_UMASK=022 ML_HUGEPAGES_TOTAL=0 MARKLOGIC_DISABLE_JVM=true MARKLOGIC_USER=marklogic_user JAVA_HOME=fakejava CLASSPATH=fakeclasspath MARKLOGIC_EC2_HOST=false [Teardown] Delete container -Verify implicit parameter overrides +D20 Verify implicit parameter overrides + [Tags] docker-run positive + [Documentation] Detailed scenario: Verify implicit parameter overrides. + ... Covers setup, execution, and expected outcome validation for this scenario. Create container with -e TZ=America/Los_Angeles ... -e MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test ... -e MARKLOGIC_UMASK=022 @@ -677,4 +805,4 @@ Verify implicit parameter overrides END Verify That marklogic.conf contains TZ=America/Los_Angeles MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test MARKLOGIC_UMASK=022 ML_HUGEPAGES_TOTAL=0 MARKLOGIC_DISABLE_JVM=true MARKLOGIC_USER=marklogic_user JAVA_HOME=fakejava CLASSPATH=fakeclasspath MARKLOGIC_EC2_HOST=false [Teardown] Delete container - \ No newline at end of file + diff --git a/test/keywords.resource b/test/keywords.resource index 418f0ba6..07cd50f3 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -13,17 +13,17 @@ ${DOCKER_PLATFORM} %{DOCKER_PLATFORM=linux/amd64} ${DEFAULT ADMIN USER} test_admin ${DEFAULT ADMIN PASS} test_admin_pass ${SPEC CHARS ADMIN PASS} Admin@2$s%^&*! -${TEST_IMAGE} %{DOCKER_TEST_IMAGE=progressofficial/marklogic-db:11.3.1-ubi-rootless-2.2.5} +${TEST_IMAGE} %{DOCKER_TEST_IMAGE=progressofficial/marklogic-db:11.3.6-ubi-rootless-2.2.6} ${UPGRADE_TEST_IMAGE} progressofficial/marklogic-db:${MARKLOGIC_VERSION}-${IMAGE_TYPE}-${MARKLOGIC_DOCKER_VERSION} ${DOCKER TIMEOUT} 300s ${LICENSE KEY} %{QA_LICENSE_KEY=none} ${LICENSEE} MarkLogic - Version 9 QA Test License -${MARKLOGIC_VERSION} 11.3.1 -${BUILD_BRANCH} release_2.2.1 +${MARKLOGIC_VERSION} 11.3.6 +${BUILD_BRANCH} release_2.2.6 ${IMAGE_TYPE} ubi-rootless ${VOL_NAME} MarkLogic_vol_1 ${VOL_INFO} src=${VOL_NAME},dst=/var/opt/MarkLogic -${MARKLOGIC_DOCKER_VERSION} 2.2.5 +${MARKLOGIC_DOCKER_VERSION} 2.2.6 ${TEST_RESULTS_DIR} test_results *** Keywords *** @@ -142,7 +142,7 @@ Start compose from @{nodes}= Split to lines ${result.stdout} IF @{nodes} == [] Fail No containers detected in ${new path}! FOR ${node} IN @{nodes} - ${node}= Get Variable Value ${node} + ${node}= Get Variable Value ${node} Compose logs should contain ${new path} *${node}*Cluster config complete, marking this container as ready.* END END @@ -321,7 +321,7 @@ Apply certificate ${templateName} on App Server ${appServer} ${port} Get CAcertificate for ${templateName} ${port} [Documentation] Uses eval endpoint to get the CA of Cert template ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} - ${header}= Create Dictionary Content-type=application/x-www-form-urlencoded Accept=multipart/mixed boundary=BOUNDARY + ${header}= Create Dictionary Content-type=application/x-www-form-urlencoded Accept=multipart/mixed boundary=BOUNDARY ${xqy_data}= Get File get_ca_xquery.xqy ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} disable_warnings=1 ${response}= POST On Session RestSession url=http://localhost:${port}/v1/eval?database=Security data=${xqy_data} headers=${header} @@ -590,7 +590,7 @@ Verify Full Cluster Restart Removes Dynamic Host Configuration ${group} Should Be Equal As Integers ${host_count} 2 Expected 2 dynamic hosts but found ${host_count} # Restart the entire cluster - Restart compose from compose-test-16.yaml + Restart compose from compose-3core-11dynamic-hosts.yaml Sleep 30s # Verify dynamic host information is removed @@ -610,6 +610,85 @@ Verify Dynamic Host Count on port ${port} for group ${group} equals ${expected_c END RETURN ${host_ids} +Get Decoded Tokens on port ${port} + [Documentation] Gets all dynamic host tokens with decode=true to include decoded JWT fields (jti, group, exp) + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${cluster_name}= Get Local Cluster Name on port ${port} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= GET On Session RestSession url=/manage/v2/clusters/${cluster_name}/dynamic-host-token?decode=true&format=json headers=${header} + RETURN ${response} + +Verify Decoded Tokens Contain Fields on port ${port} + [Documentation] Verifies that decoded tokens contain expected fields: jti, group, exp + ${response}= Get Decoded Tokens on port ${port} + ${json}= Set Variable ${response.json()} + + # Check if response has the expected structure + ${has_tokens_key}= Run Keyword And Return Status Dictionary Should Contain Key ${json} dynamic-host-tokens + IF not ${has_tokens_key} + Log No dynamic-host-tokens in response. Available keys: ${json.keys()} + Fail Response does not contain dynamic-host-tokens. This might mean no tokens exist. + END + + ${tokens}= Set Variable ${json["dynamic-host-tokens"]} + ${is_list}= Evaluate isinstance($tokens, list) + IF not ${is_list} + Fail dynamic-host-tokens is not a list: ${tokens} + END + + IF len($tokens) == 0 + Fail No tokens found in response - expected tokens to exist at this point in the test + END + + FOR ${token} IN @{tokens} + # Each token should be a dict with 'payload' containing jti, group, exp + ${payload}= Set Variable ${token["payload"]} + Dictionary Should Contain Key ${payload} jti + Dictionary Should Contain Key ${payload} group + Dictionary Should Contain Key ${payload} exp + END + Log All decoded tokens contain required fields (jti, group, exp) + +Delete Token By JTI on port ${port} with jti ${jti} + [Documentation] Deletes a specific dynamic host token by its JTI value + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${cluster_name}= Get Local Cluster Name on port ${port} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= DELETE On Session RestSession url=/manage/v2/clusters/${cluster_name}/dynamic-host-token/${jti} expected_status=204 + +Delete Token By Invalid JTI on port ${port} + [Documentation] Verifies that deleting a token with an invalid JTI returns 404 + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${cluster_name}= Get Local Cluster Name on port ${port} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= DELETE On Session RestSession url=/manage/v2/clusters/${cluster_name}/dynamic-host-token/invalid-jti-does-not-exist expected_status=404 + Log Correctly received 404 for invalid JTI deletion + +Delete Token By Host ID on port ${port} with host-id ${host_id} + [Documentation] Deletes all dynamic host tokens for a specific host-id using the /dynamic-hosts/{host-id} endpoint + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${cluster_name}= Get Local Cluster Name on port ${port} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= DELETE On Session RestSession url=/manage/v2/clusters/${cluster_name}/dynamic-hosts/${host_id} expected_status=204 + Log Successfully deleted tokens for host-id: ${host_id} + +Delete Token By Invalid Host ID on port ${port} + [Documentation] Verifies that deleting tokens with an invalid host-id returns 404 + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${cluster_name}= Get Local Cluster Name on port ${port} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= DELETE On Session RestSession url=/manage/v2/clusters/${cluster_name}/dynamic-hosts/99999999999999999999 expected_status=404 + Log Correctly received 404 for invalid host-id deletion + +Verify Invalid Cluster Name Returns 404 on port ${port} + [Documentation] Verifies that using a nonexistent cluster name returns 404 + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= GET On Session RestSession url=/manage/v2/clusters/nonexistent-cluster-name/dynamic-host-token?format=json headers=${header} expected_status=404 + Log Correctly received 404 for nonexistent cluster name + Dynamic Host Join Fails When Token Expires ${group} [Documentation] Tests that a token cannot be reused after it expires @@ -635,6 +714,93 @@ Dynamic Host Join Fails After Token Revoked ${group} ${status}= Run Keyword And Return Status Init dynamic host 8201 with token ${token} Should Be Equal ${status} ${FALSE} Join succeeded with revoked token when it should have failed +Delete Token By JTI Succeeds on ${group} + [Documentation] Tests creating a token, extracting its JTI via decode=true, then deleting it by JTI + + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment "jti-delete-test" + + # Get the JTI of the token using decode=true + ${decoded_response}= Get Decoded Tokens on port 7102 + ${json}= Set Variable ${decoded_response.json()} + + # Check if response has tokens + ${has_tokens_key}= Run Keyword And Return Status Dictionary Should Contain Key ${json} dynamic-host-tokens + IF not ${has_tokens_key} + Fail No dynamic-host-tokens in decoded response. Response keys: ${json.keys()} + END + + ${tokens}= Set Variable ${json["dynamic-host-tokens"]} + ${is_list}= Evaluate isinstance($tokens, list) + IF not ${is_list} + Fail dynamic-host-tokens is not a list. Response: ${tokens} + END + + IF len($tokens) == 0 + Fail No tokens found after creating token. Response: ${json} + END + + # Get the last token (most recently created) + ${last_token}= Set Variable ${tokens[-1]} + ${payload}= Set Variable ${last_token["payload"]} + ${jti}= Set Variable ${payload["jti"]} + + # Delete the token by JTI and verify + Delete Token By JTI on port 7102 with jti ${jti} + + # Verify the join fails with the deleted token + ${status}= Run Keyword And Return Status Init dynamic host 8201 with token ${token} + Should Be Equal ${status} ${FALSE} Join succeeded with JTI-deleted token when it should have failed + +Delete Token By Host ID Succeeds on ${group} + [Documentation] Tests creating a token, initializing a dynamic host, then deleting token by host-id via query parameter + + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment "host-id-delete-test" + + # Initialize a dynamic host with the token + Init dynamic host 8201 with token ${token} + Sleep 5s # Give the host time to fully initialize + + # Get the host-id of dynamic8 (the container behind port 8201) by its known stable hostname + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${response}= Create Digest Session RestSession url=http://localhost:7102 headers=${headers} auth=${auth} + ${response}= GET On Session RestSession url=/manage/v2/hosts?format=json&group-id=${group} headers=${header} + ${hosts}= Set Variable ${response.json()["host-default-list"]["list-items"]["list-item"]} + + # Normalise to a list (API returns a dict when only one host exists) + ${is_list}= Evaluate isinstance($hosts, list) + IF not ${is_list} + ${hosts}= Create List ${hosts} + END + + # Find dynamic8 by its known hostname (stable identifier for the host on ports 8200-8210) + ${host_id}= Set Variable ${EMPTY} + FOR ${h} IN @{hosts} + ${is_dynamic8}= Run Keyword And Return Status Should Contain ${h["nameref"]} dynamic8 + IF ${is_dynamic8} + ${host_id}= Set Variable ${h["idref"]} + BREAK + END + END + + Should Not Be Empty ${host_id} Could not find dynamic8 host-id in group ${group} + Log Found dynamic host-id for dynamic8 + + # Delete the token by host-id + Delete Token By Host ID on port 7102 with host-id ${host_id} + + # Verify deletion: dynamic8 must no longer appear in the group's host list + ${response2}= GET On Session RestSession url=/manage/v2/hosts?format=json&group-id=${group} headers=${header} + ${hosts_after}= Set Variable ${response2.json()["host-default-list"]["list-items"]["list-item"]} + ${is_list2}= Evaluate isinstance($hosts_after, list) + IF not ${is_list2} + ${hosts_after}= Create List ${hosts_after} + END + FOR ${h} IN @{hosts_after} + Should Not Contain ${h["nameref"]} dynamic8 DELETE by host-id did not remove dynamic8 from the group + END + Log Token deletion by host-id verified: dynamic8 no longer in group ${group} + Verify Dynamic Host Can Execute Query ${group} ${port} [Documentation] Verifies that a dynamic host can execute a query using the REST API. @@ -695,3 +861,116 @@ Verify That marklogic.conf contains Log STDOUT: ${output.stdout} Should Not Be Empty ${output.stdout} Variable ${variable} not found in /etc/marklogic.conf END + +Couple Cluster on port ${local_port} with Foreign Cluster on port ${foreign_port} + [Documentation] Couples the local cluster with a foreign cluster by running admin:foreign-cluster-create + ... XQuery via /v1/eval. Uses admin library functions: + ... admin:cluster-get-id, admin:cluster-get-name, admin:foreign-host, admin:foreign-cluster-create, + ... admin:save-configuration-without-restart + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header_json}= Create Dictionary Accept=application/json + + # --- Gather foreign cluster info via Management API (port = management port 8002) --- + ${foreign_mgmt}= Create Digest Session ForeignMgmt_${foreign_port} url=http://localhost:${foreign_port} headers=${headers} auth=${auth} + ${props}= GET On Session ForeignMgmt_${foreign_port} url=/manage/v2/properties?format=json headers=${header_json} + ${foreign_cluster_id}= Set Variable ${props.json()["cluster-id"]} + ${foreign_cluster_name}= Set Variable ${props.json()["cluster-name"]} + + # Get bootstrap host info - find host with 'bootstrap' in its hostname + ${hosts_resp}= GET On Session ForeignMgmt_${foreign_port} url=/manage/v2/hosts?format=json headers=${header_json} + ${hosts}= Set Variable ${hosts_resp.json()["host-default-list"]["list-items"]["list-item"]} + # Default to first host, then override if we find one named 'bootstrap' + ${foreign_host_id}= Set Variable ${hosts[0]["idref"]} + ${foreign_host_name}= Set Variable ${hosts[0]["nameref"]} + FOR ${host} IN @{hosts} + ${is_bootstrap}= Run Keyword And Return Status Should Contain ${host["nameref"]} bootstrap + IF ${is_bootstrap} + ${foreign_host_id}= Set Variable ${host["idref"]} + ${foreign_host_name}= Set Variable ${host["nameref"]} + BREAK + END + END + Log Foreign cluster: id=${foreign_cluster_id} name=${foreign_cluster_name} bootstrap=${foreign_host_name} (id=${foreign_host_id}) + + # --- Run XQuery on local cluster to register foreign cluster --- + # /v1/eval is on port 8000; management API is on 8002; eval_port = local_port - 2 + ${eval_port}= Evaluate int(${local_port}) - 2 + # Build XQuery using admin:foreign-cluster-create - SSL disabled (fn:false()) so no cert exchange needed + ${xquery}= Catenate SEPARATOR=\n + ... xquery version '1.0-ml'; + ... import module namespace admin = 'http://marklogic.com/xdmp/admin' at '/MarkLogic/admin.xqy'; + ... let $cfg := admin:get-configuration() + ... let $host := admin:foreign-host(xs:unsignedLong('${foreign_host_id}'), '${foreign_host_name}', xs:unsignedInt('7998')) + ... let $new-cfg := admin:foreign-cluster-create($cfg, xs:unsignedLong('${foreign_cluster_id}'), '${foreign_cluster_name}', xs:unsignedInt('10'), xs:unsignedInt('30'), (), fn:false(), (), (), (), $host) + ... return admin:save-configuration-without-restart($new-cfg) + + ${eval_session}= Create Digest Session LocalEval_${local_port} url=http://localhost:${eval_port} headers=${headers} auth=${auth} + ${data}= Create Dictionary xquery=${xquery} + ${resp}= POST On Session LocalEval_${local_port} url=/v1/eval data=${data} expected_status=200 + Log Registered foreign cluster '${foreign_cluster_name}' on local cluster (port ${local_port}) + Sleep 2s # Wait for configuration to propagate + RETURN ${foreign_cluster_name} + +Verify Cross Cluster API Returns Error on port ${local_port} for cluster ${foreign_cluster_name} + [Documentation] Verifies behaviour when accessing a coupled (foreign) cluster's dynamic host token API. + ... Tested endpoints: + ... GET /dynamic-host-token → 200 empty list (cluster known/coupled, no cross-cluster token visibility) + ... POST /dynamic-host-token → 400 Bad Request (cannot create tokens for foreign cluster) + ... DELETE /dynamic-host-token/{jti} → 404 Not Found (real JTI exists on foreign cluster but not visible cross-cluster) + ... DELETE /dynamic-hosts/{real-host-id} → 404 Not Found (dynamic hosts not visible cross-cluster) + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${header_post}= Create Dictionary Content-type=application/json Accept=application/json + ${response}= Create Digest Session CrossClusterSess_${local_port} url=http://localhost:${local_port} headers=${headers} auth=${auth} + + # Derive foreign management port: cluster2 = 7302, cluster1 = 7102 + ${is_cluster2}= Run Keyword And Return Status Should Contain ${foreign_cluster_name} cluster2 + ${foreign_mgmt_port}= Set Variable If ${is_cluster2} 7302 7102 + ${foreign_session}= Create Digest Session ForeignSess_${foreign_mgmt_port} url=http://localhost:${foreign_mgmt_port} headers=${headers} auth=${auth} + + # --- /dynamic-host-token endpoint --- + + # GET dynamic host tokens using foreign cluster name - returns 200 with empty list (cluster is known/coupled) + # This distinguishes from 404 (XDMP-NOSUCHCLUSTER) which would mean the clusters aren't coupled + ${response}= GET On Session CrossClusterSess_${local_port} url=/manage/v2/clusters/${foreign_cluster_name}/dynamic-host-token?format=json headers=${header} expected_status=200 + ${tokens}= Set Variable ${response.json().get("dynamic-host-tokens", [])} + Should Be Empty ${tokens} Expected no tokens visible on foreign cluster, got: ${tokens} + Log Correctly received 200 with empty tokens - cluster is coupled/known + + # POST (create) token for foreign cluster - should return 400 (Bad Request) + # Use the local cluster's own bootstrap host and container Admin port so the 400 is caused by + # the cross-cluster restriction, not an invalid/unreachable host-mapped port combination. + # is_cluster2=True means foreign=cluster2 → local=cluster1 → cluster1_bootstrap; else local=cluster2 → cluster2_bootstrap + ${local_bootstrap_name}= Set Variable If ${is_cluster2} cluster1_bootstrap cluster2_bootstrap + ${token_details}= Create Dictionary group=Default host=${local_bootstrap_name} port=8001 duration=PT10M comment=test + ${token_body}= Create Dictionary dynamic-host-token=${token_details} + ${response}= POST On Session CrossClusterSess_${local_port} url=/manage/v2/clusters/${foreign_cluster_name}/dynamic-host-token json=${token_body} headers=${header_post} expected_status=400 + Log Correctly received 400 when POST-ing token on foreign cluster + + # Create a real token on the foreign cluster (via its own API) and extract its JTI + ${foreign_cluster_name_local}= Get Local Cluster Name on port ${foreign_mgmt_port} + ${foreign_bootstrap_name}= Set Variable If ${is_cluster2} cluster2_bootstrap cluster1_bootstrap + ${token_details_foreign}= Create Dictionary group=Default host=${foreign_bootstrap_name} port=8001 duration=PT10M comment=cross-cluster-jti-test + ${token_body_foreign}= Create Dictionary dynamic-host-token=${token_details_foreign} + ${create_resp}= POST On Session ForeignSess_${foreign_mgmt_port} url=/manage/v2/clusters/${foreign_cluster_name_local}/dynamic-host-token json=${token_body_foreign} headers=${header_post} expected_status=201 + # Decode to get the JTI of the newly created token + ${decoded}= GET On Session ForeignSess_${foreign_mgmt_port} url=/manage/v2/clusters/${foreign_cluster_name_local}/dynamic-host-token?decode=true&format=json headers=${header} + ${foreign_tokens}= Set Variable ${decoded.json()["dynamic-host-tokens"]} + ${real_jti}= Set Variable ${foreign_tokens[-1]["payload"]["jti"]} + # DELETE that real JTI cross-cluster from the local cluster - returns 404 (tokens not visible cross-cluster) + ${response}= DELETE On Session CrossClusterSess_${local_port} url=/manage/v2/clusters/${foreign_cluster_name}/dynamic-host-token/${real_jti} expected_status=404 + Log Correctly received 404 when DELETE-ing real foreign JTI cross-cluster + # Cleanup: delete the token on the foreign cluster directly + ${response}= DELETE On Session ForeignSess_${foreign_mgmt_port} url=/manage/v2/clusters/${foreign_cluster_name_local}/dynamic-host-token/${real_jti} expected_status=204 + + # --- /dynamic-hosts endpoint --- + + # Get a real host-id from the foreign cluster's own Management API + ${fhosts_resp}= GET On Session ForeignSess_${foreign_mgmt_port} url=/manage/v2/hosts?format=json headers=${header} + ${fhosts}= Set Variable ${fhosts_resp.json()["host-default-list"]["list-items"]["list-item"]} + ${is_list}= Evaluate isinstance($fhosts, list) + ${real_host_id}= Set Variable If ${is_list} ${fhosts[0]["idref"]} ${fhosts["idref"]} + Log Real foreign host-id to test: ${real_host_id} + # DELETE /dynamic-hosts/{real-host-id} cross-cluster - returns 404 (dynamic hosts not visible cross-cluster) + ${response}= DELETE On Session CrossClusterSess_${local_port} url=/manage/v2/clusters/${foreign_cluster_name}/dynamic-hosts/${real_host_id} expected_status=404 + Log Correctly received 404 when DELETE-ing real foreign host-id cross-cluster diff --git a/test/structure-test.yaml b/test/structure-test.yaml index 2e44f778..e121112a 100644 --- a/test/structure-test.yaml +++ b/test/structure-test.yaml @@ -64,4 +64,12 @@ fileExistenceTests: path: '/home/marklogic_user/NOTICE.txt' shouldExist: true uid: 1000 - gid: 100 \ No newline at end of file + gid: 100 +- name: 'gdb binary exists' + path: '/usr/bin/gdb' + shouldExist: true + isExecutableBy: 'any' +- name: 'python3 binary exists' + path: '/usr/bin/python3' + shouldExist: true + isExecutableBy: 'any' From 788617032afc4382eff8a08044fdb718a9cace71 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Wed, 15 Jul 2026 15:45:14 -0700 Subject: [PATCH 19/35] adding arm imagetype in jenkins file --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70f7333c..b54d9ac0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -568,7 +568,7 @@ pipeline { parameters { string(name: 'dockerVersion', defaultValue: '2.2.6', description: 'ML Docker version. This value is used as part of the Docker image tag, which is built as ${marklogicVersion}-${dockerImageType}-${dockerVersion}', trim: true) - choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9', description: 'Platform type for Docker image. Will be made part of the docker image tag') + choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rootless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) choice(name: 'marklogicVersion', choices: '12\n11', description: 'MarkLogic Server Branch. used to pick appropriate rpm') string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true) From 4245f2d761162f00e1ebfb800004f4967c67bfb3 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 16 Jul 2026 13:21:25 -0700 Subject: [PATCH 20/35] cleanup publishing stage --- Jenkinsfile | 89 +++++++++++++++++++++-------------------------------- 1 file changed, 35 insertions(+), 54 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b54d9ac0..308a79b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -421,24 +421,6 @@ void publishToInternalRegistry() { """ } - // Publish to private ECR repository that is used by the performance team. (only ML11) - // (disabled since it's not needed) - // if ( params.marklogicVersion == "11" ) { - // withCredentials( [[ - // $class: 'AmazonWebServicesCredentialsBinding', - // credentialsId: "aws-engineering-ct-ecr", - // accessKeyVariable: 'AWS_ACCESS_KEY_ID', - // secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - // ]]) { - // sh """ - // aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com - // docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - // docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} - // docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - // docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} - // """ - // } - // } // Publish to private ACR repositories that are used by PDC. if ( params.marklogicVersion == "12" ) { @@ -453,42 +435,41 @@ void publishToInternalRegistry() { """ } } - if ( params.marklogicVersion == "11" || params.marklogicVersion == "12" ) { - // Publish to Dev PDC registry - withCredentials([usernamePassword(credentialsId: 'pdc-azure-cr', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { - sh """ - echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcDevRegistry} - docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} - docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} - """ - } + + // Publish to Dev PDC registry + withCredentials([usernamePassword(credentialsId: 'pdc-azure-cr', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { + sh """ + echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcDevRegistry} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} + docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} + """ + } + + // Publish to Kubernetes ECR for testing on EKS + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'KUBE_NINJAS_OPS_AWS_JENKINS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + // Resolve account ID via STS - no account number is hardcoded in this file. + def awsAccountId = sh(returnStdout: true, + script: 'aws sts get-caller-identity --region us-west-1 --query Account --output text').trim() + def kubeNinjasEcrRegistry = "${awsAccountId}.dkr.ecr.us-west-1.amazonaws.com" + def ecrRepo = "${kubeNinjasEcrRegistry}/jenkins-kube-ninjas/marklogic-server-${dockerImageType}" + sh """ + aws ecr get-login-password --region us-west-1 | \\ + docker login --username AWS --password-stdin ${kubeNinjasEcrRegistry} + docker tag ${builtImage} ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${builtImage} ${ecrRepo}:latest-${mlVerShort} + docker push ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker push ${ecrRepo}:latest-${mlVerShort} + """ } - if ( params.marklogicVersion == "12" ) { - // Publish to Kubernetes ECR for testing on EKS - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', - credentialsId: 'KUBE_NINJAS_OPS_AWS_JENKINS', - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - // Resolve account ID via STS — no account number is hardcoded in this file. - def awsAccountId = sh(returnStdout: true, - script: 'aws sts get-caller-identity --region us-west-1 --query Account --output text').trim() - def kubeNinjasEcrRegistry = "${awsAccountId}.dkr.ecr.us-west-1.amazonaws.com" - def ecrRepo = "${kubeNinjasEcrRegistry}/jenkins-kube-ninjas/marklogic-server-${dockerImageType}" - sh """ - aws ecr get-login-password --region us-west-1 | \\ - docker login --username AWS --password-stdin ${kubeNinjasEcrRegistry} - docker tag ${builtImage} ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${builtImage} ${ecrRepo}:latest-${mlVerShort} - docker push ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker push ${ecrRepo}:latest-${mlVerShort} - """ - } - } currentBuild.description = "Published" } + /** * Triggers a BlackDuck scan job for the published image. * Runs asynchronously (wait: false). @@ -553,14 +534,14 @@ pipeline { 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false + 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm 30 05 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true 00 06 * * * % marklogicVersion=12;dockerImageType=ubi9-arm 30 06 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true - 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency - 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : '') + 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false + 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') From 078f3ad9d8d245e2c31ed2813bdb13980422ff30 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 16 Jul 2026 13:28:20 -0700 Subject: [PATCH 21/35] remove ML11 converter installation skip --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 308a79b1..180ccaee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -261,9 +261,6 @@ void copyRPMs() { fi if [ -n "${env.ML_CONVERTERS}" ]; then wget --no-verbose "${env.ML_CONVERTERS}" - elif [ "${env.marklogicVersion}" = "11" ] && [ "${archSuffix}" = "aarch64" ]; then - # Temporary exception: remove once the default ML11 ARM converters package is published. - touch MarkLogicConverters-placeholder.rpm else if [ "${archSuffix}" = "aarch64" ]; then wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-dev-tierpoint/${RPMbranch}/converters-arm/MarkLogicConverters-${RPMversion}.\${ARM_DATE}-${armRhelSuffix}.aarch64.rpm From 27c3f94d572c4dc29be1c6bfb3e96f07979eae41 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Thu, 16 Jul 2026 13:32:56 -0700 Subject: [PATCH 22/35] GRAVITON3_AGENT shouldn't be set by default on non-arm builds --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 180ccaee..ff218396 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -533,12 +533,12 @@ pipeline { 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false - 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm - 30 05 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true - 00 06 * * * % marklogicVersion=12;dockerImageType=ubi9-arm - 30 06 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true - 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false - 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false''' : '') + 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;GRAVITON3_AGENT=true + 30 05 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;GRAVITON3_AGENT=true + 00 06 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;GRAVITON3_AGENT=true + 30 06 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;GRAVITON3_AGENT=true + 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true + 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') @@ -556,7 +556,7 @@ pipeline { booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests') string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true) booleanParam(name: 'SCAP_SCAN', defaultValue: false, description: 'Run Open SCAP scan on the image.') - booleanParam(name: 'GRAVITON3_AGENT', defaultValue: true, description: '[ARM only] Run ARM-only stages on Graviton3 agent') + booleanParam(name: 'GRAVITON3_AGENT', defaultValue: false, description: '[ARM only] Run ARM-only stages on Graviton3 agent') string(name: 'emailList', defaultValue: '', description: 'Optional override for the build notification email list. If left blank, the list is loaded from the KUBE_NINJAS_PIPELINE_EMAILS Jenkins credential file. Specify a comma-separated list only to send notifications to additional or different recipients for a specific build run.', trim: true) } From 2b5005d536aa6005301a446f4b94d3aee96920b8 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Thu, 16 Jul 2026 14:50:57 -0700 Subject: [PATCH 23/35] fix(jenkins): run post result notifications in node context --- Jenkinsfile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ff218396..8f00eb1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -859,16 +859,24 @@ pipeline { } } success { - resultNotification('✅ Success') + node('cld-docker') { + resultNotification('✅ Success') + } } failure { - resultNotification('❌ Failure') + node('cld-docker') { + resultNotification('❌ Failure') + } } unstable { - resultNotification('⚠️ Unstable') + node('cld-docker') { + resultNotification('⚠️ Unstable') + } } aborted { - resultNotification('🚫 Aborted') + node('cld-docker') { + resultNotification('🚫 Aborted') + } } } } \ No newline at end of file From 2a452d8e1dc45a13e52860f58f30941970dc0031 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Thu, 16 Jul 2026 16:06:40 -0700 Subject: [PATCH 24/35] remove ARM validation since ML10 is no longer an option. --- Jenkinsfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8f00eb1c..43f08edf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,11 +63,6 @@ void preBuildCheck() { // Initialize parameters as env variables (workaround for https://issues.jenkins-ci.org/browse/JENKINS-41929) evaluate """${ def script = ''; params.each { k, v -> script += "env.${k} = '''${v}'''\n" }; return script}""" - // Validate ARM images are only supported for MarkLogic 11 and 12 - if (env.dockerImageType.contains('arm') && !(env.marklogicVersion in ['11', '12'])) { - error "ARM images (${env.dockerImageType}) are only supported for MarkLogic 11 and 12. Current version: ${env.marklogicVersion}" - } - JIRA_ID = extractJiraID() echo 'Jira ticket number: ' + JIRA_ID @@ -878,5 +873,5 @@ pipeline { resultNotification('🚫 Aborted') } } - } + } } \ No newline at end of file From 0ca17416b51ac4e83dcde16b0b7d8e0dd786ea44 Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:06:15 -0700 Subject: [PATCH 25/35] updating latency test names as per latest naming convention Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33dc6dbe..2bfa113f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -521,14 +521,14 @@ pipeline { 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true - 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false - 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false + 00 07 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=D05 Initialized MarkLogic container with latency;PUBLISH_IMAGE=false + 00 08 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=D05 Initialized MarkLogic container with latency;PUBLISH_IMAGE=false 00 05 * * * % marklogicVersion=11;dockerImageType=ubi9-arm;GRAVITON3_AGENT=true 30 05 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;GRAVITON3_AGENT=true 00 06 * * * % marklogicVersion=12;dockerImageType=ubi9-arm;GRAVITON3_AGENT=true 30 06 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless-arm;SCAP_SCAN=true;GRAVITON3_AGENT=true - 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true - 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true''' : '') + 00 09 * * 7 % marklogicVersion=11;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=D05 Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true + 00 10 * * 7 % marklogicVersion=12;dockerImageType=ubi9-arm;DOCKER_TEST_LIST=D05 Initialized MarkLogic container with latency;PUBLISH_IMAGE=false;GRAVITON3_AGENT=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') From 2dfc79524ff5fae9a6800ef34b38ae5786168282 Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:11:04 -0700 Subject: [PATCH 26/35] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2bfa113f..0207dcb0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ upgradeDockerImage = '' /** * Determines if the current build is for an ARM image type. - * ARM workers (e.g., Graviton3) are only available for MarkLogic 11. + * ARM workers (e.g., Graviton3) are only available for selected MarkLogic versions (currently 11 and 12). * @return true if dockerImageType contains 'arm', false otherwise. */ @NonCPS From 6e32a39c050075712fd59af8b0b67ab2df08c98a Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:21:47 -0700 Subject: [PATCH 27/35] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- dockerFiles/marklogic-server-ubi9-arm:base | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerFiles/marklogic-server-ubi9-arm:base b/dockerFiles/marklogic-server-ubi9-arm:base index ad81e263..c91a3b96 100644 --- a/dockerFiles/marklogic-server-ubi9-arm:base +++ b/dockerFiles/marklogic-server-ubi9-arm:base @@ -115,8 +115,8 @@ RUN microdnf -y reinstall tzdata ############################################################### # Remove optional packages that have known vulnerabilities -############################################################### -RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ +# (Excluding python/gdb dependencies needed for stack traces) +RUN for package in vim-minimal cups-client cups-libs tar avahi-libs binutils libarchive binutils-gold; \ do rpm -e --nodeps $package || true; \ done; From 785622c84e28dcc8c25b7b14790c46987c38d7b7 Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:16:05 -0700 Subject: [PATCH 28/35] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0207dcb0..99237eca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -447,8 +447,8 @@ void publishToInternalRegistry() { sh """ aws ecr get-login-password --region us-west-1 | \\ docker login --username AWS --password-stdin ${kubeNinjasEcrRegistry} - docker tag ${builtImage} ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${builtImage} ${ecrRepo}:latest-${mlVerShort} + docker tag ${imageToPublish} ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${ecrRepo}:latest-${mlVerShort} docker push ${ecrRepo}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} docker push ${ecrRepo}:latest-${mlVerShort} """ From 8b75d6966be45cfea7b044b09c78f1e7809cfe55 Mon Sep 17 00:00:00 2001 From: barkhachoithani <40070058+barkhachoithani@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:01:53 -0700 Subject: [PATCH 29/35] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- dockerFiles/marklogic-deps-ubi9-arm:base | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base index 14b2a733..7fc89fde 100644 --- a/dockerFiles/marklogic-deps-ubi9-arm:base +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -11,15 +11,16 @@ LABEL "com.marklogic.maintainer"="docker@marklogic.com" # install libnsl rpm package ############################################################### -RUN microdnf -y update \ - && rpm -i --nodeps https://download.rockylinux.org/pub/rocky/9.8/BaseOS/aarch64/os/Packages/l/libnsl-2.34-270.el9_8.aarch64.rpm +RUN microdnf -y upgrade glibc \ + && rpm -i --nodeps https://download.rockylinux.org/pub/rocky/9.8/BaseOS/aarch64/os/Packages/l/libnsl-2.34-270.el9_8.aarch64.rpm \ + && microdnf clean all ############################################################### -# install networking, base deps and tzdata for timezone +# install gdb and dependencies for stack traces, networking, base deps and tzdata for timezone ############################################################### # hadolint ignore=DL3006 RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ - && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux hostname \ + && microdnf -y install --setopt install_weak_deps=0 gdb python3-rpm nss libcap procps-ng python3 libtool-ltdl cpio initscripts tzdata glibc libstdc++ util-linux hostname \ && microdnf clean all From db7133a4f30dea66b97d5702c35d4d3945dffc91 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Thu, 23 Jul 2026 15:21:56 -0700 Subject: [PATCH 30/35] fix for the value source for BUILD_BRANCH --- Jenkinsfile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 99237eca..7637a31d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,6 +52,20 @@ Map loadEmailConfig() { return result } +/** + * Returns the build branch value used in image metadata and tests. + * PR builds use PR-; non-PR builds use the branch name. + */ +String getBuildBranchValue() { + if (env.CHANGE_ID?.trim()) { + return "PR-${env.CHANGE_ID.trim()}" + } + if (env.BRANCH_NAME?.trim()) { + return env.BRANCH_NAME.trim() + } + return (env.GIT_BRANCH ?: 'local').toString().trim() +} + /** * Performs pre-build checks: * - Initializes parameters as environment variables. @@ -285,7 +299,8 @@ void buildDockerImage() { // Use Los Angeles time (same as ARM_DATE in copyRPMs) to ensure consistency across UTC/PST boundaries timeStamp = sh(returnStdout: true, script: "TZ=America/Los_Angeles date +%Y%m%d").trim() timestamptedTag = builtImage.replace('nightly', timeStamp) - sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}" + def buildBranchValue = getBuildBranchValue() + sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${buildBranchValue} package=${RPM} converters=${CONVERTERS}" currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})" echo "Built image: ${builtImage}" } @@ -324,6 +339,7 @@ void pullUpgradeDockerImage() { * Runs container structure tests using the 'make structure-test' target. */ void structureTests() { + def buildBranchValue = getBuildBranchValue() sh """ #install container-structure-test 1.16.0 binary (detect architecture) ARCH=\$(uname -m) @@ -333,7 +349,7 @@ void structureTests() { PLATFORM="amd64" fi curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-\${PLATFORM} && chmod +x container-structure-test-linux-\${PLATFORM} && mv container-structure-test-linux-\${PLATFORM} container-structure-test - make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true + make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${buildBranchValue} docker_image_type=${env.dockerImageType} Jenkins=true """ } @@ -341,8 +357,9 @@ void structureTests() { * Runs Docker functional tests using the 'make docker-tests' target. */ void dockerTests() { + def buildBranchValue = getBuildBranchValue() sh "make docker-test-ids" - sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType} DOCKER_TEST_LIST=\"${params.DOCKER_TEST_LIST}\"" + sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${buildBranchValue} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType} DOCKER_TEST_LIST=\"${params.DOCKER_TEST_LIST}\"" } /** From 0264f1846b8a42e56ee95f65853beb9d65587f15 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Fri, 24 Jul 2026 00:51:47 -0700 Subject: [PATCH 31/35] added skip upgrade tests for arm --- test/keywords.resource | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/keywords.resource b/test/keywords.resource index 07cd50f3..89aedd8e 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -85,6 +85,8 @@ Create test container with Create upgrade container with [Arguments] @{input parameters} [Documentation] Creates a second test container for upgrade testing. + Skip If 'arm' in '${IMAGE_TYPE}' msg=Skipping upgrade test for ARM image (no previous ARM upgrade image is available). + Should Not Be Empty ${UPGRADE_TEST_IMAGE} msg=UPGRADE_TEST_IMAGE is empty; cannot run upgrade container. ${container name}= Remove spaces from ${TEST NAME} Run Process docker run @{DOCKER DEFAULTS} @{input parameters} ... --name ${container name}-2 From 80246c22958b812119d089e8775ec8fae87cbff2 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Mon, 27 Jul 2026 17:29:54 -0700 Subject: [PATCH 32/35] fir for test: avoid false D06-D08 failures from docker pull stderr on Jenkins --- test/keywords.resource | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/keywords.resource b/test/keywords.resource index 89aedd8e..cd3971c2 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -88,13 +88,13 @@ Create upgrade container with Skip If 'arm' in '${IMAGE_TYPE}' msg=Skipping upgrade test for ARM image (no previous ARM upgrade image is available). Should Not Be Empty ${UPGRADE_TEST_IMAGE} msg=UPGRADE_TEST_IMAGE is empty; cannot run upgrade container. ${container name}= Remove spaces from ${TEST NAME} - Run Process docker run @{DOCKER DEFAULTS} @{input parameters} + ${result}= Run Process docker run @{DOCKER DEFAULTS} @{input parameters} ... --name ${container name}-2 ... --mount ${VOL_INFO} ${UPGRADE_TEST_IMAGE} ... stderr=test_results/stderr-${container name}-2.txt ... stdout=test_results/stdout-${container name}-2.txt ... timeout=${DOCKER TIMEOUT} - File Should Be Empty test_results/stderr-${container name}-2.txt + Should Be Equal As Integers ${result.rc} 0 Docker log should contain *Cluster config complete, marking this container as ready.* True Stop container From 6a7d0e5b33831af738d16b5daffa5553cedd43b9 Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Tue, 28 Jul 2026 10:43:13 -0700 Subject: [PATCH 33/35] fix for long-running tests --- test/keywords.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/keywords.resource b/test/keywords.resource index cd3971c2..e6008d47 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -51,7 +51,7 @@ Create container with latency ... --name ${container name} ... --cap-add NET_ADMIN --entrypoint /bin/bash ... ${TEST_IMAGE} - ... -c sudo microdnf -y install iproute iptables && (sudo yum install -y iproute-tc || sudo microdnf install -y iproute) && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh + ... -c sudo microdnf -y install iproute iptables iproute-tc || sudo microdnf -y install iproute iptables; TC_BIN=$(command -v tc || true); [ -n "$TC_BIN" ] || TC_BIN=/usr/sbin/tc; [ -x "$TC_BIN" ] || TC_BIN=/sbin/tc; [ -x "$TC_BIN" ] || { echo "tc command not found"; exit 1; }; sudo "$TC_BIN" qdisc add dev lo root netem delay 30000ms && sudo "$TC_BIN" qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh ... stderr=test_results/stderr-${container name}.txt ... stdout=test_results/stdout-${container name}.txt ... timeout=15000 From 7bc4f2aa30904fbbe15361321b0c467ef8e219fb Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Tue, 28 Jul 2026 11:07:15 -0700 Subject: [PATCH 34/35] fixed parsing error --- test/keywords.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/keywords.resource b/test/keywords.resource index e6008d47..a6764373 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -51,7 +51,7 @@ Create container with latency ... --name ${container name} ... --cap-add NET_ADMIN --entrypoint /bin/bash ... ${TEST_IMAGE} - ... -c sudo microdnf -y install iproute iptables iproute-tc || sudo microdnf -y install iproute iptables; TC_BIN=$(command -v tc || true); [ -n "$TC_BIN" ] || TC_BIN=/usr/sbin/tc; [ -x "$TC_BIN" ] || TC_BIN=/sbin/tc; [ -x "$TC_BIN" ] || { echo "tc command not found"; exit 1; }; sudo "$TC_BIN" qdisc add dev lo root netem delay 30000ms && sudo "$TC_BIN" qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh + ... -c sudo microdnf -y install iproute iptables iproute-tc || sudo microdnf -y install iproute iptables; (command -v tc >/dev/null && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo) || ([ -x /usr/sbin/tc ] && sudo /usr/sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /usr/sbin/tc qdisc show dev lo) || ([ -x /sbin/tc ] && sudo /sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /sbin/tc qdisc show dev lo) || (echo "tc command not found"; exit 1); /tini -- /usr/local/bin/start-marklogic.sh ... stderr=test_results/stderr-${container name}.txt ... stdout=test_results/stdout-${container name}.txt ... timeout=15000 From 4fb84ae93a544fbf1d218e0264ce66bb7ed5149d Mon Sep 17 00:00:00 2001 From: barkhachoithani Date: Tue, 28 Jul 2026 14:08:43 -0700 Subject: [PATCH 35/35] fix test(robot): harden D05 latency setup with iproute-tc fallback and strict tc/netem gating --- test/keywords.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/keywords.resource b/test/keywords.resource index a6764373..b224842a 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -51,7 +51,7 @@ Create container with latency ... --name ${container name} ... --cap-add NET_ADMIN --entrypoint /bin/bash ... ${TEST_IMAGE} - ... -c sudo microdnf -y install iproute iptables iproute-tc || sudo microdnf -y install iproute iptables; (command -v tc >/dev/null && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo) || ([ -x /usr/sbin/tc ] && sudo /usr/sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /usr/sbin/tc qdisc show dev lo) || ([ -x /sbin/tc ] && sudo /sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /sbin/tc qdisc show dev lo) || (echo "tc command not found"; exit 1); /tini -- /usr/local/bin/start-marklogic.sh + ... -c sudo microdnf -y install iproute iptables && (sudo microdnf -y install iproute-tc || (sudo curl -fsS -O https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/Packages/i/iproute-tc-6.2.0-6.el8_10.x86_64.rpm && sudo rpm -Uvh --replacepkgs iproute-tc-6.2.0-6.el8_10.x86_64.rpm)) && ((command -v tc >/dev/null && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo) || ([ -x /usr/sbin/tc ] && sudo /usr/sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /usr/sbin/tc qdisc show dev lo) || ([ -x /sbin/tc ] && sudo /sbin/tc qdisc add dev lo root netem delay 30000ms && sudo /sbin/tc qdisc show dev lo)) && /tini -- /usr/local/bin/start-marklogic.sh ... stderr=test_results/stderr-${container name}.txt ... stdout=test_results/stdout-${container name}.txt ... timeout=15000