From 831c08fbdfae1319f8e00c1aa93e848030c1c25a Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 19:25:33 +0000 Subject: [PATCH 1/4] build: pin Noah-enabled splunk-ansible --- docker-splunk/Makefile | 2 +- docker-splunk/tests/test_ansible_ref.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docker-splunk/Makefile b/docker-splunk/Makefile index 448c0ca8..ab307d32 100644 --- a/docker-splunk/Makefile +++ b/docker-splunk/Makefile @@ -4,7 +4,7 @@ NONQUOTE_IMAGE_VERSION := $(patsubst "%",%,$(IMAGE_VERSION)) DOCKER_BUILD_FLAGS ?= SPLUNK_ANSIBLE_REPO ?= https://github.com/splunk/splunk-ansible.git SPLUNK_ANSIBLE_BRANCH ?= develop -SPLUNK_ANSIBLE_REF ?= 8455e865820688d127133051d8a9705ea2d3bfcf +SPLUNK_ANSIBLE_REF ?= bb5321b2dc1ce6fbfb29937013afbf4feb5f10b5 SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml # Set Splunk version/build parameters here to define downstream URLs and file names SPLUNK_PRODUCT := splunk diff --git a/docker-splunk/tests/test_ansible_ref.py b/docker-splunk/tests/test_ansible_ref.py index 1c001ec2..5e39b199 100644 --- a/docker-splunk/tests/test_ansible_ref.py +++ b/docker-splunk/tests/test_ansible_ref.py @@ -6,7 +6,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -VALIDATED_SHC_ANSIBLE_REF = "8455e865820688d127133051d8a9705ea2d3bfcf" +NOAH_ANSIBLE_REF = "bb5321b2dc1ce6fbfb29937013afbf4feb5f10b5" def run(command, cwd, check=True): @@ -61,11 +61,11 @@ def make_ansible(self, expected_commit, check=True): check=check, ) - def test_default_ref_is_validated_shc_commit(self): + def test_default_ref_is_noah_role_commit(self): makefile = (REPOSITORY_ROOT / "Makefile").read_text(encoding="utf-8") self.assertIn( - f"SPLUNK_ANSIBLE_REF ?= {VALIDATED_SHC_ANSIBLE_REF}", + f"SPLUNK_ANSIBLE_REF ?= {NOAH_ANSIBLE_REF}", makefile, ) self.assertNotIn( @@ -73,6 +73,16 @@ def test_default_ref_is_validated_shc_commit(self): makefile, ) + def test_normal_image_build_copies_the_complete_ansible_tree(self): + dockerfile = ( + REPOSITORY_ROOT / "splunk" / "common-files" / "Dockerfile" + ).read_text(encoding="utf-8") + + self.assertIn( + "COPY splunk-ansible ${SPLUNK_ANSIBLE_HOME}", + dockerfile, + ) + def test_checks_out_and_records_exact_commit(self): self.make_ansible(self.first_commit) From fb5ff462b37f38ad8fb911c73e8db81d925dc447 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 12:49:42 -0700 Subject: [PATCH 2/4] build: install complete validated Noah Ansible tree --- docker-splunk/Dockerfile.noah-splunk | 114 +++--------------------- docker-splunk/Makefile | 2 +- docker-splunk/tests/test_ansible_ref.py | 11 ++- 3 files changed, 23 insertions(+), 104 deletions(-) diff --git a/docker-splunk/Dockerfile.noah-splunk b/docker-splunk/Dockerfile.noah-splunk index c3b18041..8e4b3c9b 100644 --- a/docker-splunk/Dockerfile.noah-splunk +++ b/docker-splunk/Dockerfile.noah-splunk @@ -1,109 +1,19 @@ FROM 667741767953.dkr.ecr.us-west-2.amazonaws.com/vivekr/splunk-cloud:10.5.2605.0-c278328a3fc1 -# Noah-patched Splunk image built on the production splunk-cloud base: -# -# - configure_noah.yml: consolidated Noah task — writes [noahService] fields (disabled=true -# for pre-auth, heartbeatPeriod, pass4SymmKey), pre-writes [general] serverName and -# shcclustering fields to make set_server_name.yml idempotent, and writes advertisedAddr -# so Noah routes distributed search to the correct pod FQDN. Replaces the scattered -# set_noah_symmkey_password.yml and set_noah_advertised_addr.yml from prior images. -# -# - main.yml (patched in-place): inserts configure_noah.yml include before enable_admin_auth. -# Surgical patch — preserves all production tasks (set_kvservice_connection_string.yml, -# set_cloud_instance.yml, etc.) that a full-file replace would silently drop. -# -# - environ.py (patched in-place): adds noah_advertised_addr = "https://:8089" -# when SPLUNK_NOAH_ENABLED is set. FQDN is derived from the already-computed server_name -# (POD_NAME + SPLUNK_HEADLESS_SERVICE_NAME + POD_NAMESPACE + cluster domain). -# -# - restart_splunk.yml: "stop && start" instead of "restart --answer-yes" to skip validatedb -# (~10 min) which exceeded the 8Gi container limit and caused OOM-kills (exit 137). +# The base image supplies the Noah-capable splunkd build. The complete, +# commit-pinned splunk-ansible checkout supplies provisioning for both classic +# and Noah roles. Keeping the whole tree preserves normal role dependencies +# and makes the image contents match the reviewed Ansible commit exactly. USER root -# 1. Drop in configure_noah.yml — the single consolidated Noah configuration task. -COPY splunk-ansible/roles/splunk_common/tasks/configure_noah.yml \ - /opt/ansible/roles/splunk_common/tasks/configure_noah.yml - -# 2. Patch main.yml in-place: insert configure_noah.yml include before enable_admin_auth.yml. -# The production image's main.yml is patched surgically so that platform-specific tasks -# (set_kvservice_connection_string.yml, set_cloud_instance.yml) are preserved unchanged. -RUN python3 - <<'PYEOF' -import sys - -path = "/opt/ansible/roles/splunk_common/tasks/main.yml" -with open(path) as f: - content = f.read() - -needle = "- include_tasks: enable_admin_auth.yml" -insert = ( - "- include_tasks: configure_noah.yml\n" - " when:\n" - " - \"'conf' in splunk and splunk.conf\"\n" - " - \"splunk.conf.server is defined\"\n" - " - \"splunk.conf.server.content is defined\"\n" - " - \"splunk.conf.server.content.noahService is defined\"\n" - "\n" -) - -if needle not in content: - print("ERROR: expected anchor not found in main.yml:", needle, file=sys.stderr) - sys.exit(1) - -if "configure_noah.yml" in content: - print("configure_noah.yml already present in main.yml, skipping patch") -else: - content = content.replace(needle, insert + needle, 1) - with open(path, "w") as f: - f.write(content) - print("Patched main.yml: inserted configure_noah.yml before enable_admin_auth.yml") -PYEOF - -# 3. Patch environ.py in-place: add noah_advertised_addr when SPLUNK_NOAH_ENABLED is set. -# The production image already sets server_name from SPLUNK_HEADLESS_SERVICE_NAME; we -# extend that block to also derive the Noah advertised address from the same FQDN. -RUN python3 - <<'PYEOF' -import sys - -path = "/opt/ansible/inventory/environ.py" -with open(path) as f: - content = f.read() - -# The production image sets server_name in this form (single assignment, no local variable). -old = ' vars_scope["splunk"]["server_name"] = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain)' -new = ( - ' vars_scope["splunk"]["server_name"] = "{}.{}.{}.svc.{}".format(podName, headlessServiceName, namespace, clusterDomain)\n' - ' if os.environ.get("SPLUNK_NOAH_ENABLED", ""):\n' - ' vars_scope["splunk"]["noah_advertised_addr"] = "https://{}:8089".format(vars_scope["splunk"]["server_name"])' -) - -if old not in content: - print("ERROR: expected pattern not found in environ.py", file=sys.stderr) - sys.exit(1) - -if "noah_advertised_addr" in content: - print("noah_advertised_addr already present in environ.py, skipping patch") -else: - content = content.replace(old, new, 1) - with open(path, "w") as f: - f.write(content) - print("Patched environ.py: added noah_advertised_addr derivation") -PYEOF - -# 4. Replace restart_splunk.yml handler: use stop+start instead of restart --answer-yes -# to skip validatedb (~10 min) which exceeded the 8Gi container memory limit and caused -# OOM-kills (exit 137) during SHC captain bootstrap, member join, and serverName changes. -COPY splunk-ansible/roles/splunk_common/handlers/restart_splunk.yml \ - /opt/ansible/roles/splunk_common/handlers/restart_splunk.yml - -RUN chmod 555 \ - /opt/ansible/roles/splunk_common/tasks/configure_noah.yml \ - /opt/ansible/roles/splunk_common/handlers/restart_splunk.yml - -# 5. Install idempotent SHC shutdown contract (SHC reliability: preStop/TERM overlap fix). -# The base splunk-cloud image has the old single-call teardown; replace both files so -# concurrent SIGTERM and preStop lifecycle hooks share one bounded, idempotent stop. +COPY splunk-ansible /opt/ansible COPY splunk/common-files/splunk-shutdown /sbin/splunk-shutdown -COPY splunk/common-files/entrypoint.sh /sbin/entrypoint.sh -RUN chmod 755 /sbin/splunk-shutdown /sbin/entrypoint.sh +COPY splunk/common-files/entrypoint.sh /sbin/entrypoint.sh + +RUN chmod -R 555 /opt/ansible \ + && chgrp ansible /opt/ansible /opt/ansible/ansible.cfg \ + && chmod 775 /opt/ansible \ + && chmod 664 /opt/ansible/ansible.cfg \ + && chmod 755 /sbin/splunk-shutdown /sbin/entrypoint.sh USER ansible diff --git a/docker-splunk/Makefile b/docker-splunk/Makefile index ab307d32..0b527490 100644 --- a/docker-splunk/Makefile +++ b/docker-splunk/Makefile @@ -4,7 +4,7 @@ NONQUOTE_IMAGE_VERSION := $(patsubst "%",%,$(IMAGE_VERSION)) DOCKER_BUILD_FLAGS ?= SPLUNK_ANSIBLE_REPO ?= https://github.com/splunk/splunk-ansible.git SPLUNK_ANSIBLE_BRANCH ?= develop -SPLUNK_ANSIBLE_REF ?= bb5321b2dc1ce6fbfb29937013afbf4feb5f10b5 +SPLUNK_ANSIBLE_REF ?= cbfb8e8dd5a5192b4a36483039172feb97b0aea4 SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml # Set Splunk version/build parameters here to define downstream URLs and file names SPLUNK_PRODUCT := splunk diff --git a/docker-splunk/tests/test_ansible_ref.py b/docker-splunk/tests/test_ansible_ref.py index 5e39b199..3710cdd8 100644 --- a/docker-splunk/tests/test_ansible_ref.py +++ b/docker-splunk/tests/test_ansible_ref.py @@ -6,7 +6,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -NOAH_ANSIBLE_REF = "bb5321b2dc1ce6fbfb29937013afbf4feb5f10b5" +NOAH_ANSIBLE_REF = "cbfb8e8dd5a5192b4a36483039172feb97b0aea4" def run(command, cwd, check=True): @@ -83,6 +83,15 @@ def test_normal_image_build_copies_the_complete_ansible_tree(self): dockerfile, ) + def test_noah_image_replaces_the_complete_ansible_tree(self): + dockerfile = (REPOSITORY_ROOT / "Dockerfile.noah-splunk").read_text( + encoding="utf-8" + ) + + self.assertIn("COPY splunk-ansible /opt/ansible", dockerfile) + self.assertNotIn("configure_noah.yml", dockerfile) + self.assertNotIn("python3 - <<", dockerfile) + def test_checks_out_and_records_exact_commit(self): self.make_ansible(self.first_commit) From 41a024af1589bcf217b5f858337ed79ead0cf134 Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 13:38:22 -0700 Subject: [PATCH 3/4] build: include pre-start SHC deployer fix --- docker-splunk/Makefile | 2 +- docker-splunk/splunk-ansible | 2 +- docker-splunk/tests/test_ansible_ref.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-splunk/Makefile b/docker-splunk/Makefile index 0b527490..cc935fc9 100644 --- a/docker-splunk/Makefile +++ b/docker-splunk/Makefile @@ -4,7 +4,7 @@ NONQUOTE_IMAGE_VERSION := $(patsubst "%",%,$(IMAGE_VERSION)) DOCKER_BUILD_FLAGS ?= SPLUNK_ANSIBLE_REPO ?= https://github.com/splunk/splunk-ansible.git SPLUNK_ANSIBLE_BRANCH ?= develop -SPLUNK_ANSIBLE_REF ?= cbfb8e8dd5a5192b4a36483039172feb97b0aea4 +SPLUNK_ANSIBLE_REF ?= 40aa2ca51eed717f71f3d2200e9c7b571ce211ce SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml # Set Splunk version/build parameters here to define downstream URLs and file names SPLUNK_PRODUCT := splunk diff --git a/docker-splunk/splunk-ansible b/docker-splunk/splunk-ansible index edca4768..40aa2ca5 160000 --- a/docker-splunk/splunk-ansible +++ b/docker-splunk/splunk-ansible @@ -1 +1 @@ -Subproject commit edca4768ebf27d28ac55b2090affaa66da1505aa +Subproject commit 40aa2ca51eed717f71f3d2200e9c7b571ce211ce diff --git a/docker-splunk/tests/test_ansible_ref.py b/docker-splunk/tests/test_ansible_ref.py index 3710cdd8..90cb92dc 100644 --- a/docker-splunk/tests/test_ansible_ref.py +++ b/docker-splunk/tests/test_ansible_ref.py @@ -6,7 +6,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -NOAH_ANSIBLE_REF = "cbfb8e8dd5a5192b4a36483039172feb97b0aea4" +NOAH_ANSIBLE_REF = "40aa2ca51eed717f71f3d2200e9c7b571ce211ce" def run(command, cwd, check=True): From bf7d32e5a96e02768973c83c22349fba2f70860f Mon Sep 17 00:00:00 2001 From: Vivek Reddy Date: Wed, 26 Aug 2026 14:07:04 -0700 Subject: [PATCH 4/4] fix: exit cleanly after SIGTERM shutdown --- docker-splunk/splunk/common-files/entrypoint.sh | 5 ++++- docker-splunk/tests/test_splunk_shutdown.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-splunk/splunk/common-files/entrypoint.sh b/docker-splunk/splunk/common-files/entrypoint.sh index 956b0726..67d35705 100755 --- a/docker-splunk/splunk/common-files/entrypoint.sh +++ b/docker-splunk/splunk/common-files/entrypoint.sh @@ -33,6 +33,10 @@ setup() { teardown() { # TERM and preStop share one idempotent, bounded local shutdown operation. /sbin/splunk-shutdown --source=term || true + # `wait` is interrupted by SIGTERM and otherwise returns 143 after this trap, + # which makes Kubernetes record an intentional pod deletion as Error. Once + # the bounded shutdown attempt has completed, terminate the container cleanly. + exit 0 } trap teardown SIGINT SIGTERM @@ -215,4 +219,3 @@ case "$1" in help $@ ;; esac - diff --git a/docker-splunk/tests/test_splunk_shutdown.py b/docker-splunk/tests/test_splunk_shutdown.py index 4c6e0730..bf7cc66f 100644 --- a/docker-splunk/tests/test_splunk_shutdown.py +++ b/docker-splunk/tests/test_splunk_shutdown.py @@ -234,6 +234,8 @@ def test_image_and_term_handler_use_stable_shutdown_contract(self): ) self.assertIn("/sbin/splunk-shutdown --source=term", entrypoint) + teardown = entrypoint.split("teardown() {", 1)[1].split("}", 1)[0] + self.assertIn("exit 0", teardown) self.assertNotIn("${SPLUNK_HOME}/bin/splunk stop || true", entrypoint) self.assertIn("SPLUNK_SHUTDOWN_TIMEOUT_SECONDS", entrypoint) self.assertIn('"splunk/common-files/splunk-shutdown"', dockerfile)