Draft: do not merge Spike/shc runtime lifecycle

Makefile

@@ -4,6 +4,7 @@ NONQUOTE_IMAGE_VERSION := $(patsubst "%",%,$(IMAGE_VERSION))
 DOCKER_BUILD_FLAGS ?=
 SPLUNK_ANSIBLE_REPO ?= https://github.com/splunk/splunk-ansible.git
 SPLUNK_ANSIBLE_BRANCH ?= develop
+SPLUNK_ANSIBLE_REF ?= $(SPLUNK_ANSIBLE_BRANCH)
 SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml
 # Set Splunk version/build parameters here to define downstream URLs and file names
 SPLUNK_PRODUCT := splunk
@@ -43,16 +44,27 @@ else
 endif
 
 
-.PHONY: tests interactive_tutorials
+.PHONY: tests interactive_tutorials test_shutdown
 
 all: splunk uf splunk-py23 uf-py23
 
 ansible:
 	@if [ -d "splunk-ansible" ]; then \
-		echo "Ansible directory exists - skipping clone"; \
+		echo "Ansible directory exists - verifying requested ref"; \
 	else \
-		git clone ${SPLUNK_ANSIBLE_REPO} --branch ${SPLUNK_ANSIBLE_BRANCH}; \
+		git clone "$(SPLUNK_ANSIBLE_REPO)" splunk-ansible; \
 	fi
+	@cd splunk-ansible && \
+		if [ -n "$$(git status --porcelain --untracked-files=all | grep -v ' version.txt$$')" ]; then \
+			echo "splunk-ansible contains local changes; refusing to replace it"; \
+			exit 1; \
+		fi && \
+		if git cat-file -e '$(SPLUNK_ANSIBLE_REF)^{commit}' 2>/dev/null; then \
+			git checkout --detach "$(SPLUNK_ANSIBLE_REF)"; \
+		else \
+			git fetch --depth 1 origin "$(SPLUNK_ANSIBLE_REF)" && \
+			git checkout --detach FETCH_HEAD; \
+		fi
 	@cd splunk-ansible && git rev-parse HEAD > version.txt
 	@cat splunk-ansible/version.txt
 
@@ -388,7 +400,10 @@ run_large_tests_redhat8:
 	@echo 'Running the super awesome large tests; RedHat 8'
 	pytest -n 2 --reruns 1 -sv tests/test_distributed_splunk_image.py --platform redhat-8 --junitxml test-results/redhat8-result/testresults_large_redhat8.xml
 
-test_setup:
+test_shutdown:
+	python3 -m unittest -v tests/test_splunk_shutdown.py
+
+test_setup: test_shutdown
 	@echo 'Install test requirements'
 	pip install --upgrade pip
 	pip install -r $(shell pwd)/tests/requirements.txt --upgrade

base/redhat-9/Dockerfile

@@ -29,7 +29,7 @@ ARG BUSYBOX_URL
 
 ENV BUSYBOX_URL=${BUSYBOX_URL} \
     PYTHON_VERSION=3.13.12 \
-    PYTHON_GPG_KEY_ID=A821E680E5FA6305
+    PYTHON_GPG_KEY_FINGERPRINT=7169605F62C751356D054A26A821E680E5FA6305
 
 COPY install.sh /install.sh
 

base/redhat-9/install.sh

@@ -44,10 +44,10 @@ microdnf -y --nodocs update gnutls kernel-headers libdnf librepo libnghttp2 nett
 PY_SHORT=${PYTHON_VERSION%.*}
 wget -O /tmp/python.tgz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
 wget -O /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz.asc
-gpg --keyserver keys.openpgp.org --recv-keys $PYTHON_GPG_KEY_ID \
-    || gpg --keyserver pool.sks-keyservers.net --recv-keys $PYTHON_GPG_KEY_ID \
-    || gpg --keyserver pgp.mit.edu --recv-keys $PYTHON_GPG_KEY_ID \
-    || gpg --keyserver keyserver.pgp.com --recv-keys $PYTHON_GPG_KEY_ID
+wget -O /tmp/python-release-key.asc \
+    https://keys.openpgp.org/vks/v1/by-fingerprint/${PYTHON_GPG_KEY_FINGERPRINT}
+gpg --batch --import /tmp/python-release-key.asc
+rm /tmp/python-release-key.asc
 gpg --verify /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc /tmp/python.tgz
 rm /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc
 mkdir -p /tmp/pyinstall

docs/SHUTDOWN_LIFECYCLE.md

@@ -0,0 +1,58 @@
+# Splunk Container Shutdown Contract
+
+The full Splunk image exposes `/sbin/splunk-shutdown` as the single supported
+local container-stop operation. PID 1 invokes it for TERM. Kubernetes `preStop`
+hooks may invoke the same executable when their image compatibility check
+confirms it is present.
+
+This contract is local to one container. It does not perform Search Head
+detention, search draining, captain transfer, cluster membership changes, or
+Kubernetes rollout orchestration. Those operations must finish before a
+controller authorizes planned Pod replacement. Forced deletion, process crash,
+OOM, and node loss may skip `preStop`; TERM therefore remains a first-class
+caller.
+
+## Interface
+
+```text
+/sbin/splunk-shutdown --source=term
+/sbin/splunk-shutdown --source=prestop
+```
+
+`--source=manual` is available for diagnostics and direct qualification.
+Unsupported arguments or source values return `2`.
+
+`SPLUNK_SHUTDOWN_TIMEOUT_SECONDS` controls the local stop deadline and defaults
+to 600 seconds. It must be a positive integer and must fit inside the
+Kubernetes termination grace period with time remaining for signal delivery
+and forced cleanup. GNU `timeout` bounds the stop command and returns `124`
+when the deadline expires.
+
+## State and ownership
+
+The operation stores bounded, non-secret evidence under
+`$CONTAINER_ARTIFACT_DIR`:
+
+- `splunk-container.state` changes atomically to `stopping` before the stop
+  command starts;
+- `splunk-shutdown.lock/owner` records the owner PID and caller source; and
+- `splunk-shutdown.lock/result` records the stop exit status.
+
+Creating `splunk-shutdown.lock` is the single-owner decision. A concurrent
+caller does not issue another stop. A later caller returns the recorded result,
+so stop failure is not silently converted into success. The entrypoint TERM
+trap logs that evidence but retains its existing best-effort container-exit
+behavior.
+
+The lock is intentionally retained for the remaining life of the container.
+The `restart` entrypoint action is a different operation: it stops and starts
+Splunk without terminating the container and does not use this terminal
+shutdown contract.
+
+## Qualification expectations
+
+Runtime qualification must cover direct TERM, preStop followed by TERM,
+concurrent callers, repeated calls, stop failure, timeout, missing tooling, and
+the stopping-state transition. Kubernetes qualification must additionally
+measure Service and EndpointSlice withdrawal, actual stop duration, grace
+expiration, force deletion, and node-loss recovery.

splunk/common-files/Dockerfile

@@ -94,7 +94,7 @@ ENV SPLUNK_ROLE=splunk_standalone \
 
 USER root
 
-COPY [ "splunk/common-files/entrypoint.sh", "splunk/common-files/createdefaults.py", "splunk/common-files/checkstate.sh", "/sbin/" ]
+COPY [ "splunk/common-files/entrypoint.sh", "splunk/common-files/createdefaults.py", "splunk/common-files/checkstate.sh", "splunk/common-files/splunk-shutdown", "/sbin/" ]
 COPY splunk-ansible ${SPLUNK_ANSIBLE_HOME}
 
 # Set sudo rights
@@ -113,7 +113,8 @@ RUN sed -i -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL\nans
     && chmod 775 ${SPLUNK_ANSIBLE_HOME} \
     && chmod 664 ${SPLUNK_ANSIBLE_HOME}/ansible.cfg \
     && sed -i '/^\[defaults\]/a\interpreter_python = /usr/bin/python3' ${SPLUNK_ANSIBLE_HOME}/ansible.cfg \
-    && chmod 755 /sbin/entrypoint.sh /sbin/createdefaults.py /sbin/checkstate.sh
+    && chmod 755 /sbin/entrypoint.sh /sbin/createdefaults.py /sbin/checkstate.sh /sbin/splunk-shutdown \
+    && command -v timeout
 
 USER ${ANSIBLE_USER}
 HEALTHCHECK --interval=30s --timeout=30s --start-period=3m --retries=5 CMD /sbin/checkstate.sh || exit 1

splunk/common-files/entrypoint.sh

@@ -31,11 +31,15 @@ setup() {
 }
 
 teardown() {
-	# Always run the stop command on termination
-	if [ `whoami` != "${SPLUNK_USER}" ]; then
-		RUN_AS_SPLUNK="sudo -u ${SPLUNK_USER}"
-	fi
-	${RUN_AS_SPLUNK} ${SPLUNK_HOME}/bin/splunk stop || true
+	# TERM and preStop share one idempotent, bounded local shutdown operation.
+	# Reset the traps before running the bounded stop so a second signal cannot
+	# enter the handler recursively. PID 1 must exit after the stop completes;
+	# otherwise Kubernetes waits for the entire termination grace period even
+	# though splunkd is already stopped.
+	trap - SIGINT SIGTERM
+	local shutdown_result=0
+	/sbin/splunk-shutdown --source=term || shutdown_result=$?
+	exit "${shutdown_result}"
 }
 
 trap teardown SIGINT SIGTERM
@@ -147,6 +151,7 @@ Environment Variables:
   * SPLUNK_USER - user under which to run Splunk (default: splunk)
   * SPLUNK_GROUP - group under which to run Splunk (default: splunk)
   * SPLUNK_HOME - home directory where Splunk gets installed (default: /opt/splunk)
+  * SPLUNK_SHUTDOWN_TIMEOUT_SECONDS - maximum time allowed for a local Splunk stop before it is terminated (default: 600)
   * SPLUNK_START_ARGS - arguments to pass into the Splunk start command; you must include '--accept-license' to start Splunk (default: none)
   * SPLUNK_GENERAL_TERMS - with the value '--accept-sgt-current-at-splunk-com', indicates acceptance of the latest Splunk General Terms: https://www.splunk.com/en_us/legal/splunk-general-terms.html (default: none)
   * SPLUNK_PASSWORD - password to log into this Splunk instance, you must include a password (default: none)
@@ -216,5 +221,3 @@ case "$1" in
 		help $@
 		;;
 esac
-
-

splunk/common-files/splunk-shutdown

@@ -0,0 +1,125 @@
+#!/bin/sh
+
+# Copyright 2026 Splunk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u
+
+source_name="unknown"
+case "${1:-}" in
+	--source=*)
+		source_name="${1#--source=}"
+		;;
+	"")
+		;;
+	*)
+		echo "splunk-shutdown: unsupported argument: $1" >&2
+		exit 2
+		;;
+esac
+
+case "${source_name}" in
+	term|prestop|manual|unknown)
+		;;
+	*)
+		echo "splunk-shutdown: unsupported source: ${source_name}" >&2
+		exit 2
+		;;
+esac
+
+timeout_seconds="${SPLUNK_SHUTDOWN_TIMEOUT_SECONDS:-600}"
+case "${timeout_seconds}" in
+	""|*[!0-9]*|0)
+		echo "splunk-shutdown: SPLUNK_SHUTDOWN_TIMEOUT_SECONDS must be a positive integer" >&2
+		exit 2
+		;;
+esac
+
+artifact_dir="${CONTAINER_ARTIFACT_DIR:-/opt/container_artifact}"
+state_file="${artifact_dir}/splunk-container.state"
+lock_dir="${artifact_dir}/splunk-shutdown.lock"
+owner_file="${lock_dir}/owner"
+result_file="${lock_dir}/result"
+
+write_atomic() {
+	target_file="$1"
+	value="$2"
+	temporary_file="${target_file}.tmp.$$"
+	if ! printf '%s\n' "${value}" > "${temporary_file}"; then
+		return 1
+	fi
+	if ! mv -f "${temporary_file}" "${target_file}"; then
+		rm -f "${temporary_file}"
+		return 1
+	fi
+}
+
+if ! mkdir -p "${artifact_dir}"; then
+	echo "splunk-shutdown: cannot create artifact directory ${artifact_dir}" >&2
+	exit 1
+fi
+
+if ! mkdir "${lock_dir}" 2>/dev/null; then
+	if [ -r "${result_file}" ]; then
+		IFS= read -r existing_result < "${result_file}"
+		echo "splunk-shutdown: shutdown already completed result=${existing_result} source=${source_name}"
+		exit "${existing_result}"
+	fi
+	echo "splunk-shutdown: shutdown already in progress source=${source_name}"
+	exit 0
+fi
+
+owner_record="pid=$$ source=${source_name}"
+if ! write_atomic "${owner_file}" "${owner_record}"; then
+	echo "splunk-shutdown: cannot record shutdown owner" >&2
+	exit 1
+fi
+
+if ! write_atomic "${state_file}" "stopping"; then
+	echo "splunk-shutdown: cannot record stopping state" >&2
+	write_atomic "${result_file}" "1" || true
+	exit 1
+fi
+
+if ! command -v timeout >/dev/null 2>&1; then
+	echo "splunk-shutdown: timeout command is unavailable" >&2
+	write_atomic "${result_file}" "127" || true
+	exit 127
+fi
+
+splunk_home="${SPLUNK_HOME:-/opt/splunk}"
+splunk_user="${SPLUNK_USER:-splunk}"
+echo "splunk-shutdown: stop started source=${source_name} timeout_seconds=${timeout_seconds}"
+
+if [ "$(id -un)" = "${splunk_user}" ]; then
+	timeout --signal=TERM --kill-after=10s "${timeout_seconds}s" \
+		"${splunk_home}/bin/splunk" stop
+	stop_result=$?
+else
+	timeout --signal=TERM --kill-after=10s "${timeout_seconds}s" \
+		sudo -u "${splunk_user}" "${splunk_home}/bin/splunk" stop
+	stop_result=$?
+fi
+
+if ! write_atomic "${result_file}" "${stop_result}"; then
+	echo "splunk-shutdown: cannot record stop result=${stop_result}" >&2
+	exit 1
+fi
+
+if [ "${stop_result}" -eq 0 ]; then
+	echo "splunk-shutdown: stop completed source=${source_name} result=0"
+else
+	echo "splunk-shutdown: stop failed source=${source_name} result=${stop_result}" >&2
+fi
+exit "${stop_result}"