From c597a73076693621bdcb0669b21a4a97ccea8c05 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 24 Aug 2026 16:26:31 -0400 Subject: [PATCH 01/12] Add filtered IoT installer variant Generate install_script_agent7_iot.sh from the common Agent template with Agent 7 metadata and a target-specific IoT mode flag. Existing common-template targets explicitly disable the mode so every generated script resolves the placeholder. Wire the ignored artifact into Make clean/all and tagged manual deployment, document its future filtered normal-Agent purpose, and teach local installation checks its package, major version, and install-info identity. Package filtering remains intentionally deferred to the follow-up tasks. --- .gitignore | 1 + .gitlab-ci.yml | 2 ++ Makefile | 20 ++++++++++++++++++-- README.md | 1 + install_script.sh.template | 2 ++ test/localtest.sh | 9 +++++++-- 6 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6a8ecac6..143b44b9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ install_script.sh install_script_docker_injection.sh install_script_agent6.sh install_script_agent7.sh +install_script_agent7_iot.sh .vscode/launch.json .vscode/settings.json .idea/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e271912..73aab911 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ generate-scripts: - install_script.sh - install_script_agent6.sh - install_script_agent7.sh + - install_script_agent7_iot.sh - install_script_docker_injection.sh - install_script_op_worker1.sh - install_script_op_worker2.sh @@ -459,6 +460,7 @@ deploy: matrix: - SCRIPT: install_script_agent6.sh - SCRIPT: install_script_agent7.sh + - SCRIPT: install_script_agent7_iot.sh - SCRIPT: install_script_docker_injection.sh - SCRIPT: install_script_op_worker1.sh - SCRIPT: install_script_op_worker2.sh diff --git a/Makefile b/Makefile index 1771b284..fee8feec 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: install_script.sh install_script_agent6.sh install_script_agent7.sh install_script_docker_injection.sh +all: install_script.sh install_script_agent6.sh install_script_agent7.sh install_script_agent7_iot.sh install_script_docker_injection.sh clean: - rm -f install_script.sh install_script_agent6.sh install_script_agent7.sh install_script_docker_injection.sh + rm -f install_script.sh install_script_agent6.sh install_script_agent7.sh install_script_agent7_iot.sh install_script_docker_injection.sh define DEPRECATION_MESSAGE \n\ @@ -26,6 +26,7 @@ install_script.sh: install_script.sh.template -e 's|INSTALL_SCRIPT_REPORT_VERSION_PLACEHOLDER|Agent|' \ -e 's|INSTALL_INFO_VERSION_PLACEHOLDER||' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER|true|' \ + -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER|echo -e "\\033[33m${DEPRECATION_MESSAGE}\\033[0m"|' \ @@ -37,6 +38,7 @@ install_script_agent6.sh: install_script.sh.template -e 's|INSTALL_SCRIPT_REPORT_VERSION_PLACEHOLDER|Agent 6|' \ -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent6|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ + -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ @@ -48,6 +50,19 @@ install_script_agent7.sh: install_script.sh.template -e 's|INSTALL_SCRIPT_REPORT_VERSION_PLACEHOLDER|Agent 7|' \ -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent7|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ + -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ + -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ + -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ + -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ + install_script.sh.template > $@ + chmod +x $@ + +install_script_agent7_iot.sh: install_script.sh.template + sed -e 's|AGENT_MAJOR_VERSION_PLACEHOLDER|7|' \ + -e 's|INSTALL_SCRIPT_REPORT_VERSION_PLACEHOLDER|Agent 7 IoT Filtered|' \ + -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent7_iot|' \ + -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ + -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|true|' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ @@ -59,6 +74,7 @@ install_script_docker_injection.sh: install_script.sh.template -e 's|INSTALL_SCRIPT_REPORT_VERSION_PLACEHOLDER|Docker Injection|' \ -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_docker_injection|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ + -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER|export DD_APM_INSTRUMENTATION_ENABLED="docker"|' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER|safe_agent_version=noagent_autoinstrumentation|' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ diff --git a/README.md b/README.md index 33ff9624..267d08db 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ This repository contains 2 basic files, `install_script.sh.template` and `Makefi * `install_script.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default and also emits a deprecation warning when run. * `install_script_agent6.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default. * `install_script_agent7.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=7` by default. +* `install_script_agent7_iot.sh` - Future Agent 7 variant intended to install a filtered normal `datadog-agent` package for IoT deployments; package filtering will be added in a later change. The generated files must never be committed to this repository. All changes must be done by modifications of the template file and Makefile. diff --git a/install_script.sh.template b/install_script.sh.template index 52ec7d90..953dad95 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -13,6 +13,8 @@ install_script_version=1.46.0.post logfile="ddagent-install.log" support_email=support@datadoghq.com variant=install_scriptINSTALL_INFO_VERSION_PLACEHOLDER +# shellcheck disable=SC2034 +iot_filtered_install=IOT_FILTERED_INSTALL_PLACEHOLDER LEGACY_ETCDIR="/etc/dd-agent" LEGACY_CONF="$LEGACY_ETCDIR/datadog.conf" diff --git a/test/localtest.sh b/test/localtest.sh index 5f78ec8a..afa3c629 100755 --- a/test/localtest.sh +++ b/test/localtest.sh @@ -15,8 +15,11 @@ elif [[ "${IMAGE}" =~ "debian:11" ]]; then cp ./test/sources11.list /etc/apt/sources.list fi -EXPECTED_FLAVOR=${DD_AGENT_FLAVOR:-datadog-agent} SCRIPT_FLAVOR=$(echo "${SCRIPT}" | sed "s|.*install_script_\(.*\).sh|\1|") +EXPECTED_FLAVOR=${DD_AGENT_FLAVOR:-datadog-agent} +if [ "${SCRIPT_FLAVOR}" == "agent7_iot" ]; then + EXPECTED_FLAVOR=datadog-agent +fi if [ "${EXPECTED_FLAVOR}" != "datadog-agent" ] && echo "${SCRIPT}" | grep "agent6.sh$" >/dev/null; then echo "[PASS] Can't install flavor '${DD_AGENT_FLAVOR}' with install_script_agent6.sh" exit 0 @@ -92,7 +95,7 @@ OS_TYPE=$(get_os_type) INSTALLED_VERSION= RESULT=0 EXPECTED_MAJOR_VERSION=6 -if [ "${SCRIPT_FLAVOR}" == "agent7" ] || [ "${EXPECTED_FLAVOR}" != "datadog-agent" ] ; then +if [ "${SCRIPT_FLAVOR}" == "agent7" ] || [ "${SCRIPT_FLAVOR}" == "agent7_iot" ] || [ "${EXPECTED_FLAVOR}" != "datadog-agent" ] ; then EXPECTED_MAJOR_VERSION=7 fi if [ "${SCRIPT_FLAVOR}" == "docker_injection" ]; then @@ -153,6 +156,8 @@ if [ "${SCRIPT_FLAVOR}" == "agent6" ]; then EXPECTED_TOOL_VERSION="install_script_agent6" elif [ "${SCRIPT_FLAVOR}" == "agent7" ]; then EXPECTED_TOOL_VERSION="install_script_agent7" +elif [ "${SCRIPT_FLAVOR}" == "agent7_iot" ]; then + EXPECTED_TOOL_VERSION="install_script_agent7_iot" elif [ "${SCRIPT_FLAVOR}" == "install_script.sh" ]; then EXPECTED_TOOL_VERSION="install_script" elif [ "${SCRIPT_FLAVOR}" == "docker_injection" ]; then From d9efc50f16fccd8717b4eb103fc43b626d67247c Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 24 Aug 2026 16:38:28 -0400 Subject: [PATCH 02/12] Validate generated installer contracts Run a lightweight contract check immediately after script generation so unresolved placeholders, incorrect IoT modes, metadata, or executable bits fail in CI. Keep the IoT output available as a generated artifact while removing it from the public deploy matrix until package filtering is implemented. --- .gitlab-ci.yml | 2 +- test/generated-script-test.sh | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 test/generated-script-test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73aab911..263f5d64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,7 @@ generate-scripts: stage: generate script: - make + - ./test/generated-script-test.sh artifacts: expire_in: 2 weeks paths: @@ -460,7 +461,6 @@ deploy: matrix: - SCRIPT: install_script_agent6.sh - SCRIPT: install_script_agent7.sh - - SCRIPT: install_script_agent7_iot.sh - SCRIPT: install_script_docker_injection.sh - SCRIPT: install_script_op_worker1.sh - SCRIPT: install_script_op_worker2.sh diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh new file mode 100755 index 00000000..77cb78de --- /dev/null +++ b/test/generated-script-test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +generated_scripts=( + install_script.sh + install_script_agent6.sh + install_script_agent7.sh + install_script_agent7_iot.sh + install_script_docker_injection.sh +) +common_scripts=( + install_script.sh + install_script_agent6.sh + install_script_agent7.sh + install_script_docker_injection.sh +) + +fail() { + printf 'generated script contract failed: %s\n' "$1" >&2 + exit 1 +} + +assert_line() { + local script=$1 + local expected=$2 + + grep -Fqx -- "$expected" "$script" || fail "$(basename "$script") is missing '$expected'" +} + +for script_name in "${generated_scripts[@]}"; do + script="$repo_root/$script_name" + [[ -x $script ]] || fail "$script_name is missing or is not executable" + if grep -Eq '[A-Z][A-Z0-9_]*_PLACEHOLDER' "$script"; then + fail "$script_name contains an unresolved template placeholder" + fi +done + +iot_script="$repo_root/install_script_agent7_iot.sh" +assert_line "$iot_script" 'iot_filtered_install=true' +assert_line "$iot_script" 'variant=install_script_agent7_iot' +assert_line "$iot_script" 'agent_major_version=7' +grep -Fq 'Datadog Agent 7 IoT Filtered install script' "$iot_script" || + fail "install_script_agent7_iot.sh is missing its report label" + +for script_name in "${common_scripts[@]}"; do + assert_line "$repo_root/$script_name" 'iot_filtered_install=false' +done + +printf 'Generated script contract passed for %d scripts.\n' "${#generated_scripts[@]}" From 87e7787d180a5e91fc40668c8e1546d7ad16e8fc Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 24 Aug 2026 17:00:11 -0400 Subject: [PATCH 03/12] Define filtered IoT install profile Add extracted helpers that validate the generated IoT mode, render deterministic dpkg filters, and derive bounded RPM exclusion prefixes from the package manifest. The helpers retain only the required Agent, data-plane, rtloader, support-view, and IoT check assets without changing any live package-manager branch.\n\nValidate filtered layouts, query installed package versions by family, and atomically write the durable install profile. Add shunit coverage for compatible and rejected options, filter ordering and retention, RPM quoting and bounds, aggregated layout failures, marker replacement failures, and package queries. --- install_script.sh.template | 345 ++++++++++++++++++++++ unit_tests/test_install_script.sh | 457 ++++++++++++++++++++++++++++++ 2 files changed, 802 insertions(+) diff --git a/install_script.sh.template b/install_script.sh.template index 953dad95..f2a24777 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -349,6 +349,351 @@ function remove_rpm_gpg_keys() { done } +function validate_iot_installer_options() { + local agent_flavor="${1-}" + local agent_major_version="${2-}" + local infrastructure_mode="${3-}" + local -a incompatible_option_names=( + DD_FIPS_MODE + DD_APM_INSTRUMENTATION_ENABLED + DD_APM_INSTRUMENTATION_LIBRARIES + DD_OTELCOLLECTOR_ENABLED + DD_REMOTE_UPDATES + DD_NO_AGENT_INSTALL + DD_UPGRADE + DD_RUNTIME_SECURITY_CONFIG_ENABLED + DD_COMPLIANCE_CONFIG_ENABLED + DD_DISCOVERY_ENABLED + DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED + DD_PRIVILEGED_LOGS_ENABLED + DD_PRIVATE_ACTION_RUNNER_ENABLED + ) + local -a incompatible_option_values=( + "${4-}" + "${5-}" + "${6-}" + "${7-}" + "${8-}" + "${9-}" + "${10-}" + "${11-}" + "${12-}" + "${13-}" + "${14-}" + "${15-}" + "${16-}" + ) + local option_index + + if [ -n "$agent_flavor" ] && [ "$agent_flavor" != "datadog-agent" ]; then + printf 'Filtered IoT installation requires the normal datadog-agent package; set DD_AGENT_FLAVOR=datadog-agent or unset it.\n' >&2 + return 1 + fi + if [ "$agent_major_version" != "7" ]; then + printf 'Filtered IoT installation requires Agent 7; set DD_AGENT_MAJOR_VERSION=7.\n' >&2 + return 1 + fi + if [ -n "$infrastructure_mode" ] && [ "$infrastructure_mode" != "iot" ]; then + printf 'Filtered IoT installation requires infrastructure mode iot; set DD_INFRASTRUCTURE_MODE=iot or unset it.\n' >&2 + return 1 + fi + + for option_index in "${!incompatible_option_names[@]}"; do + if [ -n "${incompatible_option_values[$option_index]}" ]; then + printf 'Filtered IoT installation is incompatible with %s; unset this option and retry.\n' "${incompatible_option_names[$option_index]}" >&2 + return 1 + fi + done +} + +function write_deb_iot_filter_config() { + local destination_path="${1-}" + + if [ -z "$destination_path" ]; then + printf 'A destination path is required for the filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + + cat > "$destination_path" <<'EOF' +path-exclude=/opt/datadog-agent/bin/* +path-exclude=/opt/datadog-agent/embedded/* +path-exclude=/opt/datadog-agent/python-scripts/* +path-exclude=/opt/datadog-agent/requirements/* +path-exclude=/opt/datadog-agent/requirements*.txt +path-exclude=/opt/datadog-agent/compliance/* +path-exclude=/opt/datadog-agent/runtime-security.d/* +path-exclude=/etc/datadog-agent/compliance.d/* +path-exclude=/etc/datadog-agent/runtime-security.d/* +path-exclude=/etc/datadog-agent/conf.d/* +path-include=/opt/datadog-agent/bin/agent/agent +path-include=/opt/datadog-agent/embedded/bin/agent-data-plane +path-include=/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so* +path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/etc/datadog-agent/conf.d/cpu.d/* +path-include=/etc/datadog-agent/conf.d/disk.d/* +path-include=/etc/datadog-agent/conf.d/io.d/* +path-include=/etc/datadog-agent/conf.d/load.d/* +path-include=/etc/datadog-agent/conf.d/memory.d/* +path-include=/etc/datadog-agent/conf.d/network.d/* +path-include=/etc/datadog-agent/conf.d/ntp.d/* +path-include=/etc/datadog-agent/conf.d/uptime.d/* +path-include=/etc/datadog-agent/conf.d/system_swap.d/* +path-include=/etc/datadog-agent/conf.d/systemd.d/* +path-include=/etc/datadog-agent/conf.d/jetson.d/* +EOF +} + +function rpm_iot_exclude_paths() { + local package_path="${1-}" + local package_listing + local package_entry + local relative_path + local child_path + local exclusion_prefix + local prefix_count=0 + local -a exclusion_prefixes=() + local -A seen_prefixes=() + + if [ -z "$package_path" ]; then + printf 'An Agent RPM path is required to derive filtered IoT exclusions.\n' >&2 + return 1 + fi + if ! package_listing="$(rpm -qpl "$package_path")"; then + printf 'Unable to query the Agent RPM file list: %s\n' "$package_path" >&2 + return 1 + fi + + while IFS= read -r package_entry || [ -n "$package_entry" ]; do + exclusion_prefix="" + case "$package_entry" in + /etc/datadog-agent/conf.d/*) + relative_path=${package_entry#/etc/datadog-agent/conf.d/} + child_path=${relative_path%%/*} + case "$child_path" in + cpu.d|disk.d|io.d|load.d|memory.d|network.d|ntp.d|uptime.d|system_swap.d|systemd.d|jetson.d) + ;; + ?*) + exclusion_prefix="/etc/datadog-agent/conf.d/$child_path" + ;; + esac + ;; + /etc/datadog-agent/compliance.d|/etc/datadog-agent/compliance.d/*) + exclusion_prefix="/etc/datadog-agent/compliance.d/" + ;; + /etc/datadog-agent/runtime-security.d|/etc/datadog-agent/runtime-security.d/*) + exclusion_prefix="/etc/datadog-agent/runtime-security.d/" + ;; + /opt/datadog-agent/bin/agent/dist/*) + relative_path=${package_entry#/opt/datadog-agent/bin/agent/dist/} + child_path=${relative_path%%/*} + if [ -n "$child_path" ] && [ "$child_path" != "views" ]; then + exclusion_prefix="/opt/datadog-agent/bin/agent/dist/$child_path" + fi + ;; + /opt/datadog-agent/bin/agent/jmxfetch*) + exclusion_prefix="/opt/datadog-agent/bin/agent/jmxfetch" + ;; + /opt/datadog-agent/bin/*) + relative_path=${package_entry#/opt/datadog-agent/bin/} + child_path=${relative_path%%/*} + if [ -n "$child_path" ] && [ "$child_path" != "agent" ]; then + exclusion_prefix="/opt/datadog-agent/bin/$child_path" + fi + ;; + /opt/datadog-agent/embedded/bin/*) + relative_path=${package_entry#/opt/datadog-agent/embedded/bin/} + child_path=${relative_path%%/*} + if [ -n "$child_path" ] && [ "$child_path" != "agent-data-plane" ]; then + exclusion_prefix="/opt/datadog-agent/embedded/bin/$child_path" + fi + ;; + /opt/datadog-agent/embedded/lib/*) + relative_path=${package_entry#/opt/datadog-agent/embedded/lib/} + child_path=${relative_path%%/*} + case "$child_path" in + libdatadog-agent-rtloader.so*) + ;; + ?*) + exclusion_prefix="/opt/datadog-agent/embedded/lib/$child_path" + ;; + esac + ;; + /opt/datadog-agent/embedded/msodbcsql|/opt/datadog-agent/embedded/msodbcsql/*) + exclusion_prefix="/opt/datadog-agent/embedded/msodbcsql/" + ;; + /opt/datadog-agent/embedded/sbin|/opt/datadog-agent/embedded/sbin/*) + exclusion_prefix="/opt/datadog-agent/embedded/sbin/" + ;; + /opt/datadog-agent/embedded/include|/opt/datadog-agent/embedded/include/*) + exclusion_prefix="/opt/datadog-agent/embedded/include/" + ;; + /opt/datadog-agent/embedded/share/system-probe|/opt/datadog-agent/embedded/share/system-probe/*) + exclusion_prefix="/opt/datadog-agent/embedded/share/system-probe/" + ;; + /opt/datadog-agent/embedded/share/ebpf|/opt/datadog-agent/embedded/share/ebpf/*) + exclusion_prefix="/opt/datadog-agent/embedded/share/ebpf/" + ;; + /opt/datadog-agent/embedded/share/msodbcsql*) + relative_path=${package_entry#/opt/datadog-agent/embedded/share/} + child_path=${relative_path%%/*} + exclusion_prefix="/opt/datadog-agent/embedded/share/$child_path/" + ;; + /opt/datadog-agent/python-scripts|/opt/datadog-agent/python-scripts/*) + exclusion_prefix="/opt/datadog-agent/python-scripts/" + ;; + /opt/datadog-agent/requirements|/opt/datadog-agent/requirements/*) + exclusion_prefix="/opt/datadog-agent/requirements/" + ;; + /opt/datadog-agent/requirements*.txt) + exclusion_prefix="$package_entry" + ;; + /opt/datadog-agent/compliance|/opt/datadog-agent/compliance/*) + exclusion_prefix="/opt/datadog-agent/compliance/" + ;; + /opt/datadog-agent/runtime-security.d|/opt/datadog-agent/runtime-security.d/*) + exclusion_prefix="/opt/datadog-agent/runtime-security.d/" + ;; + esac + + if [ -n "$exclusion_prefix" ] && [ -z "${seen_prefixes[$exclusion_prefix]+present}" ]; then + seen_prefixes["$exclusion_prefix"]=1 + exclusion_prefixes+=("$exclusion_prefix") + prefix_count=$((prefix_count + 1)) + if [ "$prefix_count" -gt 1024 ]; then + printf 'Filtered IoT RPM installation requires more than 1024 exclusion prefixes; refusing the package.\n' >&2 + return 1 + fi + fi + done <<< "$package_listing" + + if [ "$prefix_count" -gt 0 ]; then + printf '%s\n' "${exclusion_prefixes[@]}" | LC_ALL=C sort -u + fi +} + +function validate_iot_install_layout() { + local root_prefix="${1-}" + local rtloader_found="" + local candidate_path + local logical_path + local -a failures=() + local -a required_paths=( + /opt/datadog-agent/bin/agent/agent + /opt/datadog-agent/embedded/bin/agent-data-plane + ) + local -a disallowed_paths=( + /opt/datadog-agent/embedded/bin/process-agent + /opt/datadog-agent/embedded/bin/python3 + /opt/datadog-agent/embedded/bin/system-probe + /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar + /etc/datadog-agent/conf.d/docker.d/conf.yaml.example + ) + + if [ "$root_prefix" = "/" ]; then + root_prefix="" + else + root_prefix=${root_prefix%/} + fi + + for logical_path in "${required_paths[@]}"; do + if [ ! -e "$root_prefix$logical_path" ]; then + failures+=("missing required path: $logical_path") + fi + done + + for candidate_path in "$root_prefix"/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*; do + if [ -e "$candidate_path" ]; then + rtloader_found=true + break + fi + done + if [ -z "$rtloader_found" ]; then + failures+=("missing required path: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*") + fi + + for logical_path in "${disallowed_paths[@]}"; do + if [ -e "$root_prefix$logical_path" ] || [ -L "$root_prefix$logical_path" ]; then + failures+=("disallowed path remains: $logical_path") + fi + done + + if [ "${#failures[@]}" -ne 0 ]; then + printf 'Filtered IoT layout validation failed:\n' >&2 + printf ' - %s\n' "${failures[@]}" >&2 + return 1 + fi +} + +function write_iot_install_profile() { + local destination_path="${1-}" + local package_version="${2-}" + local destination_directory + local temporary_path + + if [ -z "$destination_path" ]; then + printf 'A destination path is required for the filtered IoT install profile.\n' >&2 + return 1 + fi + if [ -z "$package_version" ] || [[ "$package_version" == *"'"* ]] || [[ "$package_version" == *$'\n'* ]]; then + printf 'A nonempty package version without quotes or newlines is required for the filtered IoT install profile.\n' >&2 + return 1 + fi + + case "$destination_path" in + */*) + destination_directory=${destination_path%/*} + [ -n "$destination_directory" ] || destination_directory="/" + ;; + *) + destination_directory="." + ;; + esac + + if ! temporary_path="$(mktemp "$destination_directory/.install_profile.tmp.XXXXXX")"; then + printf 'Unable to create a temporary filtered IoT install profile in %s.\n' "$destination_directory" >&2 + return 1 + fi + + if ! printf '%s\n' \ + 'version: 1' \ + 'profile: iot-filtered' \ + 'manifest: iot-v1' \ + 'package: datadog-agent' \ + "package_version: '$package_version'" \ + 'installer: install_script_agent7_iot' > "$temporary_path"; then + rm -f "$temporary_path" + printf 'Unable to write the temporary filtered IoT install profile.\n' >&2 + return 1 + fi + if ! chmod 0644 "$temporary_path"; then + rm -f "$temporary_path" + printf 'Unable to set permissions on the temporary filtered IoT install profile.\n' >&2 + return 1 + fi + if ! mv -f -- "$temporary_path" "$destination_path"; then + rm -f "$temporary_path" + printf 'Unable to atomically replace the filtered IoT install profile at %s.\n' "$destination_path" >&2 + return 1 + fi +} + +function get_installed_agent_package_version() { + local package_family="${1-}" + + case "$package_family" in + deb) + dpkg-query --show '--showformat=${Version}\n' datadog-agent + ;; + rpm) + rpm -q --queryformat '%{VERSION}-%{RELEASE}\n' datadog-agent + ;; + *) + printf 'Unsupported package family for Agent version query: %s (expected deb or rpm).\n' "$package_family" >&2 + return 1 + ;; + esac +} + # Emulate hashmap with simple switch case function getMapData() { diff --git a/unit_tests/test_install_script.sh b/unit_tests/test_install_script.sh index d3d1f917..e827b74c 100755 --- a/unit_tests/test_install_script.sh +++ b/unit_tests/test_install_script.sh @@ -418,5 +418,462 @@ testParEnabledWithoutApiKeyOnlyEnrollment() { assertEquals "$(sudo yq eval '.private_action_runner.api_key_only_enrollment' $config_file)" "null" } +### Filtered IoT install helpers +assertIotContains() { + local message="$1" + local actual="$2" + local expected="$3" + + if [[ "$actual" == *"$expected"* ]]; then + assertTrue "$message" 0 + else + assertTrue "$message: expected <$actual> to contain <$expected>" 1 + fi +} + +assertIotNotContains() { + local message="$1" + local actual="$2" + local unexpected="$3" + + if [[ "$actual" == *"$unexpected"* ]]; then + assertTrue "$message: expected <$actual> not to contain <$unexpected>" 1 + else + assertTrue "$message" 0 + fi +} + +iotDpkgPathIsIncluded() { + local config_file_path="$1" + local package_path="$2" + local directive + local pattern + local decision="include" + + while IFS='=' read -r directive pattern; do + # shellcheck disable=SC2053 + if [[ "$package_path" == $pattern ]]; then + case "$directive" in + path-exclude) decision="exclude" ;; + path-include) decision="include" ;; + esac + fi + done < "$config_file_path" + + [ "$decision" = "include" ] +} + +createIotRetainedLayout() { + local root="$1" + + mkdir -p \ + "$root/opt/datadog-agent/bin/agent" \ + "$root/opt/datadog-agent/embedded/bin" \ + "$root/opt/datadog-agent/embedded/lib" + touch \ + "$root/opt/datadog-agent/bin/agent/agent" \ + "$root/opt/datadog-agent/embedded/bin/agent-data-plane" \ + "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" +} + +testIotOptionsAcceptSupportedMode() { + validate_iot_installer_options "" "7" "" "" "" "" "" "" "" "" "" "" "" "" "" "" + assertEquals "unset compatible options should be accepted" 0 $? + + validate_iot_installer_options "datadog-agent" "7" "iot" "" "" "" "" "" "" "" "" "" "" "" "" "" + assertEquals "explicit filtered IoT options should be accepted" 0 $? +} + +testIotOptionsRejectNonAgentFlavors() { + local flavor + local output + local status + + for flavor in datadog-iot-agent datadog-fips-agent datadog-dogstatsd; do + output=$(validate_iot_installer_options "$flavor" "7" "iot" "" "" "" "" "" "" "" "" "" "" "" "" "" 2>&1) + status=$? + assertNotEquals "$flavor should be rejected" 0 "$status" + assertIotContains "$flavor rejection should name DD_AGENT_FLAVOR" "$output" "DD_AGENT_FLAVOR" + assertEquals "$flavor rejection should be one actionable message" 1 "$(printf '%s\n' "$output" | wc -l | tr -d ' ')" + done +} + +testIotOptionsRejectIncompatibleValues() { + local -a option_names=( + DD_AGENT_MAJOR_VERSION + DD_INFRASTRUCTURE_MODE + DD_FIPS_MODE + DD_APM_INSTRUMENTATION_ENABLED + DD_APM_INSTRUMENTATION_LIBRARIES + DD_OTELCOLLECTOR_ENABLED + DD_REMOTE_UPDATES + DD_NO_AGENT_INSTALL + DD_UPGRADE + DD_RUNTIME_SECURITY_CONFIG_ENABLED + DD_COMPLIANCE_CONFIG_ENABLED + DD_DISCOVERY_ENABLED + DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED + DD_PRIVILEGED_LOGS_ENABLED + DD_PRIVATE_ACTION_RUNNER_ENABLED + ) + local -a incompatible_values=( + 6 + basic + true + host + java + true + true + true + true + true + true + true + true + true + true + ) + local -a arguments + local index + local output + local status + + for index in "${!option_names[@]}"; do + arguments=(datadog-agent 7 iot "" "" "" "" "" "" "" "" "" "" "" "" "") + arguments[index + 1]="${incompatible_values[$index]}" + output=$(validate_iot_installer_options "${arguments[@]}" 2>&1) + status=$? + assertNotEquals "${option_names[$index]} should be rejected" 0 "$status" + assertIotContains "rejection should name ${option_names[$index]}" "$output" "${option_names[$index]}" + done +} + +testDebIotFilterConfigIsDeterministicAndUnique() { + local test_dir + local filter_path + local expected + local duplicate_count + + test_dir=$(mktemp -d) + filter_path="$test_dir/99-datadog-iot" + write_deb_iot_filter_config "$filter_path" + assertEquals "filter writer should succeed" 0 $? + + expected='path-exclude=/opt/datadog-agent/bin/* +path-exclude=/opt/datadog-agent/embedded/* +path-exclude=/opt/datadog-agent/python-scripts/* +path-exclude=/opt/datadog-agent/requirements/* +path-exclude=/opt/datadog-agent/requirements*.txt +path-exclude=/opt/datadog-agent/compliance/* +path-exclude=/opt/datadog-agent/runtime-security.d/* +path-exclude=/etc/datadog-agent/compliance.d/* +path-exclude=/etc/datadog-agent/runtime-security.d/* +path-exclude=/etc/datadog-agent/conf.d/* +path-include=/opt/datadog-agent/bin/agent/agent +path-include=/opt/datadog-agent/embedded/bin/agent-data-plane +path-include=/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so* +path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/etc/datadog-agent/conf.d/cpu.d/* +path-include=/etc/datadog-agent/conf.d/disk.d/* +path-include=/etc/datadog-agent/conf.d/io.d/* +path-include=/etc/datadog-agent/conf.d/load.d/* +path-include=/etc/datadog-agent/conf.d/memory.d/* +path-include=/etc/datadog-agent/conf.d/network.d/* +path-include=/etc/datadog-agent/conf.d/ntp.d/* +path-include=/etc/datadog-agent/conf.d/uptime.d/* +path-include=/etc/datadog-agent/conf.d/system_swap.d/* +path-include=/etc/datadog-agent/conf.d/systemd.d/* +path-include=/etc/datadog-agent/conf.d/jetson.d/*' + assertEquals "DEB filter content and order" "$expected" "$(cat "$filter_path")" + + duplicate_count=$(sort "$filter_path" | uniq -d | wc -l | tr -d ' ') + assertEquals "each DEB filter rule should be unique" 0 "$duplicate_count" + rm -rf "$test_dir" +} + +testDebIotFilterConfigUsesLastMatchingRule() { + local test_dir + local filter_path + + test_dir=$(mktemp -d) + filter_path="$test_dir/99-datadog-iot" + write_deb_iot_filter_config "$filter_path" + + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/agent/agent" + assertEquals "normal Agent should be retained" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/datadog.yaml.example" + assertEquals "Agent configuration example should be retained" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/bin/agent-data-plane" + assertEquals "agent-data-plane should be re-included" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so.1" + assertEquals "rtloader shim should be re-included" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/agent/dist/views/flare.html" + assertEquals "support views should be re-included" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/conf.d/systemd.d/conf.yaml.example" + assertEquals "IoT check configuration should be re-included" 0 $? + + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/bin/python3" + assertNotEquals "Python should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/process-agent/process-agent" + assertNotEquals "process-agent should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/conf.d/docker.d/conf.yaml.example" + assertNotEquals "non-IoT check configuration should remain excluded" 0 $? + rm -rf "$test_dir" +} + +testRpmIotExcludePathsDerivesSortedUniquePrefixes() { + local test_dir + local package_path + local arguments_path + local output + local status + local expected + local expected_arguments + + test_dir=$(mktemp -d) + package_path="$test_dir/datadog agent [7].rpm" + arguments_path="$test_dir/rpm-arguments" + : > "$package_path" + + # shellcheck disable=SC2329 + rpm() { + printf '%s\n' "$@" > "$arguments_path" + cat <<'EOF' +/opt/datadog-agent/bin/agent/agent +/opt/datadog-agent/bin/agent/dist/views/index.html +/opt/datadog-agent/bin/agent/dist/checks/check.py +/opt/datadog-agent/bin/agent/dist/config/config.py +/opt/datadog-agent/bin/agent/dist/utils/util.py +/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar +/opt/datadog-agent/bin/process-agent/process-agent +/opt/datadog-agent/embedded/bin/agent-data-plane +/opt/datadog-agent/embedded/bin/process-agent +/opt/datadog-agent/embedded/bin/python3 +/opt/datadog-agent/embedded/bin/python3/site.py +/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so +/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so.1 +/opt/datadog-agent/embedded/lib/libpython3.12.so +/opt/datadog-agent/embedded/lib/python3.12/site-packages/yaml.py +/opt/datadog-agent/embedded/sbin/chroot +/opt/datadog-agent/embedded/include/Python.h +/opt/datadog-agent/embedded/share/system-probe/ebpf.o +/opt/datadog-agent/embedded/share/ebpf/co-re.o +/opt/datadog-agent/embedded/msodbcsql/lib64/libmsodbcsql.so +/opt/datadog-agent/python-scripts/post.py +/opt/datadog-agent/requirements-agent-release.txt +/opt/datadog-agent/compliance/rules.json +/opt/datadog-agent/runtime-security.d/policy.policy +/etc/datadog-agent/compliance.d/default.json +/etc/datadog-agent/runtime-security.d/default.policy +/etc/datadog-agent/conf.d/cpu.d/conf.yaml.example +/etc/datadog-agent/conf.d/jetson.d/conf.yaml.example +/etc/datadog-agent/conf.d/docker.d/conf.yaml.example +EOF + } + + output=$(rpm_iot_exclude_paths "$package_path" 2>&1) + status=$? + unset -f rpm + + assertEquals "RPM path derivation should succeed" 0 "$status" + expected='/etc/datadog-agent/compliance.d/ +/etc/datadog-agent/conf.d/docker.d +/etc/datadog-agent/runtime-security.d/ +/opt/datadog-agent/bin/agent/dist/checks +/opt/datadog-agent/bin/agent/dist/config +/opt/datadog-agent/bin/agent/dist/jmx +/opt/datadog-agent/bin/agent/dist/utils +/opt/datadog-agent/bin/process-agent +/opt/datadog-agent/compliance/ +/opt/datadog-agent/embedded/bin/process-agent +/opt/datadog-agent/embedded/bin/python3 +/opt/datadog-agent/embedded/include/ +/opt/datadog-agent/embedded/lib/libpython3.12.so +/opt/datadog-agent/embedded/lib/python3.12 +/opt/datadog-agent/embedded/msodbcsql/ +/opt/datadog-agent/embedded/sbin/ +/opt/datadog-agent/embedded/share/ebpf/ +/opt/datadog-agent/embedded/share/system-probe/ +/opt/datadog-agent/python-scripts/ +/opt/datadog-agent/requirements-agent-release.txt +/opt/datadog-agent/runtime-security.d/' + assertEquals "RPM exclusions should be sorted, unique, and retain only supported payloads" "$expected" "$output" + + expected_arguments="-qpl +$package_path" + assertEquals "package path should remain one quoted rpm argument" "$expected_arguments" "$(cat "$arguments_path")" + rm -rf "$test_dir" +} + +testRpmIotExcludePathsRejectsMoreThan1024Prefixes() { + local test_dir + local output + local status + + test_dir=$(mktemp -d) + # shellcheck disable=SC2329 + rpm() { + local index=0 + while [ "$index" -le 1024 ]; do + printf '/opt/datadog-agent/embedded/bin/tool-%04d\n' "$index" + index=$((index + 1)) + done + } + + output=$(rpm_iot_exclude_paths "$test_dir/agent.rpm" 2>&1) + status=$? + unset -f rpm + + assertNotEquals "more than 1024 RPM exclusions should fail" 0 "$status" + assertIotContains "bound error should be actionable" "$output" "1024" + assertIotNotContains "no partial prefix list should be printed" "$output" "/opt/datadog-agent/embedded/bin/tool-0000" + rm -rf "$test_dir" +} + +testValidateIotInstallLayoutAcceptsFilteredLayout() { + local root + + root=$(mktemp -d) + createIotRetainedLayout "$root" + validate_iot_install_layout "$root" + assertEquals "filtered layout should pass" 0 $? + rm -rf "$root" +} + +testValidateIotInstallLayoutAggregatesFailures() { + local root + local output + local status + + root=$(mktemp -d) + createIotRetainedLayout "$root" + rm \ + "$root/opt/datadog-agent/embedded/bin/agent-data-plane" \ + "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" + mkdir -p \ + "$root/opt/datadog-agent/embedded/bin" \ + "$root/opt/datadog-agent/bin/agent/dist/jmx" \ + "$root/etc/datadog-agent/conf.d/docker.d" + touch \ + "$root/opt/datadog-agent/embedded/bin/process-agent" \ + "$root/opt/datadog-agent/embedded/bin/python3" \ + "$root/opt/datadog-agent/embedded/bin/system-probe" \ + "$root/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar" \ + "$root/etc/datadog-agent/conf.d/docker.d/conf.yaml.example" + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + assertNotEquals "invalid filtered layout should fail" 0 "$status" + assertIotContains "missing ADP should be reported" "$output" "missing required path: /opt/datadog-agent/embedded/bin/agent-data-plane" + assertIotContains "missing rtloader should be reported" "$output" "missing required path: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*" + assertIotContains "process-agent should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/process-agent" + assertIotContains "Python should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/python3" + assertIotContains "system-probe should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/system-probe" + assertIotContains "JMX should be reported" "$output" "disallowed path remains: /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar" + assertIotContains "non-IoT config should be reported" "$output" "disallowed path remains: /etc/datadog-agent/conf.d/docker.d/conf.yaml.example" + rm -rf "$root" +} + +testWriteIotInstallProfileWritesExactYamlAndMode() { + local test_dir + local marker_path + local expected + + test_dir=$(mktemp -d) + marker_path="$test_dir/install_profile" + write_iot_install_profile "$marker_path" "7.72.1-1" + assertEquals "profile writer should succeed" 0 $? + + expected='version: 1 +profile: iot-filtered +manifest: iot-v1 +package: datadog-agent +package_version: '\''7.72.1-1'\'' +installer: install_script_agent7_iot' + assertEquals "profile YAML" "$expected" "$(cat "$marker_path")" + assertEquals "profile mode" 644 "$(stat -c '%a' "$marker_path")" + rm -rf "$test_dir" +} + +testWriteIotInstallProfilePreservesMarkerAndCleansTempOnFailure() { + local test_dir + local marker_path + local output + local status + local temp_count + + test_dir=$(mktemp -d) + marker_path="$test_dir/install_profile" + printf 'existing marker\n' > "$marker_path" + # shellcheck disable=SC2329 + mv() { + return 1 + } + + output=$(write_iot_install_profile "$marker_path" "7.72.1-1" 2>&1) + status=$? + unset -f mv + + assertNotEquals "failed replacement should return nonzero" 0 "$status" + assertEquals "existing marker should remain intact" "existing marker" "$(cat "$marker_path")" + temp_count=$(find "$test_dir" -maxdepth 1 -name '.install_profile.tmp.*' | wc -l | tr -d ' ') + assertEquals "failed replacement should clean its temporary file" 0 "$temp_count" + rm -rf "$test_dir" +} + +testGetInstalledAgentPackageVersionQueriesExplicitFamily() { + local test_dir + local arguments_path + local output + local status + local expected_dpkg_arguments + local expected_rpm_arguments + + test_dir=$(mktemp -d) + arguments_path="$test_dir/query-arguments" + # shellcheck disable=SC2329 + dpkg-query() { + printf '%s\n' "$@" > "$arguments_path" + printf '7.72.1-1\n' + } + output=$(get_installed_agent_package_version deb) + status=$? + unset -f dpkg-query + assertEquals "DEB version query should succeed" 0 "$status" + assertEquals "DEB installed package version" "7.72.1-1" "$output" + # shellcheck disable=SC2016 + expected_dpkg_arguments='--show +--showformat=${Version}\n +datadog-agent' + assertEquals "dpkg-query arguments" "$expected_dpkg_arguments" "$(cat "$arguments_path")" + + # shellcheck disable=SC2329 + rpm() { + printf '%s\n' "$@" > "$arguments_path" + printf '7.72.1-1\n' + } + output=$(get_installed_agent_package_version rpm) + status=$? + unset -f rpm + assertEquals "RPM version query should succeed" 0 "$status" + assertEquals "RPM installed package version" "7.72.1-1" "$output" + expected_rpm_arguments='-q +--queryformat +%{VERSION}-%{RELEASE}\n +datadog-agent' + assertEquals "rpm query arguments" "$expected_rpm_arguments" "$(cat "$arguments_path")" + rm -rf "$test_dir" +} + +testGetInstalledAgentPackageVersionRejectsUnknownFamily() { + local output + local status + + output=$(get_installed_agent_package_version apk 2>&1) + status=$? + assertNotEquals "unknown package family should fail" 0 "$status" + assertIotContains "unknown family error should be actionable" "$output" "apk" +} + # shellcheck source=/dev/null . shunit2 From abbfe218cb843541b133304f2a6752f11f84099f Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 24 Aug 2026 17:28:47 -0400 Subject: [PATCH 04/12] Harden filtered IoT layout validation Validate the full retained IoT payload and every representative pruned class while tolerating empty dpkg-preserved directories and dangling links. Use targeted DEB and RPM exclusions so Agent SSL and support assets remain available, with explicit DEB parent re-includes. Reject RPM exclusion-prefix collisions before producing output, and cover missing retained content, pruned payloads, parent rules, support retention, and collision cases with shunit tests. --- install_script.sh.template | 146 ++++++++++++++- unit_tests/test_install_script.sh | 300 +++++++++++++++++++++++++++--- 2 files changed, 406 insertions(+), 40 deletions(-) diff --git a/install_script.sh.template b/install_script.sh.template index f2a24777..72aa2bfb 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -416,7 +416,16 @@ function write_deb_iot_filter_config() { cat > "$destination_path" <<'EOF' path-exclude=/opt/datadog-agent/bin/* -path-exclude=/opt/datadog-agent/embedded/* +path-exclude=/opt/datadog-agent/embedded/bin/* +path-exclude=/opt/datadog-agent/embedded/include/* +path-exclude=/opt/datadog-agent/embedded/lib/libodbc* +path-exclude=/opt/datadog-agent/embedded/lib/libpython* +path-exclude=/opt/datadog-agent/embedded/lib/libtdsodbc* +path-exclude=/opt/datadog-agent/embedded/lib/python* +path-exclude=/opt/datadog-agent/embedded/msodbcsql/* +path-exclude=/opt/datadog-agent/embedded/sbin/* +path-exclude=/opt/datadog-agent/embedded/share/ebpf/* +path-exclude=/opt/datadog-agent/embedded/share/system-probe/* path-exclude=/opt/datadog-agent/python-scripts/* path-exclude=/opt/datadog-agent/requirements/* path-exclude=/opt/datadog-agent/requirements*.txt @@ -425,20 +434,38 @@ path-exclude=/opt/datadog-agent/runtime-security.d/* path-exclude=/etc/datadog-agent/compliance.d/* path-exclude=/etc/datadog-agent/runtime-security.d/* path-exclude=/etc/datadog-agent/conf.d/* +path-include=/opt/datadog-agent/bin +path-include=/opt/datadog-agent/bin/agent path-include=/opt/datadog-agent/bin/agent/agent +path-include=/opt/datadog-agent/bin/agent/dist +path-include=/opt/datadog-agent/bin/agent/dist/views +path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/opt/datadog-agent/embedded/bin path-include=/opt/datadog-agent/embedded/bin/agent-data-plane +path-include=/opt/datadog-agent/embedded/lib path-include=/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so* -path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/etc/datadog-agent/conf.d +path-include=/etc/datadog-agent/conf.d/cpu.d path-include=/etc/datadog-agent/conf.d/cpu.d/* +path-include=/etc/datadog-agent/conf.d/disk.d path-include=/etc/datadog-agent/conf.d/disk.d/* +path-include=/etc/datadog-agent/conf.d/io.d path-include=/etc/datadog-agent/conf.d/io.d/* +path-include=/etc/datadog-agent/conf.d/load.d path-include=/etc/datadog-agent/conf.d/load.d/* +path-include=/etc/datadog-agent/conf.d/memory.d path-include=/etc/datadog-agent/conf.d/memory.d/* +path-include=/etc/datadog-agent/conf.d/network.d path-include=/etc/datadog-agent/conf.d/network.d/* +path-include=/etc/datadog-agent/conf.d/ntp.d path-include=/etc/datadog-agent/conf.d/ntp.d/* +path-include=/etc/datadog-agent/conf.d/uptime.d path-include=/etc/datadog-agent/conf.d/uptime.d/* +path-include=/etc/datadog-agent/conf.d/system_swap.d path-include=/etc/datadog-agent/conf.d/system_swap.d/* +path-include=/etc/datadog-agent/conf.d/systemd.d path-include=/etc/datadog-agent/conf.d/systemd.d/* +path-include=/etc/datadog-agent/conf.d/jetson.d path-include=/etc/datadog-agent/conf.d/jetson.d/* EOF } @@ -450,8 +477,26 @@ function rpm_iot_exclude_paths() { local relative_path local child_path local exclusion_prefix + local retained_path local prefix_count=0 local -a exclusion_prefixes=() + local -a retained_paths=( + /opt/datadog-agent/bin/agent/agent + /opt/datadog-agent/bin/agent/dist/views + /opt/datadog-agent/embedded/bin/agent-data-plane + /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so + /etc/datadog-agent/conf.d/cpu.d + /etc/datadog-agent/conf.d/disk.d + /etc/datadog-agent/conf.d/io.d + /etc/datadog-agent/conf.d/load.d + /etc/datadog-agent/conf.d/memory.d + /etc/datadog-agent/conf.d/network.d + /etc/datadog-agent/conf.d/ntp.d + /etc/datadog-agent/conf.d/uptime.d + /etc/datadog-agent/conf.d/system_swap.d + /etc/datadog-agent/conf.d/systemd.d + /etc/datadog-agent/conf.d/jetson.d + ) local -A seen_prefixes=() if [ -z "$package_path" ]; then @@ -511,9 +556,7 @@ function rpm_iot_exclude_paths() { relative_path=${package_entry#/opt/datadog-agent/embedded/lib/} child_path=${relative_path%%/*} case "$child_path" in - libdatadog-agent-rtloader.so*) - ;; - ?*) + libodbc*|libpython*|libtdsodbc*|python*) exclusion_prefix="/opt/datadog-agent/embedded/lib/$child_path" ;; esac @@ -566,6 +609,15 @@ function rpm_iot_exclude_paths() { fi done <<< "$package_listing" + for exclusion_prefix in "${exclusion_prefixes[@]}"; do + for retained_path in "${retained_paths[@]}"; do + if [ "${retained_path:0:${#exclusion_prefix}}" = "$exclusion_prefix" ]; then + printf 'Filtered IoT RPM exclusion prefix %s would also exclude retained path %s; refusing the package.\n' "$exclusion_prefix" "$retained_path" >&2 + return 1 + fi + done + done + if [ "$prefix_count" -gt 0 ]; then printf '%s\n' "${exclusion_prefixes[@]}" | LC_ALL=C sort -u fi @@ -575,19 +627,59 @@ function validate_iot_install_layout() { local root_prefix="${1-}" local rtloader_found="" local candidate_path + local candidate_root local logical_path + local logical_pattern local -a failures=() local -a required_paths=( /opt/datadog-agent/bin/agent/agent /opt/datadog-agent/embedded/bin/agent-data-plane + /etc/datadog-agent/datadog.yaml.example + ) + local -a required_content_directories=( + /opt/datadog-agent/bin/agent/dist/views + /etc/datadog-agent/conf.d/cpu.d + /etc/datadog-agent/conf.d/disk.d + /etc/datadog-agent/conf.d/io.d + /etc/datadog-agent/conf.d/load.d + /etc/datadog-agent/conf.d/memory.d + /etc/datadog-agent/conf.d/network.d + /etc/datadog-agent/conf.d/ntp.d + /etc/datadog-agent/conf.d/uptime.d + /etc/datadog-agent/conf.d/system_swap.d + /etc/datadog-agent/conf.d/systemd.d + /etc/datadog-agent/conf.d/jetson.d ) local -a disallowed_paths=( /opt/datadog-agent/embedded/bin/process-agent - /opt/datadog-agent/embedded/bin/python3 + /opt/datadog-agent/embedded/bin/trace-agent + /opt/datadog-agent/embedded/bin/trace-loader + /opt/datadog-agent/embedded/bin/security-agent + /opt/datadog-agent/embedded/bin/privateactionrunner + /opt/datadog-agent/embedded/bin/installer /opt/datadog-agent/embedded/bin/system-probe - /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar + /opt/datadog-agent/embedded/bin/system-probe-lite /etc/datadog-agent/conf.d/docker.d/conf.yaml.example ) + local -a disallowed_file_patterns=( + '/opt/datadog-agent/embedded/bin/python*' + '/opt/datadog-agent/embedded/lib/libpython*' + '/opt/datadog-agent/embedded/lib/libodbc*' + '/opt/datadog-agent/embedded/lib/libtdsodbc*' + ) + local -a disallowed_tree_patterns=( + '/opt/datadog-agent/bin/agent/dist/jmx' + '/opt/datadog-agent/embedded/lib/python*/site-packages' + '/opt/datadog-agent/embedded/share/system-probe' + '/opt/datadog-agent/embedded/share/ebpf' + '/opt/datadog-agent/embedded/include' + '/opt/datadog-agent/embedded/msodbcsql' + '/opt/datadog-agent/embedded/sbin' + '/opt/datadog-agent/compliance' + '/opt/datadog-agent/runtime-security.d' + '/etc/datadog-agent/compliance.d' + '/etc/datadog-agent/runtime-security.d' + ) if [ "$root_prefix" = "/" ]; then root_prefix="" @@ -602,7 +694,7 @@ function validate_iot_install_layout() { done for candidate_path in "$root_prefix"/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*; do - if [ -e "$candidate_path" ]; then + if [ -f "$candidate_path" ]; then rtloader_found=true break fi @@ -611,12 +703,46 @@ function validate_iot_install_layout() { failures+=("missing required path: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*") fi + for logical_path in "${required_content_directories[@]}"; do + if [ ! -d "$root_prefix$logical_path" ]; then + failures+=("missing required directory: $logical_path") + continue + fi + candidate_path=$(find "$root_prefix$logical_path" -type f -print -quit 2>/dev/null) + if [ -z "$candidate_path" ]; then + failures+=("missing required content: $logical_path/*") + fi + done + for logical_path in "${disallowed_paths[@]}"; do - if [ -e "$root_prefix$logical_path" ] || [ -L "$root_prefix$logical_path" ]; then - failures+=("disallowed path remains: $logical_path") + candidate_path="$root_prefix$logical_path" + if [ -f "$candidate_path" ]; then + failures+=("disallowed payload remains: $logical_path") fi done + for logical_pattern in "${disallowed_file_patterns[@]}"; do + while IFS= read -r candidate_path; do + if [ -f "$candidate_path" ]; then + failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") + break + fi + done < <(compgen -G "$root_prefix$logical_pattern" || true) + done + + for logical_pattern in "${disallowed_tree_patterns[@]}"; do + while IFS= read -r candidate_root; do + if [ ! -d "$candidate_root" ] && [ ! -L "$candidate_root" ]; then + continue + fi + candidate_path=$(find "$candidate_root" -type f -print -quit 2>/dev/null) + if [ -n "$candidate_path" ]; then + failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") + break + fi + done < <(compgen -G "$root_prefix$logical_pattern" || true) + done + if [ "${#failures[@]}" -ne 0 ]; then printf 'Filtered IoT layout validation failed:\n' >&2 printf ' - %s\n' "${failures[@]}" >&2 diff --git a/unit_tests/test_install_script.sh b/unit_tests/test_install_script.sh index e827b74c..f6a77435 100755 --- a/unit_tests/test_install_script.sh +++ b/unit_tests/test_install_script.sh @@ -465,15 +465,27 @@ iotDpkgPathIsIncluded() { createIotRetainedLayout() { local root="$1" + local check_name + local -a check_names=( + cpu disk io load memory network ntp uptime system_swap systemd jetson + ) mkdir -p \ - "$root/opt/datadog-agent/bin/agent" \ + "$root/opt/datadog-agent/bin/agent/dist/views" \ "$root/opt/datadog-agent/embedded/bin" \ - "$root/opt/datadog-agent/embedded/lib" + "$root/opt/datadog-agent/embedded/lib" \ + "$root/etc/datadog-agent/conf.d" touch \ "$root/opt/datadog-agent/bin/agent/agent" \ + "$root/opt/datadog-agent/bin/agent/dist/views/index.html" \ "$root/opt/datadog-agent/embedded/bin/agent-data-plane" \ - "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" + "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" \ + "$root/etc/datadog-agent/datadog.yaml.example" + + for check_name in "${check_names[@]}"; do + mkdir -p "$root/etc/datadog-agent/conf.d/$check_name.d" + touch "$root/etc/datadog-agent/conf.d/$check_name.d/conf.yaml.example" + done } testIotOptionsAcceptSupportedMode() { @@ -560,7 +572,16 @@ testDebIotFilterConfigIsDeterministicAndUnique() { assertEquals "filter writer should succeed" 0 $? expected='path-exclude=/opt/datadog-agent/bin/* -path-exclude=/opt/datadog-agent/embedded/* +path-exclude=/opt/datadog-agent/embedded/bin/* +path-exclude=/opt/datadog-agent/embedded/include/* +path-exclude=/opt/datadog-agent/embedded/lib/libodbc* +path-exclude=/opt/datadog-agent/embedded/lib/libpython* +path-exclude=/opt/datadog-agent/embedded/lib/libtdsodbc* +path-exclude=/opt/datadog-agent/embedded/lib/python* +path-exclude=/opt/datadog-agent/embedded/msodbcsql/* +path-exclude=/opt/datadog-agent/embedded/sbin/* +path-exclude=/opt/datadog-agent/embedded/share/ebpf/* +path-exclude=/opt/datadog-agent/embedded/share/system-probe/* path-exclude=/opt/datadog-agent/python-scripts/* path-exclude=/opt/datadog-agent/requirements/* path-exclude=/opt/datadog-agent/requirements*.txt @@ -569,20 +590,38 @@ path-exclude=/opt/datadog-agent/runtime-security.d/* path-exclude=/etc/datadog-agent/compliance.d/* path-exclude=/etc/datadog-agent/runtime-security.d/* path-exclude=/etc/datadog-agent/conf.d/* +path-include=/opt/datadog-agent/bin +path-include=/opt/datadog-agent/bin/agent path-include=/opt/datadog-agent/bin/agent/agent +path-include=/opt/datadog-agent/bin/agent/dist +path-include=/opt/datadog-agent/bin/agent/dist/views +path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/opt/datadog-agent/embedded/bin path-include=/opt/datadog-agent/embedded/bin/agent-data-plane +path-include=/opt/datadog-agent/embedded/lib path-include=/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so* -path-include=/opt/datadog-agent/bin/agent/dist/views/* +path-include=/etc/datadog-agent/conf.d +path-include=/etc/datadog-agent/conf.d/cpu.d path-include=/etc/datadog-agent/conf.d/cpu.d/* +path-include=/etc/datadog-agent/conf.d/disk.d path-include=/etc/datadog-agent/conf.d/disk.d/* +path-include=/etc/datadog-agent/conf.d/io.d path-include=/etc/datadog-agent/conf.d/io.d/* +path-include=/etc/datadog-agent/conf.d/load.d path-include=/etc/datadog-agent/conf.d/load.d/* +path-include=/etc/datadog-agent/conf.d/memory.d path-include=/etc/datadog-agent/conf.d/memory.d/* +path-include=/etc/datadog-agent/conf.d/network.d path-include=/etc/datadog-agent/conf.d/network.d/* +path-include=/etc/datadog-agent/conf.d/ntp.d path-include=/etc/datadog-agent/conf.d/ntp.d/* +path-include=/etc/datadog-agent/conf.d/uptime.d path-include=/etc/datadog-agent/conf.d/uptime.d/* +path-include=/etc/datadog-agent/conf.d/system_swap.d path-include=/etc/datadog-agent/conf.d/system_swap.d/* +path-include=/etc/datadog-agent/conf.d/systemd.d path-include=/etc/datadog-agent/conf.d/systemd.d/* +path-include=/etc/datadog-agent/conf.d/jetson.d path-include=/etc/datadog-agent/conf.d/jetson.d/*' assertEquals "DEB filter content and order" "$expected" "$(cat "$filter_path")" @@ -591,14 +630,38 @@ path-include=/etc/datadog-agent/conf.d/jetson.d/*' rm -rf "$test_dir" } -testDebIotFilterConfigUsesLastMatchingRule() { +testDebIotFilterConfigExplicitlyIncludesParentsBeforeLeaves() { local test_dir local filter_path + local package_path + local check_name + local -a parent_paths=( + /opt/datadog-agent/bin + /opt/datadog-agent/bin/agent + /opt/datadog-agent/bin/agent/dist + /opt/datadog-agent/bin/agent/dist/views + /opt/datadog-agent/embedded/bin + /opt/datadog-agent/embedded/lib + /etc/datadog-agent/conf.d + ) + local -a check_names=( + cpu disk io load memory network ntp uptime system_swap systemd jetson + ) test_dir=$(mktemp -d) filter_path="$test_dir/99-datadog-iot" write_deb_iot_filter_config "$filter_path" + for package_path in "${parent_paths[@]}"; do + iotDpkgPathIsIncluded "$filter_path" "$package_path" + assertEquals "$package_path parent should be explicitly retained" 0 $? + done + for check_name in "${check_names[@]}"; do + package_path="/etc/datadog-agent/conf.d/$check_name.d" + iotDpkgPathIsIncluded "$filter_path" "$package_path" + assertEquals "$package_path parent should be explicitly retained" 0 $? + done + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/agent/agent" assertEquals "normal Agent should be retained" 0 $? iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/datadog.yaml.example" @@ -609,11 +672,31 @@ testDebIotFilterConfigUsesLastMatchingRule() { assertEquals "rtloader shim should be re-included" 0 $? iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/agent/dist/views/flare.html" assertEquals "support views should be re-included" 0 $? - iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/conf.d/systemd.d/conf.yaml.example" - assertEquals "IoT check configuration should be re-included" 0 $? + for check_name in "${check_names[@]}"; do + package_path="/etc/datadog-agent/conf.d/$check_name.d/conf.yaml.example" + iotDpkgPathIsIncluded "$filter_path" "$package_path" + assertEquals "$package_path payload should be retained" 0 $? + done + + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/ssl/certs/cacert.pem" + assertEquals "embedded SSL data should remain available to the Agent" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/lib/libssl.so.3" + assertEquals "embedded SSL libraries should remain available to the Agent" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/share/openscap/cpe.xml" + assertEquals "non-system-probe support data should remain available" 0 $? iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/bin/python3" assertNotEquals "Python should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/lib/libpython3.13.so.1.0" + assertNotEquals "libpython should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/lib/python3.13/site-packages/yaml.py" + assertNotEquals "Python site-packages should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/include/Python.h" + assertNotEquals "headers should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/lib/libodbc.so.2" + assertNotEquals "ODBC libraries should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/share/system-probe/ebpf.o" + assertNotEquals "system-probe support data should remain excluded" 0 $? iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/process-agent/process-agent" assertNotEquals "process-agent should remain excluded" 0 $? iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/conf.d/docker.d/conf.yaml.example" @@ -654,6 +737,9 @@ testRpmIotExcludePathsDerivesSortedUniquePrefixes() { /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so.1 /opt/datadog-agent/embedded/lib/libpython3.12.so /opt/datadog-agent/embedded/lib/python3.12/site-packages/yaml.py +/opt/datadog-agent/embedded/lib/libssl.so.3 +/opt/datadog-agent/embedded/ssl/certs/cacert.pem +/opt/datadog-agent/embedded/share/openscap/cpe.xml /opt/datadog-agent/embedded/sbin/chroot /opt/datadog-agent/embedded/include/Python.h /opt/datadog-agent/embedded/share/system-probe/ebpf.o @@ -705,6 +791,66 @@ $package_path" rm -rf "$test_dir" } +testRpmIotExcludePathsRejectsAgentDataPlanePrefixCollisionWithoutOutput() { + local test_dir + local output_path + local error_path + local status + + test_dir=$(mktemp -d) + output_path="$test_dir/output" + error_path="$test_dir/error" + # shellcheck disable=SC2329 + rpm() { + cat <<'EOF' +/opt/datadog-agent/embedded/bin/agent-data +/opt/datadog-agent/embedded/bin/agent-data-plane +/opt/datadog-agent/embedded/bin/process-agent +EOF + } + + rpm_iot_exclude_paths "$test_dir/agent.rpm" > "$output_path" 2> "$error_path" + status=$? + unset -f rpm + + assertNotEquals "an exclusion prefix that matches agent-data-plane should fail" 0 "$status" + assertEquals "a collision should not emit any partial exclusion output" "" "$(cat "$output_path")" + assertIotContains "collision error should identify the unsafe prefix" "$(cat "$error_path")" "/opt/datadog-agent/embedded/bin/agent-data" + assertIotContains "collision error should identify agent-data-plane" "$(cat "$error_path")" "/opt/datadog-agent/embedded/bin/agent-data-plane" + assertIotNotContains "unrelated exclusions should not leak to stderr" "$(cat "$error_path")" "/opt/datadog-agent/embedded/bin/process-agent" + rm -rf "$test_dir" +} + +testRpmIotExcludePathsRejectsSupportViewsPrefixCollisionWithoutOutput() { + local test_dir + local output_path + local error_path + local status + + test_dir=$(mktemp -d) + output_path="$test_dir/output" + error_path="$test_dir/error" + # shellcheck disable=SC2329 + rpm() { + cat <<'EOF' +/opt/datadog-agent/bin/agent/dist/view/index.html +/opt/datadog-agent/bin/agent/dist/views/index.html +/opt/datadog-agent/bin/agent/dist/checks/check.py +EOF + } + + rpm_iot_exclude_paths "$test_dir/agent.rpm" > "$output_path" 2> "$error_path" + status=$? + unset -f rpm + + assertNotEquals "an exclusion prefix that matches support views should fail" 0 "$status" + assertEquals "a collision should not emit any partial exclusion output" "" "$(cat "$output_path")" + assertIotContains "collision error should identify the unsafe prefix" "$(cat "$error_path")" "/opt/datadog-agent/bin/agent/dist/view" + assertIotContains "collision error should identify support views" "$(cat "$error_path")" "/opt/datadog-agent/bin/agent/dist/views" + assertIotNotContains "unrelated exclusions should not leak to stderr" "$(cat "$error_path")" "/opt/datadog-agent/bin/agent/dist/checks" + rm -rf "$test_dir" +} + testRpmIotExcludePathsRejectsMoreThan1024Prefixes() { local test_dir local output @@ -740,37 +886,131 @@ testValidateIotInstallLayoutAcceptsFilteredLayout() { rm -rf "$root" } -testValidateIotInstallLayoutAggregatesFailures() { +testValidateIotInstallLayoutAggregatesMissingRetainedClasses() { + local root + local output + local status + local check_name + local -a check_names=( + cpu disk io load memory network ntp uptime system_swap systemd jetson + ) + + root=$(mktemp -d) + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + + assertNotEquals "a layout missing retained classes should fail" 0 "$status" + assertIotContains "normal Agent should be required" "$output" "/opt/datadog-agent/bin/agent/agent" + assertIotContains "agent-data-plane should be required" "$output" "/opt/datadog-agent/embedded/bin/agent-data-plane" + assertIotContains "rtloader should be required" "$output" "/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*" + assertIotContains "support views should be required" "$output" "/opt/datadog-agent/bin/agent/dist/views" + assertIotContains "Agent configuration example should be required" "$output" "/etc/datadog-agent/datadog.yaml.example" + for check_name in "${check_names[@]}"; do + assertIotContains "$check_name configuration directory should be required" "$output" "/etc/datadog-agent/conf.d/$check_name.d" + done + rm -rf "$root" +} + +testValidateIotInstallLayoutRequiresRetainedDirectoryContent() { local root local output local status + local check_name + local -a check_names=( + cpu disk io load memory network ntp uptime system_swap systemd jetson + ) root=$(mktemp -d) createIotRetainedLayout "$root" - rm \ - "$root/opt/datadog-agent/embedded/bin/agent-data-plane" \ - "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" - mkdir -p \ - "$root/opt/datadog-agent/embedded/bin" \ - "$root/opt/datadog-agent/bin/agent/dist/jmx" \ - "$root/etc/datadog-agent/conf.d/docker.d" - touch \ - "$root/opt/datadog-agent/embedded/bin/process-agent" \ - "$root/opt/datadog-agent/embedded/bin/python3" \ - "$root/opt/datadog-agent/embedded/bin/system-probe" \ - "$root/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar" \ - "$root/etc/datadog-agent/conf.d/docker.d/conf.yaml.example" + rm "$root/opt/datadog-agent/bin/agent/dist/views/index.html" + for check_name in "${check_names[@]}"; do + rm "$root/etc/datadog-agent/conf.d/$check_name.d/conf.yaml.example" + done output=$(validate_iot_install_layout "$root" 2>&1) status=$? - assertNotEquals "invalid filtered layout should fail" 0 "$status" - assertIotContains "missing ADP should be reported" "$output" "missing required path: /opt/datadog-agent/embedded/bin/agent-data-plane" - assertIotContains "missing rtloader should be reported" "$output" "missing required path: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*" - assertIotContains "process-agent should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/process-agent" - assertIotContains "Python should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/python3" - assertIotContains "system-probe should be reported" "$output" "disallowed path remains: /opt/datadog-agent/embedded/bin/system-probe" - assertIotContains "JMX should be reported" "$output" "disallowed path remains: /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar" - assertIotContains "non-IoT config should be reported" "$output" "disallowed path remains: /etc/datadog-agent/conf.d/docker.d/conf.yaml.example" + assertNotEquals "empty retained directories should fail" 0 "$status" + assertIotContains "support view content should be required" "$output" "/opt/datadog-agent/bin/agent/dist/views/*" + for check_name in "${check_names[@]}"; do + assertIotContains "$check_name configuration content should be required" "$output" "/etc/datadog-agent/conf.d/$check_name.d/*" + done + rm -rf "$root" +} + +testValidateIotInstallLayoutAggregatesPrunedPayloadClasses() { + local root + local output + local status + local payload_path + local -a disallowed_payloads=( + /opt/datadog-agent/embedded/bin/process-agent + /opt/datadog-agent/embedded/bin/trace-agent + /opt/datadog-agent/embedded/bin/security-agent + /opt/datadog-agent/embedded/bin/privateactionrunner + /opt/datadog-agent/embedded/bin/installer + /opt/datadog-agent/embedded/bin/system-probe + /opt/datadog-agent/embedded/bin/python3 + /opt/datadog-agent/embedded/lib/libpython3.13.so.1.0 + /opt/datadog-agent/embedded/lib/python3.13/site-packages/yaml/__init__.py + /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar + /opt/datadog-agent/embedded/share/system-probe/ebpf.o + /opt/datadog-agent/embedded/share/ebpf/co-re.o + /opt/datadog-agent/embedded/include/Python.h + /opt/datadog-agent/embedded/lib/libodbc.so.2 + /opt/datadog-agent/embedded/msodbcsql/lib64/libmsodbcsql.so + /opt/datadog-agent/embedded/sbin/chroot + /opt/datadog-agent/compliance/rules.json + /etc/datadog-agent/compliance.d/default.rego + /opt/datadog-agent/runtime-security.d/policy.policy + /etc/datadog-agent/runtime-security.d/default.policy + /etc/datadog-agent/conf.d/docker.d/conf.yaml.example + ) + + root=$(mktemp -d) + createIotRetainedLayout "$root" + for payload_path in "${disallowed_payloads[@]}"; do + mkdir -p "$(dirname "$root$payload_path")" + touch "$root$payload_path" + done + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + assertNotEquals "a layout containing pruned payload classes should fail" 0 "$status" + for payload_path in "${disallowed_payloads[@]}"; do + assertIotContains "$payload_path should be reported" "$output" "$payload_path" + done + rm -rf "$root" +} + +testValidateIotInstallLayoutAllowsEmptyExcludedDirectoriesAndDanglingLinks() { + local root + local excluded_directory + local -a excluded_directories=( + /opt/datadog-agent/bin/agent/dist/jmx + /opt/datadog-agent/embedded/include + /opt/datadog-agent/embedded/lib/python3.13/site-packages + /opt/datadog-agent/embedded/msodbcsql + /opt/datadog-agent/embedded/sbin + /opt/datadog-agent/embedded/share/ebpf + /opt/datadog-agent/embedded/share/system-probe + /opt/datadog-agent/compliance + /opt/datadog-agent/runtime-security.d + /etc/datadog-agent/compliance.d + /etc/datadog-agent/runtime-security.d + /etc/datadog-agent/conf.d/docker.d + ) + + root=$(mktemp -d) + createIotRetainedLayout "$root" + for excluded_directory in "${excluded_directories[@]}"; do + mkdir -p "$root$excluded_directory" + done + ln -s python3.13 "$root/opt/datadog-agent/embedded/bin/python3" + ln -s libpython3.13.so.1.0 "$root/opt/datadog-agent/embedded/lib/libpython3.13.so" + ln -s missing-extension.so "$root/opt/datadog-agent/embedded/lib/python3.13/site-packages/native-extension.so" + + validate_iot_install_layout "$root" + assertEquals "empty excluded directories and dangling links should be harmless" 0 $? rm -rf "$root" } From 8e802ce2e578fdf98e4ad59b16450cc2c021fa08 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 10:07:14 -0400 Subject: [PATCH 05/12] Make filtered IoT helpers fail closed Write the dpkg filter and install profile through same-directory temporary files, replace exact destinations atomically, and reject unsafe post-replacement types. Broaden filtered layout validation across retained file types, disallowed payload classes, and live links while keeping RPM exclusion deduplication compatible with Bash 3. Extend shunit coverage for each failure mode. --- install_script.sh.template | 244 ++++++++++++++++++------- unit_tests/test_install_script.sh | 285 +++++++++++++++++++++++++++++- 2 files changed, 460 insertions(+), 69 deletions(-) diff --git a/install_script.sh.template b/install_script.sh.template index 72aa2bfb..28455860 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -408,13 +408,30 @@ function validate_iot_installer_options() { function write_deb_iot_filter_config() { local destination_path="${1-}" + local destination_directory + local temporary_path if [ -z "$destination_path" ]; then printf 'A destination path is required for the filtered IoT dpkg configuration.\n' >&2 return 1 fi - cat > "$destination_path" <<'EOF' + case "$destination_path" in + */*) + destination_directory=${destination_path%/*} + [ -n "$destination_directory" ] || destination_directory="/" + ;; + *) + destination_directory="." + ;; + esac + + if ! temporary_path="$(mktemp "$destination_directory/.datadog_iot_filter.tmp.XXXXXX")"; then + printf 'Unable to create a temporary filtered IoT dpkg configuration in %s.\n' "$destination_directory" >&2 + return 1 + fi + + if ! cat > "$temporary_path" <<'EOF' path-exclude=/opt/datadog-agent/bin/* path-exclude=/opt/datadog-agent/embedded/bin/* path-exclude=/opt/datadog-agent/embedded/include/* @@ -468,6 +485,26 @@ path-include=/etc/datadog-agent/conf.d/systemd.d/* path-include=/etc/datadog-agent/conf.d/jetson.d path-include=/etc/datadog-agent/conf.d/jetson.d/* EOF + then + rm -f "$temporary_path" + printf 'Unable to write the temporary filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + if ! chmod 0644 "$temporary_path"; then + rm -f "$temporary_path" + printf 'Unable to set permissions on the temporary filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + if ! mv -fT -- "$temporary_path" "$destination_path"; then + rm -f "$temporary_path" + printf 'Unable to atomically replace the filtered IoT dpkg configuration at %s.\n' "$destination_path" >&2 + return 1 + fi + if [ -e "$temporary_path" ] || [ ! -f "$destination_path" ] || [ -L "$destination_path" ] || [ ! -w "$destination_path" ]; then + rm -f "$temporary_path" + printf 'The filtered IoT dpkg configuration at %s is not an exact writable regular file after replacement.\n' "$destination_path" >&2 + return 1 + fi } function rpm_iot_exclude_paths() { @@ -477,6 +514,8 @@ function rpm_iot_exclude_paths() { local relative_path local child_path local exclusion_prefix + local existing_prefix + local duplicate_prefix local retained_path local prefix_count=0 local -a exclusion_prefixes=() @@ -497,7 +536,6 @@ function rpm_iot_exclude_paths() { /etc/datadog-agent/conf.d/systemd.d /etc/datadog-agent/conf.d/jetson.d ) - local -A seen_prefixes=() if [ -z "$package_path" ]; then printf 'An Agent RPM path is required to derive filtered IoT exclusions.\n' >&2 @@ -598,13 +636,21 @@ function rpm_iot_exclude_paths() { ;; esac - if [ -n "$exclusion_prefix" ] && [ -z "${seen_prefixes[$exclusion_prefix]+present}" ]; then - seen_prefixes["$exclusion_prefix"]=1 - exclusion_prefixes+=("$exclusion_prefix") - prefix_count=$((prefix_count + 1)) - if [ "$prefix_count" -gt 1024 ]; then - printf 'Filtered IoT RPM installation requires more than 1024 exclusion prefixes; refusing the package.\n' >&2 - return 1 + if [ -n "$exclusion_prefix" ]; then + duplicate_prefix="" + for existing_prefix in "${exclusion_prefixes[@]}"; do + if [ "$existing_prefix" = "$exclusion_prefix" ]; then + duplicate_prefix=true + break + fi + done + if [ -z "$duplicate_prefix" ]; then + exclusion_prefixes+=("$exclusion_prefix") + prefix_count=$((prefix_count + 1)) + if [ "$prefix_count" -gt 1024 ]; then + printf 'Filtered IoT RPM installation requires more than 1024 exclusion prefixes; refusing the package.\n' >&2 + return 1 + fi fi fi done <<< "$package_listing" @@ -628,12 +674,16 @@ function validate_iot_install_layout() { local rtloader_found="" local candidate_path local candidate_root + local relative_path + local child_path local logical_path - local logical_pattern + local disallowed_tree local -a failures=() - local -a required_paths=( + local -a required_executable_files=( /opt/datadog-agent/bin/agent/agent /opt/datadog-agent/embedded/bin/agent-data-plane + ) + local -a required_regular_files=( /etc/datadog-agent/datadog.yaml.example ) local -a required_content_directories=( @@ -650,35 +700,16 @@ function validate_iot_install_layout() { /etc/datadog-agent/conf.d/systemd.d /etc/datadog-agent/conf.d/jetson.d ) - local -a disallowed_paths=( - /opt/datadog-agent/embedded/bin/process-agent - /opt/datadog-agent/embedded/bin/trace-agent - /opt/datadog-agent/embedded/bin/trace-loader - /opt/datadog-agent/embedded/bin/security-agent - /opt/datadog-agent/embedded/bin/privateactionrunner - /opt/datadog-agent/embedded/bin/installer - /opt/datadog-agent/embedded/bin/system-probe - /opt/datadog-agent/embedded/bin/system-probe-lite - /etc/datadog-agent/conf.d/docker.d/conf.yaml.example - ) - local -a disallowed_file_patterns=( - '/opt/datadog-agent/embedded/bin/python*' - '/opt/datadog-agent/embedded/lib/libpython*' - '/opt/datadog-agent/embedded/lib/libodbc*' - '/opt/datadog-agent/embedded/lib/libtdsodbc*' - ) - local -a disallowed_tree_patterns=( - '/opt/datadog-agent/bin/agent/dist/jmx' - '/opt/datadog-agent/embedded/lib/python*/site-packages' - '/opt/datadog-agent/embedded/share/system-probe' - '/opt/datadog-agent/embedded/share/ebpf' - '/opt/datadog-agent/embedded/include' - '/opt/datadog-agent/embedded/msodbcsql' - '/opt/datadog-agent/embedded/sbin' - '/opt/datadog-agent/compliance' - '/opt/datadog-agent/runtime-security.d' - '/etc/datadog-agent/compliance.d' - '/etc/datadog-agent/runtime-security.d' + local -a disallowed_trees=( + /opt/datadog-agent/embedded/share/system-probe + /opt/datadog-agent/embedded/share/ebpf + /opt/datadog-agent/embedded/include + /opt/datadog-agent/embedded/msodbcsql + /opt/datadog-agent/embedded/sbin + /opt/datadog-agent/compliance + /opt/datadog-agent/runtime-security.d + /etc/datadog-agent/compliance.d + /etc/datadog-agent/runtime-security.d ) if [ "$root_prefix" = "/" ]; then @@ -687,60 +718,127 @@ function validate_iot_install_layout() { root_prefix=${root_prefix%/} fi - for logical_path in "${required_paths[@]}"; do - if [ ! -e "$root_prefix$logical_path" ]; then - failures+=("missing required path: $logical_path") + for logical_path in "${required_executable_files[@]}"; do + candidate_path="$root_prefix$logical_path" + if [ ! -f "$candidate_path" ] || [ -L "$candidate_path" ] || [ ! -x "$candidate_path" ]; then + failures+=("missing required executable regular file: $logical_path") + fi + done + + for logical_path in "${required_regular_files[@]}"; do + candidate_path="$root_prefix$logical_path" + if [ ! -f "$candidate_path" ] || [ -L "$candidate_path" ]; then + failures+=("missing required regular file: $logical_path") fi done for candidate_path in "$root_prefix"/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*; do - if [ -f "$candidate_path" ]; then + if [ -f "$candidate_path" ] && [ ! -L "$candidate_path" ]; then rtloader_found=true break fi done if [ -z "$rtloader_found" ]; then - failures+=("missing required path: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*") + failures+=("missing required regular file: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*") fi for logical_path in "${required_content_directories[@]}"; do - if [ ! -d "$root_prefix$logical_path" ]; then + candidate_root="$root_prefix$logical_path" + if [ ! -d "$candidate_root" ] || [ -L "$candidate_root" ]; then failures+=("missing required directory: $logical_path") continue fi - candidate_path=$(find "$root_prefix$logical_path" -type f -print -quit 2>/dev/null) + candidate_path=$(find "$candidate_root" -type f -print -quit 2>/dev/null) if [ -z "$candidate_path" ]; then failures+=("missing required content: $logical_path/*") fi done - for logical_path in "${disallowed_paths[@]}"; do - candidate_path="$root_prefix$logical_path" - if [ -f "$candidate_path" ]; then - failures+=("disallowed payload remains: $logical_path") - fi - done + candidate_root="$root_prefix/opt/datadog-agent/bin" + if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then + while IFS= read -r candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + logical_path=${candidate_path#"$root_prefix"} + case "$logical_path" in + /opt/datadog-agent/bin/agent/agent|/opt/datadog-agent/bin/agent/dist/views/*) + ;; + *) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + fi - for logical_pattern in "${disallowed_file_patterns[@]}"; do + candidate_root="$root_prefix/opt/datadog-agent/embedded/bin" + if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then while IFS= read -r candidate_path; do - if [ -f "$candidate_path" ]; then - failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") - break + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue fi - done < <(compgen -G "$root_prefix$logical_pattern" || true) - done + logical_path=${candidate_path#"$root_prefix"} + if [ "$logical_path" != "/opt/datadog-agent/embedded/bin/agent-data-plane" ]; then + failures+=("disallowed payload remains: $logical_path") + fi + done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + fi - for logical_pattern in "${disallowed_tree_patterns[@]}"; do - while IFS= read -r candidate_root; do - if [ ! -d "$candidate_root" ] && [ ! -L "$candidate_root" ]; then + candidate_root="$root_prefix/opt/datadog-agent/embedded/lib" + if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then + while IFS= read -r candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then continue fi - candidate_path=$(find "$candidate_root" -type f -print -quit 2>/dev/null) - if [ -n "$candidate_path" ]; then - failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") - break + logical_path=${candidate_path#"$root_prefix"} + relative_path=${logical_path#/opt/datadog-agent/embedded/lib/} + child_path=${relative_path%%/*} + case "$child_path" in + python*|libpython*|libodbc*|libtdsodbc*) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + fi + + candidate_root="$root_prefix/etc/datadog-agent/conf.d" + if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then + while IFS= read -r candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue fi - done < <(compgen -G "$root_prefix$logical_pattern" || true) + logical_path=${candidate_path#"$root_prefix"} + case "$logical_path" in + /etc/datadog-agent/conf.d/cpu.d/*|/etc/datadog-agent/conf.d/disk.d/*|/etc/datadog-agent/conf.d/io.d/*|/etc/datadog-agent/conf.d/load.d/*|/etc/datadog-agent/conf.d/memory.d/*|/etc/datadog-agent/conf.d/network.d/*|/etc/datadog-agent/conf.d/ntp.d/*|/etc/datadog-agent/conf.d/uptime.d/*|/etc/datadog-agent/conf.d/system_swap.d/*|/etc/datadog-agent/conf.d/systemd.d/*|/etc/datadog-agent/conf.d/jetson.d/*) + ;; + *) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + fi + + for disallowed_tree in "${disallowed_trees[@]}"; do + candidate_root="$root_prefix$disallowed_tree" + if [ ! -d "$candidate_root" ] && [ ! -f "$candidate_root" ] && [ ! -L "$candidate_root" ]; then + continue + fi + while IFS= read -r candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") + done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) done if [ "${#failures[@]}" -ne 0 ]; then @@ -775,6 +873,11 @@ function write_iot_install_profile() { ;; esac + if [ -d "$destination_path" ]; then + printf 'The filtered IoT install profile destination must not be a directory: %s.\n' "$destination_path" >&2 + return 1 + fi + if ! temporary_path="$(mktemp "$destination_directory/.install_profile.tmp.XXXXXX")"; then printf 'Unable to create a temporary filtered IoT install profile in %s.\n' "$destination_directory" >&2 return 1 @@ -796,11 +899,16 @@ function write_iot_install_profile() { printf 'Unable to set permissions on the temporary filtered IoT install profile.\n' >&2 return 1 fi - if ! mv -f -- "$temporary_path" "$destination_path"; then + if ! mv -fT -- "$temporary_path" "$destination_path"; then rm -f "$temporary_path" printf 'Unable to atomically replace the filtered IoT install profile at %s.\n' "$destination_path" >&2 return 1 fi + if [ -e "$temporary_path" ] || [ ! -f "$destination_path" ] || [ -L "$destination_path" ] || [ ! -w "$destination_path" ]; then + rm -f "$temporary_path" + printf 'The filtered IoT install profile at %s is not an exact writable regular file after replacement.\n' "$destination_path" >&2 + return 1 + fi } function get_installed_agent_package_version() { diff --git a/unit_tests/test_install_script.sh b/unit_tests/test_install_script.sh index f6a77435..f1bc335a 100755 --- a/unit_tests/test_install_script.sh +++ b/unit_tests/test_install_script.sh @@ -481,6 +481,9 @@ createIotRetainedLayout() { "$root/opt/datadog-agent/embedded/bin/agent-data-plane" \ "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" \ "$root/etc/datadog-agent/datadog.yaml.example" + chmod +x \ + "$root/opt/datadog-agent/bin/agent/agent" \ + "$root/opt/datadog-agent/embedded/bin/agent-data-plane" for check_name in "${check_names[@]}"; do mkdir -p "$root/etc/datadog-agent/conf.d/$check_name.d" @@ -627,6 +630,89 @@ path-include=/etc/datadog-agent/conf.d/jetson.d/*' duplicate_count=$(sort "$filter_path" | uniq -d | wc -l | tr -d ' ') assertEquals "each DEB filter rule should be unique" 0 "$duplicate_count" + assertEquals "DEB filter mode" 644 "$(stat -c '%a' "$filter_path")" + rm -rf "$test_dir" +} + +testDebIotFilterConfigPreservesFilterAndCleansTempOnWriteFailure() { + local test_dir + local filter_path + local output + local status + local temp_count + + test_dir=$(mktemp -d) + filter_path="$test_dir/99-datadog-iot" + mkdir "$test_dir/bin" + printf 'existing complete filter\n' > "$filter_path" + cat > "$test_dir/bin/cat" <<'EOF' +#!/bin/sh +exit 1 +EOF + chmod +x "$test_dir/bin/cat" + + output=$(PATH="$test_dir/bin:$PATH" write_deb_iot_filter_config "$filter_path" 2>&1) + status=$? + + assertNotEquals "failed temporary filter write should return nonzero" 0 "$status" + assertIotContains "write failure should be actionable" "$output" "temporary filtered IoT dpkg configuration" + assertEquals "failed write should preserve the complete filter" "existing complete filter" "$(cat "$filter_path")" + temp_count=$(find "$test_dir" -maxdepth 1 -name '.datadog_iot_filter.tmp.*' | wc -l | tr -d ' ') + assertEquals "failed write should clean its temporary file" 0 "$temp_count" + rm -rf "$test_dir" +} + +testDebIotFilterConfigPreservesFilterAndCleansTempOnReplaceFailure() { + local test_dir + local filter_path + local arguments_path + local output + local status + local temp_count + + test_dir=$(mktemp -d) + filter_path="$test_dir/99-datadog-iot" + arguments_path="$test_dir/mv-arguments" + printf 'existing complete filter\n' > "$filter_path" + # shellcheck disable=SC2329 + mv() { + printf '%s\n' "$@" > "$arguments_path" + return 1 + } + + output=$(write_deb_iot_filter_config "$filter_path" 2>&1) + status=$? + unset -f mv + + assertNotEquals "failed filter replacement should return nonzero" 0 "$status" + assertIotContains "replacement failure should be actionable" "$output" "atomically replace" + assertEquals "failed replacement should preserve the complete filter" "existing complete filter" "$(cat "$filter_path")" + assertEquals "filter replacement should use no-target-directory semantics" "-fT +--" "$(head -n 2 "$arguments_path")" + assertEquals "filter replacement destination should be exact" "$filter_path" "$(tail -n 1 "$arguments_path")" + temp_count=$(find "$test_dir" -maxdepth 1 -name '.datadog_iot_filter.tmp.*' | wc -l | tr -d ' ') + assertEquals "failed replacement should clean its temporary file" 0 "$temp_count" + rm -rf "$test_dir" +} + +testDebIotFilterConfigRejectsDirectoryDestination() { + local test_dir + local filter_path + local output + local status + local nested_count + + test_dir=$(mktemp -d) + filter_path="$test_dir/99-datadog-iot" + mkdir "$filter_path" + + output=$(write_deb_iot_filter_config "$filter_path" 2>&1) + status=$? + + assertNotEquals "directory filter destination should fail closed" 0 "$status" + assertTrue "directory destination should remain a directory" "[ -d '$filter_path' ]" + nested_count=$(find "$filter_path" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ') + assertEquals "no filter temporary file should be moved into the destination directory" 0 "$nested_count" rm -rf "$test_dir" } @@ -791,6 +877,37 @@ $package_path" rm -rf "$test_dir" } +testRpmIotExcludePathsRunsUnderSystemBashWithoutAssociativeArrays() { + local test_dir + local package_path + local helper_definition + local output + local status + + test_dir=$(mktemp -d) + package_path="$test_dir/agent.rpm" + mkdir "$test_dir/bin" + : > "$package_path" + cat > "$test_dir/bin/rpm" <<'EOF' +#!/bin/sh +printf '%s\n' \ + /opt/datadog-agent/embedded/bin/process-agent \ + /opt/datadog-agent/embedded/bin/process-agent +EOF + chmod +x "$test_dir/bin/rpm" + + helper_definition=$(declare -f rpm_iot_exclude_paths) + assertIotNotContains "RPM helper should not declare a local associative array" "$helper_definition" "local -A" + assertIotNotContains "RPM helper should not declare an associative array" "$helper_definition" "declare -A" + + output=$(PATH="$test_dir/bin:$PATH" /bin/bash -c 'source "$1"; rpm_iot_exclude_paths "$2"' _ "$dir_path/extracted_functions.sh" "$package_path" 2>&1) + status=$? + + assertEquals "RPM helper should run under the system Bash" 0 "$status" + assertEquals "Bash-compatible deduplication should emit one prefix" "/opt/datadog-agent/embedded/bin/process-agent" "$output" + rm -rf "$test_dir" +} + testRpmIotExcludePathsRejectsAgentDataPlanePrefixCollisionWithoutOutput() { local test_dir local output_path @@ -943,16 +1060,22 @@ testValidateIotInstallLayoutAggregatesPrunedPayloadClasses() { local status local payload_path local -a disallowed_payloads=( + /opt/datadog-agent/bin/process-agent/process-agent + /opt/datadog-agent/bin/agent/dist/checks/check.py + /opt/datadog-agent/bin/agent/dist/config/config.py + /opt/datadog-agent/bin/agent/dist/utils/util.py + /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar /opt/datadog-agent/embedded/bin/process-agent /opt/datadog-agent/embedded/bin/trace-agent /opt/datadog-agent/embedded/bin/security-agent /opt/datadog-agent/embedded/bin/privateactionrunner /opt/datadog-agent/embedded/bin/installer /opt/datadog-agent/embedded/bin/system-probe + /opt/datadog-agent/embedded/bin/unexpected-helper /opt/datadog-agent/embedded/bin/python3 /opt/datadog-agent/embedded/lib/libpython3.13.so.1.0 + /opt/datadog-agent/embedded/lib/python3.13/os.py /opt/datadog-agent/embedded/lib/python3.13/site-packages/yaml/__init__.py - /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar /opt/datadog-agent/embedded/share/system-probe/ebpf.o /opt/datadog-agent/embedded/share/ebpf/co-re.o /opt/datadog-agent/embedded/include/Python.h @@ -964,6 +1087,7 @@ testValidateIotInstallLayoutAggregatesPrunedPayloadClasses() { /opt/datadog-agent/runtime-security.d/policy.policy /etc/datadog-agent/runtime-security.d/default.policy /etc/datadog-agent/conf.d/docker.d/conf.yaml.example + /etc/datadog-agent/conf.d/kubelet.d/conf.yaml.example ) root=$(mktemp -d) @@ -982,6 +1106,95 @@ testValidateIotInstallLayoutAggregatesPrunedPayloadClasses() { rm -rf "$root" } +testValidateIotInstallLayoutRequiresExecutableRegularBinaries() { + local root + local output + local status + + root=$(mktemp -d) + createIotRetainedLayout "$root" + chmod -x \ + "$root/opt/datadog-agent/bin/agent/agent" \ + "$root/opt/datadog-agent/embedded/bin/agent-data-plane" + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + + assertNotEquals "non-executable retained binaries should fail" 0 "$status" + assertIotContains "normal Agent error should require an executable regular file" "$output" "/opt/datadog-agent/bin/agent/agent" + assertIotContains "agent-data-plane error should require an executable regular file" "$output" "/opt/datadog-agent/embedded/bin/agent-data-plane" + rm -rf "$root" +} + +testValidateIotInstallLayoutRejectsWrongRetainedTypes() { + local root + local output + local status + + root=$(mktemp -d) + createIotRetainedLayout "$root" + rm "$root/opt/datadog-agent/bin/agent/agent" + mkdir "$root/opt/datadog-agent/bin/agent/agent" + rm "$root/opt/datadog-agent/embedded/bin/agent-data-plane" + mkdir "$root/opt/datadog-agent/embedded/bin/agent-data-plane" + rm "$root/etc/datadog-agent/datadog.yaml.example" + mkdir "$root/etc/datadog-agent/datadog.yaml.example" + rm -rf "$root/opt/datadog-agent/bin/agent/dist/views" + touch "$root/opt/datadog-agent/bin/agent/dist/views" + rm -rf "$root/etc/datadog-agent/conf.d/cpu.d" + touch "$root/etc/datadog-agent/conf.d/cpu.d" + mv "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" "$root/opt/datadog-agent/embedded/lib/rtloader-target" + ln -s rtloader-target "$root/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so" + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + + assertNotEquals "wrong retained path types should fail" 0 "$status" + assertIotContains "normal Agent should reject a directory" "$output" "/opt/datadog-agent/bin/agent/agent" + assertIotContains "agent-data-plane should reject a directory" "$output" "/opt/datadog-agent/embedded/bin/agent-data-plane" + assertIotContains "configuration example should reject a directory" "$output" "/etc/datadog-agent/datadog.yaml.example" + assertIotContains "views should reject a regular file" "$output" "/opt/datadog-agent/bin/agent/dist/views" + assertIotContains "check directory should reject a regular file" "$output" "/etc/datadog-agent/conf.d/cpu.d" + assertIotContains "rtloader should reject a live symlink" "$output" "/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*" + rm -rf "$root" +} + +testValidateIotInstallLayoutRejectsLiveLinksInDisallowedTrees() { + local root + local output + local status + local link_path + local -a link_paths=( + /opt/datadog-agent/bin/process-agent + /opt/datadog-agent/embedded/bin/python3 + /opt/datadog-agent/embedded/lib/python3.13/os.py + /opt/datadog-agent/embedded/share/system-probe/live-data + /etc/datadog-agent/conf.d/kubelet.d + ) + + root=$(mktemp -d) + createIotRetainedLayout "$root" + mkdir -p \ + "$root/live-target-directory" \ + "$root/opt/datadog-agent/embedded/lib/python3.13" \ + "$root/opt/datadog-agent/embedded/share/system-probe" + touch "$root/live-target-file" + ln -s "$root/live-target-file" "$root/opt/datadog-agent/bin/process-agent" + ln -s "$root/live-target-directory" "$root/opt/datadog-agent/embedded/bin/python3" + ln -s "$root/live-target-file" "$root/opt/datadog-agent/embedded/lib/python3.13/os.py" + ln -s "$root/live-target-directory" "$root/opt/datadog-agent/embedded/share/system-probe/live-data" + ln -s "$root/live-target-directory" "$root/etc/datadog-agent/conf.d/kubelet.d" + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + + assertNotEquals "live links in disallowed trees should fail" 0 "$status" + for link_path in "${link_paths[@]}"; do + assertIotContains "$link_path should be reported" "$output" "$link_path" + done + rm -rf "$root" +} + testValidateIotInstallLayoutAllowsEmptyExcludedDirectoriesAndDanglingLinks() { local root local excluded_directory @@ -1061,6 +1274,76 @@ testWriteIotInstallProfilePreservesMarkerAndCleansTempOnFailure() { rm -rf "$test_dir" } +testWriteIotInstallProfileRejectsDirectoryDestinations() { + local test_dir + local marker_path + local output + local status + local nested_count + + test_dir=$(mktemp -d) + marker_path="$test_dir/install_profile" + mkdir "$marker_path" + + output=$(write_iot_install_profile "$marker_path" "7.72.1-1" 2>&1) + status=$? + + assertNotEquals "directory marker destination should fail closed" 0 "$status" + assertTrue "marker destination should remain a directory" "[ -d '$marker_path' ]" + nested_count=$(find "$marker_path" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ') + assertEquals "no marker temporary file should be moved into the destination directory" 0 "$nested_count" + rm -rf "$test_dir" +} + +testWriteIotInstallProfileRejectsSymlinkToDirectoryDestinations() { + local test_dir + local target_directory + local marker_path + local output + local status + local nested_count + + test_dir=$(mktemp -d) + target_directory="$test_dir/marker-target" + marker_path="$test_dir/install_profile" + mkdir "$target_directory" + ln -s "$target_directory" "$marker_path" + + output=$(write_iot_install_profile "$marker_path" "7.72.1-1" 2>&1) + status=$? + + assertNotEquals "symlink-to-directory marker destination should fail closed" 0 "$status" + assertTrue "marker destination should remain a symlink" "[ -L '$marker_path' ]" + nested_count=$(find "$target_directory" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ') + assertEquals "no marker temporary file should be moved through the destination symlink" 0 "$nested_count" + rm -rf "$test_dir" +} + +testWriteIotInstallProfileVerifiesExactDestinationAfterReplacement() { + local test_dir + local marker_path + local output + local status + local temp_count + + test_dir=$(mktemp -d) + marker_path="$test_dir/install_profile" + # shellcheck disable=SC2329 + mv() { + return 0 + } + + output=$(write_iot_install_profile "$marker_path" "7.72.1-1" 2>&1) + status=$? + unset -f mv + + assertNotEquals "replacement without an exact regular marker should fail" 0 "$status" + assertFalse "missing exact marker should not be accepted" "[ -e '$marker_path' ]" + temp_count=$(find "$test_dir" -maxdepth 1 -name '.install_profile.tmp.*' | wc -l | tr -d ' ') + assertEquals "failed post-replacement verification should clean its temporary file" 0 "$temp_count" + rm -rf "$test_dir" +} + testGetInstalledAgentPackageVersionQueriesExplicitFamily() { local test_dir local arguments_path From 79af48c5eee121346e548cda2f72ab63a1d80bb4 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 10:30:41 -0400 Subject: [PATCH 06/12] Align filtered IoT payload validation Apply the same Python-script, requirements, and shared MS ODBC pruning contract to DEB filters, RPM exclusion fixtures, and post-install validation. Capture every validator find traversal in a checked NUL-delimited temporary listing so inspection errors are aggregated and validation fails closed without relying on process-substitution status. --- install_script.sh.template | 185 ++++++++++++++++++++---------- unit_tests/test_install_script.sh | 61 +++++++++- 2 files changed, 185 insertions(+), 61 deletions(-) diff --git a/install_script.sh.template b/install_script.sh.template index 28455860..9032e07c 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -442,6 +442,7 @@ path-exclude=/opt/datadog-agent/embedded/lib/python* path-exclude=/opt/datadog-agent/embedded/msodbcsql/* path-exclude=/opt/datadog-agent/embedded/sbin/* path-exclude=/opt/datadog-agent/embedded/share/ebpf/* +path-exclude=/opt/datadog-agent/embedded/share/msodbcsql* path-exclude=/opt/datadog-agent/embedded/share/system-probe/* path-exclude=/opt/datadog-agent/python-scripts/* path-exclude=/opt/datadog-agent/requirements/* @@ -678,6 +679,7 @@ function validate_iot_install_layout() { local child_path local logical_path local disallowed_tree + local traversal_path="" local -a failures=() local -a required_executable_files=( /opt/datadog-agent/bin/agent/agent @@ -706,6 +708,8 @@ function validate_iot_install_layout() { /opt/datadog-agent/embedded/include /opt/datadog-agent/embedded/msodbcsql /opt/datadog-agent/embedded/sbin + /opt/datadog-agent/python-scripts + /opt/datadog-agent/requirements /opt/datadog-agent/compliance /opt/datadog-agent/runtime-security.d /etc/datadog-agent/compliance.d @@ -742,105 +746,166 @@ function validate_iot_install_layout() { failures+=("missing required regular file: /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*") fi + if ! traversal_path="$(mktemp "${TMPDIR:-/tmp}/datadog-iot-layout.XXXXXX")"; then + failures+=("unable to inspect installed payload: could not create a temporary path listing") + fi + for logical_path in "${required_content_directories[@]}"; do candidate_root="$root_prefix$logical_path" if [ ! -d "$candidate_root" ] || [ -L "$candidate_root" ]; then failures+=("missing required directory: $logical_path") continue fi - candidate_path=$(find "$candidate_root" -type f -print -quit 2>/dev/null) - if [ -z "$candidate_path" ]; then + [ -n "$traversal_path" ] || continue + if ! find "$candidate_root" -type f -print0 -quit > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect required content directory: $logical_path") + elif [ ! -s "$traversal_path" ]; then failures+=("missing required content: $logical_path/*") fi done candidate_root="$root_prefix/opt/datadog-agent/bin" - if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then - while IFS= read -r candidate_path; do - if [ -L "$candidate_path" ]; then - [ -e "$candidate_path" ] || continue - elif [ ! -f "$candidate_path" ]; then - continue - fi - logical_path=${candidate_path#"$root_prefix"} - case "$logical_path" in - /opt/datadog-agent/bin/agent/agent|/opt/datadog-agent/bin/agent/dist/views/*) - ;; - *) - failures+=("disallowed payload remains: $logical_path") - ;; - esac - done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + logical_path=/opt/datadog-agent/bin + if [ -n "$traversal_path" ] && { [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; }; then + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect payload tree: $logical_path") + else + while IFS= read -r -d '' candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + logical_path=${candidate_path#"$root_prefix"} + case "$logical_path" in + /opt/datadog-agent/bin/agent/agent|/opt/datadog-agent/bin/agent/dist/views/*) + ;; + *) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < "$traversal_path" + fi fi candidate_root="$root_prefix/opt/datadog-agent/embedded/bin" - if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then - while IFS= read -r candidate_path; do - if [ -L "$candidate_path" ]; then - [ -e "$candidate_path" ] || continue - elif [ ! -f "$candidate_path" ]; then - continue - fi - logical_path=${candidate_path#"$root_prefix"} - if [ "$logical_path" != "/opt/datadog-agent/embedded/bin/agent-data-plane" ]; then - failures+=("disallowed payload remains: $logical_path") - fi - done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + logical_path=/opt/datadog-agent/embedded/bin + if [ -n "$traversal_path" ] && { [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; }; then + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect payload tree: $logical_path") + else + while IFS= read -r -d '' candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + logical_path=${candidate_path#"$root_prefix"} + if [ "$logical_path" != "/opt/datadog-agent/embedded/bin/agent-data-plane" ]; then + failures+=("disallowed payload remains: $logical_path") + fi + done < "$traversal_path" + fi fi candidate_root="$root_prefix/opt/datadog-agent/embedded/lib" - if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then - while IFS= read -r candidate_path; do - if [ -L "$candidate_path" ]; then - [ -e "$candidate_path" ] || continue - elif [ ! -f "$candidate_path" ]; then - continue - fi - logical_path=${candidate_path#"$root_prefix"} - relative_path=${logical_path#/opt/datadog-agent/embedded/lib/} - child_path=${relative_path%%/*} - case "$child_path" in - python*|libpython*|libodbc*|libtdsodbc*) - failures+=("disallowed payload remains: $logical_path") - ;; - esac - done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + logical_path=/opt/datadog-agent/embedded/lib + if [ -n "$traversal_path" ] && { [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; }; then + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect payload tree: $logical_path") + else + while IFS= read -r -d '' candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + logical_path=${candidate_path#"$root_prefix"} + relative_path=${logical_path#/opt/datadog-agent/embedded/lib/} + child_path=${relative_path%%/*} + case "$child_path" in + python*|libpython*|libodbc*|libtdsodbc*) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < "$traversal_path" + fi fi candidate_root="$root_prefix/etc/datadog-agent/conf.d" - if [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; then - while IFS= read -r candidate_path; do + logical_path=/etc/datadog-agent/conf.d + if [ -n "$traversal_path" ] && { [ -d "$candidate_root" ] || [ -L "$candidate_root" ]; }; then + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect payload tree: $logical_path") + else + while IFS= read -r -d '' candidate_path; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + logical_path=${candidate_path#"$root_prefix"} + case "$logical_path" in + /etc/datadog-agent/conf.d/cpu.d/*|/etc/datadog-agent/conf.d/disk.d/*|/etc/datadog-agent/conf.d/io.d/*|/etc/datadog-agent/conf.d/load.d/*|/etc/datadog-agent/conf.d/memory.d/*|/etc/datadog-agent/conf.d/network.d/*|/etc/datadog-agent/conf.d/ntp.d/*|/etc/datadog-agent/conf.d/uptime.d/*|/etc/datadog-agent/conf.d/system_swap.d/*|/etc/datadog-agent/conf.d/systemd.d/*|/etc/datadog-agent/conf.d/jetson.d/*) + ;; + *) + failures+=("disallowed payload remains: $logical_path") + ;; + esac + done < "$traversal_path" + fi + fi + + for candidate_path in "$root_prefix"/opt/datadog-agent/requirements*.txt; do + if [ -L "$candidate_path" ]; then + [ -e "$candidate_path" ] || continue + elif [ ! -f "$candidate_path" ]; then + continue + fi + failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") + done + + for candidate_root in "$root_prefix"/opt/datadog-agent/embedded/share/msodbcsql*; do + if [ ! -d "$candidate_root" ] && [ ! -f "$candidate_root" ] && [ ! -L "$candidate_root" ]; then + continue + fi + [ -n "$traversal_path" ] || continue + logical_path=${candidate_root#"$root_prefix"} + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect disallowed payload tree: $logical_path") + continue + fi + while IFS= read -r -d '' candidate_path; do if [ -L "$candidate_path" ]; then [ -e "$candidate_path" ] || continue elif [ ! -f "$candidate_path" ]; then continue fi - logical_path=${candidate_path#"$root_prefix"} - case "$logical_path" in - /etc/datadog-agent/conf.d/cpu.d/*|/etc/datadog-agent/conf.d/disk.d/*|/etc/datadog-agent/conf.d/io.d/*|/etc/datadog-agent/conf.d/load.d/*|/etc/datadog-agent/conf.d/memory.d/*|/etc/datadog-agent/conf.d/network.d/*|/etc/datadog-agent/conf.d/ntp.d/*|/etc/datadog-agent/conf.d/uptime.d/*|/etc/datadog-agent/conf.d/system_swap.d/*|/etc/datadog-agent/conf.d/systemd.d/*|/etc/datadog-agent/conf.d/jetson.d/*) - ;; - *) - failures+=("disallowed payload remains: $logical_path") - ;; - esac - done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) - fi + failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") + done < "$traversal_path" + done for disallowed_tree in "${disallowed_trees[@]}"; do candidate_root="$root_prefix$disallowed_tree" if [ ! -d "$candidate_root" ] && [ ! -f "$candidate_root" ] && [ ! -L "$candidate_root" ]; then continue fi - while IFS= read -r candidate_path; do + [ -n "$traversal_path" ] || continue + if ! find "$candidate_root" \( -type f -o -type l \) -print0 > "$traversal_path" 2>/dev/null; then + failures+=("unable to inspect disallowed payload tree: $disallowed_tree") + continue + fi + while IFS= read -r -d '' candidate_path; do if [ -L "$candidate_path" ]; then [ -e "$candidate_path" ] || continue elif [ ! -f "$candidate_path" ]; then continue fi failures+=("disallowed payload remains: ${candidate_path#"$root_prefix"}") - done < <(find "$candidate_root" \( -type f -o -type l \) -print 2>/dev/null) + done < "$traversal_path" done + [ -z "$traversal_path" ] || rm -f "$traversal_path" if [ "${#failures[@]}" -ne 0 ]; then printf 'Filtered IoT layout validation failed:\n' >&2 printf ' - %s\n' "${failures[@]}" >&2 diff --git a/unit_tests/test_install_script.sh b/unit_tests/test_install_script.sh index f1bc335a..7833bcb3 100755 --- a/unit_tests/test_install_script.sh +++ b/unit_tests/test_install_script.sh @@ -584,6 +584,7 @@ path-exclude=/opt/datadog-agent/embedded/lib/python* path-exclude=/opt/datadog-agent/embedded/msodbcsql/* path-exclude=/opt/datadog-agent/embedded/sbin/* path-exclude=/opt/datadog-agent/embedded/share/ebpf/* +path-exclude=/opt/datadog-agent/embedded/share/msodbcsql* path-exclude=/opt/datadog-agent/embedded/share/system-probe/* path-exclude=/opt/datadog-agent/python-scripts/* path-exclude=/opt/datadog-agent/requirements/* @@ -783,6 +784,14 @@ testDebIotFilterConfigExplicitlyIncludesParentsBeforeLeaves() { assertNotEquals "ODBC libraries should remain excluded" 0 $? iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/share/system-probe/ebpf.o" assertNotEquals "system-probe support data should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/embedded/share/msodbcsql18/lib64/libmsodbcsql.so" + assertNotEquals "shared MS ODBC payloads should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/python-scripts/post.py" + assertNotEquals "Agent Python scripts should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/requirements/base.txt" + assertNotEquals "Agent requirement directories should remain excluded" 0 $? + iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/requirements-agent-release.txt" + assertNotEquals "Agent requirement manifests should remain excluded" 0 $? iotDpkgPathIsIncluded "$filter_path" "/opt/datadog-agent/bin/process-agent/process-agent" assertNotEquals "process-agent should remain excluded" 0 $? iotDpkgPathIsIncluded "$filter_path" "/etc/datadog-agent/conf.d/docker.d/conf.yaml.example" @@ -830,8 +839,10 @@ testRpmIotExcludePathsDerivesSortedUniquePrefixes() { /opt/datadog-agent/embedded/include/Python.h /opt/datadog-agent/embedded/share/system-probe/ebpf.o /opt/datadog-agent/embedded/share/ebpf/co-re.o +/opt/datadog-agent/embedded/share/msodbcsql18/lib64/libmsodbcsql.so /opt/datadog-agent/embedded/msodbcsql/lib64/libmsodbcsql.so /opt/datadog-agent/python-scripts/post.py +/opt/datadog-agent/requirements/base.txt /opt/datadog-agent/requirements-agent-release.txt /opt/datadog-agent/compliance/rules.json /opt/datadog-agent/runtime-security.d/policy.policy @@ -865,9 +876,11 @@ EOF /opt/datadog-agent/embedded/msodbcsql/ /opt/datadog-agent/embedded/sbin/ /opt/datadog-agent/embedded/share/ebpf/ +/opt/datadog-agent/embedded/share/msodbcsql18/ /opt/datadog-agent/embedded/share/system-probe/ /opt/datadog-agent/python-scripts/ /opt/datadog-agent/requirements-agent-release.txt +/opt/datadog-agent/requirements/ /opt/datadog-agent/runtime-security.d/' assertEquals "RPM exclusions should be sorted, unique, and retain only supported payloads" "$expected" "$output" @@ -1078,10 +1091,14 @@ testValidateIotInstallLayoutAggregatesPrunedPayloadClasses() { /opt/datadog-agent/embedded/lib/python3.13/site-packages/yaml/__init__.py /opt/datadog-agent/embedded/share/system-probe/ebpf.o /opt/datadog-agent/embedded/share/ebpf/co-re.o + /opt/datadog-agent/embedded/share/msodbcsql18/lib64/libmsodbcsql.so /opt/datadog-agent/embedded/include/Python.h /opt/datadog-agent/embedded/lib/libodbc.so.2 /opt/datadog-agent/embedded/msodbcsql/lib64/libmsodbcsql.so /opt/datadog-agent/embedded/sbin/chroot + /opt/datadog-agent/python-scripts/post.py + /opt/datadog-agent/requirements/base.txt + /opt/datadog-agent/requirements-agent-release.txt /opt/datadog-agent/compliance/rules.json /etc/datadog-agent/compliance.d/default.rego /opt/datadog-agent/runtime-security.d/policy.policy @@ -1169,6 +1186,10 @@ testValidateIotInstallLayoutRejectsLiveLinksInDisallowedTrees() { /opt/datadog-agent/embedded/bin/python3 /opt/datadog-agent/embedded/lib/python3.13/os.py /opt/datadog-agent/embedded/share/system-probe/live-data + /opt/datadog-agent/embedded/share/msodbcsql18/live-data + /opt/datadog-agent/python-scripts/live-script + /opt/datadog-agent/requirements/live-requirement + /opt/datadog-agent/requirements-agent-release.txt /etc/datadog-agent/conf.d/kubelet.d ) @@ -1177,12 +1198,19 @@ testValidateIotInstallLayoutRejectsLiveLinksInDisallowedTrees() { mkdir -p \ "$root/live-target-directory" \ "$root/opt/datadog-agent/embedded/lib/python3.13" \ - "$root/opt/datadog-agent/embedded/share/system-probe" + "$root/opt/datadog-agent/embedded/share/system-probe" \ + "$root/opt/datadog-agent/embedded/share/msodbcsql18" \ + "$root/opt/datadog-agent/python-scripts" \ + "$root/opt/datadog-agent/requirements" touch "$root/live-target-file" ln -s "$root/live-target-file" "$root/opt/datadog-agent/bin/process-agent" ln -s "$root/live-target-directory" "$root/opt/datadog-agent/embedded/bin/python3" ln -s "$root/live-target-file" "$root/opt/datadog-agent/embedded/lib/python3.13/os.py" ln -s "$root/live-target-directory" "$root/opt/datadog-agent/embedded/share/system-probe/live-data" + ln -s "$root/live-target-directory" "$root/opt/datadog-agent/embedded/share/msodbcsql18/live-data" + ln -s "$root/live-target-file" "$root/opt/datadog-agent/python-scripts/live-script" + ln -s "$root/live-target-file" "$root/opt/datadog-agent/requirements/live-requirement" + ln -s "$root/live-target-file" "$root/opt/datadog-agent/requirements-agent-release.txt" ln -s "$root/live-target-directory" "$root/etc/datadog-agent/conf.d/kubelet.d" output=$(validate_iot_install_layout "$root" 2>&1) @@ -1195,6 +1223,34 @@ testValidateIotInstallLayoutRejectsLiveLinksInDisallowedTrees() { rm -rf "$root" } +testValidateIotInstallLayoutFailsClosedWhenDisallowedTreeCannotBeInspected() { + local root + local failing_tree + local output + local status + + root=$(mktemp -d) + createIotRetainedLayout "$root" + failing_tree="$root/opt/datadog-agent/embedded/share/system-probe" + mkdir -p "$failing_tree" + touch "$failing_tree/hidden-payload" + # shellcheck disable=SC2329 + find() { + if [ "${1-}" = "$failing_tree" ]; then + return 73 + fi + command find "$@" + } + + output=$(validate_iot_install_layout "$root" 2>&1) + status=$? + unset -f find + + assertNotEquals "an unreadable disallowed tree should fail closed" 0 "$status" + assertIotContains "inspection failure should be aggregated" "$output" "unable to inspect disallowed payload tree: /opt/datadog-agent/embedded/share/system-probe" + rm -rf "$root" +} + testValidateIotInstallLayoutAllowsEmptyExcludedDirectoriesAndDanglingLinks() { local root local excluded_directory @@ -1205,7 +1261,10 @@ testValidateIotInstallLayoutAllowsEmptyExcludedDirectoriesAndDanglingLinks() { /opt/datadog-agent/embedded/msodbcsql /opt/datadog-agent/embedded/sbin /opt/datadog-agent/embedded/share/ebpf + /opt/datadog-agent/embedded/share/msodbcsql18 /opt/datadog-agent/embedded/share/system-probe + /opt/datadog-agent/python-scripts + /opt/datadog-agent/requirements /opt/datadog-agent/compliance /opt/datadog-agent/runtime-security.d /etc/datadog-agent/compliance.d From 56aa4524398ba3efa0c1717bdf93c63ddec3e25d Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 11:07:29 -0400 Subject: [PATCH 07/12] Install filtered IoT profile on Debian Wire the generated Agent 7 IoT variant into fresh-install-only option validation and force the normal datadog-agent package with IoT infrastructure mode. Install the dpkg path filter atomically before package unpack, retain the package installer only for post-install configuration, then persist the final filter, validate the retained layout, and capture the installed package version. Add orchestration, generated-script, local container, checksum, and dedicated Ubuntu/Debian CI coverage while keeping every existing generated installer byte-identical and excluding the draft from deploy. --- .gitlab-ci.yml | 15 ++ Makefile | 13 +- install_script.sh.template | 147 +++++++++++++ test/generated-script-test.sh | 55 ++++- test/localtest.sh | 199 ++++++++++++++++- unit_tests/test_iot_deb_orchestration.sh | 263 +++++++++++++++++++++++ 6 files changed, 684 insertions(+), 8 deletions(-) create mode 100755 unit_tests/test_iot_deb_orchestration.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 263f5d64..b7f0483a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -187,6 +187,21 @@ test: paths: - artifacts/ +test_iot_filtered_ubuntu_22_04: + extends: .test + variables: + IMAGE: mirror/ubuntu:22.04 + MAJOR_VERSION: 7 + SCRIPT: install_script_agent7_iot.sh + +test_iot_filtered_debian_12_pinned: + extends: .test + variables: + IMAGE: mirror/debian:12.1 + MAJOR_VERSION: 7 + MINOR_VERSION: 82 + SCRIPT: install_script_agent7_iot.sh + # These tests should not be launched on pipelines triggered by datadog-agent pipelines, but in the future we will trigger # them from the `6.53.x` branch of the datadog-agent repo test_agent6: diff --git a/Makefile b/Makefile index fee8feec..5889abca 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ endif CUR_VERSION:=$(shell awk -F "=" '/^install_script_version=/{print $$NF}' install_script.sh.template) +STRIP_FILTERED_IOT_ONLY = -e '/BEGIN GENERATED FILTERED IOT ONLY/,/END GENERATED FILTERED IOT ONLY/d' +STRIP_FILTERED_IOT_ONLY_MARKERS = -e '/BEGIN GENERATED FILTERED IOT ONLY/d' -e '/END GENERATED FILTERED IOT ONLY/d' + install_script.sh: install_script.sh.template export DEPRECATION_MESSAGE sed -e 's|AGENT_MAJOR_VERSION_PLACEHOLDER|6|' \ @@ -30,7 +33,7 @@ install_script.sh: install_script.sh.template -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER|echo -e "\\033[33m${DEPRECATION_MESSAGE}\\033[0m"|' \ - install_script.sh.template > $@ + $(STRIP_FILTERED_IOT_ONLY) install_script.sh.template > $@ chmod +x $@ install_script_agent6.sh: install_script.sh.template @@ -42,7 +45,7 @@ install_script_agent6.sh: install_script.sh.template -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ - install_script.sh.template > $@ + $(STRIP_FILTERED_IOT_ONLY) install_script.sh.template > $@ chmod +x $@ install_script_agent7.sh: install_script.sh.template @@ -54,7 +57,7 @@ install_script_agent7.sh: install_script.sh.template -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ - install_script.sh.template > $@ + $(STRIP_FILTERED_IOT_ONLY) install_script.sh.template > $@ chmod +x $@ install_script_agent7_iot.sh: install_script.sh.template @@ -66,7 +69,7 @@ install_script_agent7_iot.sh: install_script.sh.template -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ - install_script.sh.template > $@ + $(STRIP_FILTERED_IOT_ONLY_MARKERS) install_script.sh.template > $@ chmod +x $@ install_script_docker_injection.sh: install_script.sh.template @@ -78,7 +81,7 @@ install_script_docker_injection.sh: install_script.sh.template -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER|export DD_APM_INSTRUMENTATION_ENABLED="docker"|' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER|safe_agent_version=noagent_autoinstrumentation|' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ - install_script.sh.template > $@ + $(STRIP_FILTERED_IOT_ONLY) install_script.sh.template > $@ chmod +x $@ pre_release_%: diff --git a/install_script.sh.template b/install_script.sh.template index 9032e07c..b4de9f64 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -507,6 +507,122 @@ EOF return 1 fi } +# BEGIN GENERATED FILTERED IOT ONLY +function install_deb_iot_filter_config() { + local sudo_cmd="${1-}" + local destination_path="${2-}" + local retain_installer="${3-}" + local staging_directory + local staging_path + local install_status=0 + local -a root_context_command=() + + if [ -z "$destination_path" ]; then + printf 'A destination path is required to install the filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + if ! staging_directory="$(mktemp -d "${TMPDIR:-/tmp}/datadog-iot-filter.XXXXXX")"; then + printf 'Unable to create a staging directory for the filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + staging_path="$staging_directory/99-datadog-iot" + if ! chmod 0755 "$staging_directory" || ! write_deb_iot_filter_config "$staging_path"; then + rm -rf "$staging_directory" + return 1 + fi + if [ "$retain_installer" = retain-installer ]; then + if ! printf '%s\n' 'path-include=/opt/datadog-agent/embedded/bin/installer' >> "$staging_path"; then + rm -rf "$staging_directory" + return 1 + fi + fi + + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + # shellcheck disable=SC2016 + "${root_context_command[@]}" sh -c ' + set -e + source_path=$1 + destination_path=$2 + destination_directory=${destination_path%/*} + [ -n "$destination_directory" ] || destination_directory=/ + temporary_path=$(mktemp "$destination_directory/.datadog_iot_filter.root.XXXXXX") + trap '\''rm -f "$temporary_path"'\'' EXIT HUP INT TERM + cat "$source_path" > "$temporary_path" + chown root:root "$temporary_path" + chmod 0644 "$temporary_path" + mv -fT -- "$temporary_path" "$destination_path" + temporary_path= + trap - EXIT HUP INT TERM + [ -f "$destination_path" ] && [ ! -L "$destination_path" ] + [ "$(stat -c %u "$destination_path")" = 0 ] + [ "$(stat -c %g "$destination_path")" = 0 ] + [ "$(stat -c %a "$destination_path")" = 644 ] + ' sh "$staging_path" "$destination_path" || install_status=$? + + rm -rf "$staging_directory" + if [ "$install_status" -ne 0 ]; then + printf 'Unable to atomically install the filtered IoT dpkg configuration at %s.\n' "$destination_path" >&2 + return "$install_status" + fi +} + +function activate_iot_install_mode() { + local filtered_install="${1-}" + local resolved_agent_major_version="${2-}" + local agent_config_path="${3-}" + local legacy_config_path="${4-}" + local package_name + local package_status + + [ "$filtered_install" = true ] || return 0 + + validate_iot_installer_options \ + "${DD_AGENT_FLAVOR-}" \ + "$resolved_agent_major_version" \ + "${DD_INFRASTRUCTURE_MODE-}" \ + "${DD_FIPS_MODE-}" \ + "${DD_APM_INSTRUMENTATION_ENABLED-}" \ + "${DD_APM_INSTRUMENTATION_LIBRARIES-}" \ + "${DD_OTELCOLLECTOR_ENABLED-}" \ + "${DD_REMOTE_UPDATES-}" \ + "${DD_NO_AGENT_INSTALL-}" \ + "${DD_UPGRADE-}" \ + "${DD_RUNTIME_SECURITY_CONFIG_ENABLED-}" \ + "${DD_COMPLIANCE_CONFIG_ENABLED-}" \ + "${DD_DISCOVERY_ENABLED-}" \ + "${DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED-}" \ + "${DD_PRIVILEGED_LOGS_ENABLED-}" \ + "${DD_PRIVATE_ACTION_RUNNER_ENABLED-}" || return 1 + + for package_name in datadog-agent datadog-iot-agent; do + package_status= + if command -v dpkg-query >/dev/null 2>&1; then + package_status=$(dpkg-query --show '--showformat=${db:Status-Status}\n' "$package_name" 2>/dev/null) || true + if [ "$package_status" = installed ]; then + printf 'Filtered IoT installation is fresh-install-only and found existing package %s; remove it and its configuration before retrying.\n' "$package_name" >&2 + return 1 + fi + fi + if command -v rpm >/dev/null 2>&1 && rpm -q "$package_name" >/dev/null 2>&1; then + printf 'Filtered IoT installation is fresh-install-only and found existing package %s; remove it and its configuration before retrying.\n' "$package_name" >&2 + return 1 + fi + done + + for agent_config_path in "$agent_config_path" "$legacy_config_path"; do + if [ -n "$agent_config_path" ] && { [ -e "$agent_config_path" ] || [ -L "$agent_config_path" ]; }; then + printf 'Filtered IoT installation is fresh-install-only and found existing Agent configuration at %s; remove or archive it before retrying.\n' "$agent_config_path" >&2 + return 1 + fi + done + + agent_flavor=datadog-agent + infrastructure_mode=iot + nice_flavor="Datadog Agent" +} +# END GENERATED FILTERED IOT ONLY function rpm_iot_exclude_paths() { local package_path="${1-}" @@ -1785,6 +1901,9 @@ elif [ "IS_LEGACY_SCRIPT_PLACEHOLDER" == "true" ]; then agent_major_version=7 fi fi +# BEGIN GENERATED FILTERED IOT ONLY +activate_iot_install_mode "$iot_filtered_install" "$agent_major_version" "$etcdir" "$LEGACY_ETCDIR" +# END GENERATED FILTERED IOT ONLY if [ -n "$DD_AGENT_MINOR_VERSION" ]; then # Examples: @@ -1916,6 +2035,15 @@ elif [ -f /etc/Eos-release ] || [ "$DISTRIBUTION" == "Arista" ]; then elif [ -f /etc/SuSE-release ] || [ "$DISTRIBUTION" == "SUSE" ] || [ "$DISTRIBUTION" == "openSUSE" ]; then OS="SUSE" fi +# BEGIN GENERATED FILTERED IOT ONLY +if [ "$OS" != Debian ]; then + ERROR_MESSAGE="Filtered IoT installation currently supports Debian and Ubuntu only." + ERROR_CODE=$UNSUPPORTED_PLATFORM_CODE + printf "\033[31m%s\033[0m\n" "$ERROR_MESSAGE" + report_telemetry + exit 1 +fi +# END GENERATED FILTERED IOT ONLY if [[ "$agent_flavor" == "datadog-dogstatsd" ]]; then if [[ $(uname -m) == "armv7l" ]] || { [[ $(uname -m) != "x86_64" ]] && [[ "$OS" != "Debian" ]]; }; then @@ -2339,6 +2467,10 @@ If the cause is unclear, please contact Datadog support. END )" start_stage "install_agent_packages" +# BEGIN GENERATED FILTERED IOT ONLY + # The DEB post-install script invokes this binary, so retain it only while dpkg configures the package. + install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot retain-installer +# END GENERATED FILTERED IOT ONLY $sudo_cmd bash -c "DD_OTELCOLLECTOR_ENABLED='${DD_OTELCOLLECTOR_ENABLED}' DD_API_KEY='${apikey}' DD_SITE='${site}' DD_INSTALLER_REGISTRY_URL='${DD_INSTALLER_REGISTRY_URL}' POLICYRCD='${POLICYRCD}' apt-get install -o Acquire::Retries='5' -y --force-yes ${packages[*]} 2> >($sudo_cmd tee /tmp/ddog_install_error_msg >&2)" if [ -n "$DD_OTELCOLLECTOR_ENABLED" ] && [ -z "$ddot_installed_by_agent" ]; then @@ -2346,6 +2478,21 @@ END $sudo_cmd bash -c "POLICYRCD='${POLICYRCD}' apt-get install -o Acquire::Retries='5' -y --force-yes ${ddot_package} 2> >($sudo_cmd tee /tmp/ddog_install_error_msg >&2)" $sudo_cmd mv /etc/apt/sources.list.d/datadog-ddot.list /etc/apt/sources.list.d/datadog-ddot.list.disabled fi +# BEGIN GENERATED FILTERED IOT ONLY + # Persist the final filter before removing the package-configuration-only binary. + install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot + $sudo_cmd rm -f /opt/datadog-agent/embedded/bin/installer + if ! validate_iot_install_layout /; then + ERROR_MESSAGE="Filtered IoT package installation left an incomplete or unfiltered Agent layout." + ERROR_CODE=$UNABLE_TO_INSTALL_DEPENDENCY_CODE + exit 1 + fi + if ! installed_agent_package_version=$(get_installed_agent_package_version deb) || [ -z "$installed_agent_package_version" ]; then + ERROR_MESSAGE="Unable to query the installed datadog-agent package version after filtered IoT installation." + ERROR_CODE=$UNABLE_TO_INSTALL_DEPENDENCY_CODE + exit 1 + fi +# END GENERATED FILTERED IOT ONLY ERR_SUMMARY=$(grep "No space left on device" -C1 /tmp/ddog_install_error_msg || true) diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh index 77cb78de..71d62497 100755 --- a/test/generated-script-test.sh +++ b/test/generated-script-test.sh @@ -29,6 +29,19 @@ assert_line() { grep -Fqx -- "$expected" "$script" || fail "$(basename "$script") is missing '$expected'" } +assert_order() { + local script=$1 + local first=$2 + local second=$3 + local first_line + local second_line + + first_line=$(grep -Fnm1 -- "$first" "$script" | cut -d: -f1) + second_line=$(grep -Fnm1 -- "$second" "$script" | cut -d: -f1) + [[ -n $first_line && -n $second_line && $first_line -lt $second_line ]] || + fail "$(basename "$script") does not order '$first' before '$second'" +} + for script_name in "${generated_scripts[@]}"; do script="$repo_root/$script_name" [[ -x $script ]] || fail "$script_name is missing or is not executable" @@ -43,9 +56,49 @@ assert_line "$iot_script" 'variant=install_script_agent7_iot' assert_line "$iot_script" 'agent_major_version=7' grep -Fq 'Datadog Agent 7 IoT Filtered install script' "$iot_script" || fail "install_script_agent7_iot.sh is missing its report label" +# shellcheck disable=SC2016 +assert_line "$iot_script" 'activate_iot_install_mode "$iot_filtered_install" "$agent_major_version" "$etcdir" "$LEGACY_ETCDIR"' +# shellcheck disable=SC2016 +assert_line "$iot_script" ' install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot retain-installer' +# shellcheck disable=SC2016 +assert_line "$iot_script" ' install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot' +assert_line "$iot_script" ' if ! validate_iot_install_layout /; then' +# shellcheck disable=SC2016 +assert_line "$iot_script" ' if ! installed_agent_package_version=$(get_installed_agent_package_version deb) || [ -z "$installed_agent_package_version" ]; then' +# shellcheck disable=SC2016 +assert_order "$iot_script" 'activate_iot_install_mode "$iot_filtered_install"' 'if [ -n "$DD_AGENT_MINOR_VERSION" ]' +# shellcheck disable=SC2016 +assert_order "$iot_script" 'install_deb_iot_filter_config "$sudo_cmd"' "apt-get install -o Acquire::Retries='5' -y --force-yes" +assert_order "$iot_script" "apt-get install -o Acquire::Retries='5' -y --force-yes" 'validate_iot_install_layout /' +assert_order "$iot_script" 'validate_iot_install_layout /' '# Complete install_agent_packages' +[[ $(grep -Fc 'write_iot_install_profile()' "$iot_script") -eq 1 ]] || + fail "install_script_agent7_iot.sh should define but not call the final install profile writer" for script_name in "${common_scripts[@]}"; do - assert_line "$repo_root/$script_name" 'iot_filtered_install=false' + script="$repo_root/$script_name" + assert_line "$script" 'iot_filtered_install=false' + # shellcheck disable=SC2016 + if grep -Fq 'activate_iot_install_mode "$iot_filtered_install"' "$script" || + grep -Fq 'install_deb_iot_filter_config "$sudo_cmd"' "$script"; then + fail "$script_name contains filtered IoT Task 3 orchestration" + fi done +localtest="$repo_root/test/localtest.sh" +grep -Fq 'verify_iot_debsums' "$localtest" || fail "localtest is missing fail-closed filtered debsums verification" +grep -Fq 'infrastructure_mode: iot' "$localtest" || fail "localtest does not verify filtered IoT infrastructure mode" +grep -Fq 'iot-installed-bytes.txt' "$localtest" || fail "localtest does not record filtered IoT logical bytes" +grep -Fq '/etc/dpkg/dpkg.cfg.d/99-datadog-iot' "$localtest" || fail "localtest does not verify the persistent dpkg filter" +grep -Fq '/etc/datadog-agent/install_profile' "$localtest" || fail "localtest does not verify that the final profile marker is absent" + +grep -q '^test_iot_filtered_ubuntu_22_04:' "$repo_root/.gitlab-ci.yml" || + fail "GitLab CI is missing the filtered IoT Ubuntu 22.04 job" +grep -q '^test_iot_filtered_debian_12_pinned:' "$repo_root/.gitlab-ci.yml" || + fail "GitLab CI is missing the pinned filtered IoT Debian 12 job" + +if awk '/^deploy:/{in_deploy=1} /^deploy_deprecated:/{in_deploy=0} in_deploy' "$repo_root/.gitlab-ci.yml" | + grep -Fq install_script_agent7_iot.sh; then + fail "install_script_agent7_iot.sh must not be publicly deployed in this draft" +fi + printf 'Generated script contract passed for %d scripts.\n' "${#generated_scripts[@]}" diff --git a/test/localtest.sh b/test/localtest.sh index afa3c629..e8558672 100755 --- a/test/localtest.sh +++ b/test/localtest.sh @@ -8,6 +8,152 @@ function get_os_type() { fi } +function dpkg_path_is_excluded() { + local package_path=$1 + local config_path + local line + local directive + local pattern + local decision=include + local -a config_paths=(/etc/dpkg/dpkg.cfg /etc/dpkg/dpkg.cfg.d/*) + + for config_path in "${config_paths[@]}"; do + [ -f "$config_path" ] || continue + while IFS= read -r line || [ -n "$line" ]; do + line=${line#"${line%%[![:space:]]*}"} + case "$line" in + path-exclude=*|path-include=*) + directive=${line%%=*} + pattern=${line#*=} + # shellcheck disable=SC2053 + if [[ "$package_path" == $pattern ]]; then + case "$directive" in + path-exclude) decision=exclude ;; + path-include) decision=include ;; + esac + fi + ;; + esac + done < "$config_path" + done + + [ "$decision" = exclude ] +} + +function verify_iot_debsums() { + local package_name=$1 + local output_path + local debsums_status=0 + local line + local missing_path + local missing_count=0 + local failure_count=0 + + output_path=$(mktemp) + debsums -as "$package_name" > "$output_path" 2>&1 || debsums_status=$? + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "debsums: missing file "*) + missing_path=${line#debsums: missing file } + missing_path=${missing_path%% (from *} + if [[ "$missing_path" != /* ]] || ! dpkg_path_is_excluded "$missing_path"; then + echo "[FAIL] Missing package path is not excluded by the installed dpkg rules: $line" + failure_count=$((failure_count + 1)) + else + missing_count=$((missing_count + 1)) + fi + ;; + "") ;; + *) + echo "[FAIL] Retained package checksum error: $line" + failure_count=$((failure_count + 1)) + ;; + esac + done < "$output_path" + rm -f "$output_path" + + if [ "$failure_count" -ne 0 ]; then + return 1 + fi + if [ "$debsums_status" -ne 0 ] && [ "$missing_count" -eq 0 ]; then + echo "[FAIL] debsums failed without reporting an excluded missing path (status $debsums_status)" + return 1 + fi + echo "[OK] All $missing_count reported missing package paths are excluded by installed dpkg rules; retained checksums are valid" +} + +function verify_iot_filtered_layout() { + local required_path + local check_name + local unexpected_path + local result=0 + local -a required_executables=( + /opt/datadog-agent/bin/agent/agent + /opt/datadog-agent/embedded/bin/agent-data-plane + ) + local -a retained_checks=( + cpu disk io load memory network ntp uptime system_swap systemd jetson + ) + local -a excluded_roots=( + /opt/datadog-agent/embedded/include + /opt/datadog-agent/embedded/msodbcsql + /opt/datadog-agent/embedded/sbin + /opt/datadog-agent/embedded/share/ebpf + /opt/datadog-agent/embedded/share/system-probe + /opt/datadog-agent/python-scripts + /opt/datadog-agent/requirements + /opt/datadog-agent/compliance + /opt/datadog-agent/runtime-security.d + /etc/datadog-agent/compliance.d + /etc/datadog-agent/runtime-security.d + /etc/datadog-agent/conf.d/docker.d + ) + + for required_path in "${required_executables[@]}"; do + if [ ! -f "$required_path" ] || [ -L "$required_path" ] || [ ! -x "$required_path" ]; then + echo "[FAIL] Filtered layout is missing required executable $required_path" + result=1 + fi + done + if ! compgen -G '/opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so*' >/dev/null; then + echo "[FAIL] Filtered layout is missing the rtloader library" + result=1 + fi + if ! find /opt/datadog-agent/bin/agent/dist/views -type f -print -quit 2>/dev/null | grep -q .; then + echo "[FAIL] Filtered layout is missing support view content" + result=1 + fi + for check_name in "${retained_checks[@]}"; do + if ! find "/etc/datadog-agent/conf.d/$check_name.d" -type f -print -quit 2>/dev/null | grep -q .; then + echo "[FAIL] Filtered layout is missing retained $check_name check configuration" + result=1 + fi + done + for required_path in "${excluded_roots[@]}"; do + [ -e "$required_path" ] || [ -L "$required_path" ] || continue + unexpected_path=$(find "$required_path" \( -type f -o -type l \) -print -quit 2>/dev/null) || { + echo "[FAIL] Unable to inspect excluded payload root $required_path" + result=1 + continue + } + if [ -n "$unexpected_path" ] && { [ ! -L "$unexpected_path" ] || [ -e "$unexpected_path" ]; }; then + echo "[FAIL] Excluded payload remains at $unexpected_path" + result=1 + fi + done + for required_path in \ + /opt/datadog-agent/bin/process-agent \ + /opt/datadog-agent/embedded/bin/python3 \ + /opt/datadog-agent/requirements-agent-release.txt; do + if [ -e "$required_path" ]; then + echo "[FAIL] Excluded payload remains at $required_path" + result=1 + fi + done + + return "$result" +} + # Patch the sources.list file for debian. This is a workaround, we should change the image instead if [[ "${IMAGE}" =~ "debian:10" ]]; then cp ./test/sources10.list /etc/apt/sources.list @@ -36,7 +182,7 @@ fi # Override curl to capture trace payloads (only if SHOW_TRACE is enabled) if [[ "${SHOW_TRACE}" == "1" ]]; then - # shellcheck disable=SC2317 + # shellcheck disable=SC2317,SC2329 curl() { if [[ "$*" == *"instrumentation-telemetry-intake"* ]]; then echo "[TRACE CAPTURE] Intercepting telemetry submission" >&2 @@ -108,7 +254,13 @@ if [[ "$OS_TYPE" == "ubuntu" ]]; then apt-get install -y debsums if [ -z "$DD_NO_AGENT_INSTALL" ]; then - debsums -c "${EXPECTED_FLAVOR}" + if [ "${SCRIPT_FLAVOR}" = "agent7_iot" ]; then + if ! verify_iot_debsums "${EXPECTED_FLAVOR}"; then + RESULT=1 + fi + elif ! debsums -c "${EXPECTED_FLAVOR}"; then + RESULT=1 + fi INSTALLED_VERSION=$(dpkg-query -W "${EXPECTED_FLAVOR}" | cut -f2 | cut -d: -f2) elif debsums -c datadog-agent ; then echo "[FAIL] datadog-agent should not be installed" @@ -202,6 +354,49 @@ if [ "${EXPECTED_FLAVOR}" == "datadog-agent" ] && [ -z "$DD_NO_AGENT_INSTALL" ]; fi fi +if [ "${SCRIPT_FLAVOR}" = "agent7_iot" ] && [ -z "$DD_NO_AGENT_INSTALL" ]; then + iot_filter=/etc/dpkg/dpkg.cfg.d/99-datadog-iot + if [ ! -f "$iot_filter" ] || [ -L "$iot_filter" ]; then + echo "[FAIL] Persistent filtered IoT dpkg configuration is missing or is not a regular file" + RESULT=1 + elif [ "$(stat -c '%u:%g:%a' "$iot_filter")" != "0:0:644" ]; then + echo "[FAIL] Persistent filtered IoT dpkg configuration must be root:root mode 0644" + RESULT=1 + elif grep -Ev '^(path-exclude|path-include)=/' "$iot_filter" | grep -q .; then + echo "[FAIL] Persistent filtered IoT dpkg configuration contains an invalid directive" + RESULT=1 + else + echo "[OK] Persistent filtered IoT dpkg configuration has the expected ownership and mode" + fi + + if ! grep -q '^infrastructure_mode: iot$' /etc/datadog-agent/datadog.yaml; then + echo "[FAIL] Filtered IoT configuration does not set infrastructure_mode: iot" + RESULT=1 + else + echo "[OK] Filtered IoT infrastructure mode is configured" + fi + if [ -e /etc/datadog-agent/install_profile ] || [ -L /etc/datadog-agent/install_profile ]; then + echo "[FAIL] Final filtered IoT install profile marker must not be written by this draft" + RESULT=1 + else + echo "[OK] Final filtered IoT install profile marker is absent" + fi + if ! verify_iot_filtered_layout; then + RESULT=1 + else + echo "[OK] Filtered IoT retained and excluded layout is valid" + fi + if ! /opt/datadog-agent/bin/agent/agent version; then + echo "[FAIL] Filtered normal Agent version command failed" + RESULT=1 + fi + + mkdir -p "${TESTING_DIR}/artifacts" + iot_logical_bytes=$(du -sb /opt/datadog-agent /etc/datadog-agent | awk '{total += $1} END {print total}') + printf '%s\n' "$iot_logical_bytes" | tee "${TESTING_DIR}/artifacts/iot-installed-bytes.txt" + echo "[INFO] Filtered IoT installed logical bytes: $iot_logical_bytes" +fi + # Lint configuration files when they exist config_file=/etc/datadog-agent/datadog.yaml security_agent_config_file=/etc/datadog-agent/security-agent.yaml diff --git a/unit_tests/test_iot_deb_orchestration.sh b/unit_tests/test_iot_deb_orchestration.sh new file mode 100755 index 00000000..ec499e35 --- /dev/null +++ b/unit_tests/test_iot_deb_orchestration.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash + +set -u + +dir_path=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +source "${dir_path}/extracted_functions.sh" +eval "$(awk ' + /^function dpkg_path_is_excluded\(\)/ { capture=1 } + /^# Patch the sources.list file for debian/ { capture=0 } + capture { print } +' "${dir_path}/../test/localtest.sh")" + +unset_iot_options() { + unset DD_AGENT_FLAVOR + unset DD_INFRASTRUCTURE_MODE + unset DD_FIPS_MODE + unset DD_APM_INSTRUMENTATION_ENABLED + unset DD_APM_INSTRUMENTATION_LIBRARIES + unset DD_OTELCOLLECTOR_ENABLED + unset DD_REMOTE_UPDATES + unset DD_NO_AGENT_INSTALL + unset DD_UPGRADE + unset DD_RUNTIME_SECURITY_CONFIG_ENABLED + unset DD_COMPLIANCE_CONFIG_ENABLED + unset DD_DISCOVERY_ENABLED + unset DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED + unset DD_PRIVILEGED_LOGS_ENABLED + unset DD_PRIVATE_ACTION_RUNNER_ENABLED +} + +setUp() { + unset_iot_options + TEST_ROOT=$(mktemp -d) + agent_flavor=unexpected-internal-flavor + infrastructure_mode=unexpected-internal-mode + nice_flavor=Unexpected +} + +tearDown() { + unset -f dpkg-query 2>/dev/null || true + unset -f rpm 2>/dev/null || true + unset -f sudo 2>/dev/null || true + unset_iot_options + rm -rf "$TEST_ROOT" +} + +assertContains() { + local message=$1 + local actual=$2 + local expected=$3 + + case "$actual" in + *"$expected"*) assertTrue "$message" 0 ;; + *) assertTrue "$message: expected <$actual> to contain <$expected>" 1 ;; + esac +} + +testFilteredModeForcesNormalAgentAndIotInfrastructureMode() { + DD_INSTALL_ONLY=true + # shellcheck disable=SC2329 + dpkg-query() { return 1; } + # shellcheck disable=SC2329 + rpm() { return 1; } + + activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" + assertEquals "filtered activation should succeed" 0 $? + assertEquals "normal Agent package should be forced" datadog-agent "$agent_flavor" + assertEquals "IoT infrastructure mode should be forced" iot "$infrastructure_mode" + assertEquals "readable flavor should follow the forced package" "Datadog Agent" "$nice_flavor" + assertEquals "DD_INSTALL_ONLY remains supported" true "$DD_INSTALL_ONLY" +} + +testOrdinaryModeDoesNotChangeResolvedOptions() { + activate_iot_install_mode false 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" + assertEquals "ordinary activation should be a no-op" 0 $? + assertEquals unexpected-internal-flavor "$agent_flavor" + assertEquals unexpected-internal-mode "$infrastructure_mode" + assertEquals Unexpected "$nice_flavor" +} + +testFilteredModeValidatesExplicitOptionsBeforeForcingValues() { + local output + local status + + export DD_AGENT_FLAVOR=datadog-iot-agent + # shellcheck disable=SC2329 + dpkg-query() { return 1; } + output=$(activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" 2>&1) + status=$? + + assertNotEquals "an incompatible explicit flavor should be rejected" 0 "$status" + assertContains "the rejection should name the explicit option" "$output" DD_AGENT_FLAVOR +} + +testFilteredModeRejectsEachExistingDebAgentPackage() { + local existing_package + local output + local status + + for existing_package in datadog-agent datadog-iot-agent; do + # shellcheck disable=SC2329 + dpkg-query() { + if [ "${*: -1}" = "$existing_package" ]; then + printf 'installed\n' + return 0 + fi + return 1 + } + + output=$(activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" 2>&1) + status=$? + assertNotEquals "$existing_package should make the filtered draft fail closed" 0 "$status" + assertContains "the package rejection should identify $existing_package" "$output" "$existing_package" + unset -f dpkg-query + done +} + +testFilteredModeRejectsPreExistingConfigurationTrees() { + local config_path + local output + local status + + # shellcheck disable=SC2329 + dpkg-query() { return 1; } + # shellcheck disable=SC2329 + rpm() { return 1; } + + for config_path in "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent"; do + rm -rf "${TEST_ROOT:?}/etc" + mkdir -p "$config_path" + output=$(activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" 2>&1) + status=$? + assertNotEquals "$config_path should make the filtered draft fail closed" 0 "$status" + assertContains "the configuration rejection should identify $config_path" "$output" "$config_path" + done +} + +testIotDebsumsAcceptsOnlyMissingPathsCoveredByDpkgRules() { + local original_matcher + local output + local status + + original_matcher=$(declare -f dpkg_path_is_excluded) + # shellcheck disable=SC2329 + debsums() { + printf '%s\n' \ + 'debsums: missing file /opt/datadog-agent/embedded/bin/python3 (from datadog-agent package)' \ + 'debsums: missing file /opt/datadog-agent/bin/process-agent/process-agent (from datadog-agent package)' + return 2 + } + # shellcheck disable=SC2329 + dpkg_path_is_excluded() { + case "$1" in + /opt/datadog-agent/embedded/bin/python3|/opt/datadog-agent/bin/process-agent/process-agent) return 0 ;; + *) return 1 ;; + esac + } + + output=$(verify_iot_debsums datadog-agent 2>&1) + status=$? + eval "$original_matcher" + unset -f debsums + + assertEquals "excluded missing paths should be accepted" 0 "$status" + assertContains "the result should count every classified missing path" "$output" "All 2 reported missing package paths" +} + +testIotDebsumsRejectsUnexcludedMissingPathsAndRetainedChecksumErrors() { + local original_matcher + local output + local status + + original_matcher=$(declare -f dpkg_path_is_excluded) + # shellcheck disable=SC2329 + debsums() { + printf '%s\n' \ + 'debsums: missing file /opt/datadog-agent/embedded/bin/python3 (from datadog-agent package)' \ + 'debsums: changed file /opt/datadog-agent/bin/agent/agent (from datadog-agent package)' + return 2 + } + # shellcheck disable=SC2329 + dpkg_path_is_excluded() { return 1; } + + output=$(verify_iot_debsums datadog-agent 2>&1) + status=$? + eval "$original_matcher" + unset -f debsums + + assertNotEquals "unexcluded or retained checksum failures should fail closed" 0 "$status" + assertContains "unexcluded missing path should be reported" "$output" "not excluded" + assertContains "changed retained content should be fatal" "$output" "Retained package checksum error" +} + +testDebFilterRootInstallerUsesShellWrapperAndProducesRootMode0644File() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + local sudo_arguments=$TEST_ROOT/sudo-arguments + + mkdir -p "$(dirname "$destination")" + # shellcheck disable=SC2329 + sudo() { + printf '%s\n' "$@" > "$sudo_arguments" + "$@" + } + + install_deb_iot_filter_config sudo "$destination" + assertEquals "root-context installation should succeed" 0 $? + assertEquals "the wrapper should invoke a shell, not a shell function" sh "$(head -n 1 "$sudo_arguments")" + assertTrue "the persistent filter should be a regular file" "[ -f '$destination' ]" + assertFalse "the persistent filter should not be a symlink" "[ -L '$destination' ]" + assertEquals "the persistent filter mode" 644 "$(stat -c '%a' "$destination")" + assertEquals "the persistent filter owner" "$(id -u)" "$(stat -c '%u' "$destination")" + assertContains "the rendered filter should exclude Python" "$(cat "$destination")" 'path-exclude=/opt/datadog-agent/embedded/bin/*' +} + +testDebFilterRootInstallerCanRetainInstallerOnlyDuringPackageConfiguration() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + + mkdir -p "$(dirname "$destination")" + install_deb_iot_filter_config "" "$destination" retain-installer + assertEquals "install-time filter installation should succeed" 0 $? + assertEquals "the installer should be transiently retained" \ + 'path-include=/opt/datadog-agent/embedded/bin/installer' "$(tail -n 1 "$destination")" + + install_deb_iot_filter_config "" "$destination" + assertEquals "persistent filter replacement should succeed" 0 $? + assertFalse "the persistent filter should not retain the package installer" \ + "grep -q '^path-include=/opt/datadog-agent/embedded/bin/installer$' '$destination'" +} + +testDebFilterRootInstallerPreservesExistingFilterUntilReplacementSucceeds() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + local fake_bin=$TEST_ROOT/bin + local real_mv + local output + local status + + mkdir -p "$(dirname "$destination")" "$fake_bin" + printf 'existing valid filter\n' > "$destination" + real_mv=$(command -v mv) + cat > "$fake_bin/mv" <&1) + status=$? + + assertNotEquals "a failed atomic replacement should return nonzero" 0 "$status" + assertEquals "the prior complete filter should be preserved" "existing valid filter" "$(cat "$destination")" + assertEquals "same-directory root temporary files should be cleaned" 0 "$(find "$(dirname "$destination")" -name '.datadog_iot_filter.root.*' | wc -l | tr -d ' ')" +} + +# shellcheck source=/dev/null +. shunit2 From 7f099d40cf5e08bbb64c08b388f68020d2973a1e Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 11:18:45 -0400 Subject: [PATCH 08/12] Pin filtered Debian CI to Agent 7.82 Pass the installer-facing DD_AGENT_MINOR_VERSION variable so the filtered Debian job actually selects and verifies Agent 7.82. Extend the generated-script contract to catch an unconnected pinned-version variable. --- .gitlab-ci.yml | 2 +- test/generated-script-test.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7f0483a..32446499 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -199,7 +199,7 @@ test_iot_filtered_debian_12_pinned: variables: IMAGE: mirror/debian:12.1 MAJOR_VERSION: 7 - MINOR_VERSION: 82 + DD_AGENT_MINOR_VERSION: 82 SCRIPT: install_script_agent7_iot.sh # These tests should not be launched on pipelines triggered by datadog-agent pipelines, but in the future we will trigger diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh index 71d62497..c1dee081 100755 --- a/test/generated-script-test.sh +++ b/test/generated-script-test.sh @@ -95,6 +95,13 @@ grep -q '^test_iot_filtered_ubuntu_22_04:' "$repo_root/.gitlab-ci.yml" || fail "GitLab CI is missing the filtered IoT Ubuntu 22.04 job" grep -q '^test_iot_filtered_debian_12_pinned:' "$repo_root/.gitlab-ci.yml" || fail "GitLab CI is missing the pinned filtered IoT Debian 12 job" +if ! awk ' + /^test_iot_filtered_debian_12_pinned:/ { in_job=1; next } + in_job && /^[^[:space:]]/ { exit } + in_job { print } +' "$repo_root/.gitlab-ci.yml" | grep -Fqx ' DD_AGENT_MINOR_VERSION: 82'; then + fail "filtered IoT Debian 12 CI does not pin DD_AGENT_MINOR_VERSION=82" +fi if awk '/^deploy:/{in_deploy=1} /^deploy_deprecated:/{in_deploy=0} in_deploy' "$repo_root/.gitlab-ci.yml" | grep -Fq install_script_agent7_iot.sh; then From 4e04d5f59babd27a3b74da3da6c48a1ef9cfe321 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 11:37:17 -0400 Subject: [PATCH 09/12] Verify installed IoT dpkg policy Exercise the persistent Debian filter through the localtest decision helper so retained Agent, ADP, rtloader, views, and IoT check paths stay included while pruned payload classes stay excluded. Also reject the transient installer override and any retained installer binary, with a generated-script contract requiring the real-install check. --- test/generated-script-test.sh | 2 ++ test/localtest.sh | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh index c1dee081..9542a193 100755 --- a/test/generated-script-test.sh +++ b/test/generated-script-test.sh @@ -89,6 +89,8 @@ grep -Fq 'verify_iot_debsums' "$localtest" || fail "localtest is missing fail-cl grep -Fq 'infrastructure_mode: iot' "$localtest" || fail "localtest does not verify filtered IoT infrastructure mode" grep -Fq 'iot-installed-bytes.txt' "$localtest" || fail "localtest does not record filtered IoT logical bytes" grep -Fq '/etc/dpkg/dpkg.cfg.d/99-datadog-iot' "$localtest" || fail "localtest does not verify the persistent dpkg filter" +# shellcheck disable=SC2016 +grep -Fq 'if ! verify_iot_dpkg_policy "$iot_filter"; then' "$localtest" || fail "localtest does not exercise the installed filtered IoT dpkg policy" grep -Fq '/etc/datadog-agent/install_profile' "$localtest" || fail "localtest does not verify that the final profile marker is absent" grep -q '^test_iot_filtered_ubuntu_22_04:' "$repo_root/.gitlab-ci.yml" || diff --git a/test/localtest.sh b/test/localtest.sh index e8558672..f0d99f39 100755 --- a/test/localtest.sh +++ b/test/localtest.sh @@ -40,6 +40,50 @@ function dpkg_path_is_excluded() { [ "$decision" = exclude ] } +function verify_iot_dpkg_policy() { + local filter_path=$1 + local package_path + local result=0 + local -a included_paths=( + /opt/datadog-agent/bin/agent/agent + /opt/datadog-agent/embedded/bin/agent-data-plane + /opt/datadog-agent/embedded/lib/libdatadog-agent-rtloader.so + /opt/datadog-agent/bin/agent/dist/views/index.html + /etc/datadog-agent/conf.d/cpu.d/conf.yaml.example + ) + local -a excluded_paths=( + /opt/datadog-agent/embedded/bin/python3 + /opt/datadog-agent/bin/process-agent/process-agent + /opt/datadog-agent/embedded/bin/system-probe + /opt/datadog-agent/embedded/share/ebpf/co-re.o + /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar + /etc/datadog-agent/conf.d/docker.d/conf.yaml.example + ) + + for package_path in "${included_paths[@]}"; do + if dpkg_path_is_excluded "$package_path"; then + echo "[FAIL] Persistent filtered IoT dpkg policy excludes required path $package_path" + result=1 + fi + done + for package_path in "${excluded_paths[@]}"; do + if ! dpkg_path_is_excluded "$package_path"; then + echo "[FAIL] Persistent filtered IoT dpkg policy includes disallowed path $package_path" + result=1 + fi + done + if [ -f "$filter_path" ] && grep -Fqx 'path-include=/opt/datadog-agent/embedded/bin/installer' "$filter_path"; then + echo "[FAIL] Persistent filtered IoT dpkg policy retains the transient installer rule" + result=1 + fi + if [ -e /opt/datadog-agent/embedded/bin/installer ] || [ -L /opt/datadog-agent/embedded/bin/installer ]; then + echo "[FAIL] Filtered IoT install retains the package installer binary" + result=1 + fi + + return "$result" +} + function verify_iot_debsums() { local package_name=$1 local output_path @@ -368,6 +412,11 @@ if [ "${SCRIPT_FLAVOR}" = "agent7_iot" ] && [ -z "$DD_NO_AGENT_INSTALL" ]; then else echo "[OK] Persistent filtered IoT dpkg configuration has the expected ownership and mode" fi + if ! verify_iot_dpkg_policy "$iot_filter"; then + RESULT=1 + else + echo "[OK] Persistent filtered IoT dpkg policy has the expected effective decisions" + fi if ! grep -q '^infrastructure_mode: iot$' /etc/datadog-agent/datadog.yaml; then echo "[FAIL] Filtered IoT configuration does not set infrastructure_mode: iot" From 9b4ded592807aeecfbfadab1ba9853059235287a Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 12:02:54 -0400 Subject: [PATCH 10/12] Harden filtered IoT Debian transactions Force IoT infrastructure mode into the Debian package environment and normalize the fresh Agent configuration after postinst, while leaving generated ordinary installers byte-for-byte unchanged. Treat every queryable dpkg state as pre-existing and add a rollback transaction around the transient dpkg path filter. The transaction snapshots prior content, ownership, and mode in private storage, restores or removes the filter on APT and EXIT failures, and is disarmed only after the persistent policy and filtered layout validate. Strengthen filtered checksum evidence with a required dpkg md5 manifest and accurate retained-file reporting. Run orchestration regressions in GitLab, keep the pinned 7.82 job push-only, and document fresh-install and upgrade constraints plus the experimental same-version disk result. Tests cover unset infrastructure input, all dpkg states, empty checksum evidence, silent debsums success, and APT rollback with and without a prior filter. --- .gitlab-ci.yml | 3 + Makefile | 10 ++ README.md | 6 +- install_script.sh.template | 214 ++++++++++++++++++++++- test/generated-script-test.sh | 42 ++++- test/localtest.sh | 8 +- unit_tests/test_iot_deb_orchestration.sh | 208 ++++++++++++++++++++-- 7 files changed, 460 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32446499..2eea8b1a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,7 @@ unit_tests: - DD_API_KEY=123 DD_SITE="datadoghq.com" DD_INSTALL_ONLY=true bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)" - python3 unit_tests/extract_functions.py - ./unit_tests/test_install_script.sh + - ./unit_tests/test_iot_deb_orchestration.sh .test: image: registry.ddbuild.io/images/${IMAGE} @@ -196,6 +197,8 @@ test_iot_filtered_ubuntu_22_04: test_iot_filtered_debian_12_pinned: extends: .test + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' variables: IMAGE: mirror/debian:12.1 MAJOR_VERSION: 7 diff --git a/Makefile b/Makefile index 5889abca..c289da82 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,8 @@ install_script.sh: install_script.sh.template -e 's|INSTALL_INFO_VERSION_PLACEHOLDER||' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER|true|' \ -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ + -e 's|DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDER||' \ + -e 's|IOT_APT_FAILURE_PLACEHOLDER||' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER|echo -e "\\033[33m${DEPRECATION_MESSAGE}\\033[0m"|' \ @@ -42,6 +44,8 @@ install_script_agent6.sh: install_script.sh.template -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent6|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ + -e 's|DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDER||' \ + -e 's|IOT_APT_FAILURE_PLACEHOLDER||' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ @@ -54,6 +58,8 @@ install_script_agent7.sh: install_script.sh.template -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent7|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ + -e 's|DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDER||' \ + -e 's|IOT_APT_FAILURE_PLACEHOLDER||' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ @@ -66,6 +72,8 @@ install_script_agent7_iot.sh: install_script.sh.template -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_agent7_iot|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|true|' \ + -e "s|DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDER|DD_INFRASTRUCTURE_MODE='iot' |" \ + -e 's~IOT_APT_FAILURE_PLACEHOLDER~ || iot_apt_exit_code=$$?~' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER||' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER||' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ @@ -78,6 +86,8 @@ install_script_docker_injection.sh: install_script.sh.template -e 's|INSTALL_INFO_VERSION_PLACEHOLDER|_docker_injection|' \ -e 's|IS_LEGACY_SCRIPT_PLACEHOLDER||' \ -e 's|IOT_FILTERED_INSTALL_PLACEHOLDER|false|' \ + -e 's|DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDER||' \ + -e 's|IOT_APT_FAILURE_PLACEHOLDER||' \ -e 's|DD_APM_INSTRUMENTATION_ENABLED_DOCKER_PLACEHOLDER|export DD_APM_INSTRUMENTATION_ENABLED="docker"|' \ -e 's|APM_TELEMETRY_SAFE_AGENT_VERSION_OVERRIDE_PLACEHOLDER|safe_agent_version=noagent_autoinstrumentation|' \ -e 's|DEPRECATION_MESSAGE_PLACEHOLDER||' \ diff --git a/README.md b/README.md index 267d08db..ec2bc49f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,11 @@ This repository contains 2 basic files, `install_script.sh.template` and `Makefi * `install_script.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default and also emits a deprecation warning when run. * `install_script_agent6.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default. * `install_script_agent7.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=7` by default. -* `install_script_agent7_iot.sh` - Future Agent 7 variant intended to install a filtered normal `datadog-agent` package for IoT deployments; package filtering will be added in a later change. +* `install_script_agent7_iot.sh` - Experimental Agent 7 variant that installs a filesystem-filtered normal `datadog-agent` DEB or RPM for IoT deployments. + +The filtered IoT installer is fresh-install-only: it rejects an existing Agent package or configuration. Ordinary APT upgrades of a filtered installation are not supported yet. The dedicated dpkg path-filter policy intentionally remains installed after a successful DEB installation so that unsupported upgrades cannot silently restore the excluded payload. + +For scale only, a same-version Agent 7.82 DEB comparison on the validated test layout showed about a 77% reduction in installed logical bytes. This result is experimental and varies with package and platform; CI does not enforce a hard disk-reduction threshold. The generated files must never be committed to this repository. All changes must be done by modifications of the template file and Makefile. diff --git a/install_script.sh.template b/install_script.sh.template index b4de9f64..73be9ab4 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -15,6 +15,12 @@ support_email=support@datadoghq.com variant=install_scriptINSTALL_INFO_VERSION_PLACEHOLDER # shellcheck disable=SC2034 iot_filtered_install=IOT_FILTERED_INSTALL_PLACEHOLDER +# BEGIN GENERATED FILTERED IOT ONLY +iot_deb_filter_rollback_active= +iot_deb_filter_rollback_sudo_cmd= +iot_deb_filter_rollback_destination= +iot_deb_filter_rollback_directory= +# END GENERATED FILTERED IOT ONLY LEGACY_ETCDIR="/etc/dd-agent" LEGACY_CONF="$LEGACY_ETCDIR/datadog.conf" @@ -113,6 +119,9 @@ DISTRIBUTION=$(lsb_release -d 2>/dev/null | grep -Eo "$KNOWN_DISTRIBUTION" || gr function on_exit() { exit_code=$? +# BEGIN GENERATED FILTERED IOT ONLY + rollback_deb_iot_filter_transaction || true +# END GENERATED FILTERED IOT ONLY rm -f $npipe # Handle any remaining active stage @@ -568,6 +577,191 @@ function install_deb_iot_filter_config() { fi } +function begin_deb_iot_filter_transaction() { + local sudo_cmd="${1-}" + local destination_path="${2-}" + local rollback_directory + local -a root_context_command=() + + if [ -z "$destination_path" ]; then + printf 'A destination path is required to begin the filtered IoT dpkg transaction.\n' >&2 + return 1 + fi + if [ "$iot_deb_filter_rollback_active" = true ]; then + printf 'A filtered IoT dpkg transaction is already active.\n' >&2 + return 1 + fi + if ! rollback_directory="$(mktemp -d "${TMPDIR:-/tmp}/datadog-iot-filter-rollback.XXXXXX")"; then + printf 'Unable to create private rollback storage for the filtered IoT dpkg configuration.\n' >&2 + return 1 + fi + if ! chmod 0700 "$rollback_directory"; then + rm -rf "$rollback_directory" + printf 'Unable to make filtered IoT dpkg rollback storage private.\n' >&2 + return 1 + fi + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + # shellcheck disable=SC2016 + if ! "${root_context_command[@]}" sh -c ' + set -e + destination_path=$1 + rollback_directory=$2 + umask 077 + if [ -e "$destination_path" ] || [ -L "$destination_path" ]; then + [ -f "$destination_path" ] && [ ! -L "$destination_path" ] + printf "present\n" > "$rollback_directory/state" + cat -- "$destination_path" > "$rollback_directory/content" + stat -c "%a" "$destination_path" > "$rollback_directory/mode" + stat -c "%u" "$destination_path" > "$rollback_directory/uid" + stat -c "%g" "$destination_path" > "$rollback_directory/gid" + else + printf "absent\n" > "$rollback_directory/state" + fi + ' sh "$destination_path" "$rollback_directory"; then + "${root_context_command[@]}" rm -rf -- "$rollback_directory" || true + printf 'Unable to snapshot the prior filtered IoT dpkg configuration at %s.\n' "$destination_path" >&2 + return 1 + fi + + iot_deb_filter_rollback_sudo_cmd=$sudo_cmd + iot_deb_filter_rollback_destination=$destination_path + iot_deb_filter_rollback_directory=$rollback_directory + iot_deb_filter_rollback_active=true +} + +function rollback_deb_iot_filter_transaction() { + local sudo_cmd="${iot_deb_filter_rollback_sudo_cmd-}" + local destination_path="${iot_deb_filter_rollback_destination-}" + local rollback_directory="${iot_deb_filter_rollback_directory-}" + local -a root_context_command=() + + [ "${iot_deb_filter_rollback_active-}" = true ] || return 0 + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + # shellcheck disable=SC2016 + if ! "${root_context_command[@]}" sh -c ' + set -e + destination_path=$1 + rollback_directory=$2 + state=$(cat "$rollback_directory/state") + case "$state" in + present) + destination_directory=${destination_path%/*} + [ -n "$destination_directory" ] || destination_directory=/ + temporary_path=$(mktemp "$destination_directory/.datadog_iot_filter.rollback.XXXXXX") + trap '\''rm -f -- "$temporary_path"'\'' EXIT HUP INT TERM + cat -- "$rollback_directory/content" > "$temporary_path" + chown "$(cat "$rollback_directory/uid"):$(cat "$rollback_directory/gid")" "$temporary_path" + chmod "$(cat "$rollback_directory/mode")" "$temporary_path" + mv -fT -- "$temporary_path" "$destination_path" + temporary_path= + trap - EXIT HUP INT TERM + cmp -s -- "$rollback_directory/content" "$destination_path" + [ "$(stat -c "%a" "$destination_path")" = "$(cat "$rollback_directory/mode")" ] + [ "$(stat -c "%u" "$destination_path")" = "$(cat "$rollback_directory/uid")" ] + [ "$(stat -c "%g" "$destination_path")" = "$(cat "$rollback_directory/gid")" ] + ;; + absent) + [ ! -d "$destination_path" ] + rm -f -- "$destination_path" + [ ! -e "$destination_path" ] && [ ! -L "$destination_path" ] + ;; + *) + exit 1 + ;; + esac + ' sh "$destination_path" "$rollback_directory"; then + printf 'Unable to restore the prior filtered IoT dpkg configuration at %s.\n' "$destination_path" >&2 + return 1 + fi + if ! "${root_context_command[@]}" rm -rf -- "$rollback_directory"; then + printf 'Unable to remove private filtered IoT dpkg rollback storage.\n' >&2 + return 1 + fi + + iot_deb_filter_rollback_active= + iot_deb_filter_rollback_sudo_cmd= + iot_deb_filter_rollback_destination= + iot_deb_filter_rollback_directory= +} + +function commit_deb_iot_filter_transaction() { + local sudo_cmd="${iot_deb_filter_rollback_sudo_cmd-}" + local rollback_directory="${iot_deb_filter_rollback_directory-}" + local -a root_context_command=() + + [ "${iot_deb_filter_rollback_active-}" = true ] || return 0 + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + if ! "${root_context_command[@]}" rm -rf -- "$rollback_directory"; then + printf 'Unable to remove private filtered IoT dpkg rollback storage before commit.\n' >&2 + return 1 + fi + + iot_deb_filter_rollback_active= + iot_deb_filter_rollback_sudo_cmd= + iot_deb_filter_rollback_destination= + iot_deb_filter_rollback_directory= +} + +function finish_deb_iot_apt_install() { + local apt_status="${1-}" + + if [ -z "$apt_status" ]; then + printf 'An APT status is required to finish the filtered IoT dpkg transaction.\n' >&2 + return 1 + fi + [ "$apt_status" -ne 0 ] || return 0 + if ! rollback_deb_iot_filter_transaction; then + printf 'APT failed with status %s and the filtered IoT dpkg configuration could not be rolled back.\n' "$apt_status" >&2 + return 1 + fi + return "$apt_status" +} + +function ensure_iot_infrastructure_mode_config() { + local sudo_cmd="${1-}" + local config_path="${2-}" + local -a root_context_command=() + + if [ -z "$config_path" ]; then + printf 'An Agent configuration path is required to force filtered IoT infrastructure mode.\n' >&2 + return 1 + fi + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + # shellcheck disable=SC2016 + if ! "${root_context_command[@]}" sh -c ' + set -e + config_path=$1 + [ -f "$config_path" ] && [ ! -L "$config_path" ] + config_directory=${config_path%/*} + [ -n "$config_directory" ] || config_directory=/ + temporary_path=$(mktemp "$config_directory/.datadog_iot_config.XXXXXX") + trap '\''rm -f -- "$temporary_path"'\'' EXIT HUP INT TERM + awk '\'' + /^(#[[:space:]]*)?infrastructure_mode:[[:space:]]*/ { next } + { print } + END { print "infrastructure_mode: iot" } + '\'' "$config_path" > "$temporary_path" + chown --reference="$config_path" "$temporary_path" + chmod --reference="$config_path" "$temporary_path" + mv -fT -- "$temporary_path" "$config_path" + temporary_path= + trap - EXIT HUP INT TERM + [ "$(grep -Fxc "infrastructure_mode: iot" "$config_path")" = 1 ] + [ "$(grep -Ec "^(#[[:space:]]*)?infrastructure_mode:" "$config_path")" = 1 ] + ' sh "$config_path"; then + printf 'Unable to set and verify infrastructure_mode: iot in %s.\n' "$config_path" >&2 + return 1 + fi +} + function activate_iot_install_mode() { local filtered_install="${1-}" local resolved_agent_major_version="${2-}" @@ -598,12 +792,9 @@ function activate_iot_install_mode() { for package_name in datadog-agent datadog-iot-agent; do package_status= - if command -v dpkg-query >/dev/null 2>&1; then - package_status=$(dpkg-query --show '--showformat=${db:Status-Status}\n' "$package_name" 2>/dev/null) || true - if [ "$package_status" = installed ]; then - printf 'Filtered IoT installation is fresh-install-only and found existing package %s; remove it and its configuration before retrying.\n' "$package_name" >&2 - return 1 - fi + if command -v dpkg-query >/dev/null 2>&1 && package_status=$(dpkg-query --show '--showformat=${db:Status-Status}\n' "$package_name" 2>/dev/null); then + printf 'Filtered IoT installation is fresh-install-only and found existing package %s in dpkg state %s; purge it and its configuration before retrying.\n' "$package_name" "${package_status:-unknown}" >&2 + return 1 fi if command -v rpm >/dev/null 2>&1 && rpm -q "$package_name" >/dev/null 2>&1; then printf 'Filtered IoT installation is fresh-install-only and found existing package %s; remove it and its configuration before retrying.\n' "$package_name" >&2 @@ -2468,11 +2659,16 @@ END )" start_stage "install_agent_packages" # BEGIN GENERATED FILTERED IOT ONLY + begin_deb_iot_filter_transaction "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot # The DEB post-install script invokes this binary, so retain it only while dpkg configures the package. install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot retain-installer + iot_apt_exit_code=0 # END GENERATED FILTERED IOT ONLY - $sudo_cmd bash -c "DD_OTELCOLLECTOR_ENABLED='${DD_OTELCOLLECTOR_ENABLED}' DD_API_KEY='${apikey}' DD_SITE='${site}' DD_INSTALLER_REGISTRY_URL='${DD_INSTALLER_REGISTRY_URL}' POLICYRCD='${POLICYRCD}' apt-get install -o Acquire::Retries='5' -y --force-yes ${packages[*]} 2> >($sudo_cmd tee /tmp/ddog_install_error_msg >&2)" + $sudo_cmd bash -c "DD_INFRASTRUCTURE_MODE_APT_PLACEHOLDERDD_OTELCOLLECTOR_ENABLED='${DD_OTELCOLLECTOR_ENABLED}' DD_API_KEY='${apikey}' DD_SITE='${site}' DD_INSTALLER_REGISTRY_URL='${DD_INSTALLER_REGISTRY_URL}' POLICYRCD='${POLICYRCD}' apt-get install -o Acquire::Retries='5' -y --force-yes ${packages[*]} 2> >($sudo_cmd tee /tmp/ddog_install_error_msg >&2)"IOT_APT_FAILURE_PLACEHOLDER +# BEGIN GENERATED FILTERED IOT ONLY + finish_deb_iot_apt_install "$iot_apt_exit_code" || exit $? +# END GENERATED FILTERED IOT ONLY if [ -n "$DD_OTELCOLLECTOR_ENABLED" ] && [ -z "$ddot_installed_by_agent" ]; then $sudo_cmd mv /etc/apt/sources.list.d/datadog-ddot.list.disabled /etc/apt/sources.list.d/datadog-ddot.list $sudo_cmd bash -c "POLICYRCD='${POLICYRCD}' apt-get install -o Acquire::Retries='5' -y --force-yes ${ddot_package} 2> >($sudo_cmd tee /tmp/ddog_install_error_msg >&2)" @@ -2487,6 +2683,7 @@ END ERROR_CODE=$UNABLE_TO_INSTALL_DEPENDENCY_CODE exit 1 fi + commit_deb_iot_filter_transaction if ! installed_agent_package_version=$(get_installed_agent_package_version deb) || [ -z "$installed_agent_package_version" ]; then ERROR_MESSAGE="Unable to query the installed datadog-agent package version after filtered IoT installation." ERROR_CODE=$UNABLE_TO_INSTALL_DEPENDENCY_CODE @@ -3166,6 +3363,9 @@ elif [ ! "$no_agent" ]; then manage_security_config "$sudo_cmd" "$security_agent_config_file" "$DD_RUNTIME_SECURITY_CONFIG_ENABLED" "$DD_COMPLIANCE_CONFIG_ENABLED" manage_system_probe_config "$sudo_cmd" "$system_probe_config_file" "$DD_RUNTIME_SECURITY_CONFIG_ENABLED" "$DD_DISCOVERY_ENABLED" "$DD_PRIVILEGED_LOGS_ENABLED" "$usm_enabled" fi +# BEGIN GENERATED FILTERED IOT ONLY + ensure_iot_infrastructure_mode_config "$sudo_cmd" "$config_file" +# END GENERATED FILTERED IOT ONLY # DDOT configuration update # Even if DDOT is installed by the agent, update_ddot still needs to run diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh index 9542a193..4eb7d670 100755 --- a/test/generated-script-test.sh +++ b/test/generated-script-test.sh @@ -59,10 +59,22 @@ grep -Fq 'Datadog Agent 7 IoT Filtered install script' "$iot_script" || # shellcheck disable=SC2016 assert_line "$iot_script" 'activate_iot_install_mode "$iot_filtered_install" "$agent_major_version" "$etcdir" "$LEGACY_ETCDIR"' # shellcheck disable=SC2016 +assert_line "$iot_script" ' begin_deb_iot_filter_transaction "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot' +# shellcheck disable=SC2016 assert_line "$iot_script" ' install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot retain-installer' # shellcheck disable=SC2016 assert_line "$iot_script" ' install_deb_iot_filter_config "$sudo_cmd" /etc/dpkg/dpkg.cfg.d/99-datadog-iot' assert_line "$iot_script" ' if ! validate_iot_install_layout /; then' +assert_line "$iot_script" ' commit_deb_iot_filter_transaction' +grep -Fq "DD_INFRASTRUCTURE_MODE='iot'" "$iot_script" || + fail "filtered IoT APT package environment does not force DD_INFRASTRUCTURE_MODE=iot" +# shellcheck disable=SC2016 +grep -Fq 'finish_deb_iot_apt_install "$iot_apt_exit_code" || exit $?' "$iot_script" || + fail "filtered IoT APT failure does not explicitly invoke rollback" +grep -Fq 'rollback_deb_iot_filter_transaction || true' "$iot_script" || + fail "filtered IoT EXIT handler does not invoke rollback" +# shellcheck disable=SC2016 +assert_line "$iot_script" ' ensure_iot_infrastructure_mode_config "$sudo_cmd" "$config_file"' # shellcheck disable=SC2016 assert_line "$iot_script" ' if ! installed_agent_package_version=$(get_installed_agent_package_version deb) || [ -z "$installed_agent_package_version" ]; then' # shellcheck disable=SC2016 @@ -70,7 +82,8 @@ assert_order "$iot_script" 'activate_iot_install_mode "$iot_filtered_install"' ' # shellcheck disable=SC2016 assert_order "$iot_script" 'install_deb_iot_filter_config "$sudo_cmd"' "apt-get install -o Acquire::Retries='5' -y --force-yes" assert_order "$iot_script" "apt-get install -o Acquire::Retries='5' -y --force-yes" 'validate_iot_install_layout /' -assert_order "$iot_script" 'validate_iot_install_layout /' '# Complete install_agent_packages' +assert_order "$iot_script" 'validate_iot_install_layout /' ' commit_deb_iot_filter_transaction' +assert_order "$iot_script" ' commit_deb_iot_filter_transaction' '# Complete install_agent_packages' [[ $(grep -Fc 'write_iot_install_profile()' "$iot_script") -eq 1 ]] || fail "install_script_agent7_iot.sh should define but not call the final install profile writer" @@ -79,7 +92,9 @@ for script_name in "${common_scripts[@]}"; do assert_line "$script" 'iot_filtered_install=false' # shellcheck disable=SC2016 if grep -Fq 'activate_iot_install_mode "$iot_filtered_install"' "$script" || - grep -Fq 'install_deb_iot_filter_config "$sudo_cmd"' "$script"; then + grep -Fq 'install_deb_iot_filter_config "$sudo_cmd"' "$script" || + grep -Fq "DD_INFRASTRUCTURE_MODE='iot'" "$script" || + grep -Fq 'iot_deb_filter_rollback' "$script"; then fail "$script_name contains filtered IoT Task 3 orchestration" fi done @@ -97,13 +112,32 @@ grep -q '^test_iot_filtered_ubuntu_22_04:' "$repo_root/.gitlab-ci.yml" || fail "GitLab CI is missing the filtered IoT Ubuntu 22.04 job" grep -q '^test_iot_filtered_debian_12_pinned:' "$repo_root/.gitlab-ci.yml" || fail "GitLab CI is missing the pinned filtered IoT Debian 12 job" -if ! awk ' +pinned_iot_job=$(awk ' /^test_iot_filtered_debian_12_pinned:/ { in_job=1; next } in_job && /^[^[:space:]]/ { exit } in_job { print } -' "$repo_root/.gitlab-ci.yml" | grep -Fqx ' DD_AGENT_MINOR_VERSION: 82'; then +' "$repo_root/.gitlab-ci.yml") +if ! grep -Fqx ' DD_AGENT_MINOR_VERSION: 82' <<< "$pinned_iot_job"; then fail "filtered IoT Debian 12 CI does not pin DD_AGENT_MINOR_VERSION=82" fi +# shellcheck disable=SC2016 +if ! grep -Fqx ' - if: '\''$CI_PIPELINE_SOURCE == "push"'\''' <<< "$pinned_iot_job"; then + fail "filtered IoT Debian 12 pinned CI job must be push-only" +fi + +unit_test_job=$(awk ' + /^unit_tests:/ { in_job=1; next } + in_job && /^[^[:space:]]/ { exit } + in_job { print } +' "$repo_root/.gitlab-ci.yml") +if ! grep -Fqx ' - ./unit_tests/test_iot_deb_orchestration.sh' <<< "$unit_test_job"; then + fail "GitLab unit_tests does not run filtered IoT DEB orchestration tests" +fi +main_unit_line=$(grep -Fn './unit_tests/test_install_script.sh' <<< "$unit_test_job" | cut -d: -f1) +iot_unit_line=$(grep -Fn './unit_tests/test_iot_deb_orchestration.sh' <<< "$unit_test_job" | cut -d: -f1) +if [[ -z $main_unit_line || -z $iot_unit_line || $main_unit_line -ge $iot_unit_line ]]; then + fail "GitLab unit_tests must run filtered IoT DEB orchestration after the main shunit suite" +fi if awk '/^deploy:/{in_deploy=1} /^deploy_deprecated:/{in_deploy=0} in_deploy' "$repo_root/.gitlab-ci.yml" | grep -Fq install_script_agent7_iot.sh; then diff --git a/test/localtest.sh b/test/localtest.sh index f0d99f39..f78c8d61 100755 --- a/test/localtest.sh +++ b/test/localtest.sh @@ -86,6 +86,7 @@ function verify_iot_dpkg_policy() { function verify_iot_debsums() { local package_name=$1 + local md5_manifest=${2:-/var/lib/dpkg/info/${package_name}.md5sums} local output_path local debsums_status=0 local line @@ -93,6 +94,11 @@ function verify_iot_debsums() { local missing_count=0 local failure_count=0 + if [ ! -f "$md5_manifest" ] || ! grep -q '[^[:space:]]' "$md5_manifest"; then + echo "[FAIL] Expected a nonempty dpkg md5 manifest at $md5_manifest" + return 1 + fi + output_path=$(mktemp) debsums -as "$package_name" > "$output_path" 2>&1 || debsums_status=$? while IFS= read -r line || [ -n "$line" ]; do @@ -123,7 +129,7 @@ function verify_iot_debsums() { echo "[FAIL] debsums failed without reporting an excluded missing path (status $debsums_status)" return 1 fi - echo "[OK] All $missing_count reported missing package paths are excluded by installed dpkg rules; retained checksums are valid" + echo "[OK] debsums checked retained installed files from a nonempty dpkg md5 manifest; $missing_count filtered manifest paths were reported missing and validated separately against installed dpkg policy" } function verify_iot_filtered_layout() { diff --git a/unit_tests/test_iot_deb_orchestration.sh b/unit_tests/test_iot_deb_orchestration.sh index ec499e35..5aa725b4 100755 --- a/unit_tests/test_iot_deb_orchestration.sh +++ b/unit_tests/test_iot_deb_orchestration.sh @@ -35,12 +35,23 @@ setUp() { agent_flavor=unexpected-internal-flavor infrastructure_mode=unexpected-internal-mode nice_flavor=Unexpected + iot_deb_filter_rollback_active= + # Referenced by the sourced transaction helpers. + # shellcheck disable=SC2034 + iot_deb_filter_rollback_sudo_cmd= + # shellcheck disable=SC2034 + iot_deb_filter_rollback_destination= + iot_deb_filter_rollback_directory= } tearDown() { unset -f dpkg-query 2>/dev/null || true unset -f rpm 2>/dev/null || true unset -f sudo 2>/dev/null || true + unset -f debsums 2>/dev/null || true + if [ "${iot_deb_filter_rollback_active-}" = true ]; then + rollback_deb_iot_filter_transaction >/dev/null 2>&1 || true + fi unset_iot_options rm -rf "$TEST_ROOT" } @@ -93,29 +104,54 @@ testFilteredModeValidatesExplicitOptionsBeforeForcingValues() { assertContains "the rejection should name the explicit option" "$output" DD_AGENT_FLAVOR } -testFilteredModeRejectsEachExistingDebAgentPackage() { +testFilteredModeRejectsEveryExistingDpkgState() { local existing_package + local package_state local output local status + local -a package_states=( + not-installed + config-files + half-installed + unpacked + half-configured + triggers-awaited + triggers-pending + installed + ) for existing_package in datadog-agent datadog-iot-agent; do - # shellcheck disable=SC2329 - dpkg-query() { - if [ "${*: -1}" = "$existing_package" ]; then - printf 'installed\n' - return 0 - fi - return 1 - } - - output=$(activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" 2>&1) - status=$? - assertNotEquals "$existing_package should make the filtered draft fail closed" 0 "$status" - assertContains "the package rejection should identify $existing_package" "$output" "$existing_package" - unset -f dpkg-query + for package_state in "${package_states[@]}"; do + # shellcheck disable=SC2329 + dpkg-query() { + if [ "${*: -1}" = "$existing_package" ]; then + printf '%s\n' "$package_state" + return 0 + fi + return 1 + } + + output=$(activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" 2>&1) + status=$? + assertNotEquals "$existing_package in $package_state state should fail closed" 0 "$status" + assertContains "the package rejection should identify $existing_package" "$output" "$existing_package" + assertContains "the package rejection should identify $package_state" "$output" "$package_state" + unset -f dpkg-query + done done } + +testFilteredModeAcceptsOnlyAbsentDpkgPackages() { + # shellcheck disable=SC2329 + dpkg-query() { return 1; } + # shellcheck disable=SC2329 + rpm() { return 1; } + + activate_iot_install_mode true 7 "$TEST_ROOT/etc/datadog-agent" "$TEST_ROOT/etc/dd-agent" + assertEquals "only absent dpkg packages should be accepted" 0 $? +} + testFilteredModeRejectsPreExistingConfigurationTrees() { local config_path local output @@ -136,11 +172,31 @@ testFilteredModeRejectsPreExistingConfigurationTrees() { done } +testUnsetInfrastructureModeIsForcedThroughPostinstAndCommonConfig() { + local config_file=$TEST_ROOT/etc/datadog-agent/datadog.yaml + + mkdir -p "$(dirname "$config_file")" + cat > "$config_file" <<'EOF' +api_key: package-created +infrastructure_mode: full +# infrastructure_mode: legacy-comment +EOF + + ensure_iot_infrastructure_mode_config "" "$config_file" + assertEquals "a package-created config should be updated" 0 $? + assertEquals "the forced mode should have one active key" 1 "$(grep -Fxc 'infrastructure_mode: iot' "$config_file")" + assertEquals "prior active and commented values should be removed" 1 "$(grep -Ec '^(# ?)?infrastructure_mode:' "$config_file")" + assertEquals "unrelated package-created config should remain" "api_key: package-created" "$(head -n 1 "$config_file")" +} + + testIotDebsumsAcceptsOnlyMissingPathsCoveredByDpkgRules() { local original_matcher + local manifest=$TEST_ROOT/datadog-agent.md5sums local output local status + printf '%s\n' '0123456789abcdef0123456789abcdef opt/datadog-agent/bin/agent/agent' > "$manifest" original_matcher=$(declare -f dpkg_path_is_excluded) # shellcheck disable=SC2329 debsums() { @@ -157,20 +213,24 @@ testIotDebsumsAcceptsOnlyMissingPathsCoveredByDpkgRules() { esac } - output=$(verify_iot_debsums datadog-agent 2>&1) + output=$(verify_iot_debsums datadog-agent "$manifest" 2>&1) status=$? eval "$original_matcher" unset -f debsums assertEquals "excluded missing paths should be accepted" 0 "$status" - assertContains "the result should count every classified missing path" "$output" "All 2 reported missing package paths" + assertContains "the result should classify every reported missing path" "$output" "2 filtered manifest paths" + assertContains "the result should accurately describe retained checks" "$output" "retained installed files" } + testIotDebsumsRejectsUnexcludedMissingPathsAndRetainedChecksumErrors() { local original_matcher + local manifest=$TEST_ROOT/datadog-agent.md5sums local output local status + printf '%s\n' '0123456789abcdef0123456789abcdef opt/datadog-agent/bin/agent/agent' > "$manifest" original_matcher=$(declare -f dpkg_path_is_excluded) # shellcheck disable=SC2329 debsums() { @@ -182,7 +242,7 @@ testIotDebsumsRejectsUnexcludedMissingPathsAndRetainedChecksumErrors() { # shellcheck disable=SC2329 dpkg_path_is_excluded() { return 1; } - output=$(verify_iot_debsums datadog-agent 2>&1) + output=$(verify_iot_debsums datadog-agent "$manifest" 2>&1) status=$? eval "$original_matcher" unset -f debsums @@ -192,6 +252,46 @@ testIotDebsumsRejectsUnexcludedMissingPathsAndRetainedChecksumErrors() { assertContains "changed retained content should be fatal" "$output" "Retained package checksum error" } + +testIotDebsumsRejectsEmptyDpkgManifestBeforeRunningDebsums() { + local manifest=$TEST_ROOT/datadog-agent.md5sums + local output + local status + + : > "$manifest" + # shellcheck disable=SC2329 + debsums() { + fail "debsums must not run without checksum evidence" + } + + output=$(verify_iot_debsums datadog-agent "$manifest" 2>&1) + status=$? + + assertNotEquals "an empty dpkg md5 manifest should fail closed" 0 "$status" + assertContains "the missing evidence should be identified" "$output" "nonempty dpkg md5 manifest" +} + + +testIotDebsumsAcceptsEmptySuccessWithNonemptyManifestWithoutClaimingMissingPaths() { + local manifest=$TEST_ROOT/datadog-agent.md5sums + local output + local status + + printf '%s\n' '0123456789abcdef0123456789abcdef opt/datadog-agent/bin/agent/agent' > "$manifest" + # shellcheck disable=SC2329 + debsums() { return 0; } + + output=$(verify_iot_debsums datadog-agent "$manifest" 2>&1) + status=$? + + assertEquals "silent debsums success with a manifest should pass" 0 "$status" + assertContains "silent success should report retained checksum evidence" "$output" "retained installed files" + assertContains "silent success should say no filtered paths were reported" "$output" "0 filtered manifest paths" + case "$output" in + *"All 0 reported missing package paths"*) fail "empty success must not claim missing-path evidence" ;; + esac +} + testDebFilterRootInstallerUsesShellWrapperAndProducesRootMode0644File() { local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot local sudo_arguments=$TEST_ROOT/sudo-arguments @@ -228,6 +328,78 @@ testDebFilterRootInstallerCanRetainInstallerOnlyDuringPackageConfiguration() { "grep -q '^path-include=/opt/datadog-agent/embedded/bin/installer$' '$destination'" } +testAptFailureRemovesTransientFilterWhenNoPriorFilterExisted() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + local rollback_directory + local status + + mkdir -p "$(dirname "$destination")" + begin_deb_iot_filter_transaction "" "$destination" + assertEquals "filter transaction should begin" 0 $? + rollback_directory=$iot_deb_filter_rollback_directory + assertEquals "rollback evidence should be private" 700 "$(stat -c '%a' "$rollback_directory")" + install_deb_iot_filter_config "" "$destination" retain-installer + + finish_deb_iot_apt_install 42 + status=$? + + assertEquals "the apt failure status should be preserved" 42 "$status" + assertFalse "a transient filter with no predecessor should be removed" "[ -e '$destination' ] || [ -L '$destination' ]" + assertFalse "rollback evidence should be removed" "[ -e '$rollback_directory' ]" + assertEquals "rollback should be disarmed after restoration" "" "${iot_deb_filter_rollback_active-}" +} + + +testAptFailureRestoresPriorFilterContentModeAndOwnership() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + local expected=$TEST_ROOT/prior-filter + local prior_identity + local rollback_directory + local status + + mkdir -p "$(dirname "$destination")" + printf 'prior filter first line\nprior filter final line without newline' > "$expected" + cp "$expected" "$destination" + chmod 0600 "$destination" + if [ "$(id -u)" -eq 0 ]; then + chown 123:456 "$destination" + fi + prior_identity=$(stat -c '%u:%g:%a' "$destination") + + begin_deb_iot_filter_transaction "" "$destination" + assertEquals "existing filter transaction should begin" 0 $? + rollback_directory=$iot_deb_filter_rollback_directory + install_deb_iot_filter_config "" "$destination" retain-installer + + finish_deb_iot_apt_install 73 + status=$? + + assertEquals "the apt failure status should be preserved" 73 "$status" + assertTrue "the exact prior filter content should be restored" "cmp -s '$expected' '$destination'" + assertEquals "prior ownership and mode should be restored" "$prior_identity" "$(stat -c '%u:%g:%a' "$destination")" + assertFalse "rollback evidence should be removed" "[ -e '$rollback_directory' ]" +} + + +testSuccessfulFilterCommitDisarmsRollbackOnlyAfterFinalValidation() { + local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot + local rollback_directory + + mkdir -p "$(dirname "$destination")" + begin_deb_iot_filter_transaction "" "$destination" + rollback_directory=$iot_deb_filter_rollback_directory + install_deb_iot_filter_config "" "$destination" retain-installer + install_deb_iot_filter_config "" "$destination" + + commit_deb_iot_filter_transaction + assertEquals "a validated final filter should commit" 0 $? + assertTrue "the persistent filter should remain" "[ -f '$destination' ]" + assertFalse "the transient installer rule should not persist" "grep -q '^path-include=/opt/datadog-agent/embedded/bin/installer$' '$destination'" + assertFalse "committed rollback evidence should be removed" "[ -e '$rollback_directory' ]" + assertEquals "commit should disarm EXIT rollback" "" "${iot_deb_filter_rollback_active-}" +} + + testDebFilterRootInstallerPreservesExistingFilterUntilReplacementSucceeds() { local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot local fake_bin=$TEST_ROOT/bin From 508165707eb926d461deeb44d9827616aaaaee2e Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 12:20:36 -0400 Subject: [PATCH 11/12] Document filtered IoT installer scope Clarify that the experimental filtered installer currently supports Debian and Ubuntu only. Record the RPM post-install installer dependency that prevents retaining a verified RPM path in this draft, and add the feature to the unreleased changelog. --- CHANGELOG.rst | 2 ++ README.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9dec2dd1..ff5f009f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ Release Notes Unreleased ================ +- Add an experimental Debian/Ubuntu installer for a filesystem-filtered IoT Agent profile. + 1.46.0 ================ diff --git a/README.md b/README.md index ec2bc49f..9c263bc8 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,9 @@ This repository contains 2 basic files, `install_script.sh.template` and `Makefi * `install_script.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default and also emits a deprecation warning when run. * `install_script_agent6.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=6` by default. * `install_script_agent7.sh` - Install script that uses `DD_AGENT_MAJOR_VERSION=7` by default. -* `install_script_agent7_iot.sh` - Experimental Agent 7 variant that installs a filesystem-filtered normal `datadog-agent` DEB or RPM for IoT deployments. +* `install_script_agent7_iot.sh` - Experimental Agent 7 variant that installs a filesystem-filtered normal `datadog-agent` DEB for IoT deployments. + +The filtered IoT installer currently supports Debian and Ubuntu only. RPM installation remains unsupported because the Agent RPM post-install scriptlet needs the embedded installer while the final filtered layout removes it; reconciling that transient requirement with RPM's recorded file state needs a separate design. The filtered IoT installer is fresh-install-only: it rejects an existing Agent package or configuration. Ordinary APT upgrades of a filtered installation are not supported yet. The dedicated dpkg path-filter policy intentionally remains installed after a successful DEB installation so that unsupported upgrades cannot silently restore the excluded payload. From 80012a26c86ad71634ff457250fb8d3e885424d9 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 25 Aug 2026 12:34:37 -0400 Subject: [PATCH 12/12] Record filtered IoT install profile Publish the reviewed filtered IoT profile only after Debian package, layout, configuration, and service handling complete. Stage the profile outside the root context, install it atomically as root:root mode 0644, and verify its exact content against the captured package version and installer identity before success telemetry.\n\nUpdate generated-script, shunit, local container, and documentation coverage for publication ordering, controlled pre-marker failure, prior-marker preservation, and the Agent companion-guard contract while keeping ordinary generated installers byte-identical. --- README.md | 13 +++++ install_script.sh.template | 74 ++++++++++++++++++++++++ test/generated-script-test.sh | 61 ++++++++++++++++++- test/localtest.sh | 24 ++++++-- unit_tests/test_iot_deb_orchestration.sh | 61 +++++++++++++++++++ 5 files changed, 226 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9c263bc8..a759eb30 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,19 @@ The filtered IoT installer currently supports Debian and Ubuntu only. RPM instal The filtered IoT installer is fresh-install-only: it rejects an existing Agent package or configuration. Ordinary APT upgrades of a filtered installation are not supported yet. The dedicated dpkg path-filter policy intentionally remains installed after a successful DEB installation so that unsupported upgrades cannot silently restore the excluded payload. +After package, layout, configuration, and service handling succeed, the installer publishes `/etc/datadog-agent/install_profile` as root-owned mode `0644` with this schema (using the exact installed DEB version): + +```yaml +version: 1 +profile: iot-filtered +manifest: iot-v1 +package: datadog-agent +package_version: '1:7.82.0-1' +installer: install_script_agent7_iot +``` + +This marker is the installer-side contract for the normal Agent's filtered-IoT companion-process guard; the Agent-side guard is maintained in the `datadog-agent` repository rather than in this install script. + For scale only, a same-version Agent 7.82 DEB comparison on the validated test layout showed about a 77% reduction in installed logical bytes. This result is experimental and varies with package and platform; CI does not enforce a hard disk-reduction threshold. The generated files must never be committed to this repository. All changes must be done by modifications of the template file and Makefile. diff --git a/install_script.sh.template b/install_script.sh.template index 73be9ab4..b9c20ff3 100644 --- a/install_script.sh.template +++ b/install_script.sh.template @@ -1283,6 +1283,76 @@ function write_iot_install_profile() { fi } +# BEGIN GENERATED FILTERED IOT ONLY +function install_iot_install_profile() { + local sudo_cmd="${1-}" + local destination_path="${2-}" + local package_version="${3-}" + local installer_identity="${4-}" + local staging_directory + local staging_path + local install_status + local -a root_context_command=() + + if [ -z "$destination_path" ]; then + printf 'A destination path is required to install the filtered IoT install profile.\n' >&2 + return 1 + fi + if [ "$installer_identity" != install_script_agent7_iot ]; then + printf 'The filtered IoT install profile requires installer identity install_script_agent7_iot.\n' >&2 + return 1 + fi + if ! staging_directory="$(mktemp -d "${TMPDIR:-/tmp}/datadog-iot-install-profile.XXXXXX")"; then + printf 'Unable to create a staging directory for the filtered IoT install profile.\n' >&2 + return 1 + fi + staging_path="$staging_directory/install_profile" + if ! chmod 0755 "$staging_directory" || ! write_iot_install_profile "$staging_path" "$package_version"; then + rm -rf "$staging_directory" + return 1 + fi + + if [ -n "$sudo_cmd" ]; then + root_context_command=("$sudo_cmd") + fi + # shellcheck disable=SC2016 + if "${root_context_command[@]}" sh -c ' + set -e + source_path=$1 + destination_path=$2 + destination_directory=${destination_path%/*} + [ -n "$destination_directory" ] || destination_directory=/ + temporary_path= + trap '\''[ -z "$temporary_path" ] || rm -f -- "$temporary_path"'\'' EXIT HUP INT TERM + + [ -f "$source_path" ] && [ ! -L "$source_path" ] + [ -d "$destination_directory" ] && [ ! -L "$destination_directory" ] + [ ! -d "$destination_path" ] + temporary_path=$(mktemp "$destination_directory/.install_profile.root.XXXXXX") + cat -- "$source_path" > "$temporary_path" + chown root:root "$temporary_path" + chmod 0644 "$temporary_path" + mv -fT -- "$temporary_path" "$destination_path" + temporary_path= + trap - EXIT HUP INT TERM + + [ -f "$destination_path" ] && [ ! -L "$destination_path" ] + [ "$(stat -c "%u:%g:%a" "$destination_path")" = 0:0:644 ] + cmp -s -- "$source_path" "$destination_path" + ' sh "$staging_path" "$destination_path"; then + install_status=0 + else + install_status=$? + fi + + rm -rf "$staging_directory" + if [ "$install_status" -ne 0 ]; then + printf 'Unable to atomically install and verify the filtered IoT install profile at %s.\n' "$destination_path" >&2 + return "$install_status" + fi +} + +# END GENERATED FILTERED IOT ONLY function get_installed_agent_package_version() { local package_family="${1-}" @@ -3584,5 +3654,9 @@ end_stage 0 "$(cat <<-END | tr -d '\n' "suse11_mode": "$([ "$SUSE11" == "yes" ] && echo "true" || echo "false")" END )" +# BEGIN GENERATED FILTERED IOT ONLY + +install_iot_install_profile "$sudo_cmd" "$etcdir/install_profile" "$installed_agent_package_version" "$variant" +# END GENERATED FILTERED IOT ONLY report_telemetry "$install_id" "$install_type" "$install_time" diff --git a/test/generated-script-test.sh b/test/generated-script-test.sh index 4eb7d670..e7ccd736 100755 --- a/test/generated-script-test.sh +++ b/test/generated-script-test.sh @@ -85,7 +85,18 @@ assert_order "$iot_script" "apt-get install -o Acquire::Retries='5' -y --force-y assert_order "$iot_script" 'validate_iot_install_layout /' ' commit_deb_iot_filter_transaction' assert_order "$iot_script" ' commit_deb_iot_filter_transaction' '# Complete install_agent_packages' [[ $(grep -Fc 'write_iot_install_profile()' "$iot_script") -eq 1 ]] || - fail "install_script_agent7_iot.sh should define but not call the final install profile writer" + fail "install_script_agent7_iot.sh should define the reviewed install profile writer once" +[[ $(grep -Fc 'install_iot_install_profile()' "$iot_script") -eq 1 ]] || + fail "install_script_agent7_iot.sh should define the root-context profile installer once" +# shellcheck disable=SC2016 +assert_line "$iot_script" 'install_iot_install_profile "$sudo_cmd" "$etcdir/install_profile" "$installed_agent_package_version" "$variant"' +# shellcheck disable=SC2016 +assert_order "$iot_script" '# Complete service_management (final stage)' \ + 'install_iot_install_profile "$sudo_cmd" "$etcdir/install_profile"' +# shellcheck disable=SC2016 +assert_order "$iot_script" \ + 'install_iot_install_profile "$sudo_cmd" "$etcdir/install_profile"' \ + 'report_telemetry "$install_id" "$install_type" "$install_time"' for script_name in "${common_scripts[@]}"; do script="$repo_root/$script_name" @@ -93,12 +104,49 @@ for script_name in "${common_scripts[@]}"; do # shellcheck disable=SC2016 if grep -Fq 'activate_iot_install_mode "$iot_filtered_install"' "$script" || grep -Fq 'install_deb_iot_filter_config "$sudo_cmd"' "$script" || + grep -Fq 'install_iot_install_profile' "$script" || grep -Fq "DD_INFRASTRUCTURE_MODE='iot'" "$script" || grep -Fq 'iot_deb_filter_rollback' "$script"; then - fail "$script_name contains filtered IoT Task 3 orchestration" + fail "$script_name contains filtered IoT orchestration" fi done +failure_marker=$(mktemp) +rm -f "$failure_marker" +finalization_block=$(awk ' + /^# Complete service_management \(final stage\)/ { capture=1 } + capture { print } + capture && /^report_telemetry / { exit } +' "$iot_script") +set +e +# The extracted block consumes these variables and functions through eval. +# shellcheck disable=SC2034,SC2329 +( + set -e + services=() + no_start= + service_cmd=service + SUSE11= + sudo_cmd= + installed_agent_package_version=1:7.82.0-1 + variant=install_script_agent7_iot + etcdir=$(dirname "$failure_marker") + install_id=test-install-id + install_type=install_script + install_time=0 + end_stage() { return 73; } + install_iot_install_profile() { : > "$failure_marker"; } + report_telemetry() { return 0; } + eval "$finalization_block" +) +controlled_failure_status=$? +set -e +[[ $controlled_failure_status -eq 73 ]] || + fail "controlled pre-marker failure should retain its status" +[[ ! -e $failure_marker ]] || + fail "controlled pre-marker failure published an install profile" +rm -f "$failure_marker" + localtest="$repo_root/test/localtest.sh" grep -Fq 'verify_iot_debsums' "$localtest" || fail "localtest is missing fail-closed filtered debsums verification" grep -Fq 'infrastructure_mode: iot' "$localtest" || fail "localtest does not verify filtered IoT infrastructure mode" @@ -106,7 +154,14 @@ grep -Fq 'iot-installed-bytes.txt' "$localtest" || fail "localtest does not reco grep -Fq '/etc/dpkg/dpkg.cfg.d/99-datadog-iot' "$localtest" || fail "localtest does not verify the persistent dpkg filter" # shellcheck disable=SC2016 grep -Fq 'if ! verify_iot_dpkg_policy "$iot_filter"; then' "$localtest" || fail "localtest does not exercise the installed filtered IoT dpkg policy" -grep -Fq '/etc/datadog-agent/install_profile' "$localtest" || fail "localtest does not verify that the final profile marker is absent" +grep -Fq '/etc/datadog-agent/install_profile' "$localtest" || fail "localtest does not verify the final install profile" +# shellcheck disable=SC2016 +grep -Fq 'package_version: '\''$INSTALLED_PACKAGE_VERSION'\''' "$localtest" || + fail "localtest does not bind the install profile to the installed DEB version" +grep -Fq "stat -c '%u:%g:%a'" "$localtest" || + fail "localtest does not verify the install profile ownership and mode" +grep -Fq 'installer: install_script_agent7_iot' "$localtest" || + fail "localtest does not verify the install profile identity" grep -q '^test_iot_filtered_ubuntu_22_04:' "$repo_root/.gitlab-ci.yml" || fail "GitLab CI is missing the filtered IoT Ubuntu 22.04 job" diff --git a/test/localtest.sh b/test/localtest.sh index f78c8d61..5cf08036 100755 --- a/test/localtest.sh +++ b/test/localtest.sh @@ -289,6 +289,7 @@ fi OS_TYPE=$(get_os_type) INSTALLED_VERSION= +INSTALLED_PACKAGE_VERSION= RESULT=0 EXPECTED_MAJOR_VERSION=6 if [ "${SCRIPT_FLAVOR}" == "agent7" ] || [ "${SCRIPT_FLAVOR}" == "agent7_iot" ] || [ "${EXPECTED_FLAVOR}" != "datadog-agent" ] ; then @@ -311,7 +312,8 @@ if [[ "$OS_TYPE" == "ubuntu" ]]; then elif ! debsums -c "${EXPECTED_FLAVOR}"; then RESULT=1 fi - INSTALLED_VERSION=$(dpkg-query -W "${EXPECTED_FLAVOR}" | cut -f2 | cut -d: -f2) + INSTALLED_PACKAGE_VERSION=$(dpkg-query --show '--showformat=${Version}\n' "${EXPECTED_FLAVOR}") + INSTALLED_VERSION=${INSTALLED_PACKAGE_VERSION#*:} elif debsums -c datadog-agent ; then echo "[FAIL] datadog-agent should not be installed" RESULT=1 @@ -430,11 +432,25 @@ if [ "${SCRIPT_FLAVOR}" = "agent7_iot" ] && [ -z "$DD_NO_AGENT_INSTALL" ]; then else echo "[OK] Filtered IoT infrastructure mode is configured" fi - if [ -e /etc/datadog-agent/install_profile ] || [ -L /etc/datadog-agent/install_profile ]; then - echo "[FAIL] Final filtered IoT install profile marker must not be written by this draft" + iot_install_profile=/etc/datadog-agent/install_profile + expected_iot_install_profile="version: 1 +profile: iot-filtered +manifest: iot-v1 +package: datadog-agent +package_version: '$INSTALLED_PACKAGE_VERSION' +installer: install_script_agent7_iot" + if [ ! -f "$iot_install_profile" ] || [ -L "$iot_install_profile" ]; then + echo "[FAIL] Final filtered IoT install profile is missing or is not a regular file" + RESULT=1 + elif [ "$(stat -c '%u:%g:%a' "$iot_install_profile")" != "0:0:644" ]; then + echo "[FAIL] Final filtered IoT install profile must be root:root mode 0644" + RESULT=1 + elif [ "$(cat "$iot_install_profile")" != "$expected_iot_install_profile" ]; then + echo "[FAIL] Final filtered IoT install profile does not match the installed package version and reviewed schema" + diff -u <(printf '%s\n' "$expected_iot_install_profile") "$iot_install_profile" || true RESULT=1 else - echo "[OK] Final filtered IoT install profile marker is absent" + echo "[OK] Final filtered IoT install profile matches the installed package version, schema, ownership, and mode" fi if ! verify_iot_filtered_layout; then RESULT=1 diff --git a/unit_tests/test_iot_deb_orchestration.sh b/unit_tests/test_iot_deb_orchestration.sh index 5aa725b4..0b2d4806 100755 --- a/unit_tests/test_iot_deb_orchestration.sh +++ b/unit_tests/test_iot_deb_orchestration.sh @@ -328,6 +328,67 @@ testDebFilterRootInstallerCanRetainInstallerOnlyDuringPackageConfiguration() { "grep -q '^path-include=/opt/datadog-agent/embedded/bin/installer$' '$destination'" } +testInstallIotProfileUsesRootShellWrapperAndPublishesExactMarker() { + local destination=$TEST_ROOT/etc/datadog-agent/install_profile + local sudo_arguments=$TEST_ROOT/profile-sudo-arguments + local expected + + mkdir -p "$(dirname "$destination")" + # shellcheck disable=SC2329 + sudo() { + printf '%s\n' "$@" > "$sudo_arguments" + "$@" + } + + install_iot_install_profile sudo "$destination" "1:7.82.0-1" install_script_agent7_iot + assertEquals "root-context profile publication should succeed" 0 $? + assertEquals "the wrapper should invoke a shell, not a shell function" sh "$(head -n 1 "$sudo_arguments")" + assertTrue "the profile should be a regular file" "[ -f '$destination' ]" + assertFalse "the profile should not be a symlink" "[ -L '$destination' ]" + assertEquals "the profile owner and mode" "0:0:644" "$(stat -c '%u:%g:%a' "$destination")" + expected='version: 1 +profile: iot-filtered +manifest: iot-v1 +package: datadog-agent +package_version: '\''1:7.82.0-1'\'' +installer: install_script_agent7_iot' + assertEquals "the published profile should match the reviewed schema" "$expected" "$(cat "$destination")" +} + +testInstallIotProfilePreservesPreviousMarkerWhenRootReplacementFails() { + local destination=$TEST_ROOT/etc/datadog-agent/install_profile + local fake_bin=$TEST_ROOT/bin + local real_mv + local output + local status + + mkdir -p "$(dirname "$destination")" "$fake_bin" + printf 'previous marker\n' > "$destination" + chmod 0600 "$destination" + real_mv=$(command -v mv) + cat > "$fake_bin/mv" <&1) + status=$? + + assertNotEquals "a failed root replacement should return nonzero" 0 "$status" + assertEquals "a prior marker should remain intact" "previous marker" "$(cat "$destination")" + assertEquals "a prior marker should retain its mode" 600 "$(stat -c '%a' "$destination")" + assertEquals "same-directory root temporary files should be cleaned" 0 \ + "$(find "$(dirname "$destination")" -name '.install_profile.root.*' | wc -l | tr -d ' ')" +} + testAptFailureRemovesTransientFilterWhenNoPriorFilterExisted() { local destination=$TEST_ROOT/etc/dpkg/dpkg.cfg.d/99-datadog-iot local rollback_directory