diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 7d478d86f5..dce8ee3953 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -1,40 +1,88 @@ #!/usr/bin/env bash set -Eeuo pipefail - umask 022 - CONFIG_ROOT="${POWERFORGE_SERVICE_CONFIG_ROOT:-/etc/powerforge/services}" LOCK_ROOT="${POWERFORGE_SERVICE_LOCK_ROOT:-/var/lock}" TRUSTED_STAGE_ROOT="${POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT:-/var/lib/powerforge/service-deployment-staging}" -service_id="" -archive="" -metadata="" -promoted=0 -previous_target="" -release_dir="" -workflow_stage="" -trusted_stage="" - +TRANSACTION_ROOT="${POWERFORGE_SERVICE_TRANSACTION_ROOT:-/var/lib/powerforge/service-deployment-state}" +SYSTEMD_CONFIG_ROOT="${POWERFORGE_SYSTEMD_CONFIG_ROOT:-/etc/systemd/system}" +deployment_shell_pid="$BASHPID" +service_id="" archive="" metadata="" +promoted=0 previous_target="" release_dir="" candidate_link="" +workflow_stage="" trusted_stage="" +systemd_drop_in_backup="" systemd_drop_in_dir="" systemd_drop_in_path="" +systemd_drop_in_existed=0 systemd_drop_in_owner="" systemd_drop_in_group="" systemd_drop_in_mode="" +systemd_write_paths_snapshot_ready=0 systemd_transaction_path="" cleanup_staging() { [[ -z "$workflow_stage" || ! -d "$workflow_stage" ]] || rm -rf -- "$workflow_stage" [[ -z "$trusted_stage" || ! -d "$trusted_stage" ]] || rm -rf -- "$trusted_stage" } - trap cleanup_staging EXIT - log() { printf '[powerforge-service-deploy] %s\n' "$*" } - fail() { log "ERROR: $*" >&2 return 1 } - +assert_trusted_directory_chain() { + local declared_path="$1" + local description="$2" + local deployment_uid component current owner mode + local -a components + deployment_uid="$(id -u)" + current='/' + IFS='/' read -r -a components <<<"${declared_path#/}" + for component in "${components[@]}"; do + [[ -n "$component" ]] || continue + current="${current%/}/$component" + [[ -d "$current" && ! -L "$current" ]] || fail "$description must be a real directory: $current" + owner="$(stat -c '%u' -- "$current")" + mode="$(stat -c '%a' -- "$current")" + [[ "$owner" -eq 0 || "$owner" -eq "$deployment_uid" ]] || fail "$description has an untrusted owner: $current" + (( (8#$mode & 0022) == 0 )) || fail "$description must not be group/world writable: $current" + done +} +paths_overlap() { + local first="$1" + local second="$2" + [[ "$first" == "$second" || "$first" == "$second"/* || "$second" == "$first"/* ]] +} +prepare_transaction_root() { + local parent resolved + [[ "$TRANSACTION_ROOT" == /* && "$TRANSACTION_ROOT" != '/' && "$TRANSACTION_ROOT" != *[[:space:]]* ]] || + fail 'Transaction root must be an absolute non-root path without whitespace.' + parent="$(dirname -- "$TRANSACTION_ROOT")" + assert_trusted_directory_chain "$parent" 'Transaction root parent' + if [[ -e "$TRANSACTION_ROOT" || -L "$TRANSACTION_ROOT" ]]; then + [[ -d "$TRANSACTION_ROOT" && ! -L "$TRANSACTION_ROOT" ]] || fail "Transaction root must be a real directory: $TRANSACTION_ROOT" + else + install -d -m 0700 "$TRANSACTION_ROOT" + fi + assert_trusted_directory_chain "$TRANSACTION_ROOT" 'Transaction root' + resolved="$(realpath -e -- "$TRANSACTION_ROOT")" + [[ "$resolved" == "$TRANSACTION_ROOT" ]] || fail "Transaction root must be canonical and contain no symlinked components: $TRANSACTION_ROOT" + TRANSACTION_ROOT="$resolved" +} +prepare_trusted_stage_root() { + local parent resolved + [[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' && "$TRUSTED_STAGE_ROOT" != *[[:space:]]* ]] || + fail 'Trusted staging root must be an absolute non-root path without whitespace.' + parent="$(dirname -- "$TRUSTED_STAGE_ROOT")" + assert_trusted_directory_chain "$parent" 'Trusted staging root parent' + if [[ -e "$TRUSTED_STAGE_ROOT" || -L "$TRUSTED_STAGE_ROOT" ]]; then + [[ -d "$TRUSTED_STAGE_ROOT" && ! -L "$TRUSTED_STAGE_ROOT" ]] || fail "Trusted staging root must be a real directory: $TRUSTED_STAGE_ROOT" + else + install -d -m 0700 "$TRUSTED_STAGE_ROOT" + fi + assert_trusted_directory_chain "$TRUSTED_STAGE_ROOT" 'Trusted staging root' + resolved="$(realpath -e -- "$TRUSTED_STAGE_ROOT")" + [[ "$resolved" == "$TRUSTED_STAGE_ROOT" ]] || fail "Trusted staging root must be canonical and contain no symlinked components: $TRUSTED_STAGE_ROOT" + TRUSTED_STAGE_ROOT="$resolved" +} usage() { echo 'Usage: powerforge-service-deploy --service ' } - while (($# > 0)); do case "$1" in --service) @@ -51,12 +99,307 @@ while (($# > 0)); do ;; esac done - [[ "$service_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail 'Invalid service identifier.' workflow_stage="/tmp/powerforge-service-${service_id}" archive="$workflow_stage/artifact.tar" metadata="$workflow_stage/deployment.json" - +assert_trusted_systemd_path() { + local declared_path="$1" + [[ ! -L "$declared_path" ]] || fail "Systemd writable path must not be a symlink: $declared_path" + assert_trusted_directory_chain "$(dirname -- "$declared_path")" 'Systemd writable path parent' +} +prepare_systemd_drop_in_directory() { + local config_parent + config_parent="$(dirname -- "$SYSTEMD_CONFIG_ROOT")" + assert_trusted_directory_chain "$config_parent" 'Systemd config parent' || return 1 + if [[ -e "$SYSTEMD_CONFIG_ROOT" || -L "$SYSTEMD_CONFIG_ROOT" ]]; then + [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || { fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT"; return 1; } + else + install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" || return 1 + fi + assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' || return 1 + [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || { fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT"; return 1; } + systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" + systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" + if [[ -e "$systemd_drop_in_dir" || -L "$systemd_drop_in_dir" ]]; then + [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || { fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir"; return 1; } + else + install -d -m 0755 "$systemd_drop_in_dir" || return 1 + fi + assert_trusted_directory_chain "$systemd_drop_in_dir" 'Systemd drop-in directory' || return 1 +} +prepare_service_release_root() { + [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || { fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT"; return 1; } + assert_trusted_directory_chain "$SERVICE_ROOT" 'Service root' || return 1 + resolved_service_root="$(realpath -e -- "$SERVICE_ROOT")" || return 1 + [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || { fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT"; return 1; } + SERVICE_ROOT="$resolved_service_root" + resolved_release_root="${SERVICE_ROOT}/releases" + if [[ -e "$resolved_release_root" || -L "$resolved_release_root" ]]; then + [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || { fail "Release root must be a real directory: $resolved_release_root"; return 1; } + else + install -d -m 0755 "$resolved_release_root" || return 1 + fi + assert_trusted_directory_chain "$resolved_release_root" 'Release root' || return 1 + [[ "$(realpath -e -- "$resolved_release_root")" == "$resolved_release_root" ]] || fail "Release root must be canonical and contain no symlinked components: $resolved_release_root" +} +snapshot_systemd_write_paths() { + local transaction_temporary drop_in_mode + [[ ! -e "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "An incomplete systemd writable-path transaction already exists: $systemd_transaction_path" + transaction_temporary="$(mktemp -d "${TRANSACTION_ROOT}/.service-${service_id}.XXXXXXXX")" + chmod 0700 "$transaction_temporary" + systemd_drop_in_existed=0 + systemd_drop_in_owner="" + systemd_drop_in_group="" + systemd_drop_in_mode="" + printf '%s\n' "$SERVICE_ROOT" >"$transaction_temporary/service-root" + printf '%s\n' "$SYSTEMD_SERVICE" >"$transaction_temporary/systemd-service" + printf '%s\n' "$SYSTEMD_CONFIG_ROOT" >"$transaction_temporary/systemd-config-root" + printf '%s\n' "$previous_target" >"$transaction_temporary/previous-target" + if [[ -e "$systemd_drop_in_path" || -L "$systemd_drop_in_path" ]]; then + [[ -f "$systemd_drop_in_path" && ! -L "$systemd_drop_in_path" ]] || fail "PowerForge systemd drop-in must be a regular file: $systemd_drop_in_path" + if [[ "$(id -u)" -eq 0 ]]; then + [[ "$(stat -c '%u' -- "$systemd_drop_in_path")" -eq 0 ]] || fail "PowerForge systemd drop-in must be owned by root: $systemd_drop_in_path" + drop_in_mode="$(stat -c '%a' -- "$systemd_drop_in_path")" + (( (8#$drop_in_mode & 0022) == 0 )) || fail "PowerForge systemd drop-in must not be group/world writable: $systemd_drop_in_path" + fi + systemd_drop_in_owner="$(stat -c '%u' -- "$systemd_drop_in_path")" + systemd_drop_in_group="$(stat -c '%g' -- "$systemd_drop_in_path")" + systemd_drop_in_mode="$(stat -c '%a' -- "$systemd_drop_in_path")" + if ! install -m 0600 "$systemd_drop_in_path" "$transaction_temporary/drop-in"; then + rm -rf -- "$transaction_temporary" + return 1 + fi + printf 'present\n' >"$transaction_temporary/drop-in-state" + printf '%s\n' "$systemd_drop_in_owner" >"$transaction_temporary/drop-in-owner" + printf '%s\n' "$systemd_drop_in_group" >"$transaction_temporary/drop-in-group" + printf '%s\n' "$systemd_drop_in_mode" >"$transaction_temporary/drop-in-mode" + systemd_drop_in_existed=1 + else + printf 'absent\n' >"$transaction_temporary/drop-in-state" + fi + chmod 0600 "$transaction_temporary"/* + sync -f "$transaction_temporary"/* + sync -f "$transaction_temporary" + mv -- "$transaction_temporary" "$systemd_transaction_path" + sync -f "$TRANSACTION_ROOT" + systemd_drop_in_backup="${systemd_transaction_path}/drop-in" + systemd_write_paths_snapshot_ready=1 +} +load_systemd_write_paths_transaction() { + local stored_root stored_service stored_systemd_root stored_state + [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" + assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' + stored_root="$(<"$systemd_transaction_path/service-root")" + stored_service="$(<"$systemd_transaction_path/systemd-service")" + stored_systemd_root="$(<"$systemd_transaction_path/systemd-config-root")" + previous_target="$(<"$systemd_transaction_path/previous-target")" + stored_state="$(<"$systemd_transaction_path/drop-in-state")" + [[ "$stored_root" == "$SERVICE_ROOT" ]] || fail "Incomplete transaction belongs to a different service root: $stored_root" + [[ "$stored_service" == "$SYSTEMD_SERVICE" ]] || fail "Incomplete transaction belongs to a different systemd unit: $stored_service" + [[ "$stored_systemd_root" == "$SYSTEMD_CONFIG_ROOT" ]] || fail "Incomplete transaction belongs to a different systemd config root: $stored_systemd_root" + if [[ -n "$previous_target" ]]; then + [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || + fail "Incomplete transaction contains an invalid previous release: $previous_target" + fi + if [[ "$stored_state" == 'present' ]]; then + [[ -f "$systemd_transaction_path/drop-in" && ! -L "$systemd_transaction_path/drop-in" ]] || + fail 'Incomplete transaction is missing its systemd drop-in backup.' + systemd_drop_in_owner="$(<"$systemd_transaction_path/drop-in-owner")" + systemd_drop_in_group="$(<"$systemd_transaction_path/drop-in-group")" + systemd_drop_in_mode="$(<"$systemd_transaction_path/drop-in-mode")" + [[ "$systemd_drop_in_owner" =~ ^[0-9]+$ && "$systemd_drop_in_group" =~ ^[0-9]+$ && "$systemd_drop_in_mode" =~ ^[0-7]{3,4}$ ]] || + fail 'Incomplete transaction contains invalid systemd drop-in metadata.' + systemd_drop_in_backup="${systemd_transaction_path}/drop-in" + systemd_drop_in_existed=1 + elif [[ "$stored_state" == 'absent' ]]; then + systemd_drop_in_backup="" + systemd_drop_in_existed=0 + else + fail "Incomplete transaction contains an invalid drop-in state: $stored_state" + fi + systemd_write_paths_snapshot_ready=1 +} +restore_systemd_write_paths() { + local restore_temporary="" + [[ "$systemd_write_paths_snapshot_ready" == '1' ]] || return 0 + if [[ "$systemd_drop_in_existed" == '1' ]]; then + restore_temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.restore.XXXXXXXX")" || return 1 + if ! install -m "$systemd_drop_in_mode" "$systemd_drop_in_backup" "$restore_temporary" || + ! chown "$systemd_drop_in_owner:$systemd_drop_in_group" "$restore_temporary" || + ! mv -f -- "$restore_temporary" "$systemd_drop_in_path"; then + rm -f -- "$restore_temporary" + return 1 + fi + else + rm -f -- "$systemd_drop_in_path" || return 1 + fi + systemctl daemon-reload || return 1 +} +finish_systemd_write_paths_transaction() { + [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/service-*.transaction ]] || + fail "Refusing to remove an unexpected transaction path: $systemd_transaction_path" + sync_deployment_state || return 1 + rm -rf -- "$systemd_transaction_path" || return 1 + sync -f "$TRANSACTION_ROOT" || return 1 + systemd_write_paths_snapshot_ready=0 + systemd_drop_in_backup="" +} +sync_deployment_state() { + if [[ -f "$systemd_drop_in_path" ]]; then + sync -f "$systemd_drop_in_path" || return 1 + fi + sync -f "$systemd_drop_in_dir" || return 1 + sync -f "$SERVICE_ROOT" || return 1 + sync -f "$resolved_release_root" || return 1 +} +commit_systemd_write_paths() { + local committed_path="${systemd_transaction_path}.committed" + # The rename is the durable commit point; ignore catchable termination so a signal cannot run rollback after it has + # disappeared but before the in-memory state reflects that fact. + sync_deployment_state || return 1 + trap - INT TERM + mv -- "$systemd_transaction_path" "$committed_path" || return 1 + promoted=0 + systemd_write_paths_snapshot_ready=0 + systemd_drop_in_backup="" + if ! sync -f "$TRANSACTION_ROOT"; then + log "ERROR: committed transaction retained until its directory can be synchronized: $committed_path" >&2 + return 1 + fi + rm -rf -- "$committed_path" || log "WARNING: committed transaction cleanup remains at $committed_path" >&2 + sync -f "$TRANSACTION_ROOT" || log "WARNING: committed transaction cleanup was not synchronized in $TRANSACTION_ROOT" >&2 +} +report_systemd_restore_failure() { + log "ERROR: failed to restore systemd writable paths; transaction retained at $systemd_transaction_path" >&2 +} +restore_previous_current_link() { + local rollback_link="" + if [[ -n "$previous_target" ]]; then + rollback_link="$SERVICE_ROOT/.current.rollback.$$" + rm -f -- "$rollback_link" + ln -s "$previous_target" "$rollback_link" || return 1 + mv -Tf "$rollback_link" "$SERVICE_ROOT/current" || { rm -f -- "$rollback_link"; return 1; } + [[ "$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null)" == "$previous_target" ]] + else + rm -f -- "$SERVICE_ROOT/current" + [[ ! -e "$SERVICE_ROOT/current" && ! -L "$SERVICE_ROOT/current" ]] + fi +} +peek_systemd_transaction_identity() { + [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" + assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' + transaction_service_root="$(<"$systemd_transaction_path/service-root")" + transaction_systemd_service="$(<"$systemd_transaction_path/systemd-service")" + transaction_systemd_config_root="$(<"$systemd_transaction_path/systemd-config-root")" + [[ "$transaction_service_root" == /* && "$transaction_service_root" != '/' && "$transaction_service_root" != *[[:space:]]* ]] || + fail 'Incomplete transaction contains an invalid service root.' + [[ "$transaction_systemd_service" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || + fail 'Incomplete transaction contains an invalid systemd unit.' + [[ "$transaction_systemd_config_root" == /* && "$transaction_systemd_config_root" != '/' && "$transaction_systemd_config_root" != *[[:space:]]* ]] || + fail 'Incomplete transaction contains an invalid systemd config root.' +} +settle_committed_transaction() { + local committed="$1" + [[ -e "$committed" || -L "$committed" ]] || return 0 + [[ -d "$committed" && ! -L "$committed" ]] || fail "Committed transaction marker must be a real directory: $committed" + assert_trusted_directory_chain "$committed" 'Committed transaction marker' + sync -f "$TRANSACTION_ROOT" || fail "Committed transaction directory is not durable: $committed" + rm -rf -- "$committed" || fail "Committed transaction marker could not be removed: $committed" + sync -f "$TRANSACTION_ROOT" || fail "Committed transaction cleanup is not durable: $committed" +} +recover_incomplete_systemd_transaction() { + local permissions_restored=1 current_restored=1 service_safe=0 + if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then + log "Recovering incomplete systemd writable-path transaction for $SYSTEMD_SERVICE." + if ! load_systemd_write_paths_transaction; then + systemctl stop "$SYSTEMD_SERVICE" || true + fail "Incomplete deployment transaction is invalid; $SYSTEMD_SERVICE was stopped and operator recovery is required." + fi + restore_systemd_write_paths || permissions_restored=0 + restore_previous_current_link || current_restored=0 + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' && -n "$previous_target" ]]; then + if systemctl restart "$SYSTEMD_SERVICE"; then + service_safe=1 + else + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + else + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + if [[ "$permissions_restored" != '1' || "$current_restored" != '1' || "$service_safe" != '1' ]]; then + report_systemd_restore_failure + fail "Incomplete deployment recovery could not prove $SYSTEMD_SERVICE safe." + fi + finish_systemd_write_paths_transaction + log "Recovered incomplete systemd writable-path transaction for $SYSTEMD_SERVICE." + fi +} +recover_selected_transaction() { + SYSTEMD_SERVICE="$transaction_systemd_service" + SERVICE_ROOT="$transaction_service_root" + SYSTEMD_CONFIG_ROOT="$transaction_systemd_config_root" + local preparation_status=0 + prepare_service_release_root || preparation_status=$? + if [[ "$preparation_status" -eq 0 ]]; then prepare_systemd_drop_in_directory || preparation_status=$?; fi + if [[ "$preparation_status" -ne 0 ]]; then + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop recorded unit $SYSTEMD_SERVICE after recovery preparation failed." >&2 + fail "Recorded recovery paths are unavailable; $SYSTEMD_SERVICE was stopped and operator recovery is required." + fi + recover_incomplete_systemd_transaction +} +reconcile_systemd_write_paths() ( + if ((${#systemd_read_write_paths[@]} == 0)); then + [[ ! -f "$systemd_drop_in_path" ]] || rm -f -- "$systemd_drop_in_path" + systemctl daemon-reload + return 0 + fi + temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.XXXXXXXX")" + trap 'rm -f -- "$temporary"' EXIT + { + printf '[Service]\n' + for path in "${systemd_read_write_paths[@]}"; do + printf 'ReadWritePaths=%s\n' "$path" + done + } >"$temporary" + chmod 0644 "$temporary" + if [[ -f "$systemd_drop_in_path" ]] && cmp -s -- "$temporary" "$systemd_drop_in_path"; then + rm -f -- "$temporary" + else + mv -f -- "$temporary" "$systemd_drop_in_path" + fi + systemctl daemon-reload +) +mkdir -p "$LOCK_ROOT" +exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" +flock -n 9 || fail "Another deployment is active for $service_id." +prepare_transaction_root +requested_systemd_config_root="$SYSTEMD_CONFIG_ROOT" +own_transaction="${TRANSACTION_ROOT}/service-${service_id}.transaction" +own_committed="${own_transaction}.committed" +settle_committed_transaction "$own_committed" +if [[ -e "$own_transaction" || -L "$own_transaction" ]]; then + systemd_transaction_path="$own_transaction" + peek_systemd_transaction_identity + unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" + service_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" + exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n 8 || fail "Another deployment is active for systemd unit $transaction_systemd_service." + exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n 7 || fail "Another deployment is active for service root $transaction_service_root." + recover_selected_transaction + exec 8>&- 7>&- +fi +SYSTEMD_CONFIG_ROOT="$requested_systemd_config_root" +[[ "$CONFIG_ROOT" == /* && "$CONFIG_ROOT" != '/' && "$CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Service config root must be an absolute non-root path without whitespace.' +[[ -d "$CONFIG_ROOT" && ! -L "$CONFIG_ROOT" ]] || fail "Service config root must be a real directory: $CONFIG_ROOT" +assert_trusted_directory_chain "$CONFIG_ROOT" 'Service config root' +resolved_config_root="$(realpath -e -- "$CONFIG_ROOT")" +[[ "$resolved_config_root" == "$CONFIG_ROOT" ]] || fail "Service config root must be canonical and contain no symlinked components: $CONFIG_ROOT" +CONFIG_ROOT="$resolved_config_root" config_path="${CONFIG_ROOT}/${service_id}.env" [[ -f "$config_path" && ! -L "$config_path" ]] || fail "Service is not configured: $service_id" if [[ "$(id -u)" -eq 0 ]]; then @@ -64,34 +407,151 @@ if [[ "$(id -u)" -eq 0 ]]; then config_mode="$(stat -c '%a' "$config_path")" (( (8#$config_mode & 0022) == 0 )) || fail "Service config must not be group/world writable: $config_path" fi - -# The config is trusted, root-owned operator input and contains no values supplied by the workflow. +unset SERVICE_ROOT SYSTEMD_SERVICE SYSTEMD_READ_WRITE_PATHS LOCAL_HEALTH_URL RELEASES_TO_KEEP REQUIRED_RELEASE_PATHS PUBLIC_HEALTH_URLS REQUIRE_HEALTH_PROVENANCE # shellcheck disable=SC1090 source "$config_path" - -: "${SERVICE_ROOT:?SERVICE_ROOT is required in $config_path}" -: "${SYSTEMD_SERVICE:?SYSTEMD_SERVICE is required in $config_path}" -: "${LOCAL_HEALTH_URL:?LOCAL_HEALTH_URL is required in $config_path}" -: "${RELEASES_TO_KEEP:=5}" -: "${REQUIRED_RELEASE_PATHS:=}" -: "${PUBLIC_HEALTH_URLS:=}" -: "${REQUIRE_HEALTH_PROVENANCE:=1}" - -[[ "$SERVICE_ROOT" == /* && "$SERVICE_ROOT" != '/' ]] || fail 'SERVICE_ROOT must be an absolute non-root path.' -[[ "$SERVICE_ROOT" != *[[:space:]]* ]] || fail 'SERVICE_ROOT must not contain whitespace.' +: "${SERVICE_ROOT:?SERVICE_ROOT is required in $config_path}" "${SYSTEMD_SERVICE:?SYSTEMD_SERVICE is required in $config_path}" "${LOCAL_HEALTH_URL:?LOCAL_HEALTH_URL is required in $config_path}" +: "${SYSTEMD_READ_WRITE_PATHS:=}" "${RELEASES_TO_KEEP:=5}" "${REQUIRED_RELEASE_PATHS:=}" "${PUBLIC_HEALTH_URLS:=}" "${REQUIRE_HEALTH_PROVENANCE:=1}" +[[ "$SERVICE_ROOT" == /* && "$SERVICE_ROOT" != '/' && "$SERVICE_ROOT" != *[[:space:]]* ]] || fail 'SERVICE_ROOT must be an absolute non-root path without whitespace.' [[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' ]] || fail 'Trusted staging root must be an absolute non-root path.' +[[ "$SYSTEMD_CONFIG_ROOT" == /* && "$SYSTEMD_CONFIG_ROOT" != '/' && "$SYSTEMD_CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Systemd config root must be an absolute non-root path without whitespace.' [[ "$SYSTEMD_SERVICE" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || fail 'SYSTEMD_SERVICE must be a systemd service unit name.' [[ "$LOCAL_HEALTH_URL" =~ ^https?://[^[:space:]]+$ ]] || fail 'LOCAL_HEALTH_URL must be an HTTP or HTTPS URL.' [[ "$RELEASES_TO_KEEP" =~ ^[1-9][0-9]*$ ]] || fail 'RELEASES_TO_KEEP must be a positive integer.' [[ "$REQUIRE_HEALTH_PROVENANCE" == '0' || "$REQUIRE_HEALTH_PROVENANCE" == '1' ]] || fail 'REQUIRE_HEALTH_PROVENANCE must be 0 or 1.' -for health_url in $PUBLIC_HEALTH_URLS; do - [[ "$health_url" =~ ^https://[^[:space:]]+$ ]] || fail "Public health URL must use HTTPS: $health_url" +for health_url in $PUBLIC_HEALTH_URLS; do [[ "$health_url" =~ ^https://[^[:space:]]+$ ]] || fail "Public health URL must use HTTPS: $health_url"; done +read -r -a configured_systemd_read_write_paths <<<"$SYSTEMD_READ_WRITE_PATHS" +for read_write_path in "${configured_systemd_read_write_paths[@]}"; do + [[ "$read_write_path" == /* && "$read_write_path" != '/' ]] || fail 'Systemd writable paths must be absolute non-root paths.' + [[ "$read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Systemd writable path contains unsupported characters: $read_write_path" + [[ "/${read_write_path#/}/" != *'/../'* ]] || fail "Systemd writable path must not contain traversal: $read_write_path" +done +configured_service_root="$SERVICE_ROOT" +configured_service_root_identity="$(realpath -e -- "$SERVICE_ROOT" 2>/dev/null || true)" +configured_systemd_service="$SYSTEMD_SERVICE" +configured_systemd_config_root="$SYSTEMD_CONFIG_ROOT" +for candidate_committed in "$TRANSACTION_ROOT"/service-*.transaction.committed; do + [[ "$candidate_committed" != "$own_committed" && ( -e "$candidate_committed" || -L "$candidate_committed" ) ]] || continue + systemd_transaction_path="$candidate_committed" + peek_systemd_transaction_identity + if [[ "$transaction_systemd_service" == "$configured_systemd_service" || "$transaction_service_root" == "$configured_service_root" || ( -n "$configured_service_root_identity" && "$transaction_service_root" == "$configured_service_root_identity" ) ]]; then + transaction_id="${candidate_committed##*/service-}" + transaction_id="${transaction_id%.transaction.committed}" + [[ "$transaction_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail "Committed transaction has an invalid service identity: $candidate_committed" + exec {committed_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${transaction_id}.lock" + flock -n "$committed_service_lock_fd" || fail "Another deployment is active for $transaction_id." + settle_committed_transaction "$candidate_committed" + fi +done +related_transactions=() +related_units=() +related_ids=() +related_roots=() +for candidate_transaction in "$TRANSACTION_ROOT"/service-*.transaction; do + [[ -e "$candidate_transaction" || -L "$candidate_transaction" ]] || continue + systemd_transaction_path="$candidate_transaction" + peek_systemd_transaction_identity + transaction_id="${candidate_transaction##*/service-}" + transaction_id="${transaction_id%.transaction}" + [[ "$transaction_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail "Incomplete transaction has an invalid service identity: $candidate_transaction" + if [[ "$transaction_systemd_service" == "$configured_systemd_service" || "$transaction_service_root" == "$configured_service_root" || ( -n "$configured_service_root_identity" && "$transaction_service_root" == "$configured_service_root_identity" ) ]]; then + related_transactions+=("$candidate_transaction") + related_units+=("$transaction_systemd_service") + related_ids+=("$transaction_id") + related_roots+=("$transaction_service_root") + fi +done +if ((${#related_transactions[@]} > 1)); then + declare -A held_related_units=() held_related_roots=() + for index in "${!related_ids[@]}"; do + related_id="${related_ids[$index]}" + if [[ "$related_id" != "$service_id" ]]; then + exec {related_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${related_id}.lock" + flock -n "$related_service_lock_fd" || fail "Another deployment is active for $related_id." + fi + unit_lock_key="$(printf '%s' "${related_units[$index]}" | sha256sum | awk '{print $1}')" + if [[ -z "${held_related_units[$unit_lock_key]:-}" ]]; then + exec {related_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n "$related_unit_lock_fd" || fail "Another deployment is active for systemd unit ${related_units[$index]}." + held_related_units[$unit_lock_key]="$related_unit_lock_fd" + fi + service_root_lock_key="$(printf '%s' "${related_roots[$index]}" | sha256sum | awk '{print $1}')" + if [[ -z "${held_related_roots[$service_root_lock_key]:-}" ]]; then + exec {related_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n "$related_root_lock_fd" || fail "Another deployment is active for service root ${related_roots[$index]}." + held_related_roots[$service_root_lock_key]="$related_root_lock_fd" + fi + done + declare -A stopped_related_units=() + for related_unit in "${related_units[@]}"; do + [[ -n "${stopped_related_units[$related_unit]:-}" ]] && continue + systemctl stop "$related_unit" || fail "Failed to stop ambiguous recorded unit $related_unit." + stopped_related_units[$related_unit]=1 + done + fail 'Multiple incomplete transactions overlap this deployment; recorded units were stopped and operator recovery is required.' +fi +locked_systemd_service="" +locked_service_root="" +if ((${#related_transactions[@]} == 1)); then + systemd_transaction_path="${related_transactions[0]}" + peek_systemd_transaction_identity + transaction_id="${systemd_transaction_path##*/service-}" + transaction_id="${transaction_id%.transaction}" + if [[ "$transaction_id" != "$service_id" ]]; then + exec {transaction_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${transaction_id}.lock" + flock -n "$transaction_service_lock_fd" || fail "Another deployment is active for $transaction_id." + fi + unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" + service_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" + exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n 8 || fail "Another deployment is active for systemd unit $transaction_systemd_service." + exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n 7 || fail "Another deployment is active for service root $transaction_service_root." + locked_systemd_service="$transaction_systemd_service" + locked_service_root="$transaction_service_root" + recover_selected_transaction + SYSTEMD_SERVICE="$configured_systemd_service" + SERVICE_ROOT="$configured_service_root" + SYSTEMD_CONFIG_ROOT="$configured_systemd_config_root" +fi +prepare_service_release_root +for deployment_control_root in "$CONFIG_ROOT" "$SYSTEMD_CONFIG_ROOT" "$TRANSACTION_ROOT" "$TRUSTED_STAGE_ROOT" "$(realpath -e -- "$LOCK_ROOT")"; do + paths_overlap "$deployment_control_root" "$resolved_release_root" && fail "Deployment control path must not overlap release storage: $deployment_control_root" +done +prepare_trusted_stage_root +if [[ "$locked_systemd_service" != "$SYSTEMD_SERVICE" ]]; then + unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" + exec {configured_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n "$configured_unit_lock_fd" || fail "Another deployment is active for systemd unit $SYSTEMD_SERVICE." +fi +if [[ "$locked_service_root" != "$SERVICE_ROOT" ]]; then + service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" + exec {configured_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n "$configured_root_lock_fd" || fail "Another deployment is active for service root $SERVICE_ROOT." +fi +systemd_transaction_path="${TRANSACTION_ROOT}/service-${service_id}.transaction" +prepare_systemd_drop_in_directory +previous_target="" +if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then + [[ -L "$SERVICE_ROOT/current" ]] || fail 'Current release pointer must be a symlink.' + previous_target="$(readlink -f "$SERVICE_ROOT/current")" + [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || fail 'Current release pointer must resolve inside the canonical release root.' +fi +systemd_read_write_paths=() +for read_write_path in "${configured_systemd_read_write_paths[@]}"; do + [[ -d "$read_write_path" ]] || fail "Systemd writable path does not exist: $read_write_path" + assert_trusted_systemd_path "$read_write_path" + resolved_read_write_path="$(realpath -e -- "$read_write_path")" + [[ -d "$resolved_read_write_path" && "$resolved_read_write_path" != '/' ]] || fail "Systemd writable path is not a safe directory: $read_write_path" + [[ "$resolved_read_write_path" == "$read_write_path" ]] || fail "Systemd writable path must be canonical and contain no symlinked components: $read_write_path" + [[ "$resolved_read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Resolved systemd writable path contains unsupported characters: $read_write_path" + protected_roots=("$CONFIG_ROOT" "$SYSTEMD_CONFIG_ROOT" "$TRANSACTION_ROOT" "$TRUSTED_STAGE_ROOT" "$SERVICE_ROOT" "$(realpath -e -- "$LOCK_ROOT")") + for protected_root in "${protected_roots[@]}"; do + paths_overlap "$resolved_read_write_path" "$protected_root" || continue + fail "Systemd writable path must not overlap deployment control path $protected_root: $read_write_path" + done + systemd_read_write_paths+=("$resolved_read_write_path") done - -mkdir -p "$LOCK_ROOT" -exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" -flock -n 9 || fail "Another deployment is active for $service_id." - archive="$(realpath -e "$archive")" metadata="$(realpath -e "$metadata")" [[ -f "$archive" && ! -L "$archive" ]] || fail 'Artifact must be a regular file, not a symlink.' @@ -102,7 +562,6 @@ if [[ -n "${SUDO_UID:-}" ]]; then [[ "$(stat -c '%u' "$archive")" -eq "$SUDO_UID" ]] || fail 'Artifact owner does not match the invoking deployment account.' [[ "$(stat -c '%u' "$metadata")" -eq "$SUDO_UID" ]] || fail 'Metadata owner does not match the invoking deployment account.' fi - install -d -m 0700 "$TRUSTED_STAGE_ROOT" trusted_stage="$(mktemp -d "${TRUSTED_STAGE_ROOT}/${service_id}.XXXXXXXX")" chmod 0700 "$trusted_stage" @@ -110,12 +569,10 @@ install -m 0600 "$archive" "$trusted_stage/artifact.tar" install -m 0600 "$metadata" "$trusted_stage/deployment.json" archive="$trusted_stage/artifact.tar" metadata="$trusted_stage/deployment.json" - json_string() { local key="$1" sed -n "s/.*\"${key}\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\".*/\1/p" "$metadata" | head -n 1 } - source_sha="$(json_string sourceSha)" artifact_sha="$(json_string artifactSha256)" run_id="$(json_string workflowRunId)" @@ -125,28 +582,22 @@ run_attempt="$(json_string workflowRunAttempt)" [[ "$run_id" =~ ^[0-9]+$ && "$run_attempt" =~ ^[0-9]+$ ]] || fail 'Metadata workflow run identity is invalid.' actual_artifact_sha="$(sha256sum "$archive" | awk '{print $1}')" [[ "$actual_artifact_sha" == "$artifact_sha" ]] || fail 'Artifact checksum does not match deployment metadata.' - while IFS= read -r entry; do stripped="${entry#./}" [[ "$entry" != /* ]] || fail "Archive contains an absolute path: $entry" [[ "/${stripped}/" != *'/../'* ]] || fail "Archive contains path traversal: $entry" done < <(tar -tf "$archive") - while IFS= read -r listing; do entry_type="${listing:0:1}" [[ "$entry_type" == '-' || "$entry_type" == 'd' ]] || fail 'Archive contains links or special files.' done < <(tar -tvf "$archive") - -mkdir -p "$SERVICE_ROOT/releases" release_id="$(date -u +%Y%m%d%H%M%S)-${run_id}-${run_attempt}-${source_sha:0:12}" -release_dir="$SERVICE_ROOT/releases/$release_id" +release_dir="$resolved_release_root/$release_id" [[ ! -e "$release_dir" ]] || fail "Release already exists: $release_id" - health_response() { local url="$1" curl -fsS --retry 3 --retry-all-errors --max-time 30 "${url}?powerforge-deploy=${run_id}-${run_attempt}" } - verify_health() { local url response for url in "$LOCAL_HEALTH_URL" $PUBLIC_HEALTH_URLS; do @@ -158,28 +609,69 @@ verify_health() { fi done } - rollback() { - local exit_code="$1" + local exit_code="$1" permissions_restored=1 current_restored=1 service_safe=0 current_target="" set +e + if ! restore_systemd_write_paths; then + permissions_restored=0 + report_systemd_restore_failure + fi if [[ "$promoted" == '1' ]]; then if [[ -n "$previous_target" && -d "$previous_target" ]]; then log "Deployment failed; rolling back to $previous_target" - rollback_link="$SERVICE_ROOT/.current.rollback.$$" - ln -s "$previous_target" "$rollback_link" - mv -Tf "$rollback_link" "$SERVICE_ROOT/current" - systemctl restart "$SYSTEMD_SERVICE" + if ! restore_previous_current_link; then + current_restored=0 + log 'ERROR: failed to restore the previous current release link.' >&2 + fi + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' ]]; then + if systemctl restart "$SYSTEMD_SERVICE"; then + service_safe=1 + else + log "ERROR: failed to restart restored service $SYSTEMD_SERVICE; stopping it." >&2 + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + else + log 'Rollback state is unverified; stopping instead of restarting the service.' >&2 + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi else log 'Deployment failed; removing the first release from current and stopping the service.' - rm -f "$SERVICE_ROOT/current" - systemctl stop "$SYSTEMD_SERVICE" + if ! rm -f -- "$SERVICE_ROOT/current" || [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then + current_restored=0 + log 'ERROR: failed to remove the first release from current.' >&2 + fi + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + if [[ "$service_safe" != '1' ]]; then + log "CRITICAL: failed to prove $SYSTEMD_SERVICE is safely restored or stopped." >&2 + fi + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' && "$service_safe" == '1' ]]; then + finish_systemd_write_paths_transaction || log "WARNING: rollback transaction retained at $systemd_transaction_path" >&2 + else + log "Rollback transaction retained for recovery: $systemd_transaction_path" >&2 + fi + elif [[ "$systemd_write_paths_snapshot_ready" == '1' ]]; then + if [[ "$permissions_restored" == '1' ]]; then + finish_systemd_write_paths_transaction || log "WARNING: restored pre-switch transaction retained at $systemd_transaction_path" >&2 + else + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop $SYSTEMD_SERVICE after pre-switch permission rollback failure." >&2 + log "Pre-switch transaction retained for recovery: $systemd_transaction_path" >&2 + fi + fi + current_target="$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null || true)" + [[ -z "$candidate_link" || ! -L "$candidate_link" ]] || rm -f -- "$candidate_link" + if [[ -n "$release_dir" && -d "$release_dir" && "$release_dir" != "$previous_target" ]]; then + if [[ "$current_target" != "$release_dir" && ( "$promoted" != '1' || ( "$current_restored" == '1' && "$service_safe" == '1' ) ) ]]; then + rm -rf -- "$release_dir" || log "WARNING: failed to remove rejected release $release_dir" >&2 + else + log "Rejected release retained for recovery: $release_dir" >&2 fi fi - [[ -z "$release_dir" || ! -d "$release_dir" || "$release_dir" == "$previous_target" ]] || rm -rf "$release_dir" exit "$exit_code" } -trap 'rollback $?' ERR INT TERM - +trap 'exit_code=$?; if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback "$exit_code"; else exit "$exit_code"; fi' ERR +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 130; else exit 130; fi' INT +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 143; else exit 143; fi' TERM mkdir -p "$release_dir" tar --extract --file "$archive" --directory "$release_dir" --no-same-owner --no-same-permissions for required_path in $REQUIRED_RELEASE_PATHS; do @@ -188,23 +680,19 @@ for required_path in $REQUIRED_RELEASE_PATHS; do done mkdir -p "$release_dir/_powerforge" install -m 0644 "$metadata" "$release_dir/_powerforge/deployment.json" - -if [[ -L "$SERVICE_ROOT/current" ]]; then - previous_target="$(readlink -f "$SERVICE_ROOT/current")" -fi candidate_link="$SERVICE_ROOT/.current.${run_id}.${run_attempt}" ln -s "$release_dir" "$candidate_link" -mv -Tf "$candidate_link" "$SERVICE_ROOT/current" +snapshot_systemd_write_paths +reconcile_systemd_write_paths promoted=1 - +mv -Tf "$candidate_link" "$SERVICE_ROOT/current" systemctl restart "$SYSTEMD_SERVICE" verify_health - -mapfile -t old_releases < <(find "$SERVICE_ROOT/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}') +mapfile -t old_releases < <(find "$resolved_release_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}') for ((index=RELEASES_TO_KEEP; index<${#old_releases[@]}; index++)); do [[ "${old_releases[$index]}" == "$release_dir" || "${old_releases[$index]}" == "$previous_target" ]] || rm -rf "${old_releases[$index]}" done - +commit_systemd_write_paths trap - ERR INT TERM cleanup_staging trap - EXIT diff --git a/Deployment/Linux/powerforge-service.env.example b/Deployment/Linux/powerforge-service.env.example index 7d00974262..55b5c7035f 100644 --- a/Deployment/Linux/powerforge-service.env.example +++ b/Deployment/Linux/powerforge-service.env.example @@ -1,6 +1,8 @@ # Root-owned configuration for /usr/local/sbin/powerforge-service-deploy. SERVICE_ROOT=/srv/example/service SYSTEMD_SERVICE=example.service +# Optional, space-separated existing data directories that a ProtectSystem=strict unit may write. +SYSTEMD_READ_WRITE_PATHS="/var/lib/example-service" LOCAL_HEALTH_URL=http://127.0.0.1:8080/healthz PUBLIC_HEALTH_URLS="https://api.example.com/healthz https://api-alt.example.com/healthz" REQUIRED_RELEASE_PATHS="package.json src/server.mjs" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index eb8b809328..a0324889af 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -30,16 +30,28 @@ Create one root-owned configuration per service under `/etc/powerforge/services` ```bash install -d -o root -g root -m 0750 /etc/powerforge/services +install -d -o root -g root -m 0755 /srv/example/service +install -d -o example-service -g example-service -m 0750 /var/lib/example-service +install -d -o root -g root -m 0755 /var/lib/powerforge +install -d -o root -g root -m 0700 \ + /var/lib/powerforge/service-deployment-staging \ + /var/lib/powerforge/service-deployment-state install -o root -g root -m 0640 \ Deployment/Linux/powerforge-service.env.example \ /etc/powerforge/services/example.env ``` +`example-service` is the account configured by the systemd unit. Create that account +first or substitute the unit's existing `User=`/`Group=` values. `ReadWritePaths=` +opens the systemd mount namespace but does not bypass normal filesystem ownership and +mode checks. + Example configuration: ```dotenv SERVICE_ROOT=/srv/example/service SYSTEMD_SERVICE=example.service +SYSTEMD_READ_WRITE_PATHS="/var/lib/example-service" LOCAL_HEALTH_URL=http://127.0.0.1:8080/healthz PUBLIC_HEALTH_URLS="https://api.example.com/healthz https://api-alt.example.com/healthz" REQUIRED_RELEASE_PATHS="package.json src/server.mjs" @@ -47,6 +59,42 @@ RELEASES_TO_KEEP=5 REQUIRE_HEALTH_PROVENANCE=1 ``` +`SYSTEMD_READ_WRITE_PATHS` is optional. Set it to existing, space-separated absolute +data directories when the service unit uses `ProtectSystem=strict`. The root-owned +promoter writes a PowerForge-owned systemd drop-in and reloads systemd before restart, +so application releases remain immutable while declared databases, uploads, or other +mutable service state stay writable. The deployment rejects missing, relative, root, +traversal, symlinked, redirectable, release/control-plane-overlapping, or +systemd-special paths. Writable exceptions cannot contain or reside beneath the +service configuration, systemd configuration, lock, transaction, trusted staging, +or service/release roots. +Every parent must be root-owned and not group/world writable when the promoter runs +as root. Removing the setting removes only PowerForge's owned drop-in after a +successful deployment, while a failed deployment restores the previous permissions +before rolling the application back. Restoration preserves the previous drop-in owner, +group, and mode. If the permissions cannot be restored and reloaded, the promoter +keeps its recovery backup and stops instead of restarting with unverified access. +The systemd configuration root and unit drop-in directory must also be real, +root-owned, non-writable directory chains; the promoter never follows a service-owned +drop-in directory or reloads configuration from one. The same trust rule protects the +root-sourced service configuration and the canonical service/release roots. Deployments +are serialized by service id, systemd unit, and canonical service root. Cancellation +uses explicit non-zero signal exits, and rollback retains a rejected release whenever +the previous link or safe service state cannot be proven. +Immediately before switching `current`, the promoter persists the previous permission +and current-release state under `/var/lib/powerforge/service-deployment-state`, then +reloads the candidate policy. Transactions are keyed by the stable service id, so a +later invocation restores the recorded unit, service root, and systemd configuration +root even when current configuration was renamed or is temporarily unavailable. Before +validating a new service root, the promoter also scans pending transactions for another +service id that shares the configured unit or root and recovers that state under both +service locks. It proves the service restarted or stopped before accepting a new +deployment, so process termination or host loss cannot strand an uncommitted writable +policy. Restored drop-in and service-root filesystems are flushed before recovery state +is removed. Successful promotion retains a recognizable committed marker until the +transaction-directory rename is durable, then safely clears that marker on this or the +next invocation. + Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: ```sudoers @@ -152,7 +200,14 @@ The root promoter: 8. Stops the service after a failed first deployment and removes the failed release. 9. Retains the configured number of known-good releases. -The promoter never copies or removes files outside `SERVICE_ROOT`, its lock, and its deployment staging. Environment files, private keys, API credentials, queues, databases, registration stores, and other mutable state must remain external and be covered by the server-recovery manifest. +The promoter mutates only the configured `SERVICE_ROOT`, its lock and root-only +transaction/staging roots, and the PowerForge-owned +`SYSTEMD_CONFIG_ROOT/.d/powerforge-read-write-paths.conf`. It reloads systemd +after changing or restoring that drop-in. Environment files, private keys, API +credentials, queues, databases, registration stores, and other mutable state remain +external and must be covered by the server-recovery manifest. Recovery planning must +also preserve the service configuration and systemd unit/drop-ins; transaction state +is temporary and is either committed or replayed by the next promoter invocation. ## Recovery Coverage diff --git a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs index a5b81582d7..5f7ea6b07c 100644 --- a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs +++ b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs @@ -44,7 +44,16 @@ public void PromoterUsesTrustedStagingAndRollsBackSystemdService() Assert.Contains("sourceSha", script, StringComparison.Ordinal); Assert.Contains("workflowRunId", script, StringComparison.Ordinal); Assert.Contains("workflowRunAttempt", script, StringComparison.Ordinal); - Assert.True(script.IndexOf("flock -n", StringComparison.Ordinal) < script.IndexOf("realpath -e", StringComparison.Ordinal)); + Assert.Contains("assert_trusted_directory_chain \"$CONFIG_ROOT\"", script, StringComparison.Ordinal); + Assert.Contains("powerforge-systemd-${unit_lock_key}.lock", script, StringComparison.Ordinal); + Assert.Contains("powerforge-root-${service_root_lock_key}.lock", script, StringComparison.Ordinal); + Assert.Contains("service-deployment-state", script, StringComparison.Ordinal); + Assert.Contains("service-${service_id}.transaction", script, StringComparison.Ordinal); + Assert.Contains("Recovering incomplete systemd writable-path transaction", script, StringComparison.Ordinal); + Assert.Contains("sync_deployment_state", script, StringComparison.Ordinal); + Assert.Contains("must not overlap deployment control path", script, StringComparison.Ordinal); + Assert.Contains("rollback 143", script, StringComparison.Ordinal); + Assert.Contains("Rejected release retained for recovery", script, StringComparison.Ordinal); } private static string ReadRepoFile(params string[] relativePath) diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 97a1c0b775..bf630a6a52 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -3,19 +3,43 @@ set -Eeuo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" deploy_script="$repo_root/Deployment/Linux/powerforge-service-deploy.sh" -test_root="$(mktemp -d)" +test_root="$(mktemp -d "${HOME}/powerforge-service-deploy-tests.XXXXXXXX")" trap 'rm -rf "$test_root" /tmp/powerforge-service-example /tmp/powerforge-service-fresh' EXIT mkdir -p "$test_root/config" "$test_root/locks" "$test_root/bin" "$test_root/service" export POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config" export POWERFORGE_SERVICE_LOCK_ROOT="$test_root/locks" export POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT="$test_root/trusted-stage" +export POWERFORGE_SERVICE_TRANSACTION_ROOT="$test_root/transactions" +export POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd" export TEST_SYSTEMCTL_LOG="$test_root/systemctl.log" +export TEST_SYNC_LOG="$test_root/sync.log" cat >"$test_root/bin/systemctl" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$*" >>"$TEST_SYSTEMCTL_LOG" +if [[ "$*" == 'daemon-reload' && "${FAIL_DAEMON_RELOAD:-}" == '1' ]]; then + exit 1 +fi +if [[ "$*" == 'daemon-reload' && -n "${FAIL_DAEMON_RELOAD_COUNT_FILE:-}" ]]; then + count=0 + [[ ! -f "$FAIL_DAEMON_RELOAD_COUNT_FILE" ]] || count="$(cat "$FAIL_DAEMON_RELOAD_COUNT_FILE")" + count=$((count + 1)) + printf '%s\n' "$count" >"$FAIL_DAEMON_RELOAD_COUNT_FILE" + if [[ -n "${FAIL_DAEMON_RELOAD_FROM_CALL:-}" ]] && (( count >= FAIL_DAEMON_RELOAD_FROM_CALL )); then + exit 1 + fi + if [[ -n "${FAIL_DAEMON_RELOAD_ON_CALL:-}" && "$count" -eq "$FAIL_DAEMON_RELOAD_ON_CALL" ]]; then + exit 1 + fi +fi +if [[ -n "${FAIL_SYSTEMCTL_COMMAND:-}" && "$*" == "$FAIL_SYSTEMCTL_COMMAND" ]]; then + exit 1 +fi +if [[ -n "${SIGNAL_ON_SYSTEMCTL_COMMAND:-}" && "$*" == "$SIGNAL_ON_SYSTEMCTL_COMMAND" ]]; then + kill -"${SIGNAL_NAME:-TERM}" "$PPID" +fi EOF cat >"$test_root/bin/curl" <<'EOF' @@ -28,6 +52,31 @@ fi cat "$marker" EOF chmod +x "$test_root/bin/systemctl" "$test_root/bin/curl" + +cat >"$test_root/bin/mv" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +if [[ "${FAIL_ROLLBACK_LINK_MOVE:-}" == '1' && "$*" == *'.current.rollback.'* ]]; then + exit 1 +fi +exec /usr/bin/mv "$@" +EOF +chmod +x "$test_root/bin/mv" +cat >"$test_root/bin/sync" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +[[ -z "${TEST_SYNC_LOG:-}" ]] || printf '%s\n' "$*" >>"$TEST_SYNC_LOG" +if [[ -n "${FAIL_SYNC_PATH:-}" && "${*: -1}" == "$FAIL_SYNC_PATH" ]]; then + count=1 + if [[ -n "${FAIL_SYNC_COUNT_FILE:-}" ]]; then + [[ ! -f "$FAIL_SYNC_COUNT_FILE" ]] || count=$(( $(cat "$FAIL_SYNC_COUNT_FILE") + 1 )) + printf '%s\n' "$count" >"$FAIL_SYNC_COUNT_FILE" + fi + if [[ -z "${FAIL_SYNC_FROM_CALL:-}" || "$count" -ge "$FAIL_SYNC_FROM_CALL" ]]; then exit 1; fi +fi +exec /usr/bin/sync "$@" +EOF +chmod +x "$test_root/bin/sync" export PATH="$test_root/bin:$PATH" write_config() { @@ -74,6 +123,8 @@ EOF } write_config example "$test_root/service" +mkdir -p "$test_root/example-data" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" create_stage example 92001 1 1111111111111111111111111111111111111111 TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" \ --service example @@ -82,25 +133,265 @@ first_target="$(readlink -f "$test_root/service/current")" [[ -s "$first_target/package.json" ]] grep -q '1111111111111111111111111111111111111111' "$first_target/_powerforge/deployment.json" grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" +grep -q '^daemon-reload$' "$TEST_SYSTEMCTL_LOG" +grep -qxF -- "-f $test_root/service/releases" "$TEST_SYNC_LOG" +drop_in="$POWERFORGE_SYSTEMD_CONFIG_ROOT/example.service.d/powerforge-read-write-paths.conf" +transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction" +grep -qxF '[Service]' "$drop_in" +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e /tmp/powerforge-service-example ]] + +# A release-filesystem flush failure must happen before the commit marker and +# roll the current link back through the still-live transaction. +create_stage example 92012 1 cccccccccccccccccccccccccccccccccccccccc +set +e +release_sync_output="$(TEST_SERVICE_ROOT="$test_root/service" FAIL_SYNC_PATH="$test_root/service/releases" "$deploy_script" --service example 2>&1)" +release_sync_status=$? +set -e +[[ "$release_sync_status" -ne 0 ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +[[ ! -e "$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction.committed" ]] +[[ -d "$transaction_dir" ]] +grep -q 'Deployment failed; rolling back' <<<"$release_sync_output" +grep -q 'transaction retained' <<<"${release_sync_output,,}" +rm -rf -- "$transaction_dir" + +# A persisted transaction must restore the prior drop-in/current link after an +# uncatchable process or host failure, before a later deployment is considered. +mkdir -m 0700 "$transaction_dir" +printf '%s\n' "$test_root/service" >"$transaction_dir/service-root" +printf '%s\n' 'example.service' >"$transaction_dir/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$transaction_dir/systemd-config-root" +printf '%s\n' "$first_target" >"$transaction_dir/previous-target" +printf '%s\n' 'present' >"$transaction_dir/drop-in-state" +printf '%s\n' "$(stat -c '%u' "$drop_in")" >"$transaction_dir/drop-in-owner" +printf '%s\n' "$(stat -c '%g' "$drop_in")" >"$transaction_dir/drop-in-group" +printf '%s\n' "$(stat -c '%a' "$drop_in")" >"$transaction_dir/drop-in-mode" +cp "$drop_in" "$transaction_dir/drop-in" +chmod 0600 "$transaction_dir"/* +stranded_release="$test_root/service/releases/stranded-release" +cp -a "$first_target" "$stranded_release" +ln -sfn "$stranded_release" "$test_root/service/current" +printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" +mv "$test_root/config/example.env" "$test_root/config/example.env.unavailable" +set +e +recovery_output="$(POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd-renamed" TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" +recovery_status=$? +set -e +[[ "$recovery_status" -ne 0 ]] +grep -q 'Recovering incomplete systemd writable-path transaction' <<<"$recovery_output" +grep -q 'Recovered incomplete systemd writable-path transaction' <<<"$recovery_output" +grep -q 'Service is not configured' <<<"$recovery_output" +grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" +[[ ! -e "$transaction_dir" ]] +rm -f -- "$test_root/config/example.env.unavailable" +write_config example "$test_root/service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" + +alias_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-aliasowner.transaction" +mkdir -m 0700 "$alias_transaction" +printf '%s\n' "$test_root/service" >"$alias_transaction/service-root" +printf '%s\n' 'example.service' >"$alias_transaction/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$alias_transaction/systemd-config-root" +printf '%s\n' "$first_target" >"$alias_transaction/previous-target" +printf '%s\n' 'present' >"$alias_transaction/drop-in-state" +printf '%s\n' "$(stat -c '%u' "$drop_in")" >"$alias_transaction/drop-in-owner" +printf '%s\n' "$(stat -c '%g' "$drop_in")" >"$alias_transaction/drop-in-group" +printf '%s\n' "$(stat -c '%a' "$drop_in")" >"$alias_transaction/drop-in-mode" +cp "$drop_in" "$alias_transaction/drop-in" +chmod 0600 "$alias_transaction"/* +ln -sfn "$stranded_release" "$test_root/service/current" +printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" +write_config aliasconsumer "$test_root/service/." +set +e +alias_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service aliasconsumer 2>&1)" +alias_status=$? +set -e +[[ "$alias_status" -ne 0 ]] +grep -q 'Recovered incomplete systemd writable-path transaction for example.service' <<<"$alias_output" +[[ ! -e "$alias_transaction" ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" + +mkdir -p "$test_root/multi-current" +write_config multiinvoker "$test_root/multi-current" +sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=sharedmulti.service/' "$test_root/config/multiinvoker.env" +multi_transactions=() +for owner in aliasmulti1 aliasmulti2; do + multi_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-${owner}.transaction" + multi_transactions+=("$multi_transaction") + mkdir -m 0700 "$multi_transaction" + printf '%s\n' "$test_root/${owner}-root" >"$multi_transaction/service-root" + printf '%s\n' 'sharedmulti.service' >"$multi_transaction/systemd-service" + printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$multi_transaction/systemd-config-root" + chmod 0600 "$multi_transaction"/* +done +multi_lock_ready="$test_root/multi-lock-ready" +( + exec 202>"$test_root/locks/powerforge-service-aliasmulti1.lock" + flock 202 + : >"$multi_lock_ready" + sleep 30 +) & +multi_lock_holder=$! +for _ in {1..100}; do [[ -e "$multi_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$multi_lock_ready" ]] +: >"$TEST_SYSTEMCTL_LOG" +set +e +multi_busy_output="$(TEST_SERVICE_ROOT="$test_root/multi-current" "$deploy_script" --service multiinvoker 2>&1)" +multi_busy_status=$? +set -e +[[ "$multi_busy_status" -ne 0 ]] +grep -q 'Another deployment is active for aliasmulti1' <<<"$multi_busy_output" +if grep -q '^stop sharedmulti.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Ambiguous recovery stopped a unit while a transaction owner was active.' >&2 + exit 1 +fi +kill "$multi_lock_holder" 2>/dev/null || true +wait "$multi_lock_holder" 2>/dev/null || true +set +e +multi_output="$(TEST_SERVICE_ROOT="$test_root/multi-current" "$deploy_script" --service multiinvoker 2>&1)" +multi_status=$? +set -e +[[ "$multi_status" -ne 0 ]] +grep -q 'Multiple incomplete transactions overlap this deployment' <<<"$multi_output" +[[ "$(grep -c '^stop sharedmulti.service$' "$TEST_SYSTEMCTL_LOG")" -eq 1 ]] +for multi_transaction in "${multi_transactions[@]}"; do [[ -d "$multi_transaction" ]]; rm -rf -- "$multi_transaction"; done + +mkdir -p "$test_root/recovery-current-service" +write_config recoverymissing "$test_root/recovery-current-service" +missing_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-recoverymissing.transaction" +mkdir -m 0700 "$missing_transaction" +printf '%s\n' "$test_root/recovery-root-missing" >"$missing_transaction/service-root" +printf '%s\n' 'recorded.service' >"$missing_transaction/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$missing_transaction/systemd-config-root" +chmod 0600 "$missing_transaction"/* +set +e +missing_recovery_output="$(TEST_SERVICE_ROOT="$test_root/recovery-current-service" "$deploy_script" --service recoverymissing 2>&1)" +missing_recovery_status=$? +set -e +[[ "$missing_recovery_status" -ne 0 ]] +grep -q '^stop recorded.service$' "$TEST_SYSTEMCTL_LOG" +grep -q 'Recorded recovery paths are unavailable' <<<"$missing_recovery_output" +[[ -d "$missing_transaction" ]] +rm -rf -- "$missing_transaction" + +create_stage example 92010 1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +set +e +sync_failure_output="$(TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + FAIL_SYNC_PATH="$drop_in" \ + "$deploy_script" --service example 2>&1)" +sync_failure_status=$? +set -e +[[ "$sync_failure_status" -ne 0 ]] +grep -q 'rollback transaction retained' <<<"${sync_failure_output,,}" +[[ -d "$transaction_dir" ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +rm -rf -- "$transaction_dir" + +mkdir -p "$test_root/commit-service" "$test_root/commit-data" +write_config commitdurable "$test_root/commit-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/commit-data" >>"$test_root/config/commitdurable.env" +create_stage commitdurable 92011 1 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +commit_sync_count="$test_root/commit-sync-count" +set +e +commit_output="$(TEST_SERVICE_ROOT="$test_root/commit-service" \ + FAIL_SYNC_PATH="$POWERFORGE_SERVICE_TRANSACTION_ROOT" \ + FAIL_SYNC_COUNT_FILE="$commit_sync_count" \ + FAIL_SYNC_FROM_CALL=2 \ + "$deploy_script" --service commitdurable 2>&1)" +commit_status=$? +set -e +[[ "$commit_status" -ne 0 ]] +committed_marker="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-commitdurable.transaction.committed" +grep -q 'committed transaction retained' <<<"${commit_output,,}" +[[ -d "$committed_marker" ]] +grep -q 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' "$test_root/commit-service/current/_powerforge/deployment.json" +set +e +TEST_SERVICE_ROOT="$test_root/commit-service" "$deploy_script" --service commitdurable >/dev/null 2>&1 +set -e +[[ ! -e "$committed_marker" ]] if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 fi + +previous_example_target="$(readlink -f "$test_root/service/current")" +mkdir -p "$test_root/example-data-next" +write_config example "$test_root/service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data-next" >>"$test_root/config/example.env" +create_stage example 92005 1 5555555555555555555555555555555555555555 +: >"$TEST_SYSTEMCTL_LOG" +reload_count_file="$test_root/restore-reload-count" +if TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=5555555555555555555555555555555555555555 \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$reload_count_file" \ + FAIL_DAEMON_RELOAD_FROM_CALL=2 \ + "$deploy_script" --service example; then + echo 'Deployment unexpectedly restarted after permission rollback failed.' >&2 + exit 1 +fi +[[ "$(readlink -f "$test_root/service/current")" == "$previous_example_target" ]] +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" +[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -eq 1 ]] +grep -q '^stop example.service$' "$TEST_SYSTEMCTL_LOG" +[[ -d "$transaction_dir" ]] +rm -rf -- "$transaction_dir" +: >"$TEST_SYSTEMCTL_LOG" [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] create_stage example 92002 1 2222222222222222222222222222222222222222 -if TEST_SERVICE_ROOT="$test_root/service" FAIL_SOURCE_SHA=2222222222222222222222222222222222222222 "$deploy_script" \ +single_rollback_count_file="$test_root/single-rollback-count" +if TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=2222222222222222222222222222222222222222 \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$single_rollback_count_file" \ + FAIL_DAEMON_RELOAD_FROM_CALL=3 \ + "$deploy_script" \ --service example; then echo 'Deployment unexpectedly succeeded when exact provenance health failed.' >&2 exit 1 fi [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] [[ ! -e /tmp/powerforge-service-example ]] -[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -ge 3 ]] +[[ "$(cat "$single_rollback_count_file")" == '2' ]] +[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -eq 2 ]] +if grep -q '^stop example.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Top-level permission rollback unexpectedly stopped the restored healthy service.' >&2 + exit 1 +fi + +mkdir -p "$test_root/early-service" "$test_root/early-data" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d" +write_config early "$test_root/early-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/early-data" >>"$test_root/config/early.env" +printf '[Service]\nReadWritePaths=/previous\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" +create_stage early 92006 1 6666666666666666666666666666666666666666 +: >"$TEST_SYSTEMCTL_LOG" +early_reload_count_file="$test_root/early-reload-count" +if TEST_SERVICE_ROOT="$test_root/early-service" \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$early_reload_count_file" \ + FAIL_DAEMON_RELOAD_ON_CALL=1 \ + "$deploy_script" --service early; then + echo 'Pre-switch reconciliation unexpectedly ignored a failed systemd reload.' >&2 + exit 1 +fi +grep -qxF 'ReadWritePaths=/previous' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" +early_transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-early.transaction" +[[ ! -e "$early_transaction_dir" ]] +[[ ! -e "$test_root/early-service/current" ]] +if grep -q '^stop early.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Restored pre-switch state unexpectedly stopped the untouched service.' >&2 + exit 1 +fi +: >"$TEST_SYSTEMCTL_LOG" mkdir -p "$test_root/fresh-service" write_config fresh "$test_root/fresh-service" +mkdir -p "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d" +printf '[Service]\nReadWritePaths=/obsolete\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" +chmod 0640 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" create_stage fresh 92003 1 3333333333333333333333333333333333333333 if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333333333333333333333333333 "$deploy_script" \ --service fresh; then @@ -108,7 +399,285 @@ if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333 exit 1 fi [[ ! -e "$test_root/fresh-service/current" ]] +grep -qxF 'ReadWritePaths=/obsolete' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" +[[ "$(stat -c '%a' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf")" == '640' ]] grep -q '^stop fresh.service$' "$TEST_SYSTEMCTL_LOG" +create_stage fresh 92003 1 3333333333333333333333333333333333333333 +TEST_SERVICE_ROOT="$test_root/fresh-service" "$deploy_script" --service fresh +[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" ]] + +mkdir -p "$test_root/unsafe-service" +write_config unsafe "$test_root/unsafe-service" +printf 'SYSTEMD_READ_WRITE_PATHS="/"\n' >>"$test_root/config/unsafe.env" +if TEST_SERVICE_ROOT="$test_root/unsafe-service" "$deploy_script" --service unsafe; then + echo 'Deployment unexpectedly accepted the filesystem root as a writable path.' >&2 + exit 1 +fi + +control_ids=(configcontrol systemdcontrol transactioncontrol stagecontrol lockcontrol) +control_paths=("$POWERFORGE_SERVICE_CONFIG_ROOT" "$POWERFORGE_SYSTEMD_CONFIG_ROOT" "$POWERFORGE_SERVICE_TRANSACTION_ROOT" "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" "$(realpath -e "$POWERFORGE_SERVICE_LOCK_ROOT")") +for index in "${!control_ids[@]}"; do + control_id="${control_ids[$index]}" + control_path="${control_paths[$index]}" + control_service="$test_root/${control_id}-service" + mkdir -p "$control_service" + write_config "$control_id" "$control_service" + printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$control_path" >>"$test_root/config/${control_id}.env" + set +e + control_output="$(TEST_SERVICE_ROOT="$control_service" "$deploy_script" --service "$control_id" 2>&1)" + control_status=$? + set -e + if [[ "$control_status" -eq 0 ]]; then + echo "Deployment unexpectedly allowed writable access to deployment control state: $control_path" >&2 + exit 1 + fi + grep -q 'Systemd writable path must not overlap deployment control path' <<<"$control_output" +done + +# Deployment-owned state must never be placed inside prunable release storage. +mkdir -p "$test_root/control-overlap-service/releases" +write_config controloverlap "$test_root/control-overlap-service" +overlap_variables=(POWERFORGE_SERVICE_CONFIG_ROOT POWERFORGE_SYSTEMD_CONFIG_ROOT POWERFORGE_SERVICE_TRANSACTION_ROOT POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT POWERFORGE_SERVICE_LOCK_ROOT) +overlap_names=(config systemd transactions staging locks) +for index in "${!overlap_variables[@]}"; do + overlap_variable="${overlap_variables[$index]}" + overlap_root="$test_root/control-overlap-service/releases/${overlap_names[$index]}" + if [[ "$overlap_variable" == 'POWERFORGE_SERVICE_CONFIG_ROOT' ]]; then + mkdir -p "$overlap_root" + cp "$test_root/config/controloverlap.env" "$overlap_root/controloverlap.env" + fi + set +e + control_overlap_output="$(TEST_SERVICE_ROOT="$test_root/control-overlap-service" env "$overlap_variable=$overlap_root" "$deploy_script" --service controloverlap 2>&1)" + control_overlap_status=$? + set -e + [[ "$control_overlap_status" -ne 0 ]] + grep -qxF "[powerforge-service-deploy] ERROR: Deployment control path must not overlap release storage: $overlap_root" <<<"$control_overlap_output" +done + +mkdir -p "$test_root/glob-service" +write_config glob "$test_root/glob-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-*" >>"$test_root/config/glob.env" +if TEST_SERVICE_ROOT="$test_root/glob-service" "$deploy_script" --service glob; then + echo 'Deployment unexpectedly expanded a writable-path glob.' >&2 + exit 1 +fi + +mkdir -p "$test_root/overlap-service/releases/nested" +for overlap_path in \ + "$test_root/overlap-service" \ + "$test_root/overlap-service/releases" \ + "$test_root/overlap-service/releases/nested"; do + write_config overlap "$test_root/overlap-service" + printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$overlap_path" >>"$test_root/config/overlap.env" + if TEST_SERVICE_ROOT="$test_root/overlap-service" "$deploy_script" --service overlap; then + echo "Deployment unexpectedly allowed writable access to immutable release storage: $overlap_path" >&2 + exit 1 + fi +done + +mkdir -p "$test_root/symlink-target/data" "$test_root/symlink-service" +ln -s "$test_root/symlink-target" "$test_root/symlink-parent" +write_config symlink "$test_root/symlink-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/symlink-parent/data" >>"$test_root/config/symlink.env" +if TEST_SERVICE_ROOT="$test_root/symlink-service" "$deploy_script" --service symlink; then + echo 'Deployment unexpectedly accepted a symlinked writable-path parent.' >&2 + exit 1 +fi + +mkdir -p "$test_root/untrusted-parent/data" "$test_root/untrusted-service" +chmod 0777 "$test_root/untrusted-parent" +write_config untrusted "$test_root/untrusted-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/untrusted-parent/data" >>"$test_root/config/untrusted.env" +if TEST_SERVICE_ROOT="$test_root/untrusted-service" "$deploy_script" --service untrusted; then + echo 'Deployment unexpectedly accepted a writable-path parent that can be redirected.' >&2 + exit 1 +fi +chmod 0755 "$test_root/untrusted-parent" + +mkdir -p "$test_root/systemd-symlink-service" "$test_root/systemd-attacker" +write_config dirsymlink "$test_root/systemd-symlink-service" +ln -s "$test_root/systemd-attacker" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/dirsymlink.service.d" +if TEST_SERVICE_ROOT="$test_root/systemd-symlink-service" "$deploy_script" --service dirsymlink; then + echo 'Deployment unexpectedly accepted a symlinked systemd drop-in directory.' >&2 + exit 1 +fi +rm -f -- "$POWERFORGE_SYSTEMD_CONFIG_ROOT/dirsymlink.service.d" + +mkdir -p "$test_root/systemd-untrusted-service" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" +chmod 0777 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" +write_config diruntrusted "$test_root/systemd-untrusted-service" +if TEST_SERVICE_ROOT="$test_root/systemd-untrusted-service" "$deploy_script" --service diruntrusted; then + echo 'Deployment unexpectedly accepted a writable systemd drop-in directory.' >&2 + exit 1 +fi +chmod 0755 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" + +mkdir -p "$test_root/config-symlink-target" "$test_root/config-trust-service" +write_config configtrust "$test_root/config-trust-service" +cp "$test_root/config/configtrust.env" "$test_root/config-symlink-target/configtrust.env" +ln -s "$test_root/config-symlink-target" "$test_root/config-symlink" +set +e +config_symlink_output="$(POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config-symlink" TEST_SERVICE_ROOT="$test_root/config-trust-service" "$deploy_script" --service configtrust 2>&1)" +config_symlink_status=$? +set -e +if [[ "$config_symlink_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked service config root.' >&2 + exit 1 +fi +grep -q 'Service config root must be a real directory' <<<"$config_symlink_output" +mkdir -p "$test_root/config-writable" +cp "$test_root/config/configtrust.env" "$test_root/config-writable/configtrust.env" +chmod 0777 "$test_root/config-writable" +set +e +config_writable_output="$(POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config-writable" TEST_SERVICE_ROOT="$test_root/config-trust-service" "$deploy_script" --service configtrust 2>&1)" +config_writable_status=$? +set -e +if [[ "$config_writable_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a writable service config root.' >&2 + exit 1 +fi +grep -q 'Service config root must not be group/world writable' <<<"$config_writable_output" +chmod 0755 "$test_root/config-writable" + +mkdir -p "$test_root/service-root-target" "$test_root/service-root-config" +ln -s "$test_root/service-root-target" "$test_root/service-root-link" +write_config rootsymlink "$test_root/service-root-link" +set +e +root_symlink_output="$(TEST_SERVICE_ROOT="$test_root/service-root-target" "$deploy_script" --service rootsymlink 2>&1)" +root_symlink_status=$? +set -e +if [[ "$root_symlink_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked service root.' >&2 + exit 1 +fi +grep -q 'Service root must be a real, pre-provisioned directory' <<<"$root_symlink_output" +mkdir -p "$test_root/service-root-writable" +chmod 0777 "$test_root/service-root-writable" +write_config rootwritable "$test_root/service-root-writable" +set +e +root_writable_output="$(TEST_SERVICE_ROOT="$test_root/service-root-writable" "$deploy_script" --service rootwritable 2>&1)" +root_writable_status=$? +set -e +if [[ "$root_writable_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a writable service root.' >&2 + exit 1 +fi +grep -q 'Service root must not be group/world writable' <<<"$root_writable_output" +chmod 0755 "$test_root/service-root-writable" + +mkdir -p "$test_root/release-link-service" "$test_root/release-link-target" +ln -s "$test_root/release-link-target" "$test_root/release-link-service/releases" +write_config releaselink "$test_root/release-link-service" +set +e +release_link_output="$(TEST_SERVICE_ROOT="$test_root/release-link-service" "$deploy_script" --service releaselink 2>&1)" +release_link_status=$? +set -e +if [[ "$release_link_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked release root.' >&2 + exit 1 +fi +grep -q 'Release root must be a real directory' <<<"$release_link_output" + +mkdir -p "$test_root/unit-lock-service" +write_config unitalias "$test_root/unit-lock-service" +sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=example.service/' "$test_root/config/unitalias.env" +unit_lock_key="$(printf '%s' 'example.service' | sha256sum | awk '{print $1}')" +unit_lock_ready="$test_root/unit-lock-ready" +( + exec 200>"$test_root/locks/powerforge-systemd-${unit_lock_key}.lock" + flock 200 + : >"$unit_lock_ready" + sleep 30 +) & +unit_lock_holder=$! +for _ in {1..100}; do [[ -e "$unit_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$unit_lock_ready" ]] +set +e +unit_lock_output="$(TEST_SERVICE_ROOT="$test_root/unit-lock-service" "$deploy_script" --service unitalias 2>&1)" +unit_lock_status=$? +set -e +if [[ "$unit_lock_status" -eq 0 ]]; then + echo 'Deployment unexpectedly bypassed serialization for a shared systemd unit.' >&2 + kill "$unit_lock_holder" 2>/dev/null || true + wait "$unit_lock_holder" 2>/dev/null || true + exit 1 +fi +grep -q 'Another deployment is active for systemd unit example.service' <<<"$unit_lock_output" +kill "$unit_lock_holder" 2>/dev/null || true +wait "$unit_lock_holder" 2>/dev/null || true + +write_config rootalias "$test_root/service" +service_root_lock_key="$(printf '%s' "$test_root/service" | sha256sum | awk '{print $1}')" +root_lock_ready="$test_root/root-lock-ready" +( + exec 201>"$test_root/locks/powerforge-root-${service_root_lock_key}.lock" + flock 201 + : >"$root_lock_ready" + sleep 30 +) & +root_lock_holder=$! +for _ in {1..100}; do [[ -e "$root_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$root_lock_ready" ]] +set +e +root_lock_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service rootalias 2>&1)" +root_lock_status=$? +set -e +if [[ "$root_lock_status" -eq 0 ]]; then + echo 'Deployment unexpectedly bypassed serialization for a shared service root.' >&2 + kill "$root_lock_holder" 2>/dev/null || true + wait "$root_lock_holder" 2>/dev/null || true + exit 1 +fi +grep -q "Another deployment is active for service root $test_root/service" <<<"$root_lock_output" +kill "$root_lock_holder" 2>/dev/null || true +wait "$root_lock_holder" 2>/dev/null || true + +mkdir -p "$test_root/signal-service" +write_config signal "$test_root/signal-service" +create_stage signal 92007 1 7777777777777777777777777777777777777777 +set +e +TEST_SERVICE_ROOT="$test_root/signal-service" \ + SIGNAL_ON_SYSTEMCTL_COMMAND='restart signal.service' \ + SIGNAL_NAME=TERM \ + "$deploy_script" --service signal +signal_status=$? +set -e +[[ "$signal_status" -eq 143 ]] +[[ ! -e "$test_root/signal-service/current" ]] +grep -q '^stop signal.service$' "$TEST_SYSTEMCTL_LOG" + +mkdir -p "$test_root/rollback-service" +write_config rollback "$test_root/rollback-service" +create_stage rollback 92008 1 8888888888888888888888888888888888888888 +TEST_SERVICE_ROOT="$test_root/rollback-service" "$deploy_script" --service rollback +rollback_previous="$(readlink -f "$test_root/rollback-service/current")" +create_stage rollback 92009 1 9999999999999999999999999999999999999999 +set +e +rollback_output="$(TEST_SERVICE_ROOT="$test_root/rollback-service" \ + FAIL_SOURCE_SHA=9999999999999999999999999999999999999999 \ + FAIL_ROLLBACK_LINK_MOVE=1 \ + FAIL_SYSTEMCTL_COMMAND='stop rollback.service' \ + "$deploy_script" --service rollback 2>&1)" +rollback_status=$? +set -e +[[ "$rollback_status" -ne 0 ]] +grep -q 'CRITICAL: failed to prove rollback.service is safely restored or stopped.' <<<"$rollback_output" +rollback_current="$(readlink -f "$test_root/rollback-service/current")" +[[ "$rollback_current" != "$rollback_previous" && -d "$rollback_current" ]] +grep -q '9999999999999999999999999999999999999999' "$rollback_current/_powerforge/deployment.json" + +mkdir -p "$test_root/reload-service" "$test_root/reload-data" +write_config reload "$test_root/reload-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/reload-data" >>"$test_root/config/reload.env" +create_stage reload 92004 1 4444444444444444444444444444444444444444 +if TEST_SERVICE_ROOT="$test_root/reload-service" FAIL_DAEMON_RELOAD=1 "$deploy_script" --service reload; then + echo 'Deployment unexpectedly ignored a failed systemd reload.' >&2 + exit 1 +fi +[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/reload.service.d/powerforge-read-write-paths.conf" ]] +create_stage reload 92004 1 4444444444444444444444444444444444444444 +TEST_SERVICE_ROOT="$test_root/reload-service" "$deploy_script" --service reload +[[ -L "$test_root/reload-service/current" ]] if [[ -d "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" ]] && find "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" -mindepth 1 -maxdepth 1 | grep -q .; then echo 'Root-owned service deployment staging was not cleaned.' >&2