From cb4bded12dff5c745e8d6d0a03c93205ada37d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 11:04:49 +0200 Subject: [PATCH 1/8] Add declarative systemd writable paths --- Deployment/Linux/powerforge-service-deploy.sh | 48 +++++++++++++++++++ .../Linux/powerforge-service.env.example | 2 + Docs/PowerForge.Web.LinuxServiceDeployment.md | 9 ++++ .../Linux/powerforge-service-deploy.tests.sh | 41 ++++++++++++++++ 4 files changed, 100 insertions(+) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 7d478d86f5..cbc8b8c108 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -6,6 +6,7 @@ umask 022 CONFIG_ROOT="${POWERFORGE_SERVICE_CONFIG_ROOT:-/etc/powerforge/services}" LOCK_ROOT="${POWERFORGE_SERVICE_LOCK_ROOT:-/var/lock}" TRUSTED_STAGE_ROOT="${POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT:-/var/lib/powerforge/service-deployment-staging}" +SYSTEMD_CONFIG_ROOT="${POWERFORGE_SYSTEMD_CONFIG_ROOT:-/etc/systemd/system}" service_id="" archive="" metadata="" @@ -71,6 +72,7 @@ source "$config_path" : "${SERVICE_ROOT:?SERVICE_ROOT is required in $config_path}" : "${SYSTEMD_SERVICE:?SYSTEMD_SERVICE is required in $config_path}" +: "${SYSTEMD_READ_WRITE_PATHS:=}" : "${LOCAL_HEALTH_URL:?LOCAL_HEALTH_URL is required in $config_path}" : "${RELEASES_TO_KEEP:=5}" : "${REQUIRED_RELEASE_PATHS:=}" @@ -80,6 +82,8 @@ source "$config_path" [[ "$SERVICE_ROOT" == /* && "$SERVICE_ROOT" != '/' ]] || fail 'SERVICE_ROOT must be an absolute non-root path.' [[ "$SERVICE_ROOT" != *[[:space:]]* ]] || fail 'SERVICE_ROOT must not contain whitespace.' [[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' ]] || fail 'Trusted staging root must be an absolute non-root path.' +[[ "$SYSTEMD_CONFIG_ROOT" == /* && "$SYSTEMD_CONFIG_ROOT" != '/' ]] || fail 'Systemd config root must be an absolute non-root path.' +[[ "$SYSTEMD_CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Systemd config root must not contain whitespace.' [[ "$SYSTEMD_SERVICE" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || fail 'SYSTEMD_SERVICE must be a systemd service unit name.' [[ "$LOCAL_HEALTH_URL" =~ ^https?://[^[:space:]]+$ ]] || fail 'LOCAL_HEALTH_URL must be an HTTP or HTTPS URL.' [[ "$RELEASES_TO_KEEP" =~ ^[1-9][0-9]*$ ]] || fail 'RELEASES_TO_KEEP must be a positive integer.' @@ -88,10 +92,54 @@ for health_url in $PUBLIC_HEALTH_URLS; do [[ "$health_url" =~ ^https://[^[:space:]]+$ ]] || fail "Public health URL must use HTTPS: $health_url" done +read -r -a configured_systemd_read_write_paths <<<"$SYSTEMD_READ_WRITE_PATHS" +for read_write_path in "${configured_systemd_read_write_paths[@]}"; do + [[ "$read_write_path" == /* && "$read_write_path" != '/' ]] || fail 'Systemd writable paths must be absolute non-root paths.' + [[ "$read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Systemd writable path contains unsupported characters: $read_write_path" + [[ "/${read_write_path#/}/" != *'/../'* ]] || fail "Systemd writable path must not contain traversal: $read_write_path" +done + +reconcile_systemd_write_paths() ( + drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" + drop_in_path="${drop_in_dir}/powerforge-read-write-paths.conf" + if ((${#systemd_read_write_paths[@]} == 0)); then + [[ ! -f "$drop_in_path" ]] || rm -f -- "$drop_in_path" + systemctl daemon-reload + return 0 + fi + install -d -m 0755 "$drop_in_dir" + temporary="$(mktemp "${drop_in_dir}/.powerforge-read-write-paths.XXXXXXXX")" + trap 'rm -f -- "$temporary"' EXIT + { + printf '[Service]\n' + for path in "${systemd_read_write_paths[@]}"; do + printf 'ReadWritePaths=%s\n' "$path" + done + } >"$temporary" + chmod 0644 "$temporary" + if [[ -f "$drop_in_path" ]] && cmp -s -- "$temporary" "$drop_in_path"; then + rm -f -- "$temporary" + else + mv -f -- "$temporary" "$drop_in_path" + fi + systemctl daemon-reload +) + mkdir -p "$LOCK_ROOT" exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" flock -n 9 || fail "Another deployment is active for $service_id." +systemd_read_write_paths=() +for read_write_path in "${configured_systemd_read_write_paths[@]}"; do + [[ -d "$read_write_path" ]] || fail "Systemd writable path does not exist: $read_write_path" + resolved_read_write_path="$(realpath -e -- "$read_write_path")" + [[ -d "$resolved_read_write_path" && "$resolved_read_write_path" != '/' ]] || fail "Systemd writable path is not a safe directory: $read_write_path" + [[ "$resolved_read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Resolved systemd writable path contains unsupported characters: $read_write_path" + systemd_read_write_paths+=("$resolved_read_write_path") +done + +reconcile_systemd_write_paths + archive="$(realpath -e "$archive")" metadata="$(realpath -e "$metadata")" [[ -f "$archive" && ! -L "$archive" ]] || fail 'Artifact must be a regular file, not a symlink.' diff --git a/Deployment/Linux/powerforge-service.env.example b/Deployment/Linux/powerforge-service.env.example index 7d00974262..55b5c7035f 100644 --- a/Deployment/Linux/powerforge-service.env.example +++ b/Deployment/Linux/powerforge-service.env.example @@ -1,6 +1,8 @@ # Root-owned configuration for /usr/local/sbin/powerforge-service-deploy. SERVICE_ROOT=/srv/example/service SYSTEMD_SERVICE=example.service +# Optional, space-separated existing data directories that a ProtectSystem=strict unit may write. +SYSTEMD_READ_WRITE_PATHS="/var/lib/example-service" LOCAL_HEALTH_URL=http://127.0.0.1:8080/healthz PUBLIC_HEALTH_URLS="https://api.example.com/healthz https://api-alt.example.com/healthz" REQUIRED_RELEASE_PATHS="package.json src/server.mjs" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index eb8b809328..c586a51b43 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -40,6 +40,7 @@ Example configuration: ```dotenv SERVICE_ROOT=/srv/example/service SYSTEMD_SERVICE=example.service +SYSTEMD_READ_WRITE_PATHS="/var/lib/example-service" LOCAL_HEALTH_URL=http://127.0.0.1:8080/healthz PUBLIC_HEALTH_URLS="https://api.example.com/healthz https://api-alt.example.com/healthz" REQUIRED_RELEASE_PATHS="package.json src/server.mjs" @@ -47,6 +48,14 @@ RELEASES_TO_KEEP=5 REQUIRE_HEALTH_PROVENANCE=1 ``` +`SYSTEMD_READ_WRITE_PATHS` is optional. Set it to existing, space-separated absolute +data directories when the service unit uses `ProtectSystem=strict`. The root-owned +promoter writes a PowerForge-owned systemd drop-in and reloads systemd before restart, +so application releases remain immutable while declared databases, uploads, or other +mutable service state stay writable. The deployment rejects missing, relative, root, +traversal, or systemd-special paths. Removing the setting removes only PowerForge's +owned drop-in on the next deployment, so obsolete write access does not persist. + Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: ```sudoers diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 97a1c0b775..62200089b3 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -10,12 +10,16 @@ mkdir -p "$test_root/config" "$test_root/locks" "$test_root/bin" "$test_root/ser export POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config" export POWERFORGE_SERVICE_LOCK_ROOT="$test_root/locks" export POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT="$test_root/trusted-stage" +export POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd" export TEST_SYSTEMCTL_LOG="$test_root/systemctl.log" cat >"$test_root/bin/systemctl" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$*" >>"$TEST_SYSTEMCTL_LOG" +if [[ "$*" == 'daemon-reload' && "${FAIL_DAEMON_RELOAD:-}" == '1' ]]; then + exit 1 +fi EOF cat >"$test_root/bin/curl" <<'EOF' @@ -74,6 +78,8 @@ EOF } write_config example "$test_root/service" +mkdir -p "$test_root/service-data" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-data" >>"$test_root/config/example.env" create_stage example 92001 1 1111111111111111111111111111111111111111 TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" \ --service example @@ -82,6 +88,10 @@ first_target="$(readlink -f "$test_root/service/current")" [[ -s "$first_target/package.json" ]] grep -q '1111111111111111111111111111111111111111' "$first_target/_powerforge/deployment.json" grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" +grep -q '^daemon-reload$' "$TEST_SYSTEMCTL_LOG" +drop_in="$POWERFORGE_SYSTEMD_CONFIG_ROOT/example.service.d/powerforge-read-write-paths.conf" +grep -qxF '[Service]' "$drop_in" +grep -qxF "ReadWritePaths=$test_root/service-data" "$drop_in" [[ ! -e /tmp/powerforge-service-example ]] if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 @@ -101,6 +111,8 @@ fi mkdir -p "$test_root/fresh-service" write_config fresh "$test_root/fresh-service" +mkdir -p "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d" +printf '[Service]\nReadWritePaths=/obsolete\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" create_stage fresh 92003 1 3333333333333333333333333333333333333333 if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333333333333333333333333333 "$deploy_script" \ --service fresh; then @@ -108,8 +120,37 @@ if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333 exit 1 fi [[ ! -e "$test_root/fresh-service/current" ]] +[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" ]] grep -q '^stop fresh.service$' "$TEST_SYSTEMCTL_LOG" +mkdir -p "$test_root/unsafe-service" +write_config unsafe "$test_root/unsafe-service" +printf 'SYSTEMD_READ_WRITE_PATHS="/"\n' >>"$test_root/config/unsafe.env" +if TEST_SERVICE_ROOT="$test_root/unsafe-service" "$deploy_script" --service unsafe; then + echo 'Deployment unexpectedly accepted the filesystem root as a writable path.' >&2 + exit 1 +fi + +mkdir -p "$test_root/glob-service" +write_config glob "$test_root/glob-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-*" >>"$test_root/config/glob.env" +if TEST_SERVICE_ROOT="$test_root/glob-service" "$deploy_script" --service glob; then + echo 'Deployment unexpectedly expanded a writable-path glob.' >&2 + exit 1 +fi + +mkdir -p "$test_root/reload-service" "$test_root/reload-data" +write_config reload "$test_root/reload-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/reload-data" >>"$test_root/config/reload.env" +create_stage reload 92004 1 4444444444444444444444444444444444444444 +if TEST_SERVICE_ROOT="$test_root/reload-service" FAIL_DAEMON_RELOAD=1 "$deploy_script" --service reload; then + echo 'Deployment unexpectedly ignored a failed systemd reload.' >&2 + exit 1 +fi +create_stage reload 92004 1 4444444444444444444444444444444444444444 +TEST_SERVICE_ROOT="$test_root/reload-service" "$deploy_script" --service reload +[[ -L "$test_root/reload-service/current" ]] + if [[ -d "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" ]] && find "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" -mindepth 1 -maxdepth 1 | grep -q .; then echo 'Root-owned service deployment staging was not cleaned.' >&2 exit 1 From ca7783b7085566beada908ea4dbc53832dd5d753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 11:33:56 +0200 Subject: [PATCH 2/8] Harden writable path rollback --- Deployment/Linux/powerforge-service-deploy.sh | 131 ++++++++++++++++-- Docs/PowerForge.Web.LinuxServiceDeployment.md | 9 +- .../Linux/powerforge-service-deploy.tests.sh | 80 ++++++++++- 3 files changed, 204 insertions(+), 16 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index cbc8b8c108..15753734c3 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -15,6 +15,14 @@ previous_target="" release_dir="" workflow_stage="" trusted_stage="" +systemd_drop_in_backup="" +systemd_drop_in_dir="" +systemd_drop_in_path="" +systemd_drop_in_existed=0 +systemd_drop_in_owner="" +systemd_drop_in_group="" +systemd_drop_in_mode="" +systemd_write_paths_snapshot_ready=0 cleanup_staging() { [[ -z "$workflow_stage" || ! -d "$workflow_stage" ]] || rm -rf -- "$workflow_stage" @@ -99,16 +107,100 @@ for read_write_path in "${configured_systemd_read_write_paths[@]}"; do [[ "/${read_write_path#/}/" != *'/../'* ]] || fail "Systemd writable path must not contain traversal: $read_write_path" done +assert_trusted_systemd_path() { + local declared_path="$1" + local deployment_uid component current owner mode parent + local -a components + deployment_uid="$(id -u)" + [[ ! -L "$declared_path" ]] || fail "Systemd writable path must not be a symlink: $declared_path" + parent="$(dirname -- "$declared_path")" + current='/' + IFS='/' read -r -a components <<<"${parent#/}" + for component in "${components[@]}"; do + [[ -n "$component" ]] || continue + current="${current%/}/$component" + [[ -d "$current" && ! -L "$current" ]] || fail "Systemd writable path parent must be a real directory: $current" + owner="$(stat -c '%u' -- "$current")" + mode="$(stat -c '%a' -- "$current")" + [[ "$owner" -eq 0 || "$owner" -eq "$deployment_uid" ]] || fail "Systemd writable path parent has an untrusted owner: $current" + (( (8#$mode & 0022) == 0 )) || fail "Systemd writable path parent must not be group/world writable: $current" + done +} + +snapshot_systemd_write_paths() { + local backup_temporary drop_in_mode + systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" + systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" + systemd_drop_in_existed=0 + systemd_drop_in_owner="" + systemd_drop_in_group="" + systemd_drop_in_mode="" + systemd_write_paths_snapshot_ready=1 + if [[ -e "$systemd_drop_in_path" || -L "$systemd_drop_in_path" ]]; then + [[ -f "$systemd_drop_in_path" && ! -L "$systemd_drop_in_path" ]] || fail "PowerForge systemd drop-in must be a regular file: $systemd_drop_in_path" + if [[ "$(id -u)" -eq 0 ]]; then + [[ "$(stat -c '%u' -- "$systemd_drop_in_path")" -eq 0 ]] || fail "PowerForge systemd drop-in must be owned by root: $systemd_drop_in_path" + drop_in_mode="$(stat -c '%a' -- "$systemd_drop_in_path")" + (( (8#$drop_in_mode & 0022) == 0 )) || fail "PowerForge systemd drop-in must not be group/world writable: $systemd_drop_in_path" + fi + systemd_drop_in_owner="$(stat -c '%u' -- "$systemd_drop_in_path")" + systemd_drop_in_group="$(stat -c '%g' -- "$systemd_drop_in_path")" + systemd_drop_in_mode="$(stat -c '%a' -- "$systemd_drop_in_path")" + backup_temporary="$(mktemp "${LOCK_ROOT}/.powerforge-systemd-${service_id}.XXXXXXXX")" + if ! install -m 0600 "$systemd_drop_in_path" "$backup_temporary"; then + rm -f -- "$backup_temporary" + return 1 + fi + systemd_drop_in_backup="$backup_temporary" + systemd_drop_in_existed=1 + fi +} + +restore_systemd_write_paths() { + local restore_temporary="" + [[ "$systemd_write_paths_snapshot_ready" == '1' ]] || return 0 + if [[ "$systemd_drop_in_existed" == '1' ]]; then + install -d -m 0755 "$systemd_drop_in_dir" || return 1 + restore_temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.restore.XXXXXXXX")" || return 1 + if ! install -m "$systemd_drop_in_mode" "$systemd_drop_in_backup" "$restore_temporary" || + ! chown "$systemd_drop_in_owner:$systemd_drop_in_group" "$restore_temporary" || + ! mv -f -- "$restore_temporary" "$systemd_drop_in_path"; then + rm -f -- "$restore_temporary" + return 1 + fi + else + rm -f -- "$systemd_drop_in_path" || return 1 + fi + systemctl daemon-reload || return 1 + if [[ -n "$systemd_drop_in_backup" && "$BASH_SUBSHELL" -eq 0 ]]; then + rm -f -- "$systemd_drop_in_backup" || log "WARNING: restored systemd drop-in backup remains at $systemd_drop_in_backup" + systemd_drop_in_backup="" + fi + systemd_write_paths_snapshot_ready=0 +} + +commit_systemd_write_paths() { + [[ -z "$systemd_drop_in_backup" ]] || rm -f -- "$systemd_drop_in_backup" + systemd_drop_in_backup="" + systemd_write_paths_snapshot_ready=0 +} + +report_systemd_restore_failure() { + if [[ -n "$systemd_drop_in_backup" ]]; then + log "ERROR: failed to restore systemd writable paths; backup retained at $systemd_drop_in_backup" >&2 + else + log 'ERROR: failed to reload the prior systemd writable-path state.' >&2 + fi +} + reconcile_systemd_write_paths() ( - drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" - drop_in_path="${drop_in_dir}/powerforge-read-write-paths.conf" if ((${#systemd_read_write_paths[@]} == 0)); then - [[ ! -f "$drop_in_path" ]] || rm -f -- "$drop_in_path" + [[ ! -f "$systemd_drop_in_path" ]] || rm -f -- "$systemd_drop_in_path" systemctl daemon-reload return 0 fi - install -d -m 0755 "$drop_in_dir" - temporary="$(mktemp "${drop_in_dir}/.powerforge-read-write-paths.XXXXXXXX")" + install -d -m 0755 "$systemd_drop_in_dir" + temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.XXXXXXXX")" trap 'rm -f -- "$temporary"' EXIT { printf '[Service]\n' @@ -117,10 +209,10 @@ reconcile_systemd_write_paths() ( done } >"$temporary" chmod 0644 "$temporary" - if [[ -f "$drop_in_path" ]] && cmp -s -- "$temporary" "$drop_in_path"; then + if [[ -f "$systemd_drop_in_path" ]] && cmp -s -- "$temporary" "$systemd_drop_in_path"; then rm -f -- "$temporary" else - mv -f -- "$temporary" "$drop_in_path" + mv -f -- "$temporary" "$systemd_drop_in_path" fi systemctl daemon-reload ) @@ -129,15 +221,26 @@ mkdir -p "$LOCK_ROOT" exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" flock -n 9 || fail "Another deployment is active for $service_id." +mkdir -p "$SERVICE_ROOT/releases" +resolved_release_root="$(realpath -e -- "$SERVICE_ROOT/releases")" systemd_read_write_paths=() for read_write_path in "${configured_systemd_read_write_paths[@]}"; do [[ -d "$read_write_path" ]] || fail "Systemd writable path does not exist: $read_write_path" + assert_trusted_systemd_path "$read_write_path" resolved_read_write_path="$(realpath -e -- "$read_write_path")" [[ -d "$resolved_read_write_path" && "$resolved_read_write_path" != '/' ]] || fail "Systemd writable path is not a safe directory: $read_write_path" + [[ "$resolved_read_write_path" == "$read_write_path" ]] || fail "Systemd writable path must be canonical and contain no symlinked components: $read_write_path" [[ "$resolved_read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Resolved systemd writable path contains unsupported characters: $read_write_path" + if [[ "$resolved_read_write_path" == "$resolved_release_root" || + "$resolved_read_write_path" == "$resolved_release_root"/* || + "$resolved_release_root" == "$resolved_read_write_path"/* ]]; then + fail "Systemd writable path must not overlap immutable release storage: $read_write_path" + fi systemd_read_write_paths+=("$resolved_read_write_path") done +snapshot_systemd_write_paths +trap 'exit_code=$?; set +e; restore_systemd_write_paths || report_systemd_restore_failure; exit "$exit_code"' ERR INT TERM reconcile_systemd_write_paths archive="$(realpath -e "$archive")" @@ -185,7 +288,6 @@ while IFS= read -r listing; do [[ "$entry_type" == '-' || "$entry_type" == 'd' ]] || fail 'Archive contains links or special files.' done < <(tar -tvf "$archive") -mkdir -p "$SERVICE_ROOT/releases" release_id="$(date -u +%Y%m%d%H%M%S)-${run_id}-${run_attempt}-${source_sha:0:12}" release_dir="$SERVICE_ROOT/releases/$release_id" [[ ! -e "$release_dir" ]] || fail "Release already exists: $release_id" @@ -209,14 +311,24 @@ verify_health() { rollback() { local exit_code="$1" + local permissions_restored=1 set +e + if ! restore_systemd_write_paths; then + permissions_restored=0 + report_systemd_restore_failure + fi if [[ "$promoted" == '1' ]]; then if [[ -n "$previous_target" && -d "$previous_target" ]]; then log "Deployment failed; rolling back to $previous_target" rollback_link="$SERVICE_ROOT/.current.rollback.$$" ln -s "$previous_target" "$rollback_link" mv -Tf "$rollback_link" "$SERVICE_ROOT/current" - systemctl restart "$SYSTEMD_SERVICE" + if [[ "$permissions_restored" == '1' ]]; then + systemctl restart "$SYSTEMD_SERVICE" + else + log 'Permission rollback failed; stopping the service instead of restarting with unverified write access.' >&2 + systemctl stop "$SYSTEMD_SERVICE" + fi else log 'Deployment failed; removing the first release from current and stopping the service.' rm -f "$SERVICE_ROOT/current" @@ -253,6 +365,7 @@ for ((index=RELEASES_TO_KEEP; index<${#old_releases[@]}; index++)); do [[ "${old_releases[$index]}" == "$release_dir" || "${old_releases[$index]}" == "$previous_target" ]] || rm -rf "${old_releases[$index]}" done +commit_systemd_write_paths trap - ERR INT TERM cleanup_staging trap - EXIT diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index c586a51b43..e74b7afb60 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -53,8 +53,13 @@ data directories when the service unit uses `ProtectSystem=strict`. The root-own promoter writes a PowerForge-owned systemd drop-in and reloads systemd before restart, so application releases remain immutable while declared databases, uploads, or other mutable service state stay writable. The deployment rejects missing, relative, root, -traversal, or systemd-special paths. Removing the setting removes only PowerForge's -owned drop-in on the next deployment, so obsolete write access does not persist. +traversal, symlinked, redirectable, release-overlapping, or systemd-special paths. +Every parent must be root-owned and not group/world writable when the promoter runs +as root. Removing the setting removes only PowerForge's owned drop-in after a +successful deployment, while a failed deployment restores the previous permissions +before rolling the application back. Restoration preserves the previous drop-in owner, +group, and mode. If the permissions cannot be restored and reloaded, the promoter +keeps its recovery backup and stops instead of restarting with unverified access. Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 62200089b3..2aa9388056 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" deploy_script="$repo_root/Deployment/Linux/powerforge-service-deploy.sh" -test_root="$(mktemp -d)" +test_root="$(mktemp -d "${HOME}/powerforge-service-deploy-tests.XXXXXXXX")" trap 'rm -rf "$test_root" /tmp/powerforge-service-example /tmp/powerforge-service-fresh' EXIT mkdir -p "$test_root/config" "$test_root/locks" "$test_root/bin" "$test_root/service" @@ -20,6 +20,15 @@ printf '%s\n' "$*" >>"$TEST_SYSTEMCTL_LOG" if [[ "$*" == 'daemon-reload' && "${FAIL_DAEMON_RELOAD:-}" == '1' ]]; then exit 1 fi +if [[ "$*" == 'daemon-reload' && -n "${FAIL_DAEMON_RELOAD_COUNT_FILE:-}" ]]; then + count=0 + [[ ! -f "$FAIL_DAEMON_RELOAD_COUNT_FILE" ]] || count="$(cat "$FAIL_DAEMON_RELOAD_COUNT_FILE")" + count=$((count + 1)) + printf '%s\n' "$count" >"$FAIL_DAEMON_RELOAD_COUNT_FILE" + if (( count >= ${FAIL_DAEMON_RELOAD_FROM_CALL:-2} )); then + exit 1 + fi +fi EOF cat >"$test_root/bin/curl" <<'EOF' @@ -78,8 +87,8 @@ EOF } write_config example "$test_root/service" -mkdir -p "$test_root/service-data" -printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-data" >>"$test_root/config/example.env" +mkdir -p "$test_root/service/data" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service/data" >>"$test_root/config/example.env" create_stage example 92001 1 1111111111111111111111111111111111111111 TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" \ --service example @@ -91,12 +100,35 @@ grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" grep -q '^daemon-reload$' "$TEST_SYSTEMCTL_LOG" drop_in="$POWERFORGE_SYSTEMD_CONFIG_ROOT/example.service.d/powerforge-read-write-paths.conf" grep -qxF '[Service]' "$drop_in" -grep -qxF "ReadWritePaths=$test_root/service-data" "$drop_in" +grep -qxF "ReadWritePaths=$test_root/service/data" "$drop_in" [[ ! -e /tmp/powerforge-service-example ]] if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 fi + +previous_example_target="$(readlink -f "$test_root/service/current")" +mkdir -p "$test_root/service/data-next" +write_config example "$test_root/service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service/data-next" >>"$test_root/config/example.env" +create_stage example 92005 1 5555555555555555555555555555555555555555 +: >"$TEST_SYSTEMCTL_LOG" +reload_count_file="$test_root/restore-reload-count" +if TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=5555555555555555555555555555555555555555 \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$reload_count_file" \ + FAIL_DAEMON_RELOAD_FROM_CALL=2 \ + "$deploy_script" --service example; then + echo 'Deployment unexpectedly restarted after permission rollback failed.' >&2 + exit 1 +fi +[[ "$(readlink -f "$test_root/service/current")" == "$previous_example_target" ]] +grep -qxF "ReadWritePaths=$test_root/service/data" "$drop_in" +[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -eq 1 ]] +grep -q '^stop example.service$' "$TEST_SYSTEMCTL_LOG" +find "$test_root/locks" -maxdepth 1 -type f -name '.powerforge-systemd-example.*' | grep -q . +rm -f -- "$test_root/locks"/.powerforge-systemd-example.* +: >"$TEST_SYSTEMCTL_LOG" [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] create_stage example 92002 1 2222222222222222222222222222222222222222 @@ -113,6 +145,7 @@ mkdir -p "$test_root/fresh-service" write_config fresh "$test_root/fresh-service" mkdir -p "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d" printf '[Service]\nReadWritePaths=/obsolete\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" +chmod 0640 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" create_stage fresh 92003 1 3333333333333333333333333333333333333333 if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333333333333333333333333333 "$deploy_script" \ --service fresh; then @@ -120,8 +153,12 @@ if TEST_SERVICE_ROOT="$test_root/fresh-service" FAIL_SOURCE_SHA=3333333333333333 exit 1 fi [[ ! -e "$test_root/fresh-service/current" ]] -[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" ]] +grep -qxF 'ReadWritePaths=/obsolete' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" +[[ "$(stat -c '%a' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf")" == '640' ]] grep -q '^stop fresh.service$' "$TEST_SYSTEMCTL_LOG" +create_stage fresh 92003 1 3333333333333333333333333333333333333333 +TEST_SERVICE_ROOT="$test_root/fresh-service" "$deploy_script" --service fresh +[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/fresh.service.d/powerforge-read-write-paths.conf" ]] mkdir -p "$test_root/unsafe-service" write_config unsafe "$test_root/unsafe-service" @@ -139,6 +176,38 @@ if TEST_SERVICE_ROOT="$test_root/glob-service" "$deploy_script" --service glob; exit 1 fi +mkdir -p "$test_root/overlap-service/releases/nested" +for overlap_path in \ + "$test_root/overlap-service" \ + "$test_root/overlap-service/releases" \ + "$test_root/overlap-service/releases/nested"; do + write_config overlap "$test_root/overlap-service" + printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$overlap_path" >>"$test_root/config/overlap.env" + if TEST_SERVICE_ROOT="$test_root/overlap-service" "$deploy_script" --service overlap; then + echo "Deployment unexpectedly allowed writable access to immutable release storage: $overlap_path" >&2 + exit 1 + fi +done + +mkdir -p "$test_root/symlink-target/data" "$test_root/symlink-service" +ln -s "$test_root/symlink-target" "$test_root/symlink-parent" +write_config symlink "$test_root/symlink-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/symlink-parent/data" >>"$test_root/config/symlink.env" +if TEST_SERVICE_ROOT="$test_root/symlink-service" "$deploy_script" --service symlink; then + echo 'Deployment unexpectedly accepted a symlinked writable-path parent.' >&2 + exit 1 +fi + +mkdir -p "$test_root/untrusted-parent/data" "$test_root/untrusted-service" +chmod 0777 "$test_root/untrusted-parent" +write_config untrusted "$test_root/untrusted-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/untrusted-parent/data" >>"$test_root/config/untrusted.env" +if TEST_SERVICE_ROOT="$test_root/untrusted-service" "$deploy_script" --service untrusted; then + echo 'Deployment unexpectedly accepted a writable-path parent that can be redirected.' >&2 + exit 1 +fi +chmod 0755 "$test_root/untrusted-parent" + mkdir -p "$test_root/reload-service" "$test_root/reload-data" write_config reload "$test_root/reload-service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/reload-data" >>"$test_root/config/reload.env" @@ -147,6 +216,7 @@ if TEST_SERVICE_ROOT="$test_root/reload-service" FAIL_DAEMON_RELOAD=1 "$deploy_s echo 'Deployment unexpectedly ignored a failed systemd reload.' >&2 exit 1 fi +[[ ! -e "$POWERFORGE_SYSTEMD_CONFIG_ROOT/reload.service.d/powerforge-read-write-paths.conf" ]] create_stage reload 92004 1 4444444444444444444444444444444444444444 TEST_SERVICE_ROOT="$test_root/reload-service" "$deploy_script" --service reload [[ -L "$test_root/reload-service/current" ]] From 26ec01c3fa286a1170a7dd8affe1c07afa2796fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 12:08:24 +0200 Subject: [PATCH 3/8] Complete Linux promoter transaction hardening --- Deployment/Linux/powerforge-service-deploy.sh | 173 +++++++++++--- Docs/PowerForge.Web.LinuxServiceDeployment.md | 7 + .../GitHubServiceLinuxDeployWorkflowTests.cs | 6 +- .../Linux/powerforge-service-deploy.tests.sh | 223 +++++++++++++++++- 4 files changed, 367 insertions(+), 42 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 15753734c3..3cfc5087df 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -7,6 +7,7 @@ CONFIG_ROOT="${POWERFORGE_SERVICE_CONFIG_ROOT:-/etc/powerforge/services}" LOCK_ROOT="${POWERFORGE_SERVICE_LOCK_ROOT:-/var/lock}" TRUSTED_STAGE_ROOT="${POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT:-/var/lib/powerforge/service-deployment-staging}" SYSTEMD_CONFIG_ROOT="${POWERFORGE_SYSTEMD_CONFIG_ROOT:-/etc/systemd/system}" +deployment_shell_pid="$BASHPID" service_id="" archive="" metadata="" @@ -40,6 +41,25 @@ fail() { return 1 } +assert_trusted_directory_chain() { + local declared_path="$1" + local description="$2" + local deployment_uid component current owner mode + local -a components + deployment_uid="$(id -u)" + current='/' + IFS='/' read -r -a components <<<"${declared_path#/}" + for component in "${components[@]}"; do + [[ -n "$component" ]] || continue + current="${current%/}/$component" + [[ -d "$current" && ! -L "$current" ]] || fail "$description must be a real directory: $current" + owner="$(stat -c '%u' -- "$current")" + mode="$(stat -c '%a' -- "$current")" + [[ "$owner" -eq 0 || "$owner" -eq "$deployment_uid" ]] || fail "$description has an untrusted owner: $current" + (( (8#$mode & 0022) == 0 )) || fail "$description must not be group/world writable: $current" + done +} + usage() { echo 'Usage: powerforge-service-deploy --service ' } @@ -66,6 +86,9 @@ workflow_stage="/tmp/powerforge-service-${service_id}" archive="$workflow_stage/artifact.tar" metadata="$workflow_stage/deployment.json" +[[ "$CONFIG_ROOT" == /* && "$CONFIG_ROOT" != '/' && "$CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Service config root must be an absolute non-root path without whitespace.' +[[ -d "$CONFIG_ROOT" && ! -L "$CONFIG_ROOT" ]] || fail "Service config root must be a real directory: $CONFIG_ROOT" +assert_trusted_directory_chain "$CONFIG_ROOT" 'Service config root' config_path="${CONFIG_ROOT}/${service_id}.env" [[ -f "$config_path" && ! -L "$config_path" ]] || fail "Service is not configured: $service_id" if [[ "$(id -u)" -eq 0 ]]; then @@ -109,28 +132,50 @@ done assert_trusted_systemd_path() { local declared_path="$1" - local deployment_uid component current owner mode parent - local -a components - deployment_uid="$(id -u)" [[ ! -L "$declared_path" ]] || fail "Systemd writable path must not be a symlink: $declared_path" - parent="$(dirname -- "$declared_path")" - current='/' - IFS='/' read -r -a components <<<"${parent#/}" - for component in "${components[@]}"; do - [[ -n "$component" ]] || continue - current="${current%/}/$component" - [[ -d "$current" && ! -L "$current" ]] || fail "Systemd writable path parent must be a real directory: $current" - owner="$(stat -c '%u' -- "$current")" - mode="$(stat -c '%a' -- "$current")" - [[ "$owner" -eq 0 || "$owner" -eq "$deployment_uid" ]] || fail "Systemd writable path parent has an untrusted owner: $current" - (( (8#$mode & 0022) == 0 )) || fail "Systemd writable path parent must not be group/world writable: $current" - done + assert_trusted_directory_chain "$(dirname -- "$declared_path")" 'Systemd writable path parent' } -snapshot_systemd_write_paths() { - local backup_temporary drop_in_mode +prepare_systemd_drop_in_directory() { + local config_parent + config_parent="$(dirname -- "$SYSTEMD_CONFIG_ROOT")" + assert_trusted_directory_chain "$config_parent" 'Systemd config parent' + if [[ -e "$SYSTEMD_CONFIG_ROOT" || -L "$SYSTEMD_CONFIG_ROOT" ]]; then + [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT" + else + install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" + fi + assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' + systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" + if [[ -e "$systemd_drop_in_dir" || -L "$systemd_drop_in_dir" ]]; then + [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir" + else + install -d -m 0755 "$systemd_drop_in_dir" + fi + assert_trusted_directory_chain "$systemd_drop_in_dir" 'Systemd drop-in directory' +} + +prepare_service_release_root() { + [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT" + assert_trusted_directory_chain "$SERVICE_ROOT" 'Service root' + resolved_service_root="$(realpath -e -- "$SERVICE_ROOT")" + [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT" + SERVICE_ROOT="$resolved_service_root" + + resolved_release_root="${SERVICE_ROOT}/releases" + if [[ -e "$resolved_release_root" || -L "$resolved_release_root" ]]; then + [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || fail "Release root must be a real directory: $resolved_release_root" + else + install -d -m 0755 "$resolved_release_root" + fi + assert_trusted_directory_chain "$resolved_release_root" 'Release root' + [[ "$(realpath -e -- "$resolved_release_root")" == "$resolved_release_root" ]] || fail "Release root must be canonical and contain no symlinked components: $resolved_release_root" +} + +snapshot_systemd_write_paths() { + local backup_temporary drop_in_mode systemd_drop_in_existed=0 systemd_drop_in_owner="" systemd_drop_in_group="" @@ -160,7 +205,6 @@ restore_systemd_write_paths() { local restore_temporary="" [[ "$systemd_write_paths_snapshot_ready" == '1' ]] || return 0 if [[ "$systemd_drop_in_existed" == '1' ]]; then - install -d -m 0755 "$systemd_drop_in_dir" || return 1 restore_temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.restore.XXXXXXXX")" || return 1 if ! install -m "$systemd_drop_in_mode" "$systemd_drop_in_backup" "$restore_temporary" || ! chown "$systemd_drop_in_owner:$systemd_drop_in_group" "$restore_temporary" || @@ -180,9 +224,10 @@ restore_systemd_write_paths() { } commit_systemd_write_paths() { - [[ -z "$systemd_drop_in_backup" ]] || rm -f -- "$systemd_drop_in_backup" - systemd_drop_in_backup="" + local committed_backup="$systemd_drop_in_backup" systemd_write_paths_snapshot_ready=0 + systemd_drop_in_backup="" + [[ -z "$committed_backup" ]] || rm -f -- "$committed_backup" || log "WARNING: committed systemd drop-in backup remains at $committed_backup" } report_systemd_restore_failure() { @@ -199,7 +244,6 @@ reconcile_systemd_write_paths() ( systemctl daemon-reload return 0 fi - install -d -m 0755 "$systemd_drop_in_dir" temporary="$(mktemp "${systemd_drop_in_dir}/.powerforge-read-write-paths.XXXXXXXX")" trap 'rm -f -- "$temporary"' EXIT { @@ -221,8 +265,15 @@ mkdir -p "$LOCK_ROOT" exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" flock -n 9 || fail "Another deployment is active for $service_id." -mkdir -p "$SERVICE_ROOT/releases" -resolved_release_root="$(realpath -e -- "$SERVICE_ROOT/releases")" +prepare_service_release_root +unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" +service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" +exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" +flock -n 8 || fail "Another deployment is active for systemd unit $SYSTEMD_SERVICE." +exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" +flock -n 7 || fail "Another deployment is active for service root $SERVICE_ROOT." + +prepare_systemd_drop_in_directory systemd_read_write_paths=() for read_write_path in "${configured_systemd_read_write_paths[@]}"; do [[ -d "$read_write_path" ]] || fail "Systemd writable path does not exist: $read_write_path" @@ -240,7 +291,19 @@ for read_write_path in "${configured_systemd_read_write_paths[@]}"; do done snapshot_systemd_write_paths -trap 'exit_code=$?; set +e; restore_systemd_write_paths || report_systemd_restore_failure; exit "$exit_code"' ERR INT TERM +pre_promotion_failure() { + local exit_code="$1" + set +e + if ! restore_systemd_write_paths; then + report_systemd_restore_failure + log 'Permission rollback failed before promotion; stopping the service to prevent restart with unverified write access.' >&2 + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop $SYSTEMD_SERVICE after permission rollback failure." >&2 + fi + exit "$exit_code" +} +trap 'exit_code=$?; if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure "$exit_code"; else exit "$exit_code"; fi' ERR +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure 130; else exit 130; fi' INT +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure 143; else exit 143; fi' TERM reconcile_systemd_write_paths archive="$(realpath -e "$archive")" @@ -289,7 +352,7 @@ while IFS= read -r listing; do done < <(tar -tvf "$archive") release_id="$(date -u +%Y%m%d%H%M%S)-${run_id}-${run_attempt}-${source_sha:0:12}" -release_dir="$SERVICE_ROOT/releases/$release_id" +release_dir="$resolved_release_root/$release_id" [[ ! -e "$release_dir" ]] || fail "Release already exists: $release_id" health_response() { @@ -312,6 +375,10 @@ verify_health() { rollback() { local exit_code="$1" local permissions_restored=1 + local current_restored=1 + local service_safe=0 + local current_target="" + local rollback_link="" set +e if ! restore_systemd_write_paths; then permissions_restored=0 @@ -321,24 +388,50 @@ rollback() { if [[ -n "$previous_target" && -d "$previous_target" ]]; then log "Deployment failed; rolling back to $previous_target" rollback_link="$SERVICE_ROOT/.current.rollback.$$" - ln -s "$previous_target" "$rollback_link" - mv -Tf "$rollback_link" "$SERVICE_ROOT/current" - if [[ "$permissions_restored" == '1' ]]; then - systemctl restart "$SYSTEMD_SERVICE" + rm -f -- "$rollback_link" + if ! ln -s "$previous_target" "$rollback_link" || + ! mv -Tf "$rollback_link" "$SERVICE_ROOT/current" || + [[ "$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null)" != "$previous_target" ]]; then + current_restored=0 + rm -f -- "$rollback_link" + log 'ERROR: failed to restore the previous current release link.' >&2 + fi + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' ]]; then + if systemctl restart "$SYSTEMD_SERVICE"; then + service_safe=1 + else + log "ERROR: failed to restart restored service $SYSTEMD_SERVICE; stopping it." >&2 + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi else - log 'Permission rollback failed; stopping the service instead of restarting with unverified write access.' >&2 - systemctl stop "$SYSTEMD_SERVICE" + log 'Rollback state is unverified; stopping instead of restarting the service.' >&2 + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 fi else log 'Deployment failed; removing the first release from current and stopping the service.' - rm -f "$SERVICE_ROOT/current" - systemctl stop "$SYSTEMD_SERVICE" + if ! rm -f -- "$SERVICE_ROOT/current" || [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then + current_restored=0 + log 'ERROR: failed to remove the first release from current.' >&2 + fi + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + if [[ "$service_safe" != '1' ]]; then + log "CRITICAL: failed to prove $SYSTEMD_SERVICE is safely restored or stopped." >&2 + fi + fi + current_target="$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null || true)" + if [[ -n "$release_dir" && -d "$release_dir" && "$release_dir" != "$previous_target" ]]; then + if [[ "$current_target" != "$release_dir" && ( "$promoted" != '1' || ( "$current_restored" == '1' && "$service_safe" == '1' ) ) ]]; then + rm -rf -- "$release_dir" || log "WARNING: failed to remove rejected release $release_dir" >&2 + else + log "Rejected release retained for recovery: $release_dir" >&2 fi fi - [[ -z "$release_dir" || ! -d "$release_dir" || "$release_dir" == "$previous_target" ]] || rm -rf "$release_dir" exit "$exit_code" } -trap 'rollback $?' ERR INT TERM +trap 'exit_code=$?; if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback "$exit_code"; else exit "$exit_code"; fi' ERR +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 130; else exit 130; fi' INT +trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 143; else exit 143; fi' TERM mkdir -p "$release_dir" tar --extract --file "$archive" --directory "$release_dir" --no-same-owner --no-same-permissions @@ -349,24 +442,26 @@ done mkdir -p "$release_dir/_powerforge" install -m 0644 "$metadata" "$release_dir/_powerforge/deployment.json" -if [[ -L "$SERVICE_ROOT/current" ]]; then +if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then + [[ -L "$SERVICE_ROOT/current" ]] || fail 'Current release pointer must be a symlink.' previous_target="$(readlink -f "$SERVICE_ROOT/current")" + [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || fail 'Current release pointer must resolve inside the canonical release root.' fi candidate_link="$SERVICE_ROOT/.current.${run_id}.${run_attempt}" ln -s "$release_dir" "$candidate_link" -mv -Tf "$candidate_link" "$SERVICE_ROOT/current" promoted=1 +mv -Tf "$candidate_link" "$SERVICE_ROOT/current" systemctl restart "$SYSTEMD_SERVICE" verify_health -mapfile -t old_releases < <(find "$SERVICE_ROOT/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}') +mapfile -t old_releases < <(find "$resolved_release_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}') for ((index=RELEASES_TO_KEEP; index<${#old_releases[@]}; index++)); do [[ "${old_releases[$index]}" == "$release_dir" || "${old_releases[$index]}" == "$previous_target" ]] || rm -rf "${old_releases[$index]}" done -commit_systemd_write_paths trap - ERR INT TERM +commit_systemd_write_paths cleanup_staging trap - EXIT log "Promoted $service_id release $release_id from $source_sha" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index e74b7afb60..dc363ca815 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -60,6 +60,13 @@ successful deployment, while a failed deployment restores the previous permissio before rolling the application back. Restoration preserves the previous drop-in owner, group, and mode. If the permissions cannot be restored and reloaded, the promoter keeps its recovery backup and stops instead of restarting with unverified access. +The systemd configuration root and unit drop-in directory must also be real, +root-owned, non-writable directory chains; the promoter never follows a service-owned +drop-in directory or reloads configuration from one. The same trust rule protects the +root-sourced service configuration and the canonical service/release roots. Deployments +are serialized by service id, systemd unit, and canonical service root. Cancellation +uses explicit non-zero signal exits, and rollback retains a rejected release whenever +the previous link or safe service state cannot be proven. Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: diff --git a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs index a5b81582d7..7a98759e0a 100644 --- a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs +++ b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs @@ -44,7 +44,11 @@ public void PromoterUsesTrustedStagingAndRollsBackSystemdService() Assert.Contains("sourceSha", script, StringComparison.Ordinal); Assert.Contains("workflowRunId", script, StringComparison.Ordinal); Assert.Contains("workflowRunAttempt", script, StringComparison.Ordinal); - Assert.True(script.IndexOf("flock -n", StringComparison.Ordinal) < script.IndexOf("realpath -e", StringComparison.Ordinal)); + Assert.Contains("assert_trusted_directory_chain \"$CONFIG_ROOT\"", script, StringComparison.Ordinal); + Assert.Contains("powerforge-systemd-${unit_lock_key}.lock", script, StringComparison.Ordinal); + Assert.Contains("powerforge-root-${service_root_lock_key}.lock", script, StringComparison.Ordinal); + Assert.Contains("rollback 143", script, StringComparison.Ordinal); + Assert.Contains("Rejected release retained for recovery", script, StringComparison.Ordinal); } private static string ReadRepoFile(params string[] relativePath) diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 2aa9388056..c409ce3c27 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -29,6 +29,12 @@ if [[ "$*" == 'daemon-reload' && -n "${FAIL_DAEMON_RELOAD_COUNT_FILE:-}" ]]; the exit 1 fi fi +if [[ -n "${FAIL_SYSTEMCTL_COMMAND:-}" && "$*" == "$FAIL_SYSTEMCTL_COMMAND" ]]; then + exit 1 +fi +if [[ -n "${SIGNAL_ON_SYSTEMCTL_COMMAND:-}" && "$*" == "$SIGNAL_ON_SYSTEMCTL_COMMAND" ]]; then + kill -"${SIGNAL_NAME:-TERM}" "$PPID" +fi EOF cat >"$test_root/bin/curl" <<'EOF' @@ -41,6 +47,16 @@ fi cat "$marker" EOF chmod +x "$test_root/bin/systemctl" "$test_root/bin/curl" + +cat >"$test_root/bin/mv" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +if [[ "${FAIL_ROLLBACK_LINK_MOVE:-}" == '1' && "$*" == *'.current.rollback.'* ]]; then + exit 1 +fi +exec /usr/bin/mv "$@" +EOF +chmod +x "$test_root/bin/mv" export PATH="$test_root/bin:$PATH" write_config() { @@ -132,14 +148,45 @@ rm -f -- "$test_root/locks"/.powerforge-systemd-example.* [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] create_stage example 92002 1 2222222222222222222222222222222222222222 -if TEST_SERVICE_ROOT="$test_root/service" FAIL_SOURCE_SHA=2222222222222222222222222222222222222222 "$deploy_script" \ +single_rollback_count_file="$test_root/single-rollback-count" +if TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=2222222222222222222222222222222222222222 \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$single_rollback_count_file" \ + FAIL_DAEMON_RELOAD_FROM_CALL=3 \ + "$deploy_script" \ --service example; then echo 'Deployment unexpectedly succeeded when exact provenance health failed.' >&2 exit 1 fi [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] [[ ! -e /tmp/powerforge-service-example ]] -[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -ge 3 ]] +[[ "$(cat "$single_rollback_count_file")" == '2' ]] +[[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -eq 2 ]] +if grep -q '^stop example.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Top-level permission rollback unexpectedly stopped the restored healthy service.' >&2 + exit 1 +fi + +mkdir -p "$test_root/early-service" "$test_root/early-data" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d" +write_config early "$test_root/early-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/early-data" >>"$test_root/config/early.env" +printf '[Service]\nReadWritePaths=/previous\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" +create_stage early 92006 1 6666666666666666666666666666666666666666 +sed -i 's/"sourceSha": "[^"]*"/"sourceSha": "invalid"/' /tmp/powerforge-service-early/deployment.json +: >"$TEST_SYSTEMCTL_LOG" +early_reload_count_file="$test_root/early-reload-count" +if TEST_SERVICE_ROOT="$test_root/early-service" \ + FAIL_DAEMON_RELOAD_COUNT_FILE="$early_reload_count_file" \ + FAIL_DAEMON_RELOAD_FROM_CALL=2 \ + "$deploy_script" --service early; then + echo 'Pre-promotion validation unexpectedly ignored a failed permission restore.' >&2 + exit 1 +fi +grep -q '^stop early.service$' "$TEST_SYSTEMCTL_LOG" +grep -qxF 'ReadWritePaths=/previous' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" +find "$test_root/locks" -maxdepth 1 -type f -name '.powerforge-systemd-early.*' | grep -q . +rm -f -- "$test_root/locks"/.powerforge-systemd-early.* +: >"$TEST_SYSTEMCTL_LOG" mkdir -p "$test_root/fresh-service" write_config fresh "$test_root/fresh-service" @@ -208,6 +255,178 @@ if TEST_SERVICE_ROOT="$test_root/untrusted-service" "$deploy_script" --service u fi chmod 0755 "$test_root/untrusted-parent" +mkdir -p "$test_root/systemd-symlink-service" "$test_root/systemd-attacker" +write_config dirsymlink "$test_root/systemd-symlink-service" +ln -s "$test_root/systemd-attacker" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/dirsymlink.service.d" +if TEST_SERVICE_ROOT="$test_root/systemd-symlink-service" "$deploy_script" --service dirsymlink; then + echo 'Deployment unexpectedly accepted a symlinked systemd drop-in directory.' >&2 + exit 1 +fi +rm -f -- "$POWERFORGE_SYSTEMD_CONFIG_ROOT/dirsymlink.service.d" + +mkdir -p "$test_root/systemd-untrusted-service" "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" +chmod 0777 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" +write_config diruntrusted "$test_root/systemd-untrusted-service" +if TEST_SERVICE_ROOT="$test_root/systemd-untrusted-service" "$deploy_script" --service diruntrusted; then + echo 'Deployment unexpectedly accepted a writable systemd drop-in directory.' >&2 + exit 1 +fi +chmod 0755 "$POWERFORGE_SYSTEMD_CONFIG_ROOT/diruntrusted.service.d" + +mkdir -p "$test_root/config-symlink-target" "$test_root/config-trust-service" +write_config configtrust "$test_root/config-trust-service" +cp "$test_root/config/configtrust.env" "$test_root/config-symlink-target/configtrust.env" +ln -s "$test_root/config-symlink-target" "$test_root/config-symlink" +set +e +config_symlink_output="$(POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config-symlink" TEST_SERVICE_ROOT="$test_root/config-trust-service" "$deploy_script" --service configtrust 2>&1)" +config_symlink_status=$? +set -e +if [[ "$config_symlink_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked service config root.' >&2 + exit 1 +fi +grep -q 'Service config root must be a real directory' <<<"$config_symlink_output" +mkdir -p "$test_root/config-writable" +cp "$test_root/config/configtrust.env" "$test_root/config-writable/configtrust.env" +chmod 0777 "$test_root/config-writable" +set +e +config_writable_output="$(POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config-writable" TEST_SERVICE_ROOT="$test_root/config-trust-service" "$deploy_script" --service configtrust 2>&1)" +config_writable_status=$? +set -e +if [[ "$config_writable_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a writable service config root.' >&2 + exit 1 +fi +grep -q 'Service config root must not be group/world writable' <<<"$config_writable_output" +chmod 0755 "$test_root/config-writable" + +mkdir -p "$test_root/service-root-target" "$test_root/service-root-config" +ln -s "$test_root/service-root-target" "$test_root/service-root-link" +write_config rootsymlink "$test_root/service-root-link" +set +e +root_symlink_output="$(TEST_SERVICE_ROOT="$test_root/service-root-target" "$deploy_script" --service rootsymlink 2>&1)" +root_symlink_status=$? +set -e +if [[ "$root_symlink_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked service root.' >&2 + exit 1 +fi +grep -q 'Service root must be a real, pre-provisioned directory' <<<"$root_symlink_output" +mkdir -p "$test_root/service-root-writable" +chmod 0777 "$test_root/service-root-writable" +write_config rootwritable "$test_root/service-root-writable" +set +e +root_writable_output="$(TEST_SERVICE_ROOT="$test_root/service-root-writable" "$deploy_script" --service rootwritable 2>&1)" +root_writable_status=$? +set -e +if [[ "$root_writable_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a writable service root.' >&2 + exit 1 +fi +grep -q 'Service root must not be group/world writable' <<<"$root_writable_output" +chmod 0755 "$test_root/service-root-writable" + +mkdir -p "$test_root/release-link-service" "$test_root/release-link-target" +ln -s "$test_root/release-link-target" "$test_root/release-link-service/releases" +write_config releaselink "$test_root/release-link-service" +set +e +release_link_output="$(TEST_SERVICE_ROOT="$test_root/release-link-service" "$deploy_script" --service releaselink 2>&1)" +release_link_status=$? +set -e +if [[ "$release_link_status" -eq 0 ]]; then + echo 'Deployment unexpectedly accepted a symlinked release root.' >&2 + exit 1 +fi +grep -q 'Release root must be a real directory' <<<"$release_link_output" + +mkdir -p "$test_root/unit-lock-service" +write_config unitalias "$test_root/unit-lock-service" +sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=example.service/' "$test_root/config/unitalias.env" +unit_lock_key="$(printf '%s' 'example.service' | sha256sum | awk '{print $1}')" +unit_lock_ready="$test_root/unit-lock-ready" +( + exec 200>"$test_root/locks/powerforge-systemd-${unit_lock_key}.lock" + flock 200 + : >"$unit_lock_ready" + sleep 30 +) & +unit_lock_holder=$! +for _ in {1..100}; do [[ -e "$unit_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$unit_lock_ready" ]] +set +e +unit_lock_output="$(TEST_SERVICE_ROOT="$test_root/unit-lock-service" "$deploy_script" --service unitalias 2>&1)" +unit_lock_status=$? +set -e +if [[ "$unit_lock_status" -eq 0 ]]; then + echo 'Deployment unexpectedly bypassed serialization for a shared systemd unit.' >&2 + kill "$unit_lock_holder" 2>/dev/null || true + wait "$unit_lock_holder" 2>/dev/null || true + exit 1 +fi +grep -q 'Another deployment is active for systemd unit example.service' <<<"$unit_lock_output" +kill "$unit_lock_holder" 2>/dev/null || true +wait "$unit_lock_holder" 2>/dev/null || true + +write_config rootalias "$test_root/service" +service_root_lock_key="$(printf '%s' "$test_root/service" | sha256sum | awk '{print $1}')" +root_lock_ready="$test_root/root-lock-ready" +( + exec 201>"$test_root/locks/powerforge-root-${service_root_lock_key}.lock" + flock 201 + : >"$root_lock_ready" + sleep 30 +) & +root_lock_holder=$! +for _ in {1..100}; do [[ -e "$root_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$root_lock_ready" ]] +set +e +root_lock_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service rootalias 2>&1)" +root_lock_status=$? +set -e +if [[ "$root_lock_status" -eq 0 ]]; then + echo 'Deployment unexpectedly bypassed serialization for a shared service root.' >&2 + kill "$root_lock_holder" 2>/dev/null || true + wait "$root_lock_holder" 2>/dev/null || true + exit 1 +fi +grep -q "Another deployment is active for service root $test_root/service" <<<"$root_lock_output" +kill "$root_lock_holder" 2>/dev/null || true +wait "$root_lock_holder" 2>/dev/null || true + +mkdir -p "$test_root/signal-service" +write_config signal "$test_root/signal-service" +create_stage signal 92007 1 7777777777777777777777777777777777777777 +set +e +TEST_SERVICE_ROOT="$test_root/signal-service" \ + SIGNAL_ON_SYSTEMCTL_COMMAND='restart signal.service' \ + SIGNAL_NAME=TERM \ + "$deploy_script" --service signal +signal_status=$? +set -e +[[ "$signal_status" -eq 143 ]] +[[ ! -e "$test_root/signal-service/current" ]] +grep -q '^stop signal.service$' "$TEST_SYSTEMCTL_LOG" + +mkdir -p "$test_root/rollback-service" +write_config rollback "$test_root/rollback-service" +create_stage rollback 92008 1 8888888888888888888888888888888888888888 +TEST_SERVICE_ROOT="$test_root/rollback-service" "$deploy_script" --service rollback +rollback_previous="$(readlink -f "$test_root/rollback-service/current")" +create_stage rollback 92009 1 9999999999999999999999999999999999999999 +set +e +rollback_output="$(TEST_SERVICE_ROOT="$test_root/rollback-service" \ + FAIL_SOURCE_SHA=9999999999999999999999999999999999999999 \ + FAIL_ROLLBACK_LINK_MOVE=1 \ + FAIL_SYSTEMCTL_COMMAND='stop rollback.service' \ + "$deploy_script" --service rollback 2>&1)" +rollback_status=$? +set -e +[[ "$rollback_status" -ne 0 ]] +grep -q 'CRITICAL: failed to prove rollback.service is safely restored or stopped.' <<<"$rollback_output" +rollback_current="$(readlink -f "$test_root/rollback-service/current")" +[[ "$rollback_current" != "$rollback_previous" && -d "$rollback_current" ]] +grep -q '9999999999999999999999999999999999999999' "$rollback_current/_powerforge/deployment.json" + mkdir -p "$test_root/reload-service" "$test_root/reload-data" write_config reload "$test_root/reload-service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/reload-data" >>"$test_root/config/reload.env" From 73b2be9d9144bcfae370f5d5c266af8beaef2ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 12:34:24 +0200 Subject: [PATCH 4/8] Make Linux service promotion crash consistent --- Deployment/Linux/powerforge-service-deploy.sh | 223 +++++++++++++++--- Docs/PowerForge.Web.LinuxServiceDeployment.md | 25 +- .../GitHubServiceLinuxDeployWorkflowTests.cs | 3 + .../Linux/powerforge-service-deploy.tests.sh | 72 +++++- 4 files changed, 277 insertions(+), 46 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 3cfc5087df..50e797ec52 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -6,6 +6,7 @@ umask 022 CONFIG_ROOT="${POWERFORGE_SERVICE_CONFIG_ROOT:-/etc/powerforge/services}" LOCK_ROOT="${POWERFORGE_SERVICE_LOCK_ROOT:-/var/lock}" TRUSTED_STAGE_ROOT="${POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT:-/var/lib/powerforge/service-deployment-staging}" +TRANSACTION_ROOT="${POWERFORGE_SERVICE_TRANSACTION_ROOT:-/var/lib/powerforge/service-deployment-state}" SYSTEMD_CONFIG_ROOT="${POWERFORGE_SYSTEMD_CONFIG_ROOT:-/etc/systemd/system}" deployment_shell_pid="$BASHPID" service_id="" @@ -24,6 +25,7 @@ systemd_drop_in_owner="" systemd_drop_in_group="" systemd_drop_in_mode="" systemd_write_paths_snapshot_ready=0 +systemd_transaction_path="" cleanup_staging() { [[ -z "$workflow_stage" || ! -d "$workflow_stage" ]] || rm -rf -- "$workflow_stage" @@ -60,6 +62,46 @@ assert_trusted_directory_chain() { done } +paths_overlap() { + local first="$1" + local second="$2" + [[ "$first" == "$second" || "$first" == "$second"/* || "$second" == "$first"/* ]] +} + +prepare_transaction_root() { + local parent resolved + [[ "$TRANSACTION_ROOT" == /* && "$TRANSACTION_ROOT" != '/' && "$TRANSACTION_ROOT" != *[[:space:]]* ]] || + fail 'Transaction root must be an absolute non-root path without whitespace.' + parent="$(dirname -- "$TRANSACTION_ROOT")" + assert_trusted_directory_chain "$parent" 'Transaction root parent' + if [[ -e "$TRANSACTION_ROOT" || -L "$TRANSACTION_ROOT" ]]; then + [[ -d "$TRANSACTION_ROOT" && ! -L "$TRANSACTION_ROOT" ]] || fail "Transaction root must be a real directory: $TRANSACTION_ROOT" + else + install -d -m 0700 "$TRANSACTION_ROOT" + fi + assert_trusted_directory_chain "$TRANSACTION_ROOT" 'Transaction root' + resolved="$(realpath -e -- "$TRANSACTION_ROOT")" + [[ "$resolved" == "$TRANSACTION_ROOT" ]] || fail "Transaction root must be canonical and contain no symlinked components: $TRANSACTION_ROOT" + TRANSACTION_ROOT="$resolved" +} + +prepare_trusted_stage_root() { + local parent resolved + [[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' && "$TRUSTED_STAGE_ROOT" != *[[:space:]]* ]] || + fail 'Trusted staging root must be an absolute non-root path without whitespace.' + parent="$(dirname -- "$TRUSTED_STAGE_ROOT")" + assert_trusted_directory_chain "$parent" 'Trusted staging root parent' + if [[ -e "$TRUSTED_STAGE_ROOT" || -L "$TRUSTED_STAGE_ROOT" ]]; then + [[ -d "$TRUSTED_STAGE_ROOT" && ! -L "$TRUSTED_STAGE_ROOT" ]] || fail "Trusted staging root must be a real directory: $TRUSTED_STAGE_ROOT" + else + install -d -m 0700 "$TRUSTED_STAGE_ROOT" + fi + assert_trusted_directory_chain "$TRUSTED_STAGE_ROOT" 'Trusted staging root' + resolved="$(realpath -e -- "$TRUSTED_STAGE_ROOT")" + [[ "$resolved" == "$TRUSTED_STAGE_ROOT" ]] || fail "Trusted staging root must be canonical and contain no symlinked components: $TRUSTED_STAGE_ROOT" + TRUSTED_STAGE_ROOT="$resolved" +} + usage() { echo 'Usage: powerforge-service-deploy --service ' } @@ -89,6 +131,9 @@ metadata="$workflow_stage/deployment.json" [[ "$CONFIG_ROOT" == /* && "$CONFIG_ROOT" != '/' && "$CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Service config root must be an absolute non-root path without whitespace.' [[ -d "$CONFIG_ROOT" && ! -L "$CONFIG_ROOT" ]] || fail "Service config root must be a real directory: $CONFIG_ROOT" assert_trusted_directory_chain "$CONFIG_ROOT" 'Service config root' +resolved_config_root="$(realpath -e -- "$CONFIG_ROOT")" +[[ "$resolved_config_root" == "$CONFIG_ROOT" ]] || fail "Service config root must be canonical and contain no symlinked components: $CONFIG_ROOT" +CONFIG_ROOT="$resolved_config_root" config_path="${CONFIG_ROOT}/${service_id}.env" [[ -f "$config_path" && ! -L "$config_path" ]] || fail "Service is not configured: $service_id" if [[ "$(id -u)" -eq 0 ]]; then @@ -146,6 +191,7 @@ prepare_systemd_drop_in_directory() { install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" fi assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' + [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT" systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" @@ -175,12 +221,18 @@ prepare_service_release_root() { } snapshot_systemd_write_paths() { - local backup_temporary drop_in_mode + local transaction_temporary drop_in_mode + [[ ! -e "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "An incomplete systemd writable-path transaction already exists: $systemd_transaction_path" + transaction_temporary="$(mktemp -d "${TRANSACTION_ROOT}/.systemd-${unit_lock_key}.XXXXXXXX")" + chmod 0700 "$transaction_temporary" systemd_drop_in_existed=0 systemd_drop_in_owner="" systemd_drop_in_group="" systemd_drop_in_mode="" - systemd_write_paths_snapshot_ready=1 + printf '%s\n' "$SERVICE_ROOT" >"$transaction_temporary/service-root" + printf '%s\n' "$SYSTEMD_SERVICE" >"$transaction_temporary/systemd-service" + printf '%s\n' "$previous_target" >"$transaction_temporary/previous-target" if [[ -e "$systemd_drop_in_path" || -L "$systemd_drop_in_path" ]]; then [[ -f "$systemd_drop_in_path" && ! -L "$systemd_drop_in_path" ]] || fail "PowerForge systemd drop-in must be a regular file: $systemd_drop_in_path" if [[ "$(id -u)" -eq 0 ]]; then @@ -191,14 +243,59 @@ snapshot_systemd_write_paths() { systemd_drop_in_owner="$(stat -c '%u' -- "$systemd_drop_in_path")" systemd_drop_in_group="$(stat -c '%g' -- "$systemd_drop_in_path")" systemd_drop_in_mode="$(stat -c '%a' -- "$systemd_drop_in_path")" - backup_temporary="$(mktemp "${LOCK_ROOT}/.powerforge-systemd-${service_id}.XXXXXXXX")" - if ! install -m 0600 "$systemd_drop_in_path" "$backup_temporary"; then - rm -f -- "$backup_temporary" + if ! install -m 0600 "$systemd_drop_in_path" "$transaction_temporary/drop-in"; then + rm -rf -- "$transaction_temporary" return 1 fi - systemd_drop_in_backup="$backup_temporary" + printf 'present\n' >"$transaction_temporary/drop-in-state" + printf '%s\n' "$systemd_drop_in_owner" >"$transaction_temporary/drop-in-owner" + printf '%s\n' "$systemd_drop_in_group" >"$transaction_temporary/drop-in-group" + printf '%s\n' "$systemd_drop_in_mode" >"$transaction_temporary/drop-in-mode" systemd_drop_in_existed=1 + else + printf 'absent\n' >"$transaction_temporary/drop-in-state" fi + chmod 0600 "$transaction_temporary"/* + sync -f "$transaction_temporary"/* + sync -f "$transaction_temporary" + mv -- "$transaction_temporary" "$systemd_transaction_path" + sync -f "$TRANSACTION_ROOT" + systemd_drop_in_backup="${systemd_transaction_path}/drop-in" + systemd_write_paths_snapshot_ready=1 +} + +load_systemd_write_paths_transaction() { + local stored_root stored_service stored_state + [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" + assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' + stored_root="$(<"$systemd_transaction_path/service-root")" + stored_service="$(<"$systemd_transaction_path/systemd-service")" + previous_target="$(<"$systemd_transaction_path/previous-target")" + stored_state="$(<"$systemd_transaction_path/drop-in-state")" + [[ "$stored_root" == "$SERVICE_ROOT" ]] || fail "Incomplete transaction belongs to a different service root: $stored_root" + [[ "$stored_service" == "$SYSTEMD_SERVICE" ]] || fail "Incomplete transaction belongs to a different systemd unit: $stored_service" + if [[ -n "$previous_target" ]]; then + [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || + fail "Incomplete transaction contains an invalid previous release: $previous_target" + fi + if [[ "$stored_state" == 'present' ]]; then + [[ -f "$systemd_transaction_path/drop-in" && ! -L "$systemd_transaction_path/drop-in" ]] || + fail 'Incomplete transaction is missing its systemd drop-in backup.' + systemd_drop_in_owner="$(<"$systemd_transaction_path/drop-in-owner")" + systemd_drop_in_group="$(<"$systemd_transaction_path/drop-in-group")" + systemd_drop_in_mode="$(<"$systemd_transaction_path/drop-in-mode")" + [[ "$systemd_drop_in_owner" =~ ^[0-9]+$ && "$systemd_drop_in_group" =~ ^[0-9]+$ && "$systemd_drop_in_mode" =~ ^[0-7]{3,4}$ ]] || + fail 'Incomplete transaction contains invalid systemd drop-in metadata.' + systemd_drop_in_backup="${systemd_transaction_path}/drop-in" + systemd_drop_in_existed=1 + elif [[ "$stored_state" == 'absent' ]]; then + systemd_drop_in_backup="" + systemd_drop_in_existed=0 + else + fail "Incomplete transaction contains an invalid drop-in state: $stored_state" + fi + systemd_write_paths_snapshot_ready=1 } restore_systemd_write_paths() { @@ -216,25 +313,74 @@ restore_systemd_write_paths() { rm -f -- "$systemd_drop_in_path" || return 1 fi systemctl daemon-reload || return 1 - if [[ -n "$systemd_drop_in_backup" && "$BASH_SUBSHELL" -eq 0 ]]; then - rm -f -- "$systemd_drop_in_backup" || log "WARNING: restored systemd drop-in backup remains at $systemd_drop_in_backup" - systemd_drop_in_backup="" - fi +} + +finish_systemd_write_paths_transaction() { + [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/systemd-*.transaction ]] || + fail "Refusing to remove an unexpected transaction path: $systemd_transaction_path" + rm -rf -- "$systemd_transaction_path" || return 1 + sync -f "$TRANSACTION_ROOT" systemd_write_paths_snapshot_ready=0 + systemd_drop_in_backup="" } commit_systemd_write_paths() { - local committed_backup="$systemd_drop_in_backup" + local committed_path="${systemd_transaction_path}.committed.$$" + # The rename is the durable commit point. Ignore catchable termination during + # this tiny section so a signal cannot run rollback after the transaction has + # disappeared but before the in-memory state reflects that fact. + trap - INT TERM + mv -- "$systemd_transaction_path" "$committed_path" + promoted=0 systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" - [[ -z "$committed_backup" ]] || rm -f -- "$committed_backup" || log "WARNING: committed systemd drop-in backup remains at $committed_backup" + sync -f "$TRANSACTION_ROOT" || log "WARNING: failed to sync committed transaction directory $TRANSACTION_ROOT" >&2 + rm -rf -- "$committed_path" || log "WARNING: committed transaction cleanup remains at $committed_path" >&2 } report_systemd_restore_failure() { - if [[ -n "$systemd_drop_in_backup" ]]; then - log "ERROR: failed to restore systemd writable paths; backup retained at $systemd_drop_in_backup" >&2 + log "ERROR: failed to restore systemd writable paths; transaction retained at $systemd_transaction_path" >&2 +} + +restore_previous_current_link() { + local rollback_link="" + if [[ -n "$previous_target" ]]; then + rollback_link="$SERVICE_ROOT/.current.rollback.$$" + rm -f -- "$rollback_link" + ln -s "$previous_target" "$rollback_link" || return 1 + mv -Tf "$rollback_link" "$SERVICE_ROOT/current" || { rm -f -- "$rollback_link"; return 1; } + [[ "$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null)" == "$previous_target" ]] else - log 'ERROR: failed to reload the prior systemd writable-path state.' >&2 + rm -f -- "$SERVICE_ROOT/current" + [[ ! -e "$SERVICE_ROOT/current" && ! -L "$SERVICE_ROOT/current" ]] + fi +} + +recover_incomplete_systemd_transaction() { + local permissions_restored=1 current_restored=1 service_safe=0 + if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then + log "Recovering incomplete systemd writable-path transaction for $SYSTEMD_SERVICE." + if ! load_systemd_write_paths_transaction; then + systemctl stop "$SYSTEMD_SERVICE" || true + fail "Incomplete deployment transaction is invalid; $SYSTEMD_SERVICE was stopped and operator recovery is required." + fi + restore_systemd_write_paths || permissions_restored=0 + restore_previous_current_link || current_restored=0 + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' && -n "$previous_target" ]]; then + if systemctl restart "$SYSTEMD_SERVICE"; then + service_safe=1 + else + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + else + systemctl stop "$SYSTEMD_SERVICE" && service_safe=1 + fi + if [[ "$permissions_restored" != '1' || "$current_restored" != '1' || "$service_safe" != '1' ]]; then + report_systemd_restore_failure + fail "Incomplete deployment recovery could not prove $SYSTEMD_SERVICE safe." + fi + finish_systemd_write_paths_transaction + log "Recovered incomplete systemd writable-path transaction for $SYSTEMD_SERVICE." fi } @@ -266,6 +412,8 @@ exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" flock -n 9 || fail "Another deployment is active for $service_id." prepare_service_release_root +prepare_transaction_root +prepare_trusted_stage_root unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" @@ -274,6 +422,14 @@ exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" flock -n 7 || fail "Another deployment is active for service root $SERVICE_ROOT." prepare_systemd_drop_in_directory +systemd_transaction_path="${TRANSACTION_ROOT}/systemd-${unit_lock_key}.transaction" +recover_incomplete_systemd_transaction +previous_target="" +if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then + [[ -L "$SERVICE_ROOT/current" ]] || fail 'Current release pointer must be a symlink.' + previous_target="$(readlink -f "$SERVICE_ROOT/current")" + [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || fail 'Current release pointer must resolve inside the canonical release root.' +fi systemd_read_write_paths=() for read_write_path in "${configured_systemd_read_write_paths[@]}"; do [[ -d "$read_write_path" ]] || fail "Systemd writable path does not exist: $read_write_path" @@ -282,19 +438,24 @@ for read_write_path in "${configured_systemd_read_write_paths[@]}"; do [[ -d "$resolved_read_write_path" && "$resolved_read_write_path" != '/' ]] || fail "Systemd writable path is not a safe directory: $read_write_path" [[ "$resolved_read_write_path" == "$read_write_path" ]] || fail "Systemd writable path must be canonical and contain no symlinked components: $read_write_path" [[ "$resolved_read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Resolved systemd writable path contains unsupported characters: $read_write_path" - if [[ "$resolved_read_write_path" == "$resolved_release_root" || - "$resolved_read_write_path" == "$resolved_release_root"/* || - "$resolved_release_root" == "$resolved_read_write_path"/* ]]; then - fail "Systemd writable path must not overlap immutable release storage: $read_write_path" - fi + protected_roots=("$CONFIG_ROOT" "$SYSTEMD_CONFIG_ROOT" "$TRANSACTION_ROOT" "$TRUSTED_STAGE_ROOT" "$SERVICE_ROOT" "$(realpath -e -- "$LOCK_ROOT")") + for protected_root in "${protected_roots[@]}"; do + paths_overlap "$resolved_read_write_path" "$protected_root" || continue + fail "Systemd writable path must not overlap deployment control path $protected_root: $read_write_path" + done systemd_read_write_paths+=("$resolved_read_write_path") done snapshot_systemd_write_paths pre_promotion_failure() { local exit_code="$1" + local permissions_restored=1 set +e - if ! restore_systemd_write_paths; then + restore_systemd_write_paths || permissions_restored=0 + if [[ "$permissions_restored" == '1' ]]; then + finish_systemd_write_paths_transaction || permissions_restored=0 + fi + if [[ "$permissions_restored" != '1' ]]; then report_systemd_restore_failure log 'Permission rollback failed before promotion; stopping the service to prevent restart with unverified write access.' >&2 systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop $SYSTEMD_SERVICE after permission rollback failure." >&2 @@ -378,7 +539,6 @@ rollback() { local current_restored=1 local service_safe=0 local current_target="" - local rollback_link="" set +e if ! restore_systemd_write_paths; then permissions_restored=0 @@ -387,13 +547,8 @@ rollback() { if [[ "$promoted" == '1' ]]; then if [[ -n "$previous_target" && -d "$previous_target" ]]; then log "Deployment failed; rolling back to $previous_target" - rollback_link="$SERVICE_ROOT/.current.rollback.$$" - rm -f -- "$rollback_link" - if ! ln -s "$previous_target" "$rollback_link" || - ! mv -Tf "$rollback_link" "$SERVICE_ROOT/current" || - [[ "$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null)" != "$previous_target" ]]; then + if ! restore_previous_current_link; then current_restored=0 - rm -f -- "$rollback_link" log 'ERROR: failed to restore the previous current release link.' >&2 fi if [[ "$permissions_restored" == '1' && "$current_restored" == '1' ]]; then @@ -418,6 +573,11 @@ rollback() { if [[ "$service_safe" != '1' ]]; then log "CRITICAL: failed to prove $SYSTEMD_SERVICE is safely restored or stopped." >&2 fi + if [[ "$permissions_restored" == '1' && "$current_restored" == '1' && "$service_safe" == '1' ]]; then + finish_systemd_write_paths_transaction || log "WARNING: rollback transaction retained at $systemd_transaction_path" >&2 + else + log "Rollback transaction retained for recovery: $systemd_transaction_path" >&2 + fi fi current_target="$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null || true)" if [[ -n "$release_dir" && -d "$release_dir" && "$release_dir" != "$previous_target" ]]; then @@ -442,11 +602,6 @@ done mkdir -p "$release_dir/_powerforge" install -m 0644 "$metadata" "$release_dir/_powerforge/deployment.json" -if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then - [[ -L "$SERVICE_ROOT/current" ]] || fail 'Current release pointer must be a symlink.' - previous_target="$(readlink -f "$SERVICE_ROOT/current")" - [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || fail 'Current release pointer must resolve inside the canonical release root.' -fi candidate_link="$SERVICE_ROOT/.current.${run_id}.${run_attempt}" ln -s "$release_dir" "$candidate_link" promoted=1 @@ -460,8 +615,8 @@ for ((index=RELEASES_TO_KEEP; index<${#old_releases[@]}; index++)); do [[ "${old_releases[$index]}" == "$release_dir" || "${old_releases[$index]}" == "$previous_target" ]] || rm -rf "${old_releases[$index]}" done -trap - ERR INT TERM commit_systemd_write_paths +trap - ERR INT TERM cleanup_staging trap - EXIT log "Promoted $service_id release $release_id from $source_sha" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index dc363ca815..f2a14c5c06 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -30,6 +30,12 @@ Create one root-owned configuration per service under `/etc/powerforge/services` ```bash install -d -o root -g root -m 0750 /etc/powerforge/services +install -d -o root -g root -m 0755 /srv/example/service +install -d -o root -g root -m 0755 /var/lib/example-service +install -d -o root -g root -m 0755 /var/lib/powerforge +install -d -o root -g root -m 0700 \ + /var/lib/powerforge/service-deployment-staging \ + /var/lib/powerforge/service-deployment-state install -o root -g root -m 0640 \ Deployment/Linux/powerforge-service.env.example \ /etc/powerforge/services/example.env @@ -53,7 +59,10 @@ data directories when the service unit uses `ProtectSystem=strict`. The root-own promoter writes a PowerForge-owned systemd drop-in and reloads systemd before restart, so application releases remain immutable while declared databases, uploads, or other mutable service state stay writable. The deployment rejects missing, relative, root, -traversal, symlinked, redirectable, release-overlapping, or systemd-special paths. +traversal, symlinked, redirectable, release/control-plane-overlapping, or +systemd-special paths. Writable exceptions cannot contain or reside beneath the +service configuration, systemd configuration, lock, transaction, trusted staging, +or service/release roots. Every parent must be root-owned and not group/world writable when the promoter runs as root. Removing the setting removes only PowerForge's owned drop-in after a successful deployment, while a failed deployment restores the previous permissions @@ -67,6 +76,11 @@ root-sourced service configuration and the canonical service/release roots. Depl are serialized by service id, systemd unit, and canonical service root. Cancellation uses explicit non-zero signal exits, and rollback retains a rejected release whenever the previous link or safe service state cannot be proven. +Before changing a drop-in, the promoter persists the previous permission and current- +release state under `/var/lib/powerforge/service-deployment-state`. A later invocation +restores that state and proves the service restarted or stopped before accepting a new +deployment, so process termination or host loss cannot strand an uncommitted writable +policy. Recovery state is removed only at the successful deployment commit point. Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: @@ -173,7 +187,14 @@ The root promoter: 8. Stops the service after a failed first deployment and removes the failed release. 9. Retains the configured number of known-good releases. -The promoter never copies or removes files outside `SERVICE_ROOT`, its lock, and its deployment staging. Environment files, private keys, API credentials, queues, databases, registration stores, and other mutable state must remain external and be covered by the server-recovery manifest. +The promoter mutates only the configured `SERVICE_ROOT`, its lock and root-only +transaction/staging roots, and the PowerForge-owned +`SYSTEMD_CONFIG_ROOT/.d/powerforge-read-write-paths.conf`. It reloads systemd +after changing or restoring that drop-in. Environment files, private keys, API +credentials, queues, databases, registration stores, and other mutable state remain +external and must be covered by the server-recovery manifest. Recovery planning must +also preserve the service configuration and systemd unit/drop-ins; transaction state +is temporary and is either committed or replayed by the next promoter invocation. ## Recovery Coverage diff --git a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs index 7a98759e0a..b1a861c4e3 100644 --- a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs +++ b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs @@ -47,6 +47,9 @@ public void PromoterUsesTrustedStagingAndRollsBackSystemdService() Assert.Contains("assert_trusted_directory_chain \"$CONFIG_ROOT\"", script, StringComparison.Ordinal); Assert.Contains("powerforge-systemd-${unit_lock_key}.lock", script, StringComparison.Ordinal); Assert.Contains("powerforge-root-${service_root_lock_key}.lock", script, StringComparison.Ordinal); + Assert.Contains("service-deployment-state", script, StringComparison.Ordinal); + Assert.Contains("Recovering incomplete systemd writable-path transaction", script, StringComparison.Ordinal); + Assert.Contains("must not overlap deployment control path", script, StringComparison.Ordinal); Assert.Contains("rollback 143", script, StringComparison.Ordinal); Assert.Contains("Rejected release retained for recovery", script, StringComparison.Ordinal); } diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index c409ce3c27..9b8f29cab3 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -10,6 +10,7 @@ mkdir -p "$test_root/config" "$test_root/locks" "$test_root/bin" "$test_root/ser export POWERFORGE_SERVICE_CONFIG_ROOT="$test_root/config" export POWERFORGE_SERVICE_LOCK_ROOT="$test_root/locks" export POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT="$test_root/trusted-stage" +export POWERFORGE_SERVICE_TRANSACTION_ROOT="$test_root/transactions" export POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd" export TEST_SYSTEMCTL_LOG="$test_root/systemctl.log" @@ -103,8 +104,8 @@ EOF } write_config example "$test_root/service" -mkdir -p "$test_root/service/data" -printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service/data" >>"$test_root/config/example.env" +mkdir -p "$test_root/example-data" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" create_stage example 92001 1 1111111111111111111111111111111111111111 TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" \ --service example @@ -116,17 +117,46 @@ grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" grep -q '^daemon-reload$' "$TEST_SYSTEMCTL_LOG" drop_in="$POWERFORGE_SYSTEMD_CONFIG_ROOT/example.service.d/powerforge-read-write-paths.conf" grep -qxF '[Service]' "$drop_in" -grep -qxF "ReadWritePaths=$test_root/service/data" "$drop_in" +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e /tmp/powerforge-service-example ]] + +# A persisted transaction must restore the prior drop-in/current link after an +# uncatchable process or host failure, before a later deployment is considered. +transaction_key="$(printf '%s' 'example.service' | sha256sum | awk '{print $1}')" +transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/systemd-${transaction_key}.transaction" +mkdir -m 0700 "$transaction_dir" +printf '%s\n' "$test_root/service" >"$transaction_dir/service-root" +printf '%s\n' 'example.service' >"$transaction_dir/systemd-service" +printf '%s\n' "$first_target" >"$transaction_dir/previous-target" +printf '%s\n' 'present' >"$transaction_dir/drop-in-state" +printf '%s\n' "$(stat -c '%u' "$drop_in")" >"$transaction_dir/drop-in-owner" +printf '%s\n' "$(stat -c '%g' "$drop_in")" >"$transaction_dir/drop-in-group" +printf '%s\n' "$(stat -c '%a' "$drop_in")" >"$transaction_dir/drop-in-mode" +cp "$drop_in" "$transaction_dir/drop-in" +chmod 0600 "$transaction_dir"/* +stranded_release="$test_root/service/releases/stranded-release" +cp -a "$first_target" "$stranded_release" +ln -sfn "$stranded_release" "$test_root/service/current" +printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" +set +e +recovery_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" +recovery_status=$? +set -e +[[ "$recovery_status" -ne 0 ]] +grep -q 'Recovering incomplete systemd writable-path transaction' <<<"$recovery_output" +grep -q 'Recovered incomplete systemd writable-path transaction' <<<"$recovery_output" +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" +[[ ! -e "$transaction_dir" ]] if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 fi previous_example_target="$(readlink -f "$test_root/service/current")" -mkdir -p "$test_root/service/data-next" +mkdir -p "$test_root/example-data-next" write_config example "$test_root/service" -printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service/data-next" >>"$test_root/config/example.env" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data-next" >>"$test_root/config/example.env" create_stage example 92005 1 5555555555555555555555555555555555555555 : >"$TEST_SYSTEMCTL_LOG" reload_count_file="$test_root/restore-reload-count" @@ -139,11 +169,11 @@ if TEST_SERVICE_ROOT="$test_root/service" \ exit 1 fi [[ "$(readlink -f "$test_root/service/current")" == "$previous_example_target" ]] -grep -qxF "ReadWritePaths=$test_root/service/data" "$drop_in" +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ "$(grep -c '^restart example.service$' "$TEST_SYSTEMCTL_LOG")" -eq 1 ]] grep -q '^stop example.service$' "$TEST_SYSTEMCTL_LOG" -find "$test_root/locks" -maxdepth 1 -type f -name '.powerforge-systemd-example.*' | grep -q . -rm -f -- "$test_root/locks"/.powerforge-systemd-example.* +[[ -d "$transaction_dir" ]] +rm -rf -- "$transaction_dir" : >"$TEST_SYSTEMCTL_LOG" [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] @@ -184,8 +214,10 @@ if TEST_SERVICE_ROOT="$test_root/early-service" \ fi grep -q '^stop early.service$' "$TEST_SYSTEMCTL_LOG" grep -qxF 'ReadWritePaths=/previous' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" -find "$test_root/locks" -maxdepth 1 -type f -name '.powerforge-systemd-early.*' | grep -q . -rm -f -- "$test_root/locks"/.powerforge-systemd-early.* +early_transaction_key="$(printf '%s' 'early.service' | sha256sum | awk '{print $1}')" +early_transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/systemd-${early_transaction_key}.transaction" +[[ -d "$early_transaction_dir" ]] +rm -rf -- "$early_transaction_dir" : >"$TEST_SYSTEMCTL_LOG" mkdir -p "$test_root/fresh-service" @@ -215,6 +247,26 @@ if TEST_SERVICE_ROOT="$test_root/unsafe-service" "$deploy_script" --service unsa exit 1 fi +control_ids=(configcontrol systemdcontrol transactioncontrol stagecontrol lockcontrol) +control_paths=("$POWERFORGE_SERVICE_CONFIG_ROOT" "$POWERFORGE_SYSTEMD_CONFIG_ROOT" "$POWERFORGE_SERVICE_TRANSACTION_ROOT" "$POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT" "$(realpath -e "$POWERFORGE_SERVICE_LOCK_ROOT")") +for index in "${!control_ids[@]}"; do + control_id="${control_ids[$index]}" + control_path="${control_paths[$index]}" + control_service="$test_root/${control_id}-service" + mkdir -p "$control_service" + write_config "$control_id" "$control_service" + printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$control_path" >>"$test_root/config/${control_id}.env" + set +e + control_output="$(TEST_SERVICE_ROOT="$control_service" "$deploy_script" --service "$control_id" 2>&1)" + control_status=$? + set -e + if [[ "$control_status" -eq 0 ]]; then + echo "Deployment unexpectedly allowed writable access to deployment control state: $control_path" >&2 + exit 1 + fi + grep -q 'Systemd writable path must not overlap deployment control path' <<<"$control_output" +done + mkdir -p "$test_root/glob-service" write_config glob "$test_root/glob-service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-*" >>"$test_root/config/glob.env" From 7ffc38a212b0b7fd8e583d9a07b38b6586bbb2e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 12:55:41 +0200 Subject: [PATCH 5/8] Close Linux promotion recovery windows --- Deployment/Linux/powerforge-service-deploy.sh | 88 +++++++++++++------ Docs/PowerForge.Web.LinuxServiceDeployment.md | 14 +-- .../GitHubServiceLinuxDeployWorkflowTests.cs | 2 + .../Linux/powerforge-service-deploy.tests.sh | 29 +++--- 4 files changed, 92 insertions(+), 41 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 50e797ec52..f5f42fe965 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -15,6 +15,7 @@ metadata="" promoted=0 previous_target="" release_dir="" +candidate_link="" workflow_stage="" trusted_stage="" systemd_drop_in_backup="" @@ -224,7 +225,7 @@ snapshot_systemd_write_paths() { local transaction_temporary drop_in_mode [[ ! -e "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || fail "An incomplete systemd writable-path transaction already exists: $systemd_transaction_path" - transaction_temporary="$(mktemp -d "${TRANSACTION_ROOT}/.systemd-${unit_lock_key}.XXXXXXXX")" + transaction_temporary="$(mktemp -d "${TRANSACTION_ROOT}/.service-${service_id}.XXXXXXXX")" chmod 0700 "$transaction_temporary" systemd_drop_in_existed=0 systemd_drop_in_owner="" @@ -316,19 +317,27 @@ restore_systemd_write_paths() { } finish_systemd_write_paths_transaction() { - [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/systemd-*.transaction ]] || + [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/service-*.transaction ]] || fail "Refusing to remove an unexpected transaction path: $systemd_transaction_path" + sync_deployment_state rm -rf -- "$systemd_transaction_path" || return 1 sync -f "$TRANSACTION_ROOT" systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" } +sync_deployment_state() { + [[ ! -f "$systemd_drop_in_path" ]] || sync -f "$systemd_drop_in_path" + sync -f "$systemd_drop_in_dir" + sync -f "$SERVICE_ROOT" +} + commit_systemd_write_paths() { local committed_path="${systemd_transaction_path}.committed.$$" # The rename is the durable commit point. Ignore catchable termination during # this tiny section so a signal cannot run rollback after the transaction has # disappeared but before the in-memory state reflects that fact. + sync_deployment_state trap - INT TERM mv -- "$systemd_transaction_path" "$committed_path" promoted=0 @@ -356,6 +365,18 @@ restore_previous_current_link() { fi } +peek_systemd_transaction_identity() { + [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || + fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" + assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' + transaction_service_root="$(<"$systemd_transaction_path/service-root")" + transaction_systemd_service="$(<"$systemd_transaction_path/systemd-service")" + [[ "$transaction_service_root" == /* && "$transaction_service_root" != '/' && "$transaction_service_root" != *[[:space:]]* ]] || + fail 'Incomplete transaction contains an invalid service root.' + [[ "$transaction_systemd_service" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || + fail 'Incomplete transaction contains an invalid systemd unit.' +} + recover_incomplete_systemd_transaction() { local permissions_restored=1 current_restored=1 service_safe=0 if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then @@ -414,6 +435,15 @@ flock -n 9 || fail "Another deployment is active for $service_id." prepare_service_release_root prepare_transaction_root prepare_trusted_stage_root +configured_service_root="$SERVICE_ROOT" +configured_release_root="$resolved_release_root" +configured_systemd_service="$SYSTEMD_SERVICE" +systemd_transaction_path="${TRANSACTION_ROOT}/service-${service_id}.transaction" +transaction_service_root="" +transaction_systemd_service="" +if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then + peek_systemd_transaction_identity +fi unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" @@ -421,9 +451,28 @@ flock -n 8 || fail "Another deployment is active for systemd unit $SYSTEMD_SERVI exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" flock -n 7 || fail "Another deployment is active for service root $SERVICE_ROOT." +if [[ -n "$transaction_systemd_service" && "$transaction_systemd_service" != "$configured_systemd_service" ]]; then + transaction_unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" + exec {transaction_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${transaction_unit_lock_key}.lock" + flock -n "$transaction_unit_lock_fd" || fail "Another deployment is active for prior systemd unit $transaction_systemd_service." +fi +if [[ -n "$transaction_service_root" && "$transaction_service_root" != "$configured_service_root" ]]; then + transaction_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" + exec {transaction_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${transaction_root_lock_key}.lock" + flock -n "$transaction_root_lock_fd" || fail "Another deployment is active for prior service root $transaction_service_root." +fi + +if [[ -n "$transaction_systemd_service" ]]; then + SYSTEMD_SERVICE="$transaction_systemd_service" + SERVICE_ROOT="$transaction_service_root" + prepare_service_release_root + prepare_systemd_drop_in_directory + recover_incomplete_systemd_transaction + SYSTEMD_SERVICE="$configured_systemd_service" + SERVICE_ROOT="$configured_service_root" + resolved_release_root="$configured_release_root" +fi prepare_systemd_drop_in_directory -systemd_transaction_path="${TRANSACTION_ROOT}/systemd-${unit_lock_key}.transaction" -recover_incomplete_systemd_transaction previous_target="" if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then [[ -L "$SERVICE_ROOT/current" ]] || fail 'Current release pointer must be a symlink.' @@ -446,27 +495,6 @@ for read_write_path in "${configured_systemd_read_write_paths[@]}"; do systemd_read_write_paths+=("$resolved_read_write_path") done -snapshot_systemd_write_paths -pre_promotion_failure() { - local exit_code="$1" - local permissions_restored=1 - set +e - restore_systemd_write_paths || permissions_restored=0 - if [[ "$permissions_restored" == '1' ]]; then - finish_systemd_write_paths_transaction || permissions_restored=0 - fi - if [[ "$permissions_restored" != '1' ]]; then - report_systemd_restore_failure - log 'Permission rollback failed before promotion; stopping the service to prevent restart with unverified write access.' >&2 - systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop $SYSTEMD_SERVICE after permission rollback failure." >&2 - fi - exit "$exit_code" -} -trap 'exit_code=$?; if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure "$exit_code"; else exit "$exit_code"; fi' ERR -trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure 130; else exit 130; fi' INT -trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then pre_promotion_failure 143; else exit 143; fi' TERM -reconcile_systemd_write_paths - archive="$(realpath -e "$archive")" metadata="$(realpath -e "$metadata")" [[ -f "$archive" && ! -L "$archive" ]] || fail 'Artifact must be a regular file, not a symlink.' @@ -578,8 +606,16 @@ rollback() { else log "Rollback transaction retained for recovery: $systemd_transaction_path" >&2 fi + elif [[ "$systemd_write_paths_snapshot_ready" == '1' ]]; then + if [[ "$permissions_restored" == '1' ]]; then + finish_systemd_write_paths_transaction || log "WARNING: restored pre-switch transaction retained at $systemd_transaction_path" >&2 + else + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop $SYSTEMD_SERVICE after pre-switch permission rollback failure." >&2 + log "Pre-switch transaction retained for recovery: $systemd_transaction_path" >&2 + fi fi current_target="$(readlink -f "$SERVICE_ROOT/current" 2>/dev/null || true)" + [[ -z "$candidate_link" || ! -L "$candidate_link" ]] || rm -f -- "$candidate_link" if [[ -n "$release_dir" && -d "$release_dir" && "$release_dir" != "$previous_target" ]]; then if [[ "$current_target" != "$release_dir" && ( "$promoted" != '1' || ( "$current_restored" == '1' && "$service_safe" == '1' ) ) ]]; then rm -rf -- "$release_dir" || log "WARNING: failed to remove rejected release $release_dir" >&2 @@ -604,6 +640,8 @@ install -m 0644 "$metadata" "$release_dir/_powerforge/deployment.json" candidate_link="$SERVICE_ROOT/.current.${run_id}.${run_attempt}" ln -s "$release_dir" "$candidate_link" +snapshot_systemd_write_paths +reconcile_systemd_write_paths promoted=1 mv -Tf "$candidate_link" "$SERVICE_ROOT/current" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index f2a14c5c06..003716dd1a 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -76,11 +76,15 @@ root-sourced service configuration and the canonical service/release roots. Depl are serialized by service id, systemd unit, and canonical service root. Cancellation uses explicit non-zero signal exits, and rollback retains a rejected release whenever the previous link or safe service state cannot be proven. -Before changing a drop-in, the promoter persists the previous permission and current- -release state under `/var/lib/powerforge/service-deployment-state`. A later invocation -restores that state and proves the service restarted or stopped before accepting a new -deployment, so process termination or host loss cannot strand an uncommitted writable -policy. Recovery state is removed only at the successful deployment commit point. +Immediately before switching `current`, the promoter persists the previous permission +and current-release state under `/var/lib/powerforge/service-deployment-state`, then +reloads the candidate policy. Transactions are keyed by the stable service id, so a +later invocation restores the recorded unit and service root even when configuration +was renamed in the meantime. It proves the service restarted or stopped before +accepting a new deployment, so process termination or host loss cannot strand an +uncommitted writable policy. Restored drop-in and service-root filesystems are flushed +before recovery state is removed; successful promotion uses the same durable commit +boundary. Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: diff --git a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs index b1a861c4e3..5f7ea6b07c 100644 --- a/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs +++ b/PowerForge.Tests/GitHubServiceLinuxDeployWorkflowTests.cs @@ -48,7 +48,9 @@ public void PromoterUsesTrustedStagingAndRollsBackSystemdService() Assert.Contains("powerforge-systemd-${unit_lock_key}.lock", script, StringComparison.Ordinal); Assert.Contains("powerforge-root-${service_root_lock_key}.lock", script, StringComparison.Ordinal); Assert.Contains("service-deployment-state", script, StringComparison.Ordinal); + Assert.Contains("service-${service_id}.transaction", script, StringComparison.Ordinal); Assert.Contains("Recovering incomplete systemd writable-path transaction", script, StringComparison.Ordinal); + Assert.Contains("sync_deployment_state", script, StringComparison.Ordinal); Assert.Contains("must not overlap deployment control path", script, StringComparison.Ordinal); Assert.Contains("rollback 143", script, StringComparison.Ordinal); Assert.Contains("Rejected release retained for recovery", script, StringComparison.Ordinal); diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 9b8f29cab3..fb7eb3def2 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -26,7 +26,10 @@ if [[ "$*" == 'daemon-reload' && -n "${FAIL_DAEMON_RELOAD_COUNT_FILE:-}" ]]; the [[ ! -f "$FAIL_DAEMON_RELOAD_COUNT_FILE" ]] || count="$(cat "$FAIL_DAEMON_RELOAD_COUNT_FILE")" count=$((count + 1)) printf '%s\n' "$count" >"$FAIL_DAEMON_RELOAD_COUNT_FILE" - if (( count >= ${FAIL_DAEMON_RELOAD_FROM_CALL:-2} )); then + if [[ -n "${FAIL_DAEMON_RELOAD_FROM_CALL:-}" ]] && (( count >= FAIL_DAEMON_RELOAD_FROM_CALL )); then + exit 1 + fi + if [[ -n "${FAIL_DAEMON_RELOAD_ON_CALL:-}" && "$count" -eq "$FAIL_DAEMON_RELOAD_ON_CALL" ]]; then exit 1 fi fi @@ -122,8 +125,7 @@ grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" # A persisted transaction must restore the prior drop-in/current link after an # uncatchable process or host failure, before a later deployment is considered. -transaction_key="$(printf '%s' 'example.service' | sha256sum | awk '{print $1}')" -transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/systemd-${transaction_key}.transaction" +transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction" mkdir -m 0700 "$transaction_dir" printf '%s\n' "$test_root/service" >"$transaction_dir/service-root" printf '%s\n' 'example.service' >"$transaction_dir/systemd-service" @@ -138,6 +140,7 @@ stranded_release="$test_root/service/releases/stranded-release" cp -a "$first_target" "$stranded_release" ln -sfn "$stranded_release" "$test_root/service/current" printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" +sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=renamed.service/' "$test_root/config/example.env" set +e recovery_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" recovery_status=$? @@ -145,9 +148,12 @@ set -e [[ "$recovery_status" -ne 0 ]] grep -q 'Recovering incomplete systemd writable-path transaction' <<<"$recovery_output" grep -q 'Recovered incomplete systemd writable-path transaction' <<<"$recovery_output" +grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e "$transaction_dir" ]] +write_config example "$test_root/service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 @@ -202,22 +208,23 @@ write_config early "$test_root/early-service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/early-data" >>"$test_root/config/early.env" printf '[Service]\nReadWritePaths=/previous\n' >"$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" create_stage early 92006 1 6666666666666666666666666666666666666666 -sed -i 's/"sourceSha": "[^"]*"/"sourceSha": "invalid"/' /tmp/powerforge-service-early/deployment.json : >"$TEST_SYSTEMCTL_LOG" early_reload_count_file="$test_root/early-reload-count" if TEST_SERVICE_ROOT="$test_root/early-service" \ FAIL_DAEMON_RELOAD_COUNT_FILE="$early_reload_count_file" \ - FAIL_DAEMON_RELOAD_FROM_CALL=2 \ + FAIL_DAEMON_RELOAD_ON_CALL=1 \ "$deploy_script" --service early; then - echo 'Pre-promotion validation unexpectedly ignored a failed permission restore.' >&2 + echo 'Pre-switch reconciliation unexpectedly ignored a failed systemd reload.' >&2 exit 1 fi -grep -q '^stop early.service$' "$TEST_SYSTEMCTL_LOG" grep -qxF 'ReadWritePaths=/previous' "$POWERFORGE_SYSTEMD_CONFIG_ROOT/early.service.d/powerforge-read-write-paths.conf" -early_transaction_key="$(printf '%s' 'early.service' | sha256sum | awk '{print $1}')" -early_transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/systemd-${early_transaction_key}.transaction" -[[ -d "$early_transaction_dir" ]] -rm -rf -- "$early_transaction_dir" +early_transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-early.transaction" +[[ ! -e "$early_transaction_dir" ]] +[[ ! -e "$test_root/early-service/current" ]] +if grep -q '^stop early.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Restored pre-switch state unexpectedly stopped the untouched service.' >&2 + exit 1 +fi : >"$TEST_SYSTEMCTL_LOG" mkdir -p "$test_root/fresh-service" From 0a717ea78dbb7ab2946f5d3746b9976600509a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 13:17:16 +0200 Subject: [PATCH 6/8] Finish durable Linux recovery handling --- Deployment/Linux/powerforge-service-deploy.sh | 87 +++++++++++++------ Docs/PowerForge.Web.LinuxServiceDeployment.md | 7 +- .../Linux/powerforge-service-deploy.tests.sh | 44 +++++++++- 3 files changed, 111 insertions(+), 27 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index f5f42fe965..305c68a387 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -185,39 +185,57 @@ assert_trusted_systemd_path() { prepare_systemd_drop_in_directory() { local config_parent config_parent="$(dirname -- "$SYSTEMD_CONFIG_ROOT")" - assert_trusted_directory_chain "$config_parent" 'Systemd config parent' + assert_trusted_directory_chain "$config_parent" 'Systemd config parent' || return 1 if [[ -e "$SYSTEMD_CONFIG_ROOT" || -L "$SYSTEMD_CONFIG_ROOT" ]]; then - [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT" + [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || { + fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT" + return 1 + } else - install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" + install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" || return 1 fi - assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' - [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT" + assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' || return 1 + [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || { + fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT" + return 1 + } systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" if [[ -e "$systemd_drop_in_dir" || -L "$systemd_drop_in_dir" ]]; then - [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir" + [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || { + fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir" + return 1 + } else - install -d -m 0755 "$systemd_drop_in_dir" + install -d -m 0755 "$systemd_drop_in_dir" || return 1 fi - assert_trusted_directory_chain "$systemd_drop_in_dir" 'Systemd drop-in directory' + assert_trusted_directory_chain "$systemd_drop_in_dir" 'Systemd drop-in directory' || return 1 } prepare_service_release_root() { - [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT" - assert_trusted_directory_chain "$SERVICE_ROOT" 'Service root' - resolved_service_root="$(realpath -e -- "$SERVICE_ROOT")" - [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT" + [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || { + fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT" + return 1 + } + assert_trusted_directory_chain "$SERVICE_ROOT" 'Service root' || return 1 + resolved_service_root="$(realpath -e -- "$SERVICE_ROOT")" || return 1 + [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || { + fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT" + return 1 + } SERVICE_ROOT="$resolved_service_root" resolved_release_root="${SERVICE_ROOT}/releases" if [[ -e "$resolved_release_root" || -L "$resolved_release_root" ]]; then - [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || fail "Release root must be a real directory: $resolved_release_root" + [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || { + fail "Release root must be a real directory: $resolved_release_root" + return 1 + } else - install -d -m 0755 "$resolved_release_root" + install -d -m 0755 "$resolved_release_root" || return 1 fi - assert_trusted_directory_chain "$resolved_release_root" 'Release root' + assert_trusted_directory_chain "$resolved_release_root" 'Release root' || return 1 [[ "$(realpath -e -- "$resolved_release_root")" == "$resolved_release_root" ]] || fail "Release root must be canonical and contain no symlinked components: $resolved_release_root" } @@ -233,6 +251,7 @@ snapshot_systemd_write_paths() { systemd_drop_in_mode="" printf '%s\n' "$SERVICE_ROOT" >"$transaction_temporary/service-root" printf '%s\n' "$SYSTEMD_SERVICE" >"$transaction_temporary/systemd-service" + printf '%s\n' "$SYSTEMD_CONFIG_ROOT" >"$transaction_temporary/systemd-config-root" printf '%s\n' "$previous_target" >"$transaction_temporary/previous-target" if [[ -e "$systemd_drop_in_path" || -L "$systemd_drop_in_path" ]]; then [[ -f "$systemd_drop_in_path" && ! -L "$systemd_drop_in_path" ]] || fail "PowerForge systemd drop-in must be a regular file: $systemd_drop_in_path" @@ -266,16 +285,18 @@ snapshot_systemd_write_paths() { } load_systemd_write_paths_transaction() { - local stored_root stored_service stored_state + local stored_root stored_service stored_systemd_root stored_state [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' stored_root="$(<"$systemd_transaction_path/service-root")" stored_service="$(<"$systemd_transaction_path/systemd-service")" + stored_systemd_root="$(<"$systemd_transaction_path/systemd-config-root")" previous_target="$(<"$systemd_transaction_path/previous-target")" stored_state="$(<"$systemd_transaction_path/drop-in-state")" [[ "$stored_root" == "$SERVICE_ROOT" ]] || fail "Incomplete transaction belongs to a different service root: $stored_root" [[ "$stored_service" == "$SYSTEMD_SERVICE" ]] || fail "Incomplete transaction belongs to a different systemd unit: $stored_service" + [[ "$stored_systemd_root" == "$SYSTEMD_CONFIG_ROOT" ]] || fail "Incomplete transaction belongs to a different systemd config root: $stored_systemd_root" if [[ -n "$previous_target" ]]; then [[ -d "$previous_target" && "$previous_target" == "$resolved_release_root"/* ]] || fail "Incomplete transaction contains an invalid previous release: $previous_target" @@ -319,17 +340,19 @@ restore_systemd_write_paths() { finish_systemd_write_paths_transaction() { [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/service-*.transaction ]] || fail "Refusing to remove an unexpected transaction path: $systemd_transaction_path" - sync_deployment_state + sync_deployment_state || return 1 rm -rf -- "$systemd_transaction_path" || return 1 - sync -f "$TRANSACTION_ROOT" + sync -f "$TRANSACTION_ROOT" || return 1 systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" } sync_deployment_state() { - [[ ! -f "$systemd_drop_in_path" ]] || sync -f "$systemd_drop_in_path" - sync -f "$systemd_drop_in_dir" - sync -f "$SERVICE_ROOT" + if [[ -f "$systemd_drop_in_path" ]]; then + sync -f "$systemd_drop_in_path" || return 1 + fi + sync -f "$systemd_drop_in_dir" || return 1 + sync -f "$SERVICE_ROOT" || return 1 } commit_systemd_write_paths() { @@ -337,9 +360,9 @@ commit_systemd_write_paths() { # The rename is the durable commit point. Ignore catchable termination during # this tiny section so a signal cannot run rollback after the transaction has # disappeared but before the in-memory state reflects that fact. - sync_deployment_state + sync_deployment_state || return 1 trap - INT TERM - mv -- "$systemd_transaction_path" "$committed_path" + mv -- "$systemd_transaction_path" "$committed_path" || return 1 promoted=0 systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" @@ -371,10 +394,13 @@ peek_systemd_transaction_identity() { assert_trusted_directory_chain "$systemd_transaction_path" 'Systemd writable-path transaction' transaction_service_root="$(<"$systemd_transaction_path/service-root")" transaction_systemd_service="$(<"$systemd_transaction_path/systemd-service")" + transaction_systemd_config_root="$(<"$systemd_transaction_path/systemd-config-root")" [[ "$transaction_service_root" == /* && "$transaction_service_root" != '/' && "$transaction_service_root" != *[[:space:]]* ]] || fail 'Incomplete transaction contains an invalid service root.' [[ "$transaction_systemd_service" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || fail 'Incomplete transaction contains an invalid systemd unit.' + [[ "$transaction_systemd_config_root" == /* && "$transaction_systemd_config_root" != '/' && "$transaction_systemd_config_root" != *[[:space:]]* ]] || + fail 'Incomplete transaction contains an invalid systemd config root.' } recover_incomplete_systemd_transaction() { @@ -438,9 +464,11 @@ prepare_trusted_stage_root configured_service_root="$SERVICE_ROOT" configured_release_root="$resolved_release_root" configured_systemd_service="$SYSTEMD_SERVICE" +configured_systemd_config_root="$SYSTEMD_CONFIG_ROOT" systemd_transaction_path="${TRANSACTION_ROOT}/service-${service_id}.transaction" transaction_service_root="" transaction_systemd_service="" +transaction_systemd_config_root="" if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then peek_systemd_transaction_identity fi @@ -465,12 +493,21 @@ fi if [[ -n "$transaction_systemd_service" ]]; then SYSTEMD_SERVICE="$transaction_systemd_service" SERVICE_ROOT="$transaction_service_root" - prepare_service_release_root - prepare_systemd_drop_in_directory + SYSTEMD_CONFIG_ROOT="$transaction_systemd_config_root" + recovery_preparation_status=0 + prepare_service_release_root || recovery_preparation_status=$? + if [[ "$recovery_preparation_status" -eq 0 ]]; then + prepare_systemd_drop_in_directory || recovery_preparation_status=$? + fi + if [[ "$recovery_preparation_status" -ne 0 ]]; then + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop recorded unit $SYSTEMD_SERVICE after recovery preparation failed." >&2 + fail "Recorded recovery paths are unavailable; $SYSTEMD_SERVICE was stopped and operator recovery is required." + fi recover_incomplete_systemd_transaction SYSTEMD_SERVICE="$configured_systemd_service" SERVICE_ROOT="$configured_service_root" resolved_release_root="$configured_release_root" + SYSTEMD_CONFIG_ROOT="$configured_systemd_config_root" fi prepare_systemd_drop_in_directory previous_target="" diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index 003716dd1a..34e4623843 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -31,7 +31,7 @@ Create one root-owned configuration per service under `/etc/powerforge/services` ```bash install -d -o root -g root -m 0750 /etc/powerforge/services install -d -o root -g root -m 0755 /srv/example/service -install -d -o root -g root -m 0755 /var/lib/example-service +install -d -o example-service -g example-service -m 0750 /var/lib/example-service install -d -o root -g root -m 0755 /var/lib/powerforge install -d -o root -g root -m 0700 \ /var/lib/powerforge/service-deployment-staging \ @@ -41,6 +41,11 @@ install -o root -g root -m 0640 \ /etc/powerforge/services/example.env ``` +`example-service` is the account configured by the systemd unit. Create that account +first or substitute the unit's existing `User=`/`Group=` values. `ReadWritePaths=` +opens the systemd mount namespace but does not bypass normal filesystem ownership and +mode checks. + Example configuration: ```dotenv diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index fb7eb3def2..e5f7e3004c 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -61,6 +61,15 @@ fi exec /usr/bin/mv "$@" EOF chmod +x "$test_root/bin/mv" +cat >"$test_root/bin/sync" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +if [[ -n "${FAIL_SYNC_PATH:-}" && "$*" == *"$FAIL_SYNC_PATH"* ]]; then + exit 1 +fi +exec /usr/bin/sync "$@" +EOF +chmod +x "$test_root/bin/sync" export PATH="$test_root/bin:$PATH" write_config() { @@ -129,6 +138,7 @@ transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transactio mkdir -m 0700 "$transaction_dir" printf '%s\n' "$test_root/service" >"$transaction_dir/service-root" printf '%s\n' 'example.service' >"$transaction_dir/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$transaction_dir/systemd-config-root" printf '%s\n' "$first_target" >"$transaction_dir/previous-target" printf '%s\n' 'present' >"$transaction_dir/drop-in-state" printf '%s\n' "$(stat -c '%u' "$drop_in")" >"$transaction_dir/drop-in-owner" @@ -142,7 +152,7 @@ ln -sfn "$stranded_release" "$test_root/service/current" printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=renamed.service/' "$test_root/config/example.env" set +e -recovery_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" +recovery_output="$(POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd-renamed" TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" recovery_status=$? set -e [[ "$recovery_status" -ne 0 ]] @@ -154,6 +164,38 @@ grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e "$transaction_dir" ]] write_config example "$test_root/service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" + +mkdir -p "$test_root/recovery-current-service" +write_config recoverymissing "$test_root/recovery-current-service" +missing_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-recoverymissing.transaction" +mkdir -m 0700 "$missing_transaction" +printf '%s\n' "$test_root/recovery-root-missing" >"$missing_transaction/service-root" +printf '%s\n' 'recorded.service' >"$missing_transaction/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$missing_transaction/systemd-config-root" +chmod 0600 "$missing_transaction"/* +set +e +missing_recovery_output="$(TEST_SERVICE_ROOT="$test_root/recovery-current-service" "$deploy_script" --service recoverymissing 2>&1)" +missing_recovery_status=$? +set -e +[[ "$missing_recovery_status" -ne 0 ]] +grep -q '^stop recorded.service$' "$TEST_SYSTEMCTL_LOG" +grep -q 'Recorded recovery paths are unavailable' <<<"$missing_recovery_output" +[[ -d "$missing_transaction" ]] +rm -rf -- "$missing_transaction" + +create_stage example 92010 1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +set +e +sync_failure_output="$(TEST_SERVICE_ROOT="$test_root/service" \ + FAIL_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + FAIL_SYNC_PATH="$drop_in" \ + "$deploy_script" --service example 2>&1)" +sync_failure_status=$? +set -e +[[ "$sync_failure_status" -ne 0 ]] +grep -q 'rollback transaction retained' <<<"${sync_failure_output,,}" +[[ -d "$transaction_dir" ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +rm -rf -- "$transaction_dir" if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 From ca05310545966f77512323f9db4b4b83f587992a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 13:53:03 +0200 Subject: [PATCH 7/8] Complete cross-identity deployment recovery --- Deployment/Linux/powerforge-service-deploy.sh | 360 +++++++++--------- Docs/PowerForge.Web.LinuxServiceDeployment.md | 16 +- .../Linux/powerforge-service-deploy.tests.sh | 106 +++++- 3 files changed, 294 insertions(+), 188 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 305c68a387..09e3d5b7d8 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -1,49 +1,30 @@ #!/usr/bin/env bash set -Eeuo pipefail - umask 022 - CONFIG_ROOT="${POWERFORGE_SERVICE_CONFIG_ROOT:-/etc/powerforge/services}" LOCK_ROOT="${POWERFORGE_SERVICE_LOCK_ROOT:-/var/lock}" TRUSTED_STAGE_ROOT="${POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT:-/var/lib/powerforge/service-deployment-staging}" TRANSACTION_ROOT="${POWERFORGE_SERVICE_TRANSACTION_ROOT:-/var/lib/powerforge/service-deployment-state}" SYSTEMD_CONFIG_ROOT="${POWERFORGE_SYSTEMD_CONFIG_ROOT:-/etc/systemd/system}" deployment_shell_pid="$BASHPID" -service_id="" -archive="" -metadata="" -promoted=0 -previous_target="" -release_dir="" -candidate_link="" -workflow_stage="" -trusted_stage="" -systemd_drop_in_backup="" -systemd_drop_in_dir="" -systemd_drop_in_path="" -systemd_drop_in_existed=0 -systemd_drop_in_owner="" -systemd_drop_in_group="" -systemd_drop_in_mode="" -systemd_write_paths_snapshot_ready=0 -systemd_transaction_path="" - +service_id="" archive="" metadata="" +promoted=0 previous_target="" release_dir="" candidate_link="" +workflow_stage="" trusted_stage="" +systemd_drop_in_backup="" systemd_drop_in_dir="" systemd_drop_in_path="" +systemd_drop_in_existed=0 systemd_drop_in_owner="" systemd_drop_in_group="" systemd_drop_in_mode="" +systemd_write_paths_snapshot_ready=0 systemd_transaction_path="" cleanup_staging() { [[ -z "$workflow_stage" || ! -d "$workflow_stage" ]] || rm -rf -- "$workflow_stage" [[ -z "$trusted_stage" || ! -d "$trusted_stage" ]] || rm -rf -- "$trusted_stage" } - trap cleanup_staging EXIT - log() { printf '[powerforge-service-deploy] %s\n' "$*" } - fail() { log "ERROR: $*" >&2 return 1 } - assert_trusted_directory_chain() { local declared_path="$1" local description="$2" @@ -62,13 +43,11 @@ assert_trusted_directory_chain() { (( (8#$mode & 0022) == 0 )) || fail "$description must not be group/world writable: $current" done } - paths_overlap() { local first="$1" local second="$2" [[ "$first" == "$second" || "$first" == "$second"/* || "$second" == "$first"/* ]] } - prepare_transaction_root() { local parent resolved [[ "$TRANSACTION_ROOT" == /* && "$TRANSACTION_ROOT" != '/' && "$TRANSACTION_ROOT" != *[[:space:]]* ]] || @@ -85,7 +64,6 @@ prepare_transaction_root() { [[ "$resolved" == "$TRANSACTION_ROOT" ]] || fail "Transaction root must be canonical and contain no symlinked components: $TRANSACTION_ROOT" TRANSACTION_ROOT="$resolved" } - prepare_trusted_stage_root() { local parent resolved [[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' && "$TRUSTED_STAGE_ROOT" != *[[:space:]]* ]] || @@ -102,11 +80,9 @@ prepare_trusted_stage_root() { [[ "$resolved" == "$TRUSTED_STAGE_ROOT" ]] || fail "Trusted staging root must be canonical and contain no symlinked components: $TRUSTED_STAGE_ROOT" TRUSTED_STAGE_ROOT="$resolved" } - usage() { echo 'Usage: powerforge-service-deploy --service ' } - while (($# > 0)); do case "$1" in --service) @@ -123,122 +99,50 @@ while (($# > 0)); do ;; esac done - [[ "$service_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail 'Invalid service identifier.' workflow_stage="/tmp/powerforge-service-${service_id}" archive="$workflow_stage/artifact.tar" metadata="$workflow_stage/deployment.json" - -[[ "$CONFIG_ROOT" == /* && "$CONFIG_ROOT" != '/' && "$CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Service config root must be an absolute non-root path without whitespace.' -[[ -d "$CONFIG_ROOT" && ! -L "$CONFIG_ROOT" ]] || fail "Service config root must be a real directory: $CONFIG_ROOT" -assert_trusted_directory_chain "$CONFIG_ROOT" 'Service config root' -resolved_config_root="$(realpath -e -- "$CONFIG_ROOT")" -[[ "$resolved_config_root" == "$CONFIG_ROOT" ]] || fail "Service config root must be canonical and contain no symlinked components: $CONFIG_ROOT" -CONFIG_ROOT="$resolved_config_root" -config_path="${CONFIG_ROOT}/${service_id}.env" -[[ -f "$config_path" && ! -L "$config_path" ]] || fail "Service is not configured: $service_id" -if [[ "$(id -u)" -eq 0 ]]; then - [[ "$(stat -c '%u' "$config_path")" -eq 0 ]] || fail "Service config must be owned by root: $config_path" - config_mode="$(stat -c '%a' "$config_path")" - (( (8#$config_mode & 0022) == 0 )) || fail "Service config must not be group/world writable: $config_path" -fi - -# The config is trusted, root-owned operator input and contains no values supplied by the workflow. -# shellcheck disable=SC1090 -source "$config_path" - -: "${SERVICE_ROOT:?SERVICE_ROOT is required in $config_path}" -: "${SYSTEMD_SERVICE:?SYSTEMD_SERVICE is required in $config_path}" -: "${SYSTEMD_READ_WRITE_PATHS:=}" -: "${LOCAL_HEALTH_URL:?LOCAL_HEALTH_URL is required in $config_path}" -: "${RELEASES_TO_KEEP:=5}" -: "${REQUIRED_RELEASE_PATHS:=}" -: "${PUBLIC_HEALTH_URLS:=}" -: "${REQUIRE_HEALTH_PROVENANCE:=1}" - -[[ "$SERVICE_ROOT" == /* && "$SERVICE_ROOT" != '/' ]] || fail 'SERVICE_ROOT must be an absolute non-root path.' -[[ "$SERVICE_ROOT" != *[[:space:]]* ]] || fail 'SERVICE_ROOT must not contain whitespace.' -[[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' ]] || fail 'Trusted staging root must be an absolute non-root path.' -[[ "$SYSTEMD_CONFIG_ROOT" == /* && "$SYSTEMD_CONFIG_ROOT" != '/' ]] || fail 'Systemd config root must be an absolute non-root path.' -[[ "$SYSTEMD_CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Systemd config root must not contain whitespace.' -[[ "$SYSTEMD_SERVICE" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || fail 'SYSTEMD_SERVICE must be a systemd service unit name.' -[[ "$LOCAL_HEALTH_URL" =~ ^https?://[^[:space:]]+$ ]] || fail 'LOCAL_HEALTH_URL must be an HTTP or HTTPS URL.' -[[ "$RELEASES_TO_KEEP" =~ ^[1-9][0-9]*$ ]] || fail 'RELEASES_TO_KEEP must be a positive integer.' -[[ "$REQUIRE_HEALTH_PROVENANCE" == '0' || "$REQUIRE_HEALTH_PROVENANCE" == '1' ]] || fail 'REQUIRE_HEALTH_PROVENANCE must be 0 or 1.' -for health_url in $PUBLIC_HEALTH_URLS; do - [[ "$health_url" =~ ^https://[^[:space:]]+$ ]] || fail "Public health URL must use HTTPS: $health_url" -done - -read -r -a configured_systemd_read_write_paths <<<"$SYSTEMD_READ_WRITE_PATHS" -for read_write_path in "${configured_systemd_read_write_paths[@]}"; do - [[ "$read_write_path" == /* && "$read_write_path" != '/' ]] || fail 'Systemd writable paths must be absolute non-root paths.' - [[ "$read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Systemd writable path contains unsupported characters: $read_write_path" - [[ "/${read_write_path#/}/" != *'/../'* ]] || fail "Systemd writable path must not contain traversal: $read_write_path" -done - assert_trusted_systemd_path() { local declared_path="$1" [[ ! -L "$declared_path" ]] || fail "Systemd writable path must not be a symlink: $declared_path" assert_trusted_directory_chain "$(dirname -- "$declared_path")" 'Systemd writable path parent' } - prepare_systemd_drop_in_directory() { local config_parent config_parent="$(dirname -- "$SYSTEMD_CONFIG_ROOT")" assert_trusted_directory_chain "$config_parent" 'Systemd config parent' || return 1 if [[ -e "$SYSTEMD_CONFIG_ROOT" || -L "$SYSTEMD_CONFIG_ROOT" ]]; then - [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || { - fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT" - return 1 - } + [[ -d "$SYSTEMD_CONFIG_ROOT" && ! -L "$SYSTEMD_CONFIG_ROOT" ]] || { fail "Systemd config root must be a real directory: $SYSTEMD_CONFIG_ROOT"; return 1; } else install -d -m 0755 "$SYSTEMD_CONFIG_ROOT" || return 1 fi assert_trusted_directory_chain "$SYSTEMD_CONFIG_ROOT" 'Systemd config root' || return 1 - [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || { - fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT" - return 1 - } - + [[ "$(realpath -e -- "$SYSTEMD_CONFIG_ROOT")" == "$SYSTEMD_CONFIG_ROOT" ]] || { fail "Systemd config root must be canonical and contain no symlinked components: $SYSTEMD_CONFIG_ROOT"; return 1; } systemd_drop_in_dir="${SYSTEMD_CONFIG_ROOT}/${SYSTEMD_SERVICE}.d" systemd_drop_in_path="${systemd_drop_in_dir}/powerforge-read-write-paths.conf" if [[ -e "$systemd_drop_in_dir" || -L "$systemd_drop_in_dir" ]]; then - [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || { - fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir" - return 1 - } + [[ -d "$systemd_drop_in_dir" && ! -L "$systemd_drop_in_dir" ]] || { fail "Systemd drop-in directory must be a real directory: $systemd_drop_in_dir"; return 1; } else install -d -m 0755 "$systemd_drop_in_dir" || return 1 fi assert_trusted_directory_chain "$systemd_drop_in_dir" 'Systemd drop-in directory' || return 1 } - prepare_service_release_root() { - [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || { - fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT" - return 1 - } + [[ -d "$SERVICE_ROOT" && ! -L "$SERVICE_ROOT" ]] || { fail "Service root must be a real, pre-provisioned directory: $SERVICE_ROOT"; return 1; } assert_trusted_directory_chain "$SERVICE_ROOT" 'Service root' || return 1 resolved_service_root="$(realpath -e -- "$SERVICE_ROOT")" || return 1 - [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || { - fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT" - return 1 - } + [[ "$resolved_service_root" == "$SERVICE_ROOT" ]] || { fail "Service root must be canonical and contain no symlinked components: $SERVICE_ROOT"; return 1; } SERVICE_ROOT="$resolved_service_root" - resolved_release_root="${SERVICE_ROOT}/releases" if [[ -e "$resolved_release_root" || -L "$resolved_release_root" ]]; then - [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || { - fail "Release root must be a real directory: $resolved_release_root" - return 1 - } + [[ -d "$resolved_release_root" && ! -L "$resolved_release_root" ]] || { fail "Release root must be a real directory: $resolved_release_root"; return 1; } else install -d -m 0755 "$resolved_release_root" || return 1 fi assert_trusted_directory_chain "$resolved_release_root" 'Release root' || return 1 [[ "$(realpath -e -- "$resolved_release_root")" == "$resolved_release_root" ]] || fail "Release root must be canonical and contain no symlinked components: $resolved_release_root" } - snapshot_systemd_write_paths() { local transaction_temporary drop_in_mode [[ ! -e "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || @@ -283,7 +187,6 @@ snapshot_systemd_write_paths() { systemd_drop_in_backup="${systemd_transaction_path}/drop-in" systemd_write_paths_snapshot_ready=1 } - load_systemd_write_paths_transaction() { local stored_root stored_service stored_systemd_root stored_state [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || @@ -319,7 +222,6 @@ load_systemd_write_paths_transaction() { fi systemd_write_paths_snapshot_ready=1 } - restore_systemd_write_paths() { local restore_temporary="" [[ "$systemd_write_paths_snapshot_ready" == '1' ]] || return 0 @@ -336,7 +238,6 @@ restore_systemd_write_paths() { fi systemctl daemon-reload || return 1 } - finish_systemd_write_paths_transaction() { [[ "$systemd_transaction_path" == "$TRANSACTION_ROOT"/service-*.transaction ]] || fail "Refusing to remove an unexpected transaction path: $systemd_transaction_path" @@ -346,7 +247,6 @@ finish_systemd_write_paths_transaction() { systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" } - sync_deployment_state() { if [[ -f "$systemd_drop_in_path" ]]; then sync -f "$systemd_drop_in_path" || return 1 @@ -354,11 +254,9 @@ sync_deployment_state() { sync -f "$systemd_drop_in_dir" || return 1 sync -f "$SERVICE_ROOT" || return 1 } - commit_systemd_write_paths() { - local committed_path="${systemd_transaction_path}.committed.$$" - # The rename is the durable commit point. Ignore catchable termination during - # this tiny section so a signal cannot run rollback after the transaction has + local committed_path="${systemd_transaction_path}.committed" + # The rename is the durable commit point; ignore catchable termination so a signal cannot run rollback after it has # disappeared but before the in-memory state reflects that fact. sync_deployment_state || return 1 trap - INT TERM @@ -366,14 +264,16 @@ commit_systemd_write_paths() { promoted=0 systemd_write_paths_snapshot_ready=0 systemd_drop_in_backup="" - sync -f "$TRANSACTION_ROOT" || log "WARNING: failed to sync committed transaction directory $TRANSACTION_ROOT" >&2 + if ! sync -f "$TRANSACTION_ROOT"; then + log "ERROR: committed transaction retained until its directory can be synchronized: $committed_path" >&2 + return 1 + fi rm -rf -- "$committed_path" || log "WARNING: committed transaction cleanup remains at $committed_path" >&2 + sync -f "$TRANSACTION_ROOT" || log "WARNING: committed transaction cleanup was not synchronized in $TRANSACTION_ROOT" >&2 } - report_systemd_restore_failure() { log "ERROR: failed to restore systemd writable paths; transaction retained at $systemd_transaction_path" >&2 } - restore_previous_current_link() { local rollback_link="" if [[ -n "$previous_target" ]]; then @@ -387,7 +287,6 @@ restore_previous_current_link() { [[ ! -e "$SERVICE_ROOT/current" && ! -L "$SERVICE_ROOT/current" ]] fi } - peek_systemd_transaction_identity() { [[ -d "$systemd_transaction_path" && ! -L "$systemd_transaction_path" ]] || fail "Systemd writable-path transaction must be a real directory: $systemd_transaction_path" @@ -402,7 +301,15 @@ peek_systemd_transaction_identity() { [[ "$transaction_systemd_config_root" == /* && "$transaction_systemd_config_root" != '/' && "$transaction_systemd_config_root" != *[[:space:]]* ]] || fail 'Incomplete transaction contains an invalid systemd config root.' } - +settle_committed_transaction() { + local committed="$1" + [[ -e "$committed" || -L "$committed" ]] || return 0 + [[ -d "$committed" && ! -L "$committed" ]] || fail "Committed transaction marker must be a real directory: $committed" + assert_trusted_directory_chain "$committed" 'Committed transaction marker' + sync -f "$TRANSACTION_ROOT" || fail "Committed transaction directory is not durable: $committed" + rm -rf -- "$committed" || fail "Committed transaction marker could not be removed: $committed" + sync -f "$TRANSACTION_ROOT" || fail "Committed transaction cleanup is not durable: $committed" +} recover_incomplete_systemd_transaction() { local permissions_restored=1 current_restored=1 service_safe=0 if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then @@ -430,7 +337,19 @@ recover_incomplete_systemd_transaction() { log "Recovered incomplete systemd writable-path transaction for $SYSTEMD_SERVICE." fi } - +recover_selected_transaction() { + SYSTEMD_SERVICE="$transaction_systemd_service" + SERVICE_ROOT="$transaction_service_root" + SYSTEMD_CONFIG_ROOT="$transaction_systemd_config_root" + local preparation_status=0 + prepare_service_release_root || preparation_status=$? + if [[ "$preparation_status" -eq 0 ]]; then prepare_systemd_drop_in_directory || preparation_status=$?; fi + if [[ "$preparation_status" -ne 0 ]]; then + systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop recorded unit $SYSTEMD_SERVICE after recovery preparation failed." >&2 + fail "Recorded recovery paths are unavailable; $SYSTEMD_SERVICE was stopped and operator recovery is required." + fi + recover_incomplete_systemd_transaction +} reconcile_systemd_write_paths() ( if ((${#systemd_read_write_paths[@]} == 0)); then [[ ! -f "$systemd_drop_in_path" ]] || rm -f -- "$systemd_drop_in_path" @@ -453,62 +372,160 @@ reconcile_systemd_write_paths() ( fi systemctl daemon-reload ) - mkdir -p "$LOCK_ROOT" exec 9>"${LOCK_ROOT}/powerforge-service-${service_id}.lock" flock -n 9 || fail "Another deployment is active for $service_id." - -prepare_service_release_root prepare_transaction_root -prepare_trusted_stage_root +requested_systemd_config_root="$SYSTEMD_CONFIG_ROOT" +own_transaction="${TRANSACTION_ROOT}/service-${service_id}.transaction" +own_committed="${own_transaction}.committed" +settle_committed_transaction "$own_committed" +if [[ -e "$own_transaction" || -L "$own_transaction" ]]; then + systemd_transaction_path="$own_transaction" + peek_systemd_transaction_identity + unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" + service_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" + exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n 8 || fail "Another deployment is active for systemd unit $transaction_systemd_service." + exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n 7 || fail "Another deployment is active for service root $transaction_service_root." + recover_selected_transaction + exec 8>&- 7>&- +fi +SYSTEMD_CONFIG_ROOT="$requested_systemd_config_root" +[[ "$CONFIG_ROOT" == /* && "$CONFIG_ROOT" != '/' && "$CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Service config root must be an absolute non-root path without whitespace.' +[[ -d "$CONFIG_ROOT" && ! -L "$CONFIG_ROOT" ]] || fail "Service config root must be a real directory: $CONFIG_ROOT" +assert_trusted_directory_chain "$CONFIG_ROOT" 'Service config root' +resolved_config_root="$(realpath -e -- "$CONFIG_ROOT")" +[[ "$resolved_config_root" == "$CONFIG_ROOT" ]] || fail "Service config root must be canonical and contain no symlinked components: $CONFIG_ROOT" +CONFIG_ROOT="$resolved_config_root" +config_path="${CONFIG_ROOT}/${service_id}.env" +[[ -f "$config_path" && ! -L "$config_path" ]] || fail "Service is not configured: $service_id" +if [[ "$(id -u)" -eq 0 ]]; then + [[ "$(stat -c '%u' "$config_path")" -eq 0 ]] || fail "Service config must be owned by root: $config_path" + config_mode="$(stat -c '%a' "$config_path")" + (( (8#$config_mode & 0022) == 0 )) || fail "Service config must not be group/world writable: $config_path" +fi +unset SERVICE_ROOT SYSTEMD_SERVICE SYSTEMD_READ_WRITE_PATHS LOCAL_HEALTH_URL RELEASES_TO_KEEP REQUIRED_RELEASE_PATHS PUBLIC_HEALTH_URLS REQUIRE_HEALTH_PROVENANCE +# shellcheck disable=SC1090 +source "$config_path" +: "${SERVICE_ROOT:?SERVICE_ROOT is required in $config_path}" "${SYSTEMD_SERVICE:?SYSTEMD_SERVICE is required in $config_path}" "${LOCAL_HEALTH_URL:?LOCAL_HEALTH_URL is required in $config_path}" +: "${SYSTEMD_READ_WRITE_PATHS:=}" "${RELEASES_TO_KEEP:=5}" "${REQUIRED_RELEASE_PATHS:=}" "${PUBLIC_HEALTH_URLS:=}" "${REQUIRE_HEALTH_PROVENANCE:=1}" +[[ "$SERVICE_ROOT" == /* && "$SERVICE_ROOT" != '/' && "$SERVICE_ROOT" != *[[:space:]]* ]] || fail 'SERVICE_ROOT must be an absolute non-root path without whitespace.' +[[ "$TRUSTED_STAGE_ROOT" == /* && "$TRUSTED_STAGE_ROOT" != '/' ]] || fail 'Trusted staging root must be an absolute non-root path.' +[[ "$SYSTEMD_CONFIG_ROOT" == /* && "$SYSTEMD_CONFIG_ROOT" != '/' && "$SYSTEMD_CONFIG_ROOT" != *[[:space:]]* ]] || fail 'Systemd config root must be an absolute non-root path without whitespace.' +[[ "$SYSTEMD_SERVICE" =~ ^[A-Za-z0-9_.@-]+\.service$ ]] || fail 'SYSTEMD_SERVICE must be a systemd service unit name.' +[[ "$LOCAL_HEALTH_URL" =~ ^https?://[^[:space:]]+$ ]] || fail 'LOCAL_HEALTH_URL must be an HTTP or HTTPS URL.' +[[ "$RELEASES_TO_KEEP" =~ ^[1-9][0-9]*$ ]] || fail 'RELEASES_TO_KEEP must be a positive integer.' +[[ "$REQUIRE_HEALTH_PROVENANCE" == '0' || "$REQUIRE_HEALTH_PROVENANCE" == '1' ]] || fail 'REQUIRE_HEALTH_PROVENANCE must be 0 or 1.' +for health_url in $PUBLIC_HEALTH_URLS; do [[ "$health_url" =~ ^https://[^[:space:]]+$ ]] || fail "Public health URL must use HTTPS: $health_url"; done +read -r -a configured_systemd_read_write_paths <<<"$SYSTEMD_READ_WRITE_PATHS" +for read_write_path in "${configured_systemd_read_write_paths[@]}"; do + [[ "$read_write_path" == /* && "$read_write_path" != '/' ]] || fail 'Systemd writable paths must be absolute non-root paths.' + [[ "$read_write_path" =~ ^/[A-Za-z0-9._@:+,-]+(/[A-Za-z0-9._@:+,-]+)*$ ]] || fail "Systemd writable path contains unsupported characters: $read_write_path" + [[ "/${read_write_path#/}/" != *'/../'* ]] || fail "Systemd writable path must not contain traversal: $read_write_path" +done configured_service_root="$SERVICE_ROOT" -configured_release_root="$resolved_release_root" +configured_service_root_identity="$(realpath -e -- "$SERVICE_ROOT" 2>/dev/null || true)" configured_systemd_service="$SYSTEMD_SERVICE" configured_systemd_config_root="$SYSTEMD_CONFIG_ROOT" -systemd_transaction_path="${TRANSACTION_ROOT}/service-${service_id}.transaction" -transaction_service_root="" -transaction_systemd_service="" -transaction_systemd_config_root="" -if [[ -e "$systemd_transaction_path" || -L "$systemd_transaction_path" ]]; then +for candidate_committed in "$TRANSACTION_ROOT"/service-*.transaction.committed; do + [[ "$candidate_committed" != "$own_committed" && ( -e "$candidate_committed" || -L "$candidate_committed" ) ]] || continue + systemd_transaction_path="$candidate_committed" peek_systemd_transaction_identity -fi -unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" -service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" -exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" -flock -n 8 || fail "Another deployment is active for systemd unit $SYSTEMD_SERVICE." -exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" -flock -n 7 || fail "Another deployment is active for service root $SERVICE_ROOT." - -if [[ -n "$transaction_systemd_service" && "$transaction_systemd_service" != "$configured_systemd_service" ]]; then - transaction_unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" - exec {transaction_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${transaction_unit_lock_key}.lock" - flock -n "$transaction_unit_lock_fd" || fail "Another deployment is active for prior systemd unit $transaction_systemd_service." -fi -if [[ -n "$transaction_service_root" && "$transaction_service_root" != "$configured_service_root" ]]; then - transaction_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" - exec {transaction_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${transaction_root_lock_key}.lock" - flock -n "$transaction_root_lock_fd" || fail "Another deployment is active for prior service root $transaction_service_root." -fi - -if [[ -n "$transaction_systemd_service" ]]; then - SYSTEMD_SERVICE="$transaction_systemd_service" - SERVICE_ROOT="$transaction_service_root" - SYSTEMD_CONFIG_ROOT="$transaction_systemd_config_root" - recovery_preparation_status=0 - prepare_service_release_root || recovery_preparation_status=$? - if [[ "$recovery_preparation_status" -eq 0 ]]; then - prepare_systemd_drop_in_directory || recovery_preparation_status=$? + if [[ "$transaction_systemd_service" == "$configured_systemd_service" || "$transaction_service_root" == "$configured_service_root" || ( -n "$configured_service_root_identity" && "$transaction_service_root" == "$configured_service_root_identity" ) ]]; then + transaction_id="${candidate_committed##*/service-}" + transaction_id="${transaction_id%.transaction.committed}" + [[ "$transaction_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail "Committed transaction has an invalid service identity: $candidate_committed" + exec {committed_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${transaction_id}.lock" + flock -n "$committed_service_lock_fd" || fail "Another deployment is active for $transaction_id." + settle_committed_transaction "$candidate_committed" fi - if [[ "$recovery_preparation_status" -ne 0 ]]; then - systemctl stop "$SYSTEMD_SERVICE" || log "CRITICAL: failed to stop recorded unit $SYSTEMD_SERVICE after recovery preparation failed." >&2 - fail "Recorded recovery paths are unavailable; $SYSTEMD_SERVICE was stopped and operator recovery is required." +done +related_transactions=() +related_units=() +related_ids=() +related_roots=() +for candidate_transaction in "$TRANSACTION_ROOT"/service-*.transaction; do + [[ -e "$candidate_transaction" || -L "$candidate_transaction" ]] || continue + systemd_transaction_path="$candidate_transaction" + peek_systemd_transaction_identity + transaction_id="${candidate_transaction##*/service-}" + transaction_id="${transaction_id%.transaction}" + [[ "$transaction_id" =~ ^[a-z0-9][a-z0-9.-]{0,62}$ ]] || fail "Incomplete transaction has an invalid service identity: $candidate_transaction" + if [[ "$transaction_systemd_service" == "$configured_systemd_service" || "$transaction_service_root" == "$configured_service_root" || ( -n "$configured_service_root_identity" && "$transaction_service_root" == "$configured_service_root_identity" ) ]]; then + related_transactions+=("$candidate_transaction") + related_units+=("$transaction_systemd_service") + related_ids+=("$transaction_id") + related_roots+=("$transaction_service_root") fi - recover_incomplete_systemd_transaction +done +if ((${#related_transactions[@]} > 1)); then + declare -A held_related_units=() held_related_roots=() + for index in "${!related_ids[@]}"; do + related_id="${related_ids[$index]}" + if [[ "$related_id" != "$service_id" ]]; then + exec {related_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${related_id}.lock" + flock -n "$related_service_lock_fd" || fail "Another deployment is active for $related_id." + fi + unit_lock_key="$(printf '%s' "${related_units[$index]}" | sha256sum | awk '{print $1}')" + if [[ -z "${held_related_units[$unit_lock_key]:-}" ]]; then + exec {related_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n "$related_unit_lock_fd" || fail "Another deployment is active for systemd unit ${related_units[$index]}." + held_related_units[$unit_lock_key]="$related_unit_lock_fd" + fi + service_root_lock_key="$(printf '%s' "${related_roots[$index]}" | sha256sum | awk '{print $1}')" + if [[ -z "${held_related_roots[$service_root_lock_key]:-}" ]]; then + exec {related_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n "$related_root_lock_fd" || fail "Another deployment is active for service root ${related_roots[$index]}." + held_related_roots[$service_root_lock_key]="$related_root_lock_fd" + fi + done + declare -A stopped_related_units=() + for related_unit in "${related_units[@]}"; do + [[ -n "${stopped_related_units[$related_unit]:-}" ]] && continue + systemctl stop "$related_unit" || fail "Failed to stop ambiguous recorded unit $related_unit." + stopped_related_units[$related_unit]=1 + done + fail 'Multiple incomplete transactions overlap this deployment; recorded units were stopped and operator recovery is required.' +fi +locked_systemd_service="" +locked_service_root="" +if ((${#related_transactions[@]} == 1)); then + systemd_transaction_path="${related_transactions[0]}" + peek_systemd_transaction_identity + transaction_id="${systemd_transaction_path##*/service-}" + transaction_id="${transaction_id%.transaction}" + if [[ "$transaction_id" != "$service_id" ]]; then + exec {transaction_service_lock_fd}>"${LOCK_ROOT}/powerforge-service-${transaction_id}.lock" + flock -n "$transaction_service_lock_fd" || fail "Another deployment is active for $transaction_id." + fi + unit_lock_key="$(printf '%s' "$transaction_systemd_service" | sha256sum | awk '{print $1}')" + service_root_lock_key="$(printf '%s' "$transaction_service_root" | sha256sum | awk '{print $1}')" + exec 8>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n 8 || fail "Another deployment is active for systemd unit $transaction_systemd_service." + exec 7>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n 7 || fail "Another deployment is active for service root $transaction_service_root." + locked_systemd_service="$transaction_systemd_service" + locked_service_root="$transaction_service_root" + recover_selected_transaction SYSTEMD_SERVICE="$configured_systemd_service" SERVICE_ROOT="$configured_service_root" - resolved_release_root="$configured_release_root" SYSTEMD_CONFIG_ROOT="$configured_systemd_config_root" fi +prepare_service_release_root +prepare_trusted_stage_root +if [[ "$locked_systemd_service" != "$SYSTEMD_SERVICE" ]]; then + unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" + exec {configured_unit_lock_fd}>"${LOCK_ROOT}/powerforge-systemd-${unit_lock_key}.lock" + flock -n "$configured_unit_lock_fd" || fail "Another deployment is active for systemd unit $SYSTEMD_SERVICE." +fi +if [[ "$locked_service_root" != "$SERVICE_ROOT" ]]; then + service_root_lock_key="$(printf '%s' "$SERVICE_ROOT" | sha256sum | awk '{print $1}')" + exec {configured_root_lock_fd}>"${LOCK_ROOT}/powerforge-root-${service_root_lock_key}.lock" + flock -n "$configured_root_lock_fd" || fail "Another deployment is active for service root $SERVICE_ROOT." +fi +systemd_transaction_path="${TRANSACTION_ROOT}/service-${service_id}.transaction" prepare_systemd_drop_in_directory previous_target="" if [[ -e "$SERVICE_ROOT/current" || -L "$SERVICE_ROOT/current" ]]; then @@ -531,7 +548,6 @@ for read_write_path in "${configured_systemd_read_write_paths[@]}"; do done systemd_read_write_paths+=("$resolved_read_write_path") done - archive="$(realpath -e "$archive")" metadata="$(realpath -e "$metadata")" [[ -f "$archive" && ! -L "$archive" ]] || fail 'Artifact must be a regular file, not a symlink.' @@ -542,7 +558,6 @@ if [[ -n "${SUDO_UID:-}" ]]; then [[ "$(stat -c '%u' "$archive")" -eq "$SUDO_UID" ]] || fail 'Artifact owner does not match the invoking deployment account.' [[ "$(stat -c '%u' "$metadata")" -eq "$SUDO_UID" ]] || fail 'Metadata owner does not match the invoking deployment account.' fi - install -d -m 0700 "$TRUSTED_STAGE_ROOT" trusted_stage="$(mktemp -d "${TRUSTED_STAGE_ROOT}/${service_id}.XXXXXXXX")" chmod 0700 "$trusted_stage" @@ -550,12 +565,10 @@ install -m 0600 "$archive" "$trusted_stage/artifact.tar" install -m 0600 "$metadata" "$trusted_stage/deployment.json" archive="$trusted_stage/artifact.tar" metadata="$trusted_stage/deployment.json" - json_string() { local key="$1" sed -n "s/.*\"${key}\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\".*/\1/p" "$metadata" | head -n 1 } - source_sha="$(json_string sourceSha)" artifact_sha="$(json_string artifactSha256)" run_id="$(json_string workflowRunId)" @@ -565,27 +578,22 @@ run_attempt="$(json_string workflowRunAttempt)" [[ "$run_id" =~ ^[0-9]+$ && "$run_attempt" =~ ^[0-9]+$ ]] || fail 'Metadata workflow run identity is invalid.' actual_artifact_sha="$(sha256sum "$archive" | awk '{print $1}')" [[ "$actual_artifact_sha" == "$artifact_sha" ]] || fail 'Artifact checksum does not match deployment metadata.' - while IFS= read -r entry; do stripped="${entry#./}" [[ "$entry" != /* ]] || fail "Archive contains an absolute path: $entry" [[ "/${stripped}/" != *'/../'* ]] || fail "Archive contains path traversal: $entry" done < <(tar -tf "$archive") - while IFS= read -r listing; do entry_type="${listing:0:1}" [[ "$entry_type" == '-' || "$entry_type" == 'd' ]] || fail 'Archive contains links or special files.' done < <(tar -tvf "$archive") - release_id="$(date -u +%Y%m%d%H%M%S)-${run_id}-${run_attempt}-${source_sha:0:12}" release_dir="$resolved_release_root/$release_id" [[ ! -e "$release_dir" ]] || fail "Release already exists: $release_id" - health_response() { local url="$1" curl -fsS --retry 3 --retry-all-errors --max-time 30 "${url}?powerforge-deploy=${run_id}-${run_attempt}" } - verify_health() { local url response for url in "$LOCAL_HEALTH_URL" $PUBLIC_HEALTH_URLS; do @@ -597,7 +605,6 @@ verify_health() { fi done } - rollback() { local exit_code="$1" local permissions_restored=1 @@ -665,7 +672,6 @@ rollback() { trap 'exit_code=$?; if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback "$exit_code"; else exit "$exit_code"; fi' ERR trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 130; else exit 130; fi' INT trap 'if [[ "$BASHPID" == "$deployment_shell_pid" ]]; then rollback 143; else exit 143; fi' TERM - mkdir -p "$release_dir" tar --extract --file "$archive" --directory "$release_dir" --no-same-owner --no-same-permissions for required_path in $REQUIRED_RELEASE_PATHS; do @@ -674,22 +680,18 @@ for required_path in $REQUIRED_RELEASE_PATHS; do done mkdir -p "$release_dir/_powerforge" install -m 0644 "$metadata" "$release_dir/_powerforge/deployment.json" - candidate_link="$SERVICE_ROOT/.current.${run_id}.${run_attempt}" ln -s "$release_dir" "$candidate_link" snapshot_systemd_write_paths reconcile_systemd_write_paths promoted=1 mv -Tf "$candidate_link" "$SERVICE_ROOT/current" - systemctl restart "$SYSTEMD_SERVICE" verify_health - mapfile -t old_releases < <(find "$resolved_release_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | awk '{print $2}') for ((index=RELEASES_TO_KEEP; index<${#old_releases[@]}; index++)); do [[ "${old_releases[$index]}" == "$release_dir" || "${old_releases[$index]}" == "$previous_target" ]] || rm -rf "${old_releases[$index]}" done - commit_systemd_write_paths trap - ERR INT TERM cleanup_staging diff --git a/Docs/PowerForge.Web.LinuxServiceDeployment.md b/Docs/PowerForge.Web.LinuxServiceDeployment.md index 34e4623843..a0324889af 100644 --- a/Docs/PowerForge.Web.LinuxServiceDeployment.md +++ b/Docs/PowerForge.Web.LinuxServiceDeployment.md @@ -84,12 +84,16 @@ the previous link or safe service state cannot be proven. Immediately before switching `current`, the promoter persists the previous permission and current-release state under `/var/lib/powerforge/service-deployment-state`, then reloads the candidate policy. Transactions are keyed by the stable service id, so a -later invocation restores the recorded unit and service root even when configuration -was renamed in the meantime. It proves the service restarted or stopped before -accepting a new deployment, so process termination or host loss cannot strand an -uncommitted writable policy. Restored drop-in and service-root filesystems are flushed -before recovery state is removed; successful promotion uses the same durable commit -boundary. +later invocation restores the recorded unit, service root, and systemd configuration +root even when current configuration was renamed or is temporarily unavailable. Before +validating a new service root, the promoter also scans pending transactions for another +service id that shares the configured unit or root and recovers that state under both +service locks. It proves the service restarted or stopped before accepting a new +deployment, so process termination or host loss cannot strand an uncommitted writable +policy. Restored drop-in and service-root filesystems are flushed before recovery state +is removed. Successful promotion retains a recognizable committed marker until the +transaction-directory rename is durable, then safely clears that marker on this or the +next invocation. Give the dedicated deployment account only the exact promoter command it needs. Keep the service identifier fixed in sudoers rather than granting general root shell or `systemctl` access: diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index e5f7e3004c..7890a5c25f 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -64,8 +64,13 @@ chmod +x "$test_root/bin/mv" cat >"$test_root/bin/sync" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail -if [[ -n "${FAIL_SYNC_PATH:-}" && "$*" == *"$FAIL_SYNC_PATH"* ]]; then - exit 1 +if [[ -n "${FAIL_SYNC_PATH:-}" && "${*: -1}" == "$FAIL_SYNC_PATH" ]]; then + count=1 + if [[ -n "${FAIL_SYNC_COUNT_FILE:-}" ]]; then + [[ ! -f "$FAIL_SYNC_COUNT_FILE" ]] || count=$(( $(cat "$FAIL_SYNC_COUNT_FILE") + 1 )) + printf '%s\n' "$count" >"$FAIL_SYNC_COUNT_FILE" + fi + if [[ -z "${FAIL_SYNC_FROM_CALL:-}" || "$count" -ge "$FAIL_SYNC_FROM_CALL" ]]; then exit 1; fi fi exec /usr/bin/sync "$@" EOF @@ -150,7 +155,7 @@ stranded_release="$test_root/service/releases/stranded-release" cp -a "$first_target" "$stranded_release" ln -sfn "$stranded_release" "$test_root/service/current" printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" -sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=renamed.service/' "$test_root/config/example.env" +mv "$test_root/config/example.env" "$test_root/config/example.env.unavailable" set +e recovery_output="$(POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd-renamed" TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example 2>&1)" recovery_status=$? @@ -158,13 +163,85 @@ set -e [[ "$recovery_status" -ne 0 ]] grep -q 'Recovering incomplete systemd writable-path transaction' <<<"$recovery_output" grep -q 'Recovered incomplete systemd writable-path transaction' <<<"$recovery_output" +grep -q 'Service is not configured' <<<"$recovery_output" grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e "$transaction_dir" ]] +rm -f -- "$test_root/config/example.env.unavailable" write_config example "$test_root/service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/example-data" >>"$test_root/config/example.env" +alias_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-aliasowner.transaction" +mkdir -m 0700 "$alias_transaction" +printf '%s\n' "$test_root/service" >"$alias_transaction/service-root" +printf '%s\n' 'example.service' >"$alias_transaction/systemd-service" +printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$alias_transaction/systemd-config-root" +printf '%s\n' "$first_target" >"$alias_transaction/previous-target" +printf '%s\n' 'present' >"$alias_transaction/drop-in-state" +printf '%s\n' "$(stat -c '%u' "$drop_in")" >"$alias_transaction/drop-in-owner" +printf '%s\n' "$(stat -c '%g' "$drop_in")" >"$alias_transaction/drop-in-group" +printf '%s\n' "$(stat -c '%a' "$drop_in")" >"$alias_transaction/drop-in-mode" +cp "$drop_in" "$alias_transaction/drop-in" +chmod 0600 "$alias_transaction"/* +ln -sfn "$stranded_release" "$test_root/service/current" +printf '[Service]\nReadWritePaths=%s\n' "$test_root/stranded-data" >"$drop_in" +write_config aliasconsumer "$test_root/service/." +set +e +alias_output="$(TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service aliasconsumer 2>&1)" +alias_status=$? +set -e +[[ "$alias_status" -ne 0 ]] +grep -q 'Recovered incomplete systemd writable-path transaction for example.service' <<<"$alias_output" +[[ ! -e "$alias_transaction" ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" + +mkdir -p "$test_root/multi-current" +write_config multiinvoker "$test_root/multi-current" +sed -i 's/^SYSTEMD_SERVICE=.*/SYSTEMD_SERVICE=sharedmulti.service/' "$test_root/config/multiinvoker.env" +multi_transactions=() +for owner in aliasmulti1 aliasmulti2; do + multi_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-${owner}.transaction" + multi_transactions+=("$multi_transaction") + mkdir -m 0700 "$multi_transaction" + printf '%s\n' "$test_root/${owner}-root" >"$multi_transaction/service-root" + printf '%s\n' 'sharedmulti.service' >"$multi_transaction/systemd-service" + printf '%s\n' "$POWERFORGE_SYSTEMD_CONFIG_ROOT" >"$multi_transaction/systemd-config-root" + chmod 0600 "$multi_transaction"/* +done +multi_lock_ready="$test_root/multi-lock-ready" +( + exec 202>"$test_root/locks/powerforge-service-aliasmulti1.lock" + flock 202 + : >"$multi_lock_ready" + sleep 30 +) & +multi_lock_holder=$! +for _ in {1..100}; do [[ -e "$multi_lock_ready" ]] && break; sleep 0.05; done +[[ -e "$multi_lock_ready" ]] +: >"$TEST_SYSTEMCTL_LOG" +set +e +multi_busy_output="$(TEST_SERVICE_ROOT="$test_root/multi-current" "$deploy_script" --service multiinvoker 2>&1)" +multi_busy_status=$? +set -e +[[ "$multi_busy_status" -ne 0 ]] +grep -q 'Another deployment is active for aliasmulti1' <<<"$multi_busy_output" +if grep -q '^stop sharedmulti.service$' "$TEST_SYSTEMCTL_LOG"; then + echo 'Ambiguous recovery stopped a unit while a transaction owner was active.' >&2 + exit 1 +fi +kill "$multi_lock_holder" 2>/dev/null || true +wait "$multi_lock_holder" 2>/dev/null || true +set +e +multi_output="$(TEST_SERVICE_ROOT="$test_root/multi-current" "$deploy_script" --service multiinvoker 2>&1)" +multi_status=$? +set -e +[[ "$multi_status" -ne 0 ]] +grep -q 'Multiple incomplete transactions overlap this deployment' <<<"$multi_output" +[[ "$(grep -c '^stop sharedmulti.service$' "$TEST_SYSTEMCTL_LOG")" -eq 1 ]] +for multi_transaction in "${multi_transactions[@]}"; do [[ -d "$multi_transaction" ]]; rm -rf -- "$multi_transaction"; done + mkdir -p "$test_root/recovery-current-service" write_config recoverymissing "$test_root/recovery-current-service" missing_transaction="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-recoverymissing.transaction" @@ -196,6 +273,29 @@ grep -q 'rollback transaction retained' <<<"${sync_failure_output,,}" [[ -d "$transaction_dir" ]] [[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] rm -rf -- "$transaction_dir" + +mkdir -p "$test_root/commit-service" "$test_root/commit-data" +write_config commitdurable "$test_root/commit-service" +printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/commit-data" >>"$test_root/config/commitdurable.env" +create_stage commitdurable 92011 1 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +commit_sync_count="$test_root/commit-sync-count" +set +e +commit_output="$(TEST_SERVICE_ROOT="$test_root/commit-service" \ + FAIL_SYNC_PATH="$POWERFORGE_SERVICE_TRANSACTION_ROOT" \ + FAIL_SYNC_COUNT_FILE="$commit_sync_count" \ + FAIL_SYNC_FROM_CALL=2 \ + "$deploy_script" --service commitdurable 2>&1)" +commit_status=$? +set -e +[[ "$commit_status" -ne 0 ]] +committed_marker="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-commitdurable.transaction.committed" +grep -q 'committed transaction retained' <<<"${commit_output,,}" +[[ -d "$committed_marker" ]] +grep -q 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' "$test_root/commit-service/current/_powerforge/deployment.json" +set +e +TEST_SERVICE_ROOT="$test_root/commit-service" "$deploy_script" --service commitdurable >/dev/null 2>&1 +set -e +[[ ! -e "$committed_marker" ]] if TEST_SERVICE_ROOT="$test_root/service" "$deploy_script" --service example --archive /etc/passwd; then echo 'Promoter unexpectedly accepted a caller-controlled archive path.' >&2 exit 1 From 7f006153305210c6d34e162d2cbd8aac1a947c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20K=C5=82ys?= Date: Thu, 20 Aug 2026 14:19:42 +0200 Subject: [PATCH 8/8] Close release storage durability boundary --- Deployment/Linux/powerforge-service-deploy.sh | 10 ++--- .../Linux/powerforge-service-deploy.tests.sh | 40 ++++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/Deployment/Linux/powerforge-service-deploy.sh b/Deployment/Linux/powerforge-service-deploy.sh index 09e3d5b7d8..dce8ee3953 100755 --- a/Deployment/Linux/powerforge-service-deploy.sh +++ b/Deployment/Linux/powerforge-service-deploy.sh @@ -253,6 +253,7 @@ sync_deployment_state() { fi sync -f "$systemd_drop_in_dir" || return 1 sync -f "$SERVICE_ROOT" || return 1 + sync -f "$resolved_release_root" || return 1 } commit_systemd_write_paths() { local committed_path="${systemd_transaction_path}.committed" @@ -514,6 +515,9 @@ if ((${#related_transactions[@]} == 1)); then SYSTEMD_CONFIG_ROOT="$configured_systemd_config_root" fi prepare_service_release_root +for deployment_control_root in "$CONFIG_ROOT" "$SYSTEMD_CONFIG_ROOT" "$TRANSACTION_ROOT" "$TRUSTED_STAGE_ROOT" "$(realpath -e -- "$LOCK_ROOT")"; do + paths_overlap "$deployment_control_root" "$resolved_release_root" && fail "Deployment control path must not overlap release storage: $deployment_control_root" +done prepare_trusted_stage_root if [[ "$locked_systemd_service" != "$SYSTEMD_SERVICE" ]]; then unit_lock_key="$(printf '%s' "$SYSTEMD_SERVICE" | sha256sum | awk '{print $1}')" @@ -606,11 +610,7 @@ verify_health() { done } rollback() { - local exit_code="$1" - local permissions_restored=1 - local current_restored=1 - local service_safe=0 - local current_target="" + local exit_code="$1" permissions_restored=1 current_restored=1 service_safe=0 current_target="" set +e if ! restore_systemd_write_paths; then permissions_restored=0 diff --git a/Tests/Linux/powerforge-service-deploy.tests.sh b/Tests/Linux/powerforge-service-deploy.tests.sh index 7890a5c25f..bf630a6a52 100755 --- a/Tests/Linux/powerforge-service-deploy.tests.sh +++ b/Tests/Linux/powerforge-service-deploy.tests.sh @@ -13,6 +13,7 @@ export POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT="$test_root/trusted-stage" export POWERFORGE_SERVICE_TRANSACTION_ROOT="$test_root/transactions" export POWERFORGE_SYSTEMD_CONFIG_ROOT="$test_root/systemd" export TEST_SYSTEMCTL_LOG="$test_root/systemctl.log" +export TEST_SYNC_LOG="$test_root/sync.log" cat >"$test_root/bin/systemctl" <<'EOF' #!/usr/bin/env bash @@ -64,6 +65,7 @@ chmod +x "$test_root/bin/mv" cat >"$test_root/bin/sync" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail +[[ -z "${TEST_SYNC_LOG:-}" ]] || printf '%s\n' "$*" >>"$TEST_SYNC_LOG" if [[ -n "${FAIL_SYNC_PATH:-}" && "${*: -1}" == "$FAIL_SYNC_PATH" ]]; then count=1 if [[ -n "${FAIL_SYNC_COUNT_FILE:-}" ]]; then @@ -132,14 +134,30 @@ first_target="$(readlink -f "$test_root/service/current")" grep -q '1111111111111111111111111111111111111111' "$first_target/_powerforge/deployment.json" grep -q '^restart example.service$' "$TEST_SYSTEMCTL_LOG" grep -q '^daemon-reload$' "$TEST_SYSTEMCTL_LOG" +grep -qxF -- "-f $test_root/service/releases" "$TEST_SYNC_LOG" drop_in="$POWERFORGE_SYSTEMD_CONFIG_ROOT/example.service.d/powerforge-read-write-paths.conf" +transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction" grep -qxF '[Service]' "$drop_in" grep -qxF "ReadWritePaths=$test_root/example-data" "$drop_in" [[ ! -e /tmp/powerforge-service-example ]] +# A release-filesystem flush failure must happen before the commit marker and +# roll the current link back through the still-live transaction. +create_stage example 92012 1 cccccccccccccccccccccccccccccccccccccccc +set +e +release_sync_output="$(TEST_SERVICE_ROOT="$test_root/service" FAIL_SYNC_PATH="$test_root/service/releases" "$deploy_script" --service example 2>&1)" +release_sync_status=$? +set -e +[[ "$release_sync_status" -ne 0 ]] +[[ "$(readlink -f "$test_root/service/current")" == "$first_target" ]] +[[ ! -e "$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction.committed" ]] +[[ -d "$transaction_dir" ]] +grep -q 'Deployment failed; rolling back' <<<"$release_sync_output" +grep -q 'transaction retained' <<<"${release_sync_output,,}" +rm -rf -- "$transaction_dir" + # A persisted transaction must restore the prior drop-in/current link after an # uncatchable process or host failure, before a later deployment is considered. -transaction_dir="$POWERFORGE_SERVICE_TRANSACTION_ROOT/service-example.transaction" mkdir -m 0700 "$transaction_dir" printf '%s\n' "$test_root/service" >"$transaction_dir/service-root" printf '%s\n' 'example.service' >"$transaction_dir/systemd-service" @@ -416,6 +434,26 @@ for index in "${!control_ids[@]}"; do grep -q 'Systemd writable path must not overlap deployment control path' <<<"$control_output" done +# Deployment-owned state must never be placed inside prunable release storage. +mkdir -p "$test_root/control-overlap-service/releases" +write_config controloverlap "$test_root/control-overlap-service" +overlap_variables=(POWERFORGE_SERVICE_CONFIG_ROOT POWERFORGE_SYSTEMD_CONFIG_ROOT POWERFORGE_SERVICE_TRANSACTION_ROOT POWERFORGE_SERVICE_TRUSTED_STAGE_ROOT POWERFORGE_SERVICE_LOCK_ROOT) +overlap_names=(config systemd transactions staging locks) +for index in "${!overlap_variables[@]}"; do + overlap_variable="${overlap_variables[$index]}" + overlap_root="$test_root/control-overlap-service/releases/${overlap_names[$index]}" + if [[ "$overlap_variable" == 'POWERFORGE_SERVICE_CONFIG_ROOT' ]]; then + mkdir -p "$overlap_root" + cp "$test_root/config/controloverlap.env" "$overlap_root/controloverlap.env" + fi + set +e + control_overlap_output="$(TEST_SERVICE_ROOT="$test_root/control-overlap-service" env "$overlap_variable=$overlap_root" "$deploy_script" --service controloverlap 2>&1)" + control_overlap_status=$? + set -e + [[ "$control_overlap_status" -ne 0 ]] + grep -qxF "[powerforge-service-deploy] ERROR: Deployment control path must not overlap release storage: $overlap_root" <<<"$control_overlap_output" +done + mkdir -p "$test_root/glob-service" write_config glob "$test_root/glob-service" printf 'SYSTEMD_READ_WRITE_PATHS="%s"\n' "$test_root/service-*" >>"$test_root/config/glob.env"