Skip to content

fix(install): stop waiting on a gateway unit that has failed #4092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Yi-111-a wants to merge 1 commit into from
Closed

fix(install): stop waiting on a gateway unit that has failed #4092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -929,6 +929,10 @@ start_user_gateway() {
return 0
fi

# NRestarts is never reset by a restart, so read it before restarting the
# unit to tell a failure from this install apart from an earlier one.
GATEWAY_SERVICE_RESTART_BASELINE="$(gateway_user_service_field NRestarts)"

as_target_user systemctl --user enable openshell-gateway
as_target_user systemctl --user restart openshell-gateway
as_target_user systemctl --user is-active --quiet openshell-gateway
Expand Down Expand Up @@ -1015,10 +1019,53 @@ dump_user_service_gateway_diagnostics() {
fi
}

gateway_user_service_field() {
as_target_user systemctl --user show openshell-gateway -p "$1" --value 2>/dev/null || true
}

# The gateway runs as a systemd user service with Restart=on-failure and
# RestartSec=5s, so a gateway that fails during startup never settles in a final
# "failed" state: systemd keeps cycling it through "auto-restart", and 5s of
# delay never trips the start limit that would eventually leave it failed.
# Waiting on the listener alone therefore burns the whole timeout on a gateway
# that cannot start. Report a final "failed" unit, and a restart loop that grew
# past GATEWAY_SERVICE_RESTART_BASELINE, as a failed service.
#
# The baseline is the restart count from before the installer restarted the
# unit, so a unit that only failed during an earlier run keeps being waited on.
gateway_user_service_failed() {
case "${PLATFORM:-$(detect_platform)}" in
linux) ;;
*) return 1 ;;
esac
# Snap owns its own gateway service and its own listener probe.
[ "${LINUX_INSTALL_METHOD:-}" != "snap" ] || return 1

case "$(gateway_user_service_field ActiveState)" in
failed) return 0 ;;
esac

case "$(gateway_user_service_field SubState)" in
auto-restart) ;;
*) return 1 ;;
esac

_restarts="$(gateway_user_service_field NRestarts)"
case "$_restarts" in
"" | *[!0-9]*) return 1 ;;
esac
case "${GATEWAY_SERVICE_RESTART_BASELINE:-}" in
"" | *[!0-9]*) return 1 ;;
esac

[ "$_restarts" -gt "$GATEWAY_SERVICE_RESTART_BASELINE" ]
}

wait_for_local_gateway_listener() {
_timeout="${OPENSHELL_INSTALL_GATEWAY_TIMEOUT:-30}"
_elapsed=0
_last_output=""
_unit_failed=0
_probe_url="$(local_gateway_endpoint)/"
_mtls_dir="${TARGET_HOME}/.config/openshell/gateways/openshell/mtls"

Expand All @@ -1030,12 +1077,21 @@ wait_for_local_gateway_listener() {
info "local gateway listener is reachable"
return 0
fi

if gateway_user_service_failed; then
_unit_failed=1
break
fi

sleep 1
_elapsed=$((_elapsed + 1))
done

[ -z "$_last_output" ] || printf '%s\n' "$_last_output" >&2
dump_local_gateway_diagnostics
if [ "$_unit_failed" -eq 1 ]; then
error "the openshell-gateway service failed to start; retry it with: systemctl --user restart openshell-gateway"
fi
error "local gateway listener did not become reachable at ${_probe_url} within ${_timeout}s"
}

Expand Down
127 changes: 127 additions & 0 deletions tasks/scripts/test-install-sh.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,133 @@ if [ "$(PLATFORM=linux local_gateway_endpoint)" != "https://127.0.0.1:17670" ];
exit 1
fi

# The gateway user service is Restart=on-failure with RestartSec=5s, so a
# gateway that fails during startup never settles in a final "failed" state:
# systemd keeps cycling it through "auto-restart". The installer has to
# recognise that loop instead of waiting out its whole listener timeout.
assert_gateway_service_failure() {
local name=$1
local expected=$2
local active_state=$3
local sub_state=$4
local nrestarts=$5
local baseline=$6
local platform=$7
local method=$8
local actual

actual="$(
as_target_user() {
case "$*" in
*"-p ActiveState --value"*) printf '%s\n' "$MOCK_ACTIVE_STATE" ;;
*"-p SubState --value"*) printf '%s\n' "$MOCK_SUB_STATE" ;;
*"-p NRestarts --value"*) printf '%s\n' "$MOCK_NRESTARTS" ;;
*) return 1 ;;
esac
}
MOCK_ACTIVE_STATE="$active_state"
MOCK_SUB_STATE="$sub_state"
MOCK_NRESTARTS="$nrestarts"
PLATFORM="$platform"
LINUX_INSTALL_METHOD="$method"
GATEWAY_SERVICE_RESTART_BASELINE="$baseline"
if gateway_user_service_failed; then printf 'failed\n'; else printf 'waiting\n'; fi
)"
if [ "$actual" != "$expected" ]; then
echo "FAIL: ${name}: expected ${expected}, got ${actual}" >&2
exit 1
fi
}

assert_gateway_service_failure "a failed unit is reported" failed \
failed failed 0 0 linux deb
assert_gateway_service_failure "a restart loop past the baseline is reported" failed \
activating auto-restart 1 0 linux deb
assert_gateway_service_failure "a restart loop that predates the restart keeps waiting" waiting \
activating auto-restart 0 0 linux deb
assert_gateway_service_failure "a starting unit keeps waiting" waiting \
activating start 0 0 linux deb
assert_gateway_service_failure "a running unit keeps waiting" waiting \
active running 0 0 linux deb
assert_gateway_service_failure "an unknown restart count keeps waiting" waiting \
activating auto-restart "" 0 linux deb
assert_gateway_service_failure "a missing baseline keeps waiting" waiting \
activating auto-restart 3 "" linux deb
assert_gateway_service_failure "macOS keeps waiting on its Homebrew service" waiting \
activating auto-restart 3 0 darwin deb
assert_gateway_service_failure "snap keeps waiting on its own service" waiting \
activating auto-restart 3 0 linux snap

# The probe must give up within seconds of the unit failing, and the service
# failure has to be the last line rather than a bare listener timeout.
started_at=$SECONDS
if (
as_target_user() {
case "$*" in
"systemctl --user show"*) printf 'failed\n' ;;
*) return 1 ;;
esac
}
dump_local_gateway_diagnostics() { :; }
info() { :; }
PLATFORM=linux
TARGET_HOME="${tmpdir}/no-gateway-home"
GATEWAY_SERVICE_RESTART_BASELINE=0
wait_for_local_gateway_listener
) >"$out" 2>"$err"; then
echo "FAIL: the listener probe should fail when the gateway unit has failed" >&2
cat "$err" >&2 || true
exit 1
fi
elapsed=$((SECONDS - started_at))
if [ "$elapsed" -ge 5 ]; then
echo "FAIL: the listener probe waited ${elapsed}s on a failed gateway unit" >&2
exit 1
fi
expected_last_line="openshell: error: the openshell-gateway service failed to start; retry it with: systemctl --user restart openshell-gateway"
if [ "$(tail -n 1 "$err")" != "$expected_last_line" ]; then
echo "FAIL: the failed gateway service must be the installer's last line" >&2
tail -n 1 "$err" >&2 || true
exit 1
fi
if grep -Fq "did not become reachable at https://127.0.0.1:17670/ within 30s" "$err"; then
echo "FAIL: a failed gateway unit must not be reported as a listener timeout" >&2
cat "$err" >&2 || true
exit 1
fi

# A unit that is only starting must keep the probe waiting out its timeout.
probe_deadline=$((SECONDS + 3))
if (
as_target_user() {
case "$*" in
"systemctl --user show"*) printf 'starting\n' ;;
*) return 1 ;;
esac
}
dump_local_gateway_diagnostics() { :; }
info() { :; }
sleep() { :; }
PLATFORM=linux
TARGET_HOME="${tmpdir}/no-gateway-home"
GATEWAY_SERVICE_RESTART_BASELINE=0
OPENSHELL_INSTALL_GATEWAY_TIMEOUT=2
wait_for_local_gateway_listener
) >"$out" 2>"$err"; then
echo "FAIL: the listener probe should fail when nothing ever listens" >&2
cat "$err" >&2 || true
exit 1
fi
if [ "$(tail -n 1 "$err")" != "openshell: error: local gateway listener did not become reachable at https://127.0.0.1:17670/ within 2s" ]; then
echo "FAIL: a starting unit must not be reported as a failed service" >&2
tail -n 1 "$err" >&2 || true
exit 1
fi
if [ "$SECONDS" -ge "$probe_deadline" ]; then
echo "FAIL: the listener probe slept for real instead of respecting its timeout" >&2
exit 1
fi

cat >"${tmpdir}/checksums" <<'EOF'
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa openshell-dev-x86_64.rpm
bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb openshell-gateway-dev-x86_64.rpm
Expand Down
Loading