From 81d19c20a53c7bb52a3897d51acacb1e699e6f46 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 11:22:55 -0700 Subject: [PATCH 1/2] fix(local-dev): retry Gateway route validation Wait for Envoy to finish reconciling the sample route before local bootstrap fails. Closes #1364 Signed-off-by: Stephanie Baum --- tools/ncp-local-cluster/AGENTS.md | 1 + tools/ncp-local-cluster/Makefile | 6 +- .../scripts/validate-gateway-route.sh | 31 +++++++++ .../tests/test-validate-gateway-route.sh | 68 +++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100755 tools/ncp-local-cluster/scripts/validate-gateway-route.sh create mode 100755 tools/ncp-local-cluster/tests/test-validate-gateway-route.sh diff --git a/tools/ncp-local-cluster/AGENTS.md b/tools/ncp-local-cluster/AGENTS.md index efce1c983..79ec770cf 100644 --- a/tools/ncp-local-cluster/AGENTS.md +++ b/tools/ncp-local-cluster/AGENTS.md @@ -17,6 +17,7 @@ Run Makefile-only validation from `tools/ncp-local-cluster`: make validate-compute-clusters make print-compute-clusters make test-multicluster-make +tests/test-validate-gateway-route.sh ``` Cluster lifecycle targets require local tools such as `k3d`, `kubectl`, `helm`, and Docker. diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index 2b41a681c..f6a41e98e 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -46,6 +46,8 @@ COMPUTE_CLUSTER_NAME ?= $(COMPUTE_CLUSTER_PREFIX)-1 COMPUTE_PLANE_CONFIG_FILE ?= k3d-config-compute-plane.yaml COMPUTE_CLUSTERS ?= GATEWAY_HTTP_PORT ?= $(CONTROL_PLANE_HTTP_PORT) +GATEWAY_ROUTE_TIMEOUT_SECONDS ?= 60 +GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS ?= 2 # === Go Credential Provider Configuration === GO_PROVIDER_PATH := ./credential-provider-go @@ -421,7 +423,9 @@ validate-gateway: deploy-nginx wait-for-nginx wait-for-gateway ## Validate Gatew @echo ">>> Checking HTTPRoute status..." @kubectl get httproute nginx-route -n sample -o jsonpath='{.status}' || echo "HTTPRoute status not available" @echo ">>> Validating Gateway route (nginx.localhost)..." - @if curl -sSf http://nginx.localhost:$(GATEWAY_HTTP_PORT)/ >/dev/null 2>&1; then \ + @if GATEWAY_ROUTE_TIMEOUT_SECONDS=$(GATEWAY_ROUTE_TIMEOUT_SECONDS) \ + GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=$(GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS) \ + bash ./scripts/validate-gateway-route.sh http://nginx.localhost:$(GATEWAY_HTTP_PORT)/; then \ echo "OK Gateway route reachable at http://nginx.localhost:$(GATEWAY_HTTP_PORT)"; \ else \ echo "ERROR Gateway route validation failed"; \ diff --git a/tools/ncp-local-cluster/scripts/validate-gateway-route.sh b/tools/ncp-local-cluster/scripts/validate-gateway-route.sh new file mode 100755 index 000000000..1766b1a8a --- /dev/null +++ b/tools/ncp-local-cluster/scripts/validate-gateway-route.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +url="${1:?gateway route URL is required}" +timeout_seconds="${GATEWAY_ROUTE_TIMEOUT_SECONDS:-60}" +retry_interval_seconds="${GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS:-2}" + +if ! [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR GATEWAY_ROUTE_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 2 +fi +if ! [[ "$retry_interval_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS must be a positive integer" >&2 + exit 2 +fi + +elapsed=0 +while ((elapsed < timeout_seconds)); do + if curl -sSf --connect-timeout 5 --max-time 10 "$url" >/dev/null 2>&1; then + exit 0 + fi + echo "INFO Gateway route not reachable yet, waiting... (${elapsed}/${timeout_seconds} seconds)" + sleep "$retry_interval_seconds" + elapsed=$((elapsed + retry_interval_seconds)) +done + +echo "ERROR Gateway route did not become reachable within ${timeout_seconds} seconds" >&2 +exit 1 diff --git a/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh b/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh new file mode 100755 index 000000000..0684cd1ee --- /dev/null +++ b/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +test_dir="$(mktemp -d)" +trap 'rm -rf "$test_dir"' EXIT + +mkdir -p "$test_dir/bin" + +cat >"$test_dir/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +count=0 +if [[ -f "$TEST_CURL_STATE" ]]; then + count="$(<"$TEST_CURL_STATE")" +fi +count=$((count + 1)) +printf '%s\n' "$count" >"$TEST_CURL_STATE" +if ((count < TEST_CURL_SUCCEED_ON)); then + exit 22 +fi +EOF +chmod +x "$test_dir/bin/curl" + +cat >"$test_dir/bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +chmod +x "$test_dir/bin/sleep" + +export PATH="$test_dir/bin:$PATH" +export TEST_CURL_STATE="$test_dir/curl-count" +export TEST_CURL_SUCCEED_ON=3 + +GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \ +GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \ + "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \ + >"$test_dir/success.out" + +[[ "$(<"$TEST_CURL_STATE")" == "3" ]] +grep -q "Gateway route not reachable yet" "$test_dir/success.out" + +printf '0\n' >"$TEST_CURL_STATE" +export TEST_CURL_SUCCEED_ON=99 +if GATEWAY_ROUTE_TIMEOUT_SECONDS=2 \ + GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \ + "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \ + >"$test_dir/failure.out" 2>"$test_dir/failure.err"; then + echo "expected Gateway route validation to time out" >&2 + exit 1 +fi + +[[ "$(<"$TEST_CURL_STATE")" == "2" ]] +grep -q "did not become reachable within 2 seconds" "$test_dir/failure.err" + +for name in GATEWAY_ROUTE_TIMEOUT_SECONDS GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS; do + if env "$name=0" "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \ + >"$test_dir/invalid.out" 2>"$test_dir/invalid.err"; then + echo "expected $name=0 to fail validation" >&2 + exit 1 + fi + grep -q "$name must be a positive integer" "$test_dir/invalid.err" +done + +echo "Gateway route retry tests passed." From 74640c2484a47025aaaf3bb058c3cf127cd64d37 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 11:56:33 -0700 Subject: [PATCH 2/2] fix(local-dev): bound Gateway validation by wall clock Count failed request time toward the configured timeout and cap each request and sleep to the remaining deadline. Refs #1364 Signed-off-by: Stephanie Baum --- tools/ncp-local-cluster/AGENTS.md | 2 +- tools/ncp-local-cluster/Makefile | 5 +- .../scripts/validate-gateway-route.sh | 29 ++++++-- .../tests/test-validate-gateway-route.sh | 68 ++++++++++++++++++- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/tools/ncp-local-cluster/AGENTS.md b/tools/ncp-local-cluster/AGENTS.md index 79ec770cf..3916decb9 100644 --- a/tools/ncp-local-cluster/AGENTS.md +++ b/tools/ncp-local-cluster/AGENTS.md @@ -17,7 +17,7 @@ Run Makefile-only validation from `tools/ncp-local-cluster`: make validate-compute-clusters make print-compute-clusters make test-multicluster-make -tests/test-validate-gateway-route.sh +make test-validate-gateway-route ``` Cluster lifecycle targets require local tools such as `k3d`, `kubectl`, `helm`, and Docker. diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index f6a41e98e..ad7a0bfcd 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch +.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch # === Cluster Configuration === CLUSTER_NAME := ncp-local @@ -303,6 +303,9 @@ test-multicluster-make: ## Run dry Makefile tests for multi-cluster name derivat test-destroy-all-ncp-local: ## Run dry Makefile tests for host-wide local-cluster cleanup @tests/test-destroy-all-ncp-local.sh +test-validate-gateway-route: ## Run Gateway route retry tests + @tests/test-validate-gateway-route.sh + # --- Load Balancer & Gateway API Setup --- setup-gateway-api: $(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/) diff --git a/tools/ncp-local-cluster/scripts/validate-gateway-route.sh b/tools/ncp-local-cluster/scripts/validate-gateway-route.sh index 1766b1a8a..c35f615bb 100755 --- a/tools/ncp-local-cluster/scripts/validate-gateway-route.sh +++ b/tools/ncp-local-cluster/scripts/validate-gateway-route.sh @@ -17,14 +17,33 @@ if ! [[ "$retry_interval_seconds" =~ ^[1-9][0-9]*$ ]]; then exit 2 fi -elapsed=0 -while ((elapsed < timeout_seconds)); do - if curl -sSf --connect-timeout 5 --max-time 10 "$url" >/dev/null 2>&1; then +start_time="$(date +%s)" +deadline=$((start_time + timeout_seconds)) + +while :; do + now="$(date +%s)" + remaining=$((deadline - now)) + if ((remaining <= 0)); then + break + fi + + if curl -sSf --connect-timeout 5 --max-time "$remaining" "$url" >/dev/null 2>&1; then exit 0 fi + + now="$(date +%s)" + remaining=$((deadline - now)) + if ((remaining <= 0)); then + break + fi + + elapsed=$((timeout_seconds - remaining)) echo "INFO Gateway route not reachable yet, waiting... (${elapsed}/${timeout_seconds} seconds)" - sleep "$retry_interval_seconds" - elapsed=$((elapsed + retry_interval_seconds)) + sleep_seconds="$retry_interval_seconds" + if ((sleep_seconds > remaining)); then + sleep_seconds="$remaining" + fi + sleep "$sleep_seconds" done echo "ERROR Gateway route did not become reachable within ${timeout_seconds} seconds" >&2 diff --git a/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh b/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh index 0684cd1ee..66982e47a 100755 --- a/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh +++ b/tools/ncp-local-cluster/tests/test-validate-gateway-route.sh @@ -10,15 +10,47 @@ trap 'rm -rf "$test_dir"' EXIT mkdir -p "$test_dir/bin" +cat >"$test_dir/bin/date" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "${1:-}" == "+%s" ]] +IFS= read -r now <"$TEST_CLOCK_STATE" +printf '%s\n' "$now" +EOF +chmod +x "$test_dir/bin/date" + cat >"$test_dir/bin/curl" <<'EOF' #!/usr/bin/env bash set -euo pipefail +max_time="" +while (($# > 0)); do + case "$1" in + --max-time) + max_time="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +[[ -n "$max_time" ]] +printf '%s\n' "$max_time" >>"$TEST_CURL_MAX_TIME_LOG" + count=0 if [[ -f "$TEST_CURL_STATE" ]]; then count="$(<"$TEST_CURL_STATE")" fi count=$((count + 1)) printf '%s\n' "$count" >"$TEST_CURL_STATE" + +now="$(<"$TEST_CLOCK_STATE")" +duration="${TEST_CURL_DURATION_SECONDS:-0}" +if ((duration > max_time)); then + duration="$max_time" +fi +printf '%s\n' "$((now + duration))" >"$TEST_CLOCK_STATE" + if ((count < TEST_CURL_SUCCEED_ON)); then exit 22 fi @@ -27,13 +59,22 @@ chmod +x "$test_dir/bin/curl" cat >"$test_dir/bin/sleep" <<'EOF' #!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$1" >>"$TEST_SLEEP_LOG" +now="$(<"$TEST_CLOCK_STATE")" +printf '%s\n' "$((now + $1))" >"$TEST_CLOCK_STATE" exit 0 EOF chmod +x "$test_dir/bin/sleep" export PATH="$test_dir/bin:$PATH" +export TEST_CLOCK_STATE="$test_dir/clock" export TEST_CURL_STATE="$test_dir/curl-count" +export TEST_CURL_MAX_TIME_LOG="$test_dir/curl-max-time" +export TEST_SLEEP_LOG="$test_dir/sleep" export TEST_CURL_SUCCEED_ON=3 +export TEST_CURL_DURATION_SECONDS=0 +printf '100\n' >"$TEST_CLOCK_STATE" GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \ GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \ @@ -43,10 +84,14 @@ GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \ [[ "$(<"$TEST_CURL_STATE")" == "3" ]] grep -q "Gateway route not reachable yet" "$test_dir/success.out" +printf '100\n' >"$TEST_CLOCK_STATE" printf '0\n' >"$TEST_CURL_STATE" +: >"$TEST_CURL_MAX_TIME_LOG" +: >"$TEST_SLEEP_LOG" export TEST_CURL_SUCCEED_ON=99 -if GATEWAY_ROUTE_TIMEOUT_SECONDS=2 \ - GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \ +export TEST_CURL_DURATION_SECONDS=2 +if GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \ + GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=2 \ "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \ >"$test_dir/failure.out" 2>"$test_dir/failure.err"; then echo "expected Gateway route validation to time out" >&2 @@ -54,7 +99,24 @@ if GATEWAY_ROUTE_TIMEOUT_SECONDS=2 \ fi [[ "$(<"$TEST_CURL_STATE")" == "2" ]] -grep -q "did not become reachable within 2 seconds" "$test_dir/failure.err" +[[ "$(<"$TEST_CLOCK_STATE")" == "105" ]] +[[ "$(tr '\n' ' ' <"$TEST_CURL_MAX_TIME_LOG")" == "5 1 " ]] +grep -q "did not become reachable within 5 seconds" "$test_dir/failure.err" + +printf '100\n' >"$TEST_CLOCK_STATE" +printf '0\n' >"$TEST_CURL_STATE" +: >"$TEST_CURL_MAX_TIME_LOG" +: >"$TEST_SLEEP_LOG" +if GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \ + GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=4 \ + "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \ + >"$test_dir/capped-sleep.out" 2>"$test_dir/capped-sleep.err"; then + echo "expected Gateway route validation to time out" >&2 + exit 1 +fi + +[[ "$(<"$TEST_CLOCK_STATE")" == "105" ]] +[[ "$(<"$TEST_SLEEP_LOG")" == "3" ]] for name in GATEWAY_ROUTE_TIMEOUT_SECONDS GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS; do if env "$name=0" "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \