fix(local-dev): retry Gateway route validation

tools/ncp-local-cluster/AGENTS.md

@@ -17,6 +17,7 @@ Run Makefile-only validation from `tools/ncp-local-cluster`:
 make validate-compute-clusters
 make print-compute-clusters
 make test-multicluster-make
+make test-validate-gateway-route
 ```
 
 Cluster lifecycle targets require local tools such as `k3d`, `kubectl`, `helm`, and Docker.

tools/ncp-local-cluster/Makefile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch
+.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch
 
 # === Cluster Configuration ===
 CLUSTER_NAME := ncp-local
@@ -46,6 +46,8 @@ COMPUTE_CLUSTER_NAME ?= $(COMPUTE_CLUSTER_PREFIX)-1
 COMPUTE_PLANE_CONFIG_FILE ?= k3d-config-compute-plane.yaml
 COMPUTE_CLUSTERS ?=
 GATEWAY_HTTP_PORT ?= $(CONTROL_PLANE_HTTP_PORT)
+GATEWAY_ROUTE_TIMEOUT_SECONDS ?= 60
+GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS ?= 2
 
 # === Go Credential Provider Configuration ===
 GO_PROVIDER_PATH         := ./credential-provider-go
@@ -301,6 +303,9 @@ test-multicluster-make: ## Run dry Makefile tests for multi-cluster name derivat
 test-destroy-all-ncp-local: ## Run dry Makefile tests for host-wide local-cluster cleanup
 	@tests/test-destroy-all-ncp-local.sh
 
+test-validate-gateway-route: ## Run Gateway route retry tests
+	@tests/test-validate-gateway-route.sh
+
 # --- Load Balancer & Gateway API Setup ---
 setup-gateway-api:
 	$(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/)
@@ -421,7 +426,9 @@ validate-gateway: deploy-nginx wait-for-nginx wait-for-gateway ## Validate Gatew
 	@echo ">>> Checking HTTPRoute status..."
 	@kubectl get httproute nginx-route -n sample -o jsonpath='{.status}' || echo "HTTPRoute status not available"
 	@echo ">>> Validating Gateway route (nginx.localhost)..."
-	@if curl -sSf http://nginx.localhost:$(GATEWAY_HTTP_PORT)/ >/dev/null 2>&1; then \
+	@if GATEWAY_ROUTE_TIMEOUT_SECONDS=$(GATEWAY_ROUTE_TIMEOUT_SECONDS) \
+		GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=$(GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS) \
+		bash ./scripts/validate-gateway-route.sh http://nginx.localhost:$(GATEWAY_HTTP_PORT)/; then \
 		echo "OK Gateway route reachable at http://nginx.localhost:$(GATEWAY_HTTP_PORT)"; \
 	else \
 		echo "ERROR Gateway route validation failed"; \

tools/ncp-local-cluster/scripts/validate-gateway-route.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+url="${1:?gateway route URL is required}"
+timeout_seconds="${GATEWAY_ROUTE_TIMEOUT_SECONDS:-60}"
+retry_interval_seconds="${GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS:-2}"
+
+if ! [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR GATEWAY_ROUTE_TIMEOUT_SECONDS must be a positive integer" >&2
+  exit 2
+fi
+if ! [[ "$retry_interval_seconds" =~ ^[1-9][0-9]*$ ]]; then
+  echo "ERROR GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS must be a positive integer" >&2
+  exit 2
+fi
+
+start_time="$(date +%s)"
+deadline=$((start_time + timeout_seconds))
+
+while :; do
+  now="$(date +%s)"
+  remaining=$((deadline - now))
+  if ((remaining <= 0)); then
+    break
+  fi
+
+  if curl -sSf --connect-timeout 5 --max-time "$remaining" "$url" >/dev/null 2>&1; then
+    exit 0
+  fi
+
+  now="$(date +%s)"
+  remaining=$((deadline - now))
+  if ((remaining <= 0)); then
+    break
+  fi
+
+  elapsed=$((timeout_seconds - remaining))
+  echo "INFO Gateway route not reachable yet, waiting... (${elapsed}/${timeout_seconds} seconds)"
+  sleep_seconds="$retry_interval_seconds"
+  if ((sleep_seconds > remaining)); then
+    sleep_seconds="$remaining"
+  fi
+  sleep "$sleep_seconds"
+done
+
+echo "ERROR Gateway route did not become reachable within ${timeout_seconds} seconds" >&2
+exit 1

tools/ncp-local-cluster/tests/test-validate-gateway-route.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+test_dir="$(mktemp -d)"
+trap 'rm -rf "$test_dir"' EXIT
+
+mkdir -p "$test_dir/bin"
+
+cat >"$test_dir/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "${1:-}" == "+%s" ]]
+IFS= read -r now <"$TEST_CLOCK_STATE"
+printf '%s\n' "$now"
+EOF
+chmod +x "$test_dir/bin/date"
+
+cat >"$test_dir/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+max_time=""
+while (($# > 0)); do
+  case "$1" in
+    --max-time)
+      max_time="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+[[ -n "$max_time" ]]
+printf '%s\n' "$max_time" >>"$TEST_CURL_MAX_TIME_LOG"
+
+count=0
+if [[ -f "$TEST_CURL_STATE" ]]; then
+  count="$(<"$TEST_CURL_STATE")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" >"$TEST_CURL_STATE"
+
+now="$(<"$TEST_CLOCK_STATE")"
+duration="${TEST_CURL_DURATION_SECONDS:-0}"
+if ((duration > max_time)); then
+  duration="$max_time"
+fi
+printf '%s\n' "$((now + duration))" >"$TEST_CLOCK_STATE"
+
+if ((count < TEST_CURL_SUCCEED_ON)); then
+  exit 22
+fi
+EOF
+chmod +x "$test_dir/bin/curl"
+
+cat >"$test_dir/bin/sleep" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$1" >>"$TEST_SLEEP_LOG"
+now="$(<"$TEST_CLOCK_STATE")"
+printf '%s\n' "$((now + $1))" >"$TEST_CLOCK_STATE"
+exit 0
+EOF
+chmod +x "$test_dir/bin/sleep"
+
+export PATH="$test_dir/bin:$PATH"
+export TEST_CLOCK_STATE="$test_dir/clock"
+export TEST_CURL_STATE="$test_dir/curl-count"
+export TEST_CURL_MAX_TIME_LOG="$test_dir/curl-max-time"
+export TEST_SLEEP_LOG="$test_dir/sleep"
+export TEST_CURL_SUCCEED_ON=3
+export TEST_CURL_DURATION_SECONDS=0
+printf '100\n' >"$TEST_CLOCK_STATE"
+
+GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \
+GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=1 \
+  "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \
+  >"$test_dir/success.out"
+
+[[ "$(<"$TEST_CURL_STATE")" == "3" ]]
+grep -q "Gateway route not reachable yet" "$test_dir/success.out"
+
+printf '100\n' >"$TEST_CLOCK_STATE"
+printf '0\n' >"$TEST_CURL_STATE"
+: >"$TEST_CURL_MAX_TIME_LOG"
+: >"$TEST_SLEEP_LOG"
+export TEST_CURL_SUCCEED_ON=99
+export TEST_CURL_DURATION_SECONDS=2
+if GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \
+  GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=2 \
+  "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \
+  >"$test_dir/failure.out" 2>"$test_dir/failure.err"; then
+  echo "expected Gateway route validation to time out" >&2
+  exit 1
+fi
+
+[[ "$(<"$TEST_CURL_STATE")" == "2" ]]
+[[ "$(<"$TEST_CLOCK_STATE")" == "105" ]]
+[[ "$(tr '\n' ' ' <"$TEST_CURL_MAX_TIME_LOG")" == "5 1 " ]]
+grep -q "did not become reachable within 5 seconds" "$test_dir/failure.err"
+
+printf '100\n' >"$TEST_CLOCK_STATE"
+printf '0\n' >"$TEST_CURL_STATE"
+: >"$TEST_CURL_MAX_TIME_LOG"
+: >"$TEST_SLEEP_LOG"
+if GATEWAY_ROUTE_TIMEOUT_SECONDS=5 \
+  GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS=4 \
+  "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \
+  >"$test_dir/capped-sleep.out" 2>"$test_dir/capped-sleep.err"; then
+  echo "expected Gateway route validation to time out" >&2
+  exit 1
+fi
+
+[[ "$(<"$TEST_CLOCK_STATE")" == "105" ]]
+[[ "$(<"$TEST_SLEEP_LOG")" == "3" ]]
+
+for name in GATEWAY_ROUTE_TIMEOUT_SECONDS GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS; do
+  if env "$name=0" "$repo_dir/scripts/validate-gateway-route.sh" http://nginx.localhost:8080/ \
+    >"$test_dir/invalid.out" 2>"$test_dir/invalid.err"; then
+    echo "expected $name=0 to fail validation" >&2
+    exit 1
+  fi
+  grep -q "$name must be a positive integer" "$test_dir/invalid.err"
+done
+
+echo "Gateway route retry tests passed."