fix: hook-pod-logs waits for pod restart count > 0 before checking -p logs

Author: svennam92

website/docs/fastpaths/developer/amazon-eks-pod-identity/tests/hook-pod-logs.sh

@@ -5,22 +5,25 @@ before() {
 }
 
 after() {
-  # The pod may need extra time to crash and have previous logs available
-  # Retry up to 240s re-fetching logs until we find the expected error
-  LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
+  # Wait for the carts pod to crash and restart at least once
+  # then check previous container logs for the expected credential error
+  echo "Waiting for carts pod to crash and restart..."
 
-  for i in $(seq 1 24); do
-    LOG_OUTPUT=$(kubectl logs -n carts -p "$LATEST_POD" 2>/dev/null || true)
-    if [[ "$LOG_OUTPUT" == *"Unable to load credentials"* ]]; then
-      echo "Found expected credential error in logs"
-      return 0
+  for i in $(seq 1 36); do
+    RESTARTS=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0")
+    if [ "$RESTARTS" -gt 0 ] 2>/dev/null; then
+      LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
+      LOG_OUTPUT=$(kubectl logs -n carts -p "$LATEST_POD" 2>/dev/null || true)
+      if [[ "$LOG_OUTPUT" == *"Unable to load credentials"* ]]; then
+        echo "Found expected credential error after $i attempts (restarts=$RESTARTS)"
+        return 0
+      fi
     fi
-    echo "Attempt $i: credential error not found yet, waiting..."
+    echo "Attempt $i: restarts=$RESTARTS, waiting..."
     sleep 10
   done
 
-  echo "Failed to find expected credential error after retries"
-  echo "Last log output: ${LOG_OUTPUT:-empty}"
+  echo "Failed to find expected credential error after 360s"
   exit 1
 }
 

website/docs/fastpaths/developer/amazon-eks-pod-identity/understanding.md

@@ -7,7 +7,7 @@ The first place to look for the issue is the logs of the `carts` service:
 
 ```bash hook=pod-logs timeout=480
 $ LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
-sleep 180
+sleep 60
 kubectl logs -n carts -p $LATEST_POD
 [...]
 software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(credentialsProviders=[SystemPropertyCredentialsProvider(), EnvironmentVariableCredentialsProvider(), WebIdentityTokenCredentialsProvider(), ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])), ContainerCredentialsProvider(), InstanceProfileCredentialsProvider()]) : [SystemPropertyCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., EnvironmentVariableCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., WebIdentityTokenCredentialsProvider(): Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set., ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])): Profile file contained no credentials for profile 'default': ProfileFile(sections=[]), ContainerCredentialsProvider(): Cannot fetch credentials from container - neither AWS_CONTAINER_CREDENTIALS_FULL_URI or AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variables are set., InstanceProfileCredentialsProvider(): Failed to load credentials from IMDS.]

website/docs/fastpaths/operator/amazon-eks-pod-identity/tests/hook-pod-logs.sh

@@ -5,22 +5,25 @@ before() {
 }
 
 after() {
-  # The pod may need extra time to crash and have previous logs available
-  # Retry up to 240s re-fetching logs until we find the expected error
-  LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
+  # Wait for the carts pod to crash and restart at least once
+  # then check previous container logs for the expected credential error
+  echo "Waiting for carts pod to crash and restart..."
 
-  for i in $(seq 1 24); do
-    LOG_OUTPUT=$(kubectl logs -n carts -p "$LATEST_POD" 2>/dev/null || true)
-    if [[ "$LOG_OUTPUT" == *"Unable to load credentials"* ]]; then
-      echo "Found expected credential error in logs"
-      return 0
+  for i in $(seq 1 36); do
+    RESTARTS=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0")
+    if [ "$RESTARTS" -gt 0 ] 2>/dev/null; then
+      LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
+      LOG_OUTPUT=$(kubectl logs -n carts -p "$LATEST_POD" 2>/dev/null || true)
+      if [[ "$LOG_OUTPUT" == *"Unable to load credentials"* ]]; then
+        echo "Found expected credential error after $i attempts (restarts=$RESTARTS)"
+        return 0
+      fi
     fi
-    echo "Attempt $i: credential error not found yet, waiting..."
+    echo "Attempt $i: restarts=$RESTARTS, waiting..."
     sleep 10
   done
 
-  echo "Failed to find expected credential error after retries"
-  echo "Last log output: ${LOG_OUTPUT:-empty}"
+  echo "Failed to find expected credential error after 360s"
   exit 1
 }
 

website/docs/fastpaths/operator/amazon-eks-pod-identity/understanding.md

@@ -7,7 +7,7 @@ The first place to look for the issue is the logs of the `carts` service:
 
 ```bash hook=pod-logs timeout=480
 $ LATEST_POD=$(kubectl get pods -n carts -l app.kubernetes.io/component=service --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
-sleep 180
+sleep 60
 kubectl logs -n carts -p $LATEST_POD
 [...]
 software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(credentialsProviders=[SystemPropertyCredentialsProvider(), EnvironmentVariableCredentialsProvider(), WebIdentityTokenCredentialsProvider(), ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])), ContainerCredentialsProvider(), InstanceProfileCredentialsProvider()]) : [SystemPropertyCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., EnvironmentVariableCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., WebIdentityTokenCredentialsProvider(): Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set., ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])): Profile file contained no credentials for profile 'default': ProfileFile(sections=[]), ContainerCredentialsProvider(): Cannot fetch credentials from container - neither AWS_CONTAINER_CREDENTIALS_FULL_URI or AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variables are set., InstanceProfileCredentialsProvider(): Failed to load credentials from IMDS.]