test(bdd): add autoscaler and vanity smoke coverage

deploy/stacks/self-managed/tests/observability-autoscaler.sh

@@ -70,6 +70,10 @@ write_autoscaler_values() {
 write_autoscaler_values "$work_dir/autoscaler-values.yaml"
 
 autoscaler_values="$work_dir/autoscaler-values.yaml"
+autoscaler_tag="$(yq -r '.functionautoscaler.image.tag // ""' "$autoscaler_values")"
+if [[ -n "$autoscaler_tag" ]]; then
+  fail "stack duplicated the function autoscaler chart-owned image tag"
+fi
 for expected in \
   'CASSANDRA__CONTACT_POINTS: cassandra.cassandra-system.svc.cluster.local' \
   'CASSANDRA__IS_DEVELOPMENT: "false"' \
@@ -90,7 +94,7 @@ helm template function-autoscaler "$repo_dir/deploy/helm/function-autoscaler" \
 
 autoscaler_manifests="$work_dir/autoscaler-manifests.yaml"
 grep -q 'image: nvcr.io/YOUR_ORG/YOUR_TEAM/nvcf-function-autoscaler:1.19.0' "$autoscaler_manifests" ||
-  fail "self-managed stack did not pin the autoscaler image"
+  fail "self-managed stack did not render the chart-owned autoscaler image"
 test "$(grep -c '^kind: ConfigMap$' "$autoscaler_manifests")" = "2" ||
   fail "autoscaler chart did not render only its env and Vault template ConfigMaps"
 grep -q 'name: function-autoscaler-env' "$autoscaler_manifests" ||

tests/bdd/PLAN.md

@@ -155,10 +155,11 @@ original order. Repeated options and empty values are preserved.
 | `Given I use NVCF CLI config {string}` | Interpolates and stores the supplied config argument without resolving or checking the path. Later lifecycle steps pass it to `--config`. |
 | `When I successfully create function {string} from image {string} with CLI options:` | Runs `function create --name <name> --image <image>` followed by the option rows. |
 | `When I successfully deploy the function selected by NVCF CLI with options:` | Runs `function deploy create` followed by the option rows. Function selection remains owned by CLI state. |
-| `When I successfully generate a function API key with CLI options:` | Runs `api-key generate --for function` followed by the option rows. |
+| `When I successfully generate a function API key with CLI options:` | Runs `api-key generate --for function` followed by the option rows and suppresses secret-bearing stdout. |
 | `When I successfully invoke the function selected by NVCF CLI over HTTP with timeout {string} seconds and poll duration {string} seconds:` (JSON docstring) | Runs `function invoke` with the exact request body, timeout, and poll duration. |
 | `When I successfully invoke the function selected by NVCF CLI over plaintext gRPC service {string} method {string} with timeout {string} seconds and poll duration {string} seconds:` (JSON docstring) | Runs `function invoke --grpc --grpc-plaintext` with the visible service, method, request, timeout, and poll duration. |
 | `When I successfully invoke model {string} at {string} with timeout {string} seconds:` (JSON docstring) | Runs `function invoke` with the visible model, inference URL, exact request body, and timeout. |
+| `When I successfully invoke the function selected by NVCF CLI through Vanity Gateway host {string} path {string} with timeout {string} seconds:` (JSON docstring) | Sends an exact-host HTTP request through the local Envoy listener with the saved function API key passed over sensitive stdin, never argv or command logs. |
 | `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. |
 
 ### Assertions (Then / And)

tests/bdd/dsl/manifests.go

@@ -48,10 +48,10 @@ func RenderedManifestsContainResource(root string, resource KubernetesResource)
 		return fmt.Errorf("rendered manifests directory is empty")
 	}
 	if resource.Kind == "" {
-		return fmt.Errorf("Kubernetes resource kind is empty")
+		return fmt.Errorf("kubernetes resource kind is empty")
 	}
 	if resource.Name == "" {
-		return fmt.Errorf("Kubernetes resource name is empty")
+		return fmt.Errorf("kubernetes resource name is empty")
 	}
 	info, err := os.Stat(root)
 	if err != nil {

tests/bdd/features/observability-all.feature

@@ -22,7 +22,6 @@ Feature: Install local Helmfile observability for both planes
       | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
       | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
       | observability.profile           | all                                  |
-      | functionAutoscaler.image.tag    | 1.18.10                              |
     # Give the shared observability Helmfile the same named environment.
     And I prepare Helmfile environment "local-bdd-observability-all" for stack "observability" from fixture "tests/bdd/fixtures/self-managed-local-bdd.yaml" with values:
       | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
@@ -62,41 +61,36 @@ Feature: Install local Helmfile observability for both planes
       | monitoring       |
 
   Scenario: All profile installs one shared stack with both monitor families
-    When I run command:
+    When I successfully run command:
       """
       make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml
       """
-    Then the command exit code should be 0
 
-    When I run command:
+    When I successfully run command:
       """
       env KUBECONFIG=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-observability-all control-plane profile export --cluster-name ncp-local
       """
-    Then the command exit code should be 0
-    And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist
+    Then file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist
 
-    When I run command:
+    When I successfully run command:
       """
       ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init
       """
-    Then the command exit code should be 0
 
-    When I run command:
+    When I successfully run command:
       """
       make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
       """
-    Then the command exit code should be 0
-    And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist
+    Then file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist
     And yaml file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should have non-empty keys:
       | key            |
       | clusterID      |
       | clusterGroupID |
 
-    When I run command:
+    When I successfully run command:
       """
       make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
       """
-    Then the command exit code should be 0
 
     # Self-hosted NVCA intentionally creates an empty NGC service-key secret.
     # Supply the existing local credential so the NVCA collector can start,
@@ -119,6 +113,7 @@ Feature: Install local Helmfile observability for both planes
       | victoria-metrics         | monitoring    | 1        |
       | otel-collector           | monitoring    | 1        |
       | default-monitors         | monitoring    | 1        |
+      | function-autoscaler      | nvcf          | 1        |
       | nvca-operator            | nvca-operator | 1        |
 
     Then deployment "nvca-operator" in namespace "nvca-operator" using context "k3d-ncp-local" should complete rollout within "10m"
@@ -148,3 +143,85 @@ Feature: Install local Helmfile observability for both planes
           enabled: true
           imageRepository: nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector
       """
+
+  # The autoscaler is enabled by both control and all profiles. This functional
+  # smoke stays in all because it requires the registered compute plane from
+  # the preceding scenario. It is not a standalone tag target.
+  @function-autoscaler @function-lifecycle
+  Scenario: Autoscaler starts an idle function to serve its first request
+    Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml"
+
+    When I successfully create function "bdd-autoscaled-load-tester-supreme" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options:
+      | option           | value   |
+      | --inference-url  | /echo   |
+      | --inference-port | 8000    |
+      | --health-uri     | /health |
+      | --health-port    | 8000    |
+      | --health-timeout | PT30S   |
+
+    And I successfully deploy the function selected by NVCF CLI with options:
+      | option          | value               |
+      | --gpu           | H100                |
+      | --instance-type | NCP.GPU.H100_1x     |
+      | --backend       | ncp-local           |
+      | --regions       | us-west-1           |
+      | --min-instances | 0                   |
+      | --max-instances | 1                   |
+      | --timeout       | 900                 |
+
+    And I successfully generate a function API key with CLI options:
+      | option        | value                                                                       |
+      | --description | bdd-autoscaled-load-tester-supreme                                         |
+      | --scopes      | invoke_function,list_functions,queue_details,list_functions_details         |
+
+    # Prove the function is idle before the first request creates demand. The
+    # compute-plane CLI lists only scheduled functions, so no matching entry
+    # also represents zero instances.
+    # TODO(https://github.com/NVIDIA/nvcf/issues/1419): replace this shell
+    # command with a selected-function instance-count DSL assertion.
+    When I successfully run command:
+      """
+      /bin/bash -c 'set -euo pipefail
+      status="$("$1" --config "$2" status --json)"
+      function_id="$(jq -er ".currentFunction | select(.hasFunction == true) | .functionId" <<<"$status")"
+      version_id="$(jq -er ".currentFunction | select(.hasFunction == true) | .versionId" <<<"$status")"
+      functions="$("$1" --config "$2" cluster agent list-functions --compute-plane-context "$3" --kubeconfig "$4" --json)"
+      jq -e --arg function_id "$function_id" --arg version_id "$version_id" "[.[] | select(.functionId == \$function_id and .functionVersionId == \$version_id)] | all(.instanceCount == 0)" <<<"$functions" >/dev/null' bdd-autoscaler-zero ${NVCF_CLI} ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml k3d-ncp-local ${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml
+      """
+
+    # The invocation plane returns after its short hold-open window while the
+    # autoscaler and compute plane complete a cold start. A successful response
+    # or the expected 504 proves the first request reached the invocation path.
+    When I run command:
+      """
+      ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke --request-body '{"message":"bdd-autoscaler-echo","repeats":1}' --timeout 60 --poll-duration 5
+      """
+    Then the command output should contain one of:
+      | text                 |
+      | bdd-autoscaler-echo  |
+      | API error 504         |
+
+    # TODO(https://github.com/NVIDIA/nvcf/issues/1419): replace this polling
+    # loop with a selected-function instance-readiness DSL assertion.
+    When I successfully run command:
+      """
+      /bin/bash -c 'set -euo pipefail
+      status="$("$1" --config "$2" status --json)"
+      function_id="$(jq -er ".currentFunction | select(.hasFunction == true) | .functionId" <<<"$status")"
+      version_id="$(jq -er ".currentFunction | select(.hasFunction == true) | .versionId" <<<"$status")"
+      for attempt in {1..120}; do
+        if "$1" --config "$2" cluster agent get-function "$function_id" "$version_id" --compute-plane-context "$3" --kubeconfig "$4" --json | jq -e ".instanceCount == 1 and ([.instances[] | select((.status | ascii_downcase) == \"running\")] | length == 1)" >/dev/null; then
+          exit 0
+        fi
+        sleep 5
+      done
+      exit 1' bdd-autoscaler ${NVCF_CLI} ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml k3d-ncp-local ${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml
+      """
+
+    And I successfully invoke the function selected by NVCF CLI over HTTP with timeout "600" seconds and poll duration "5" seconds:
+      """
+      {"message":"bdd-autoscaler-echo","repeats":1}
+      """
+    Then the command output should contain "bdd-autoscaler-echo"
+
+    And I successfully undeploy the function selected by NVCF CLI

tests/bdd/features/observability-control.feature

@@ -19,7 +19,6 @@ Feature: Install local Helmfile observability with the control profile
       | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
       | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
       | observability.profile           | control                              |
-      | functionAutoscaler.image.tag    | 1.18.10                              |
     # Set the shared observability stack environment.
     And I prepare Helmfile environment "local-bdd-observability-control" for stack "observability" from fixture "tests/bdd/fixtures/self-managed-local-bdd.yaml" with values:
       | global.imagePullSecrets[0].name | nvcr-pull-secret                     |