From 06eaa0723186245b651fa2959aa6b769ef7f78cb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 09:36:20 -0500 Subject: [PATCH 01/37] fix(onboard): require managed images for stock agents --- .github/workflows/e2e-standard-profile.yaml | 42 +-- .github/workflows/e2e.yaml | 173 ++++++------- docs/reference/commands.mdx | 4 +- .../onboard-orchestration.test.ts | 33 ++- .../managed-workload/onboard-orchestration.ts | 11 +- .../sandbox-workload-preparation.test.ts | 24 +- src/lib/onboard/workload/preparation.ts | 21 +- test/e2e/README.md | 47 ++-- test/e2e/fixtures/availability-env.ts | 1 + test/e2e/fixtures/clients/host.ts | 25 +- test/e2e/fixtures/managed-image-receipt.ts | 112 ++++++++ test/e2e/fixtures/workload-source-env.ts | 32 +-- test/e2e/live/cloud-onboard.test.ts | 7 + test/e2e/live/full-e2e-workload-evidence.ts | 32 ++- test/e2e/live/hermes-e2e.test.ts | 7 + test/e2e/live/hermes-gpu-startup.test.ts | 7 + test/e2e/live/jetson-nvmap-gpu.test.ts | 7 + test/e2e/live/mcp-bridge-onboard-env.ts | 10 +- test/e2e/live/messaging-providers.test.ts | 7 + ...mage-publication-workflow-boundary.test.ts | 28 +- .../support/base-image-publication.test.ts | 42 +++ ...time-compatible-anthropic-progress.test.ts | 34 ++- .../support/exact-artifact-download.test.ts | 15 ++ .../managed-image-cohort-contract.test.ts | 135 ++++++++++ .../e2e/support/managed-image-receipt.test.ts | 149 +++++++++++ .../support/mcp-bridge-onboard-env.test.ts | 9 + .../e2e/support/mcp-workflow-boundary.test.ts | 2 +- ...ck-managed-image-workflow-boundary.test.ts | 82 ++++++ test/e2e/support/workload-source-env.test.ts | 38 +-- ...nboard-managed-image-buildless-e2e.test.ts | 4 +- tools/e2e/base-image-publication.mts | 36 ++- tools/e2e/cli-artifact-workflow-boundary.mts | 17 +- tools/e2e/exact-artifact-download.mts | 52 +++- .../hermes-gpu-startup-workflow-boundary.mts | 6 +- tools/e2e/managed-image-cohort-contract.mts | 240 ++++++++++++++++++ .../e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/mcp-workflow-boundary.mts | 16 +- tools/e2e/operations-workflow-boundary.mts | 148 +++++++---- .../standard-profile-workflow-boundary.mts | 52 +--- .../trusted-hermes-swap-workflow-boundary.mts | 2 +- tools/e2e/workflow-boundary.mts | 60 +---- 41 files changed, 1325 insertions(+), 446 deletions(-) create mode 100644 test/e2e/fixtures/managed-image-receipt.ts create mode 100644 test/e2e/support/managed-image-cohort-contract.test.ts create mode 100644 test/e2e/support/managed-image-receipt.test.ts create mode 100644 test/e2e/support/stock-managed-image-workflow-boundary.test.ts create mode 100644 tools/e2e/managed-image-cohort-contract.mts diff --git a/.github/workflows/e2e-standard-profile.yaml b/.github/workflows/e2e-standard-profile.yaml index 64eb1918244..5a17e9c2464 100644 --- a/.github/workflows/e2e-standard-profile.yaml +++ b/.github/workflows/e2e-standard-profile.yaml @@ -21,7 +21,7 @@ on: cli_artifact_provenance: required: true type: string - managed_image_catalog: + managed_image_revision: required: true type: string credential_boundary: @@ -109,6 +109,7 @@ jobs: E2E_TARGET_ID: ${{ inputs.target_id }} NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.candidate_sha }} + E2E_MANAGED_IMAGE_REVISION: ${{ inputs.managed_image_revision }} NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.risk_signal_correlation_id }} NEMOCLAW_E2E_RISK_SIGNAL_EXPECTED_SHA: ${{ inputs.risk_signal_expected_sha }} NEMOCLAW_LLAMA_CPP_QUALIFICATION_HEAD_SHA: ${{ inputs.candidate_sha }} @@ -392,45 +393,6 @@ jobs: with: provenance-json: ${{ inputs.cli_artifact_provenance }} - - name: Materialize temporary managed-image catalog - if: ${{ inputs.managed_image_catalog != '' }} - shell: /bin/bash --noprofile --norc -e -o pipefail {0} - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - MANAGED_IMAGE_CATALOG: ${{ inputs.managed_image_catalog }} - RESTORE_CLI: ${{ inputs.restore_cli && 'true' || 'false' }} - run: | - set -euo pipefail - catalog_path="${RUNNER_TEMP}/e2e-managed-image-catalog.json" - jq -e --arg revision "$CANDIDATE_SHA" ' - type == "object" and length > 0 and - all(.[]; - .source.revision == $revision and - (.source.release | type == "string" and length > 0) and - (.source.cohort | type == "string" and length > 0) - ) and - ([.[].source.release] | unique | length) == 1 and - ([.[].source.cohort] | unique | length) == 1 - ' <<<"$MANAGED_IMAGE_CATALOG" >/dev/null || { - echo "::error::managed-image catalog source identity does not match the candidate" >&2 - exit 1 - } - if [[ "$RESTORE_CLI" == "true" ]]; then - candidate_release="v$(jq -r '.nemoclawVersion' dist/build-identity.json)" - jq -e --arg release "$candidate_release" ' - all(.[]; .source.release == $release) - ' <<<"$MANAGED_IMAGE_CATALOG" >/dev/null || { - echo "::error::managed-image catalog release does not match the restored CLI" >&2 - exit 1 - } - fi - jq -c . <<<"$MANAGED_IMAGE_CATALOG" >"$catalog_path" - [[ -s "$catalog_path" && ! -L "$catalog_path" ]] || { - echo "::error::temporary managed-image catalog is invalid" >&2 - exit 1 - } - printf 'NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG=%s\n' "$catalog_path" >>"$GITHUB_ENV" - - name: Install reviewed cloudflared if: ${{ inputs.cloudflared }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 36479b85fe5..fefe21a5efe 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -99,9 +99,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 55 outputs: - dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract || steps.validate_reused_dcode_base.outputs.contract }} - dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref || steps.validate_reused_dcode_base.outputs.base_ref }} - managed_image_revision: ${{ steps.publication.outputs.head_sha || (steps.publication_mode.outputs.reuse == '1' && 'e38db201413b457614904187377ed9fd002d281d') || inputs.checkout_sha || github.sha }} + dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract }} + dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }} + managed_image_artifact_provenance: ${{ steps.download_managed_cohort.outputs.provenance }} + managed_image_cohort: ${{ steps.validate_managed_cohort.outputs.cohort }} + managed_image_revision: ${{ steps.validate_managed_cohort.outputs.revision }} + managed_image_run_attempt: ${{ steps.validate_managed_cohort.outputs.run_attempt }} + managed_image_run_id: ${{ steps.validate_managed_cohort.outputs.run_id }} permissions: actions: read contents: read @@ -110,31 +114,39 @@ jobs: name: Classify base-image publication requirement env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} + BASE_SHA: ${{ inputs.base_sha }} EVENT_NAME: ${{ github.event_name }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} + WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail - reuse=0 case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) - required=1 + expected_sha="$WORKFLOW_SHA" + allow_non_head=0 + select_nearest_successful=0 ;; NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller) - required=0 - reuse=1 + [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::manual PR publication selection requires an exact base SHA" >&2 + exit 1 + } + expected_sha="$BASE_SHA" + allow_non_head=1 + select_nearest_successful=1 ;; *) echo "::error::base-image publication mode is not trusted" >&2 exit 1 ;; esac - printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}" - printf 'reuse=%s\n' "${reuse}" >> "${GITHUB_OUTPUT}" + printf 'allow_non_head=%s\n' "${allow_non_head}" >> "${GITHUB_OUTPUT}" + printf 'expected_sha=%s\n' "${expected_sha}" >> "${GITHUB_OUTPUT}" + printf 'select_nearest_successful=%s\n' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}" - name: Check out trusted E2E workflow - if: ${{ steps.publication_mode.outputs.required == '1' || steps.publication_mode.outputs.reuse == '1' }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -142,27 +154,30 @@ jobs: persist-credentials: false - name: Set up Node for publication verification - if: ${{ steps.publication_mode.outputs.required == '1' || steps.publication_mode.outputs.reuse == '1' }} uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 - id: publication - name: Verify applicable base-image publication - if: ${{ steps.publication_mode.outputs.required == '1' }} + name: Select complete base and managed-image publication env: - EXPECTED_SHA: ${{ inputs.checkout_sha || github.sha }} + EXPECTED_SHA: ${{ steps.publication_mode.outputs.expected_sha }} GITHUB_TOKEN: ${{ github.token }} + PUBLICATION_HISTORY_ALLOW_NON_HEAD: ${{ steps.publication_mode.outputs.allow_non_head }} REQUIRE_MANAGED_IMAGE_PUBLICATION: "1" + SELECT_NEAREST_SUCCESSFUL_PUBLICATION: ${{ steps.publication_mode.outputs.select_nearest_successful }} shell: bash run: | set -euo pipefail export GITHUB_REF=refs/heads/main export GITHUB_SHA="$EXPECTED_SHA" - node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30 + wait_seconds=3000 + if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then + wait_seconds=0 + fi + node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30 - name: Download immutable Deep Agents Code base contract - if: ${{ steps.publication_mode.outputs.required == '1' }} env: GITHUB_TOKEN: ${{ github.token }} PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} @@ -170,34 +185,34 @@ jobs: PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract" - - name: Download reused Deep Agents Code base contract - if: ${{ steps.publication_mode.outputs.reuse == '1' }} - env: - GITHUB_TOKEN: ${{ github.token }} - PUBLICATION_HEAD_SHA: e38db201413b457614904187377ed9fd002d281d - PUBLICATION_RUN_ATTEMPT: "1" - PUBLICATION_RUN_ID: "32544159037" - run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract-reused" - - id: validate_dcode_base name: Validate immutable Deep Agents Code base - if: ${{ steps.publication_mode.outputs.required == '1' }} env: PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json" - - id: validate_reused_dcode_base - name: Validate reused Deep Agents Code base - if: ${{ steps.publication_mode.outputs.reuse == '1' }} + - id: download_managed_cohort + name: Download immutable managed-image cohort contract env: - PUBLICATION_HEAD_SHA: e38db201413b457614904187377ed9fd002d281d - PUBLICATION_RUN_ATTEMPT: "1" - PUBLICATION_RUN_ID: "32544159037" - run: node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract-reused/contract.json" + GITHUB_TOKEN: ${{ github.token }} + PUBLICATION_ARTIFACT_KIND: managed-image-cohort + PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} + PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} + PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} + run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/managed-image-cohort" + + - id: validate_managed_cohort + name: Validate immutable managed-image cohort contract + env: + PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} + PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} + PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} + run: node --experimental-strip-types --no-warnings tools/e2e/managed-image-cohort-contract.mts "${RUNNER_TEMP}/managed-image-cohort/cohort.json" generate-matrix: + needs: base-image-publication runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -206,7 +221,6 @@ jobs: pull-requests: read outputs: cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }} - managed_image_catalog: ${{ steps.package_cli_artifact.outputs.managed_image_catalog }} e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }} matrix: ${{ steps.matrix.outputs.matrix }} test_matrix: ${{ steps.matrix.outputs.test_matrix }} @@ -574,16 +588,6 @@ jobs: fi fi - - name: Resolve exact PR managed-image catalog - if: ${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }} - env: - BASE_SHA: ${{ inputs.base_sha }} - CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} - CANDIDATE_SHA: ${{ inputs.checkout_sha }} - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ inputs.pr_number }} - run: node --experimental-strip-types --no-warnings tools/e2e/pr-managed-image-publication.mts "${RUNNER_TEMP}/pr-managed-image-catalog.json" - - name: Check out E2E candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }} @@ -700,21 +704,6 @@ jobs: ' dist/build-identity.json >/dev/null || { echo "::error::candidate CLI build identity does not match the candidate commit SHA"; exit 1; } - managed_catalog="${RUNNER_TEMP}/pr-managed-image-catalog.json" - rm -f -- dist/e2e-managed-image-catalog.json - if [[ -e "$managed_catalog" ]]; then - [[ -f "$managed_catalog" && ! -L "$managed_catalog" && -s "$managed_catalog" ]] || - { echo "::error::trusted PR managed-image catalog is not a nonempty regular file"; exit 1; } - candidate_release="v$(jq -r '.nemoclawVersion' dist/build-identity.json)" - jq -e --arg release "$candidate_release" --arg revision "$CANDIDATE_SHA" ' - type == "object" and length > 0 and - all(.[]; .source.revision == $revision and .source.release == $release) - ' "$managed_catalog" >/dev/null || - { echo "::error::managed-image catalog source identity does not match the candidate"; exit 1; } - install -m 0600 "$managed_catalog" dist/e2e-managed-image-catalog.json - printf 'managed_image_catalog=%s\n' "$(jq -c . "$managed_catalog")" >>"$GITHUB_OUTPUT" - fi - artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" install -d -m 0700 "$artifact_dir" payload="$artifact_dir/nemoclaw-cli.tar" @@ -2707,6 +2696,7 @@ jobs: matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.base-image-publication.outputs.dcode_base_ref }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -2984,7 +2974,7 @@ jobs: catalogue-standard: name: ${{ matrix.display_name }} - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_standard_matrix != '[]' }} strategy: fail-fast: false @@ -2997,7 +2987,7 @@ jobs: risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - managed_image_catalog: ${{ needs.generate-matrix.outputs.managed_image_catalog }} + managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} credential_boundary: no provider credential target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3024,7 +3014,7 @@ jobs: catalogue-nvidia-api: name: ${{ matrix.display_name }} - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_api_matrix != '[]' }} strategy: fail-fast: false @@ -3037,7 +3027,7 @@ jobs: risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - managed_image_catalog: ${{ needs.generate-matrix.outputs.managed_image_catalog }} + managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} credential_boundary: NVIDIA API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3065,7 +3055,7 @@ jobs: catalogue-nvidia-inference: name: ${{ matrix.display_name }} - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix != '[]' }} strategy: fail-fast: false @@ -3078,7 +3068,7 @@ jobs: risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - managed_image_catalog: ${{ needs.generate-matrix.outputs.managed_image_catalog }} + managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} credential_boundary: NVIDIA inference API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3106,7 +3096,7 @@ jobs: catalogue-github-read: name: ${{ matrix.display_name }} - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_github_read_matrix != '[]' }} strategy: fail-fast: false @@ -3119,7 +3109,7 @@ jobs: risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - managed_image_catalog: ${{ needs.generate-matrix.outputs.managed_image_catalog }} + managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} credential_boundary: GitHub read token target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3146,7 +3136,7 @@ jobs: catalogue-brave-nvidia-inference: name: ${{ matrix.display_name }} - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix != '[]' }} strategy: fail-fast: false @@ -3160,7 +3150,7 @@ jobs: risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - managed_image_catalog: ${{ needs.generate-matrix.outputs.managed_image_catalog }} + managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} credential_boundary: Brave and NVIDIA inference API keys target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3266,7 +3256,7 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh mcp-bridge: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge') }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }} permissions: @@ -3289,6 +3279,7 @@ jobs: agent_runtime: langchain-deepagents-code coverage_variant: deepagents env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge" E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" @@ -3611,7 +3602,7 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh openshell-credential-generation-window: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} runs-on: ubuntu-latest permissions: @@ -3621,6 +3612,7 @@ jobs: # MCP lifecycle without sharing destructive sandbox state. timeout-minutes: 90 env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "openshell-credential-generation-window" E2E_AGENT_RUNTIME: "openclaw" @@ -3772,7 +3764,7 @@ jobs: path: ${{ runner.temp }}/openshell-dev-artifact/ mcp-bridge-dev: - needs: [generate-matrix, openshell-dev-artifact] + needs: [base-image-publication, generate-matrix, openshell-dev-artifact] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }} runs-on: ubuntu-latest permissions: @@ -3793,6 +3785,7 @@ jobs: agent_runtime: langchain-deepagents-code coverage_variant: deepagents env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge-dev" E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" @@ -4957,11 +4950,12 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh hermes-e2e: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }} timeout-minutes: 85 env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "hermes-e2e" E2E_AGENT_RUNTIME: "hermes" @@ -5047,7 +5041,7 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh hermes-gpu-startup: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'hermes-gpu-startup') }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 90 @@ -5069,6 +5063,7 @@ jobs: observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" coverage_variant: compatibility-only env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "hermes-gpu-startup" E2E_AGENT_RUNTIME: "hermes" @@ -5368,11 +5363,12 @@ jobs: path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/ cloud-onboard: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'cloud-onboard') }} runs-on: ubuntu-latest timeout-minutes: 70 env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "cloud-onboard" E2E_AGENT_RUNTIME: "openclaw" @@ -5429,30 +5425,6 @@ jobs: with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - - name: Materialize cloud-onboard managed-image catalog - if: ${{ needs.generate-matrix.outputs.managed_image_catalog != '' }} - env: - CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} - MANAGED_IMAGE_CATALOG: ${{ needs.generate-matrix.outputs.managed_image_catalog }} - shell: bash - run: | - set -euo pipefail - candidate_release="v$(jq -r '.nemoclawVersion' dist/build-identity.json)" - catalog_path="${RUNNER_TEMP}/e2e-managed-image-catalog.json" - jq -e --arg release "$candidate_release" --arg revision "$CANDIDATE_SHA" ' - type == "object" and length > 0 and - all(.[]; .source.revision == $revision and .source.release == $release) - ' <<<"$MANAGED_IMAGE_CATALOG" >/dev/null || { - echo "::error::managed-image catalog source identity does not match the candidate" >&2 - exit 1 - } - jq -c . <<<"$MANAGED_IMAGE_CATALOG" >"$catalog_path" - [[ -s "$catalog_path" && ! -L "$catalog_path" ]] || { - echo "::error::temporary managed-image catalog is invalid" >&2 - exit 1 - } - printf 'NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG=%s\n' "$catalog_path" >>"$GITHUB_ENV" - - name: Install OpenShell CLI run: bash scripts/install-openshell.sh @@ -5516,11 +5488,12 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh messaging-providers: - needs: generate-matrix + needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'messaging-providers') }} runs-on: ubuntu-latest timeout-minutes: 90 env: + E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_JOB: "1" E2E_TARGET_ID: "messaging-providers" E2E_AGENT_RUNTIME: "openclaw" diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bb6515d0980..c8d94e18b1d 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -932,8 +932,8 @@ The poll count is clamped to a minimum of `1` so the health probe always runs at Without `--from`, onboarding through the OpenShell Docker driver for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. NemoClaw validates the complete three-agent publication cohort before selecting any member. -If registry or catalog availability prevents resolution, the ordinary `prefer-managed` path builds the shipped, reviewed repository Dockerfile instead; it never selects an unpinned `:latest` image. -Available catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent fails closed before sandbox creation. +If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. +Catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent also fails closed before sandbox creation. The portable experimental profile and native Podman are not part of this activation. Build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index 665470dc7f2..d0bedccdc40 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -173,20 +173,13 @@ describe("managed workload onboard orchestration", () => { ).toBe(false); }); - it("uses the trusted Dockerfile when the stock managed-image catalog is unavailable", async () => { + it("rejects stock onboarding when the managed-image catalog is unavailable", async () => { const { runtime } = createFreshOnboardingRuntime( {}, { stockManagedRuntime: true, unavailableCatalog: true }, ); - await expect(runtime.ensurePreparedWorkload()).resolves.toMatchObject({ - source: { - kind: "legacy-dockerfile", - dockerfilePath: "agents/openclaw/Dockerfile", - reason: "contract-unavailable", - }, - fallbackDiagnostic: expect.stringContaining("registry offline"), - }); + await expect(runtime.ensurePreparedWorkload()).rejects.toThrow("registry offline"); }); it("rejects an unavailable catalog for explicit temporary managed-image onboarding", async () => { @@ -256,10 +249,13 @@ describe("managed workload onboard orchestration", () => { it("retains the live qualification catalog revision during fresh onboarding (#9385)", async () => { const catalogRevision = "a".repeat(40); - const { prepared, runtime } = createFreshOnboardingRuntime({ - GITHUB_ACTIONS: "true", - E2E_MANAGED_IMAGE_REVISION: catalogRevision, - }); + const { prepared, runtime } = createFreshOnboardingRuntime( + { + GITHUB_ACTIONS: "true", + E2E_MANAGED_IMAGE_REVISION: catalogRevision, + }, + { stockManagedRuntime: true }, + ); await expect(runtime.ensurePreparedWorkload()).resolves.toBe(prepared); expect(prepareSandboxWorkloadSource).toHaveBeenCalledExactlyOnceWith( @@ -267,6 +263,17 @@ describe("managed workload onboard orchestration", () => { ); }); + it("does not apply the stock cohort revision outside stock onboarding", async () => { + const { prepared, runtime } = createFreshOnboardingRuntime({ + GITHUB_ACTIONS: "true", + E2E_MANAGED_IMAGE_REVISION: "a".repeat(40), + }); + + await expect(runtime.ensurePreparedWorkload()).resolves.toBe(prepared); + expect(prepareSandboxWorkloadSource).toHaveBeenCalledOnce(); + expect(prepareSandboxWorkloadSource.mock.calls[0]?.[0]).not.toHaveProperty("catalogRevision"); + }); + it("binds fresh onboarding to the exact PR catalog (#9464)", async () => { const catalogRevision = "b".repeat(40); const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-live-e2e-catalog-")); diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 50d6dda2cbc..9b6830624eb 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -229,12 +229,6 @@ export function createManagedWorkloadOnboardRuntime( managedImageSelectionPolicy: "prefer-managed" as const, managedImages: null, }; - const stockManagedImagePolicy = - input.stockManagedRuntime && - !strictManagedRuntime && - discoveredRuntimeCapabilities.legacyDockerfileBuilds - ? ("prefer-managed" as const) - : null; const runtimeProvider = resolveRuntimeProviderBundle( input.computePlan.driverName, CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -244,7 +238,9 @@ export function createManagedWorkloadOnboardRuntime( let preparedProfile: BuiltManagedStartupOnboardProfile | null = null; const ensurePreparedWorkload = async (): Promise => { - const catalogRevision = liveE2eManagedImageRevision(input.startupProfile.environment); + const catalogRevision = input.stockManagedRuntime + ? liveE2eManagedImageRevision(input.startupProfile.environment) + : null; const liveCatalog = liveE2eManagedImageCatalog(input.startupProfile.environment); if (catalogRevision && liveCatalog) { throw new Error("live E2E managed-image revision and catalog authority conflict"); @@ -264,7 +260,6 @@ export function createManagedWorkloadOnboardRuntime( runtime: runtimeCapabilities, version: getVersion({ rootDir: input.rootDir }), catalogPath: input.tempManagedRuntimeCatalog ?? liveCatalog?.path ?? null, - ...(stockManagedImagePolicy ? { policy: stockManagedImagePolicy } : {}), ...(liveCatalog ? { expectedCatalogRevision: liveCatalog.revision } : {}), ...(catalogRevision ? { catalogRevision } : {}), acceptedCandidateContract: isCandidateAgent(input.agentName) diff --git a/src/lib/onboard/sandbox-workload-preparation.test.ts b/src/lib/onboard/sandbox-workload-preparation.test.ts index 33f5ea53097..72accefb9f5 100644 --- a/src/lib/onboard/sandbox-workload-preparation.test.ts +++ b/src/lib/onboard/sandbox-workload-preparation.test.ts @@ -216,7 +216,7 @@ describe("sandbox workload preparation", () => { catalogPath, expectedCatalogRevision: "b".repeat(40), }), - ).rejects.toThrow("does not match the live E2E candidate revision"); + ).rejects.toThrow("does not match the trusted catalog revision"); } finally { fs.rmSync(fixtureRoot, { force: true, recursive: true }); } @@ -244,6 +244,28 @@ describe("sandbox workload preparation", () => { } }); + it("rejects a registry catalog that does not match its trusted revision", async () => { + await expect( + prepareSandboxWorkloadSource( + { + ...input("hermes"), + version: "0.1.0", + catalogRevision: "b".repeat(40), + }, + { resolveCatalog: async () => CATALOG }, + ), + ).rejects.toThrow("does not match the trusted catalog revision"); + }); + + it("rejects a cross-release catalog without an exact trusted revision", async () => { + await expect( + prepareSandboxWorkloadSource( + { ...input("langchain-deepagents-code"), version: "0.1.0" }, + { resolveCatalog: async () => CATALOG }, + ), + ).rejects.toThrow("belongs to 'v0.0.97', not 'v0.1.0'"); + }); + it("loads an exact local all-agent catalog without using the registry resolver (#7744)", async () => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-catalog-")); const catalogPath = path.join(fixtureRoot, "catalog.json"); diff --git a/src/lib/onboard/workload/preparation.ts b/src/lib/onboard/workload/preparation.ts index fdb84a758da..fac65524a93 100644 --- a/src/lib/onboard/workload/preparation.ts +++ b/src/lib/onboard/workload/preparation.ts @@ -231,7 +231,7 @@ function requireCompleteManagedImageCatalog( } if (expectedRevision !== null && cohortRevision !== expectedRevision) { throw new SandboxWorkloadPreparationError( - "managed image catalog source revision does not match the live E2E candidate revision", + "managed image catalog source revision does not match the trusted catalog revision", ); } return { release: cohortRelease!, revision: cohortRevision! }; @@ -318,6 +318,22 @@ export async function prepareSandboxWorkloadSource( ); } + const trustedCatalogRevision = input.expectedCatalogRevision ?? input.catalogRevision ?? null; + if ( + input.expectedCatalogRevision && + input.catalogRevision && + input.expectedCatalogRevision !== input.catalogRevision + ) { + throw new SandboxWorkloadPreparationError( + "managed image catalog has conflicting trusted revision authorities", + ); + } + if (trustedCatalogRevision !== null && !/^[0-9a-f]{40}$/u.test(trustedCatalogRevision)) { + throw new SandboxWorkloadPreparationError( + "managed image catalog trusted revision must be a lowercase 40-character SHA", + ); + } + let release: string; try { release = normalizeManagedImageRelease(input.version); @@ -365,9 +381,6 @@ export async function prepareSandboxWorkloadSource( acceptedCandidateContract, ); } else { - const trustedCatalogRevision = input.catalogPath - ? (input.expectedCatalogRevision ?? null) - : (input.catalogRevision ?? null); const catalogIdentity = requireCompleteManagedImageCatalog( catalog, release, diff --git a/test/e2e/README.md b/test/e2e/README.md index da1770b6af7..bdce05810f2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -90,16 +90,13 @@ After the checks pass, the action restores root `dist/` and `nemoclaw/dist/share If the version command fails, the action stops before the live test runs. This boundary keeps candidate source separate from the trusted workflow implementation. -For a same-repository PR that changes a managed-image workflow path, the trusted planner also -requires one successful `Images / Build, Test, and Publish Managed Images` run for the candidate commit. Before candidate -checkout, the planner downloads the three nonexpired contract artifacts by immutable artifact ID. -It verifies each artifact digest, producer run, attempt, and candidate commit. The planner rejects a -missing, incomplete, or mixed all-agent publication before E2E jobs start. - -The planner adds the exact all-agent catalog to `dist/` after the candidate CLI build completes. -Each live E2E consumer verifies that the catalog source revision matches `checkout_sha`. A PR that -does not change a managed-image workflow path keeps the released catalog behavior. The GitHub token -is available only to the trusted planner job and is not included in the candidate CLI artifact. +The `base-image-publication` job selects the nearest fully successful publication on the PR base first-parent history. +It downloads the complete cohort contract and Deep Agents Code base contract by immutable artifact ID. +It binds each artifact to the selected workflow run, attempt, revision, artifact ID, and artifact digest. +The cohort validator requires OpenClaw, Hermes, and LangChain Deep Agents Code on `linux/amd64` and `linux/arm64` before it emits `managed_image_revision`. +`generate-matrix` and every stock-onboarding job depend on this job, so a missing, failed, incomplete, or mixed publication starts no onboarding consumer. +Each stock-onboarding job receives the selected revision through `E2E_MANAGED_IMAGE_REVISION` and asserts the matching durable `managed-image` receipt before later probes. +The candidate CLI artifact contains no managed-image catalog. The same-repository `Images / Build, Test, and Publish Managed Images` PR workflow also runs the OpenClaw managed-image MCP discovery and lifecycle scope in two independent matrix jobs. Each job assembles one exact candidate @@ -125,8 +122,7 @@ This baseline measures only the replaced build step. Artifact upload, download, validation, and the dependency on `generate-matrix` add runtime and can affect the workflow critical path. Do not use the build-step median to claim savings in runner time or workflow elapsed time. -A manual PR E2E run tests candidate code but executes `.github/workflows/e2e.yaml` from `main`. -The PR run cannot measure this workflow change before merge. +A same-repository manual PR E2E run tests candidate code and executes `.github/workflows/e2e.yaml` from the PR branch at the exact latest PR commit. After merge, use a passing `main` run and complete these steps: 1. Match the job selection, runner labels, and first attempt to the baseline. @@ -565,8 +561,7 @@ to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map before checking out test code, and it consumes the variable only when the workflow repository is `NVIDIA/NemoClaw`, the ref is `refs/heads/main`, and no alternate checkout SHA is requested. Manual PR E2E dispatches therefore remain on -standard runners even though they use the trusted workflow definition from -`main`. +standard runners because an alternate candidate cannot select the administrator-managed label. Manual PR E2E dispatches and direct push or manual `main` runs use a bounded swap fallback for eligible hosted Hermes image-building lanes. The @@ -1263,11 +1258,18 @@ The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. Jetson and Launchable dispatch additionally require the PR branch to be in `NVIDIA/NemoClaw`; their operator and image-producer backends do not accept a sibling-repository candidate. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. +For an NVIDIA-owned PR, the same-repository PR branch supplies the workflow definition and `workflow_sha` must match the latest PR commit. +The workflow binds the latest PR commit to the current PR base SHA. It does not run GitHub's synthetic merge commit. Before candidate execution, the workflow uploads a `nemoclaw-e2e-dispatch-v2` receipt for the trusted manual run. The full-main `Release qualification` aggregate does not use this receipt. +The `base-image-publication` job selects the nearest fully successful base and managed-image publication on the PR base first-parent history. +It binds the selected run ID, attempt, revision, cohort contract artifact ID, and artifact digest before it emits `managed_image_revision`. +The job validates the complete three-agent, two-architecture cohort artifact and the immutable Deep Agents Code base artifact from that workflow attempt. +`generate-matrix` and every stock-onboarding job depend on this publication job, so incomplete publication creates no onboarding fanout. +Direct `main` runs use the same publication workflow and artifact contract. + PR Review Advisor maps changes to either of these shared journaled-recreation handlers to recommended E2E coverage: - `src/lib/onboard/machine/handlers/sandbox-resume.ts`. @@ -1340,7 +1342,7 @@ For a manual PR run, provide these inputs: - The lowercase 40-character SHA of the latest PR commit. - The PR source repository. - The lowercase 40-character PR base SHA. -- The exact SHA of the trusted workflow commit on `main`. +- The exact SHA of the workflow commit on the same-repository PR branch, equal to the latest PR commit. For the default NVIDIA-owned PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. @@ -1354,7 +1356,7 @@ To select native runtime qualification evidence production, set `jobs=native-run Leave `targets` empty and keep `include_staging_brev_launchable=false`. For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -A trusted `main` workflow pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. +A trusted controller pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. That ownership authorizes the full ordinary plan and credential profiles; external sources retain the bounded controller plan. A second validation after checkout rejects a changed candidate commit, base commit, PR source repository, or NVIDIA ownership before preparation. Candidate runs cannot publish release qualification. @@ -1408,12 +1410,11 @@ No PR E2E controller dispatches the risk plan. The `full-e2e` target enforces a separate hard acceptance contract for the first fresh onboarding path in that job. It measures from the onboard root span (a conservative anchor before wizard step `[1/8]`) through the first non-empty -agent response and reads the registered workload receipt. A `legacy-dockerfile` -receipt requires the local BuildKit prebuild without a gateway-builder fallback. -A `managed-image` receipt instead requires an exact digest that matches the -registered sandbox image tag, a non-empty publication cohort, and an exact -40-character source revision, and it forbids a local BuildKit prebuild. Both -paths enforce the calibrated root and phase limits in the budget file and limit +agent response and reads the registered workload receipt. The receipt must be +`managed-image`, use an exact digest that matches the registered sandbox image +tag, identify the selected publication cohort and exact source revision, and +forbid a local BuildKit prebuild. The path enforces the calibrated root and phase +limits in the budget file and limits the longest onboard output gap to 60 seconds. A violation fails `full-e2e`, and the target writes its evidence to `onboard-progress-budget.json`. The artifact records the first-turn command wall clock and OpenClaw's internal diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index 29736629163..213b4e189ca 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -10,6 +10,7 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_API_VERSION", + "E2E_MANAGED_IMAGE_REVISION", "GITHUB_WORKSPACE", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", diff --git a/test/e2e/fixtures/clients/host.ts b/test/e2e/fixtures/clients/host.ts index 5d1bcbb123b..eefec95c3b3 100644 --- a/test/e2e/fixtures/clients/host.ts +++ b/test/e2e/fixtures/clients/host.ts @@ -4,6 +4,10 @@ import { isAbsolute } from "node:path"; import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { + assertStockManagedImageReceipt, + shouldAssertStockManagedImageReceipt, +} from "../managed-image-receipt.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { trustedShellCommand } from "../shell-probe.ts"; import { @@ -48,7 +52,7 @@ export class HostCliClient { return this.openshellPath; } - command( + async command( command: string, args: string[] = [], options: ShellProbeRunOptions = {}, @@ -57,7 +61,7 @@ export class HostCliClient { if (this.cwd && !merged.cwd) { merged.cwd = this.cwd; } - return this.runner.run( + const result = await this.runner.run( trustedShellCommand({ command, args, @@ -65,6 +69,23 @@ export class HostCliClient { }), merged, ); + const environment = merged.env ?? {}; + if ( + result.exitCode === 0 && + shouldAssertStockManagedImageReceipt(command, args, environment) + ) { + const sandboxName = environment.NEMOCLAW_SANDBOX_NAME?.trim(); + if (!sandboxName) { + throw new Error("stock managed-image receipt assertion requires a sandbox name"); + } + assertStockManagedImageReceipt({ + commandOutput: resultText(result), + environment, + expectedAgent: environment.NEMOCLAW_AGENT?.trim(), + sandboxName, + }); + } + return result; } async isCommandAvailable(command: string, options: ShellProbeRunOptions = {}): Promise { diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts new file mode 100644 index 00000000000..ee625d4424c --- /dev/null +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; +import path from "node:path"; + +import { DEFAULT_GATEWAY_PORT } from "../../../src/lib/core/ports.ts"; +import { readManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts"; +import { readConfigFile } from "../../../src/lib/state/config-io.ts"; +import { parseSandboxRegistryEntries } from "../../../src/lib/state/registry-normalization.ts"; +import { cloneSandboxWorkloadReceipt } from "../../../src/lib/state/registry/workload.ts"; +import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; + +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const FALLBACK_DIAGNOSTIC = "Managed image unavailable; using the trusted Dockerfile recipe."; + +export interface StockManagedImageReceiptEvidence { + readonly agent: string; + readonly reference: string; + readonly sourceCohort: string; + readonly sourceRevision: string; +} + +function gatewayPort(environment: NodeJS.ProcessEnv): number { + const raw = environment.NEMOCLAW_GATEWAY_PORT?.trim(); + if (!raw) return DEFAULT_GATEWAY_PORT; + if (!/^[1-9][0-9]{0,4}$/u.test(raw)) { + throw new Error("stock managed-image receipt assertion requires a valid gateway port"); + } + const port = Number(raw); + if (!Number.isSafeInteger(port) || port > 65_535) { + throw new Error("stock managed-image receipt assertion requires a valid gateway port"); + } + return port; +} + +/** Assert the durable receipt before an E2E test begins post-onboarding probes. */ +export function assertStockManagedImageReceipt(options: { + readonly commandOutput?: string; + readonly environment?: NodeJS.ProcessEnv; + readonly expectedAgent?: string; + readonly sandboxName: string; +}): StockManagedImageReceiptEvidence { + const environment = options.environment ?? process.env; + const revision = environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; + if (!REVISION_PATTERN.test(revision)) { + throw new Error("stock onboarding requires one exact managed-image cohort revision"); + } + if (options.commandOutput?.includes(FALLBACK_DIAGNOSTIC)) { + throw new Error("stock onboarding emitted a legacy Dockerfile fallback diagnostic"); + } + + const home = environment.HOME?.trim() || os.homedir(); + const registryPath = path.join( + nemoclawStateRoot(home, gatewayPort(environment)), + "sandboxes.json", + ); + const registry = readConfigFile(registryPath, { sandboxes: {} }); + const sandboxes = + registry && typeof registry === "object" && !Array.isArray(registry) + ? (registry as { sandboxes?: unknown }).sandboxes + : undefined; + const entry = parseSandboxRegistryEntries(sandboxes).find( + ([name]) => name === options.sandboxName, + )?.[1]; + if (!entry) { + throw new Error(`stock sandbox '${options.sandboxName}' is missing from the durable registry`); + } + const authority = readManagedWorkloadAuthority(entry); + if (!authority) { + const receipt = cloneSandboxWorkloadReceipt(entry.workload); + throw new Error( + `stock sandbox '${options.sandboxName}' must record a managed-image receipt, got '${receipt?.kind ?? "missing"}'`, + ); + } + if (authority.receipt.sourceRevision !== revision) { + throw new Error( + `stock sandbox '${options.sandboxName}' managed-image revision does not match the selected cohort`, + ); + } + if (options.expectedAgent && authority.agent !== options.expectedAgent) { + throw new Error(`stock sandbox '${options.sandboxName}' managed-image agent does not match`); + } + return { + agent: authority.agent, + reference: authority.receipt.reference, + sourceCohort: authority.receipt.sourceCohort, + sourceRevision: authority.receipt.sourceRevision, + }; +} + +export function shouldAssertStockManagedImageReceipt( + command: string, + args: readonly string[], + environment: NodeJS.ProcessEnv, +): boolean { + if (!environment.E2E_MANAGED_IMAGE_REVISION?.trim()) return false; + if (environment.NEMOCLAW_FROM_DOCKERFILE?.trim()) return false; + const executable = path.basename(command); + let onboardArgumentIndex = -1; + if (executable === "nemoclaw" || executable === "nemoclaw.js") { + onboardArgumentIndex = args[0] === "onboard" ? 0 : -1; + } + if (executable === "node" || executable === "nodejs") { + onboardArgumentIndex = + path.basename(args[0] ?? "") === "nemoclaw.js" && args[1] === "onboard" ? 1 : -1; + } + if (onboardArgumentIndex < 0) return false; + return !args + .slice(onboardArgumentIndex + 1) + .some((argument) => argument === "--from" || argument.startsWith("--from=")); +} diff --git a/test/e2e/fixtures/workload-source-env.ts b/test/e2e/fixtures/workload-source-env.ts index 23f363adc93..52ca5f2b1e6 100644 --- a/test/e2e/fixtures/workload-source-env.ts +++ b/test/e2e/fixtures/workload-source-env.ts @@ -1,40 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import path from "node:path"; - -import { REPO_ROOT } from "./paths.ts"; - -const LEGACY_DOCKERFILE_BY_AGENT = { - openclaw: "Dockerfile", - hermes: "agents/hermes/Dockerfile", - "langchain-deepagents-code": "agents/langchain-deepagents-code/Dockerfile", -} as const; - -type LegacyDockerfileAgent = keyof typeof LEGACY_DOCKERFILE_BY_AGENT; - -function legacyDockerfileAgent(env: NodeJS.ProcessEnv): LegacyDockerfileAgent { - const agent = env.NEMOCLAW_AGENT ?? "openclaw"; - if (agent in LEGACY_DOCKERFILE_BY_AGENT) return agent as LegacyDockerfileAgent; - return "openclaw"; -} - /** - * Existing live E2E targets retain the product's default legacy-Dockerfile - * path. Targets that must select a source explicitly do so through the - * provider-neutral E2E_WORKLOAD_SOURCE contract. - * - * This is applied at the final fixture spawn boundary so an agent selected by - * a test command receives its own Dockerfile rather than an OpenClaw default. + * Reject automatic legacy source selection at the final E2E spawn boundary. + * An explicit custom Dockerfile remains a separate user-supplied input. */ export function resolveLiveE2eWorkloadSourceEnv(input: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const targetId = input.E2E_TARGET_ID; const source = input.E2E_WORKLOAD_SOURCE; if (!targetId || source !== "legacy-dockerfile") return input; if (input.NEMOCLAW_FROM_DOCKERFILE) return input; - const agent = legacyDockerfileAgent(input); - return { - ...input, - NEMOCLAW_FROM_DOCKERFILE: path.join(REPO_ROOT, LEGACY_DOCKERFILE_BY_AGENT[agent]), - }; + throw new Error(`live E2E target '${targetId}' cannot select a stock legacy Dockerfile`); } diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index d54fdf3a619..29298776e77 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -16,6 +16,7 @@ import { } from "../fixtures/corporate-ca.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -221,6 +222,12 @@ test("cloud onboard: public installer creates healthy sandbox with security chec }, ); expect(install.exitCode, resultText(install)).toBe(0); + assertStockManagedImageReceipt({ + commandOutput: resultText(install), + environment: testEnv(), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }); expect(resultText(install)).toContain("Installing NemoClaw from GitHub"); expect(resultText(install)).toContain("Cloning NemoClaw source"); expect(resultText(install)).toContain( diff --git a/test/e2e/live/full-e2e-workload-evidence.ts b/test/e2e/live/full-e2e-workload-evidence.ts index 874734f5dfe..8d7f6263250 100644 --- a/test/e2e/live/full-e2e-workload-evidence.ts +++ b/test/e2e/live/full-e2e-workload-evidence.ts @@ -7,6 +7,7 @@ import { load as loadSandboxRegistry } from "../../../src/lib/state/registry/per export function readFullE2eColdWorkloadEvidence( sandboxName: string, usedBuildKitPrebuild: boolean, + environment: NodeJS.ProcessEnv = process.env, ) { const entry = loadSandboxRegistry().sandboxes[sandboxName]; if (!entry) { @@ -14,26 +15,23 @@ export function readFullE2eColdWorkloadEvidence( } const managedAuthority = readManagedWorkloadAuthority(entry); - if (managedAuthority) { - if (usedBuildKitPrebuild) { - throw new Error("managed-image cold onboarding must not use a local BuildKit prebuild"); - } - return { - kind: managedAuthority.receipt.kind, - reference: managedAuthority.receipt.reference, - sourceCohort: managedAuthority.receipt.sourceCohort, - sourceRevision: managedAuthority.receipt.sourceRevision, - } as const; + if (!managedAuthority) { + throw new Error("full E2E cold onboarding must register a managed-image workload receipt"); } - - if (entry.workload?.kind !== "legacy-dockerfile") { - throw new Error("full E2E cold onboarding must register a supported workload receipt"); + if (usedBuildKitPrebuild) { + throw new Error("managed-image cold onboarding must not use a local BuildKit prebuild"); + } + const expectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; + if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { + throw new Error("full E2E cold onboarding requires one exact managed-image cohort revision"); } - if (!usedBuildKitPrebuild) { - throw new Error("legacy Dockerfile cold onboarding must use the local BuildKit prebuild"); + if (managedAuthority.receipt.sourceRevision !== expectedRevision) { + throw new Error("full E2E cold onboarding did not use the selected managed-image cohort"); } return { - kind: entry.workload.kind, - reference: entry.workload.reference, + kind: managedAuthority.receipt.kind, + reference: managedAuthority.receipt.reference, + sourceCohort: managedAuthority.receipt.sourceCohort, + sourceRevision: managedAuthority.receipt.sourceRevision, } as const; } diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 8b9875f9eea..8dffd809adf 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { HERMES_E2E_TEST_TIMEOUT_MS } from "../../../tools/e2e/hermes-timeout-contract.mts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { resultText, shellQuote } from "../fixtures/clients/command.ts"; import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -336,6 +337,12 @@ test( ), )); expect(install.exitCode, resultText(install)).toBe(0); + assertStockManagedImageReceipt({ + commandOutput: resultText(install), + environment: env, + expectedAgent: "hermes", + sandboxName: SANDBOX_NAME, + }); const cliProbe = await host.command( "bash", diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index d7ad88ec214..1b79ef8df6a 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { cleanupUnlessVerified } from "../fixtures/cleanup-resources.ts"; import { type HostCliClient, @@ -457,6 +458,12 @@ test( ? captureFailedGpuContainer(host, gpuDiagnosticsDir) : Promise.resolve()); expect(install.exitCode, resultText(install)).toBe(0); + assertStockManagedImageReceipt({ + commandOutput: resultText(install), + environment: env, + expectedAgent: "hermes", + sandboxName: SANDBOX_NAME, + }); const verifyFallback = async (wrapper: ReturnType) => { const fallbackEvents = readHermesGpuFallbackEvents(wrapper.eventsPath); diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index 6bd5181bd7c..3c69a38c70a 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -13,6 +13,7 @@ import type { HostCliClient } from "../fixtures/clients/host.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import type { FakeOpenAiCompatibleRequest } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; @@ -261,6 +262,12 @@ fi`, }); await artifacts.writeText("install-jetson-nvmap.log", resultText(install)); expect(install.exitCode, resultText(install)).toBe(0); + assertStockManagedImageReceipt({ + commandOutput: resultText(install), + environment: env(inferenceEnv), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }); const inferenceRoute = await host.command( "bash", diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index c75d35273f8..a91152c8221 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -11,6 +11,7 @@ const EXACT_MAIN_OVERLAY_KEYS = new Set([ ]); const MCP_BRIDGE_QUALIFICATION_ENV_KEYS = [ + "E2E_MANAGED_IMAGE_REVISION", "NEMOCLAW_E2E_EXPECTED_SHA", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG", "NEMOCLAW_RUN_LIVE_E2E", @@ -44,11 +45,14 @@ export function assertMcpBridgeManagedImageReceipt(options: { workload?: Record; }): void { const environment = options.environment ?? process.env; - if (!environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG?.trim()) return; + const selectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim(); + const exactCandidateCatalog = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG?.trim(); + if (!selectedRevision && !exactCandidateCatalog) return; - const expectedRevision = environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; + const expectedRevision = + selectedRevision ?? environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { - throw new Error("managed-image MCP qualification requires an exact candidate revision"); + throw new Error("managed-image MCP qualification requires an exact cohort revision"); } if ( options.workload?.kind !== "managed-image" || diff --git a/test/e2e/live/messaging-providers.test.ts b/test/e2e/live/messaging-providers.test.ts index c81e3fa14b0..2ed5a1a4690 100644 --- a/test/e2e/live/messaging-providers.test.ts +++ b/test/e2e/live/messaging-providers.test.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import { testTimeoutOptions } from "../../helpers/timeouts"; import { test } from "../fixtures/e2e-test.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { accountBool, accountString, @@ -166,6 +167,12 @@ test( return; } expectExitZero(install, "M0: install.sh completed"); + assertStockManagedImageReceipt({ + commandOutput: outputText(install), + environment: state.env, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }); const openshellVersion = await runHost(host, "openshell", ["--version"], { artifactName: "openshell-version-messaging-providers", diff --git a/test/e2e/support/base-image-publication-workflow-boundary.test.ts b/test/e2e/support/base-image-publication-workflow-boundary.test.ts index 52221a79491..f11ad107dfc 100644 --- a/test/e2e/support/base-image-publication-workflow-boundary.test.ts +++ b/test/e2e/support/base-image-publication-workflow-boundary.test.ts @@ -68,10 +68,12 @@ function gateStep(value: MutableWorkflow, name: string): MutableStep { } function runClassifier(environment: { + baseSha?: string; checkoutSha: string; eventName: string; ref: string; repository: string; + workflowSha?: string; }): { output: string; status: number | null } { const source = required( gateSteps(workflow())[0]?.run, @@ -83,11 +85,13 @@ function runClassifier(environment: { const result = spawnSync("/bin/bash", ["-c", source], { encoding: "utf8", env: { + BASE_SHA: environment.baseSha ?? "b".repeat(40), CHECKOUT_SHA: environment.checkoutSha, EVENT_NAME: environment.eventName, GITHUB_OUTPUT: outputPath, REF: environment.ref, REPOSITORY: environment.repository, + WORKFLOW_SHA: environment.workflowSha ?? "c".repeat(40), }, }); return { @@ -107,14 +111,15 @@ describe("base-image publication workflow boundary (#7372)", () => { }); it.each([ - ["push to main", "push", "", "refs/heads/main", "1", "0"], - ["manual main", "workflow_dispatch", "", "refs/heads/main", "1", "0"], + ["push to main", "push", "", "refs/heads/main", "0", "c".repeat(40), "0"], + ["manual main", "workflow_dispatch", "", "refs/heads/main", "0", "c".repeat(40), "0"], [ "controller-selected PR", "workflow_dispatch", "a".repeat(40), "refs/heads/candidate", - "0", + "1", + "b".repeat(40), "1", ], [ @@ -122,12 +127,13 @@ describe("base-image publication workflow boundary (#7372)", () => { "workflow_dispatch", "a4f9b59aa64f88532a3e64e949dd1b4068aa1f1e", "refs/heads/candidate", - "0", + "1", + "b".repeat(40), "1", ], ])( "classifies %s without executing untrusted code (#7372)", - (_case, eventName, checkoutSha, ref, required, reuse) => { + (_case, eventName, checkoutSha, ref, allowNonHead, expectedSha, selectNearest) => { expect( runClassifier({ checkoutSha, @@ -135,7 +141,10 @@ describe("base-image publication workflow boundary (#7372)", () => { ref, repository: "NVIDIA/NemoClaw", }), - ).toEqual({ output: `required=${required}\nreuse=${reuse}\n`, status: 0 }); + ).toEqual({ + output: `allow_non_head=${allowNonHead}\nexpected_sha=${expectedSha}\nselect_nearest_successful=${selectNearest}\n`, + status: 0, + }); }, ); @@ -179,7 +188,10 @@ describe("base-image publication workflow boundary (#7372)", () => { [ "classifier outcome", (value) => { - gateSteps(value)[0].run = gateSteps(value)[0].run!.replace("required=1", "required=0"); + gateSteps(value)[0].run = gateSteps(value)[0].run!.replace( + "select_nearest_successful=0", + "select_nearest_successful=1", + ); }, ], ["checkout condition", (value) => (gateSteps(value)[1].if = "${{ always() }}")], @@ -228,7 +240,7 @@ describe("base-image publication workflow boundary (#7372)", () => { ["step count", (value) => gateSteps(value).push({ name: "Unreviewed step", run: "true" })], [ "matrix publication dependency", - (value) => (value.jobs["generate-matrix"].needs = "base-image-publication"), + (value) => (value.jobs["generate-matrix"].needs = []), ], ["live publication dependency", (value) => (value.jobs.live.needs = ["generate-matrix"])], [ diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index a1ed7f41ef4..69285ba7e7f 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -480,6 +480,48 @@ describe("base-image publication evidence", () => { }); }); + it("selects the nearest fully successful trusted run for branch reuse", () => { + const failedRunId = RUN_ID + 1; + const selection = selectPublicationRun( + runsPayload([ + workflowRun({ + id: failedRunId, + head_sha: DESCENDANT_SHA, + conclusion: "failure", + html_url: `${RUN_URL_ROOT}/${failedRunId}`, + }), + workflowRun(), + ]), + history(), + WORKFLOW_ID, + { completedSuccessOnly: true }, + ); + + expect(selection).toMatchObject({ + state: "selected", + run: { id: RUN_ID, headSha: RELEVANT_SHA, conclusion: "success" }, + }); + }); + + it("does not select an incomplete or failed publication for branch reuse", () => { + expect( + selectPublicationRun( + runsPayload([ + workflowRun({ status: "in_progress", conclusion: null }), + workflowRun({ + id: RUN_ID + 1, + head_sha: DESCENDANT_SHA, + conclusion: "failure", + html_url: `${RUN_URL_ROOT}/${RUN_ID + 1}`, + }), + ]), + history(), + WORKFLOW_ID, + { completedSuccessOnly: true }, + ), + ).toEqual({ state: "missing" }); + }); + it("ignores pre-rename workflow metadata outside the eligible history (#7372)", () => { const selection = selectPublicationRun( runsPayload([ diff --git a/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts index a70a95dee41..c8abc433dc8 100644 --- a/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts +++ b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts @@ -8,7 +8,6 @@ import path from "node:path"; import { afterEach, describe, expect, it, onTestFinished } from "vitest"; import { ArtifactSink } from "../fixtures/artifacts.ts"; -import { REPO_ROOT } from "../fixtures/paths.ts"; import { startTestProgress } from "../fixtures/progress.ts"; import { SNAPSHOT_DATA_PREFIX } from "../live/bedrock-runtime-compatible-anthropic-leaks.ts"; @@ -55,26 +54,25 @@ afterEach(async () => { }); describe("Bedrock raw-command progress", () => { - it("applies the provider-neutral workload source at the raw spawn boundary", async () => { + it("rejects a stock legacy Dockerfile at the raw spawn boundary", async () => { const artifacts = await artifactSink("bedrock-workload-source"); const observation = progressProbe(); - const result = await runRawCommand( - process.execPath, - ["-e", "process.stdout.write(process.env.NEMOCLAW_FROM_DOCKERFILE ?? '')"], - { - artifactName: "bedrock-workload-source", - artifacts, - env: { - E2E_TARGET_ID: "inference-routing", - E2E_WORKLOAD_SOURCE: "legacy-dockerfile", - NEMOCLAW_AGENT: "langchain-deepagents-code", + await expect( + runRawCommand( + process.execPath, + ["-e", "process.stdout.write(process.env.NEMOCLAW_FROM_DOCKERFILE ?? '')"], + { + artifactName: "bedrock-workload-source", + artifacts, + env: { + E2E_TARGET_ID: "inference-routing", + E2E_WORKLOAD_SOURCE: "legacy-dockerfile", + NEMOCLAW_AGENT: "langchain-deepagents-code", + }, + progress: observation.progress, }, - progress: observation.progress, - }, - ); - - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(path.join(REPO_ROOT, "agents/langchain-deepagents-code/Dockerfile")); + ), + ).rejects.toThrow("cannot select a stock legacy Dockerfile"); }); it("reports timestamp-only output activity without forwarding child payloads", async () => { diff --git a/test/e2e/support/exact-artifact-download.test.ts b/test/e2e/support/exact-artifact-download.test.ts index c6d83782481..4e7577f7c8b 100644 --- a/test/e2e/support/exact-artifact-download.test.ts +++ b/test/e2e/support/exact-artifact-download.test.ts @@ -13,7 +13,9 @@ import { bindNamedExactArtifact, downloadBoundArtifact, exactArtifactName, + exactManagedImageCohortArtifactName, materializeContractArchive, + materializeExactJsonArchive, type BoundArtifactIdentity, type ExactArtifactExpectation, } from "../../../tools/e2e/exact-artifact-download.mts"; @@ -93,6 +95,19 @@ describe("exact artifact download (#9340)", () => { }); }); + it("derives and materializes the bound managed-image cohort artifact", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cohort-artifact-")); + const bytes = artifactZip([{ name: "cohort.json", contents: '{"contractVersion":2}\n' }]); + try { + expect(exactManagedImageCohortArtifactName(EXPECTED)).toBe("managed-image-cohort-7001-2"); + const cohortPath = materializeExactJsonArchive(bytes, directory, "cohort.json"); + expect(JSON.parse(fs.readFileSync(cohortPath, "utf8"))).toEqual({ contractVersion: 2 }); + expect(fs.statSync(cohortPath).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + it.each([ ["expired", { expired: true }, "non-expired"], ["artifact id URL", { id: 9002 }, "archive URL does not match artifact id"], diff --git a/test/e2e/support/managed-image-cohort-contract.test.ts b/test/e2e/support/managed-image-cohort-contract.test.ts new file mode 100644 index 00000000000..80efc634e8d --- /dev/null +++ b/test/e2e/support/managed-image-cohort-contract.test.ts @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + MANAGED_IMAGE_REPOSITORIES, + SHIPPED_MANAGED_IMAGE_AGENTS, +} from "../../../src/lib/onboard/managed-image/contract"; +import { validateManagedImageCohort } from "../../../tools/e2e/managed-image-cohort-contract.mts"; + +const REVISION = "a".repeat(40); +const RUN_ID = 32707920950; +const RUN_ATTEMPT = 1; +const COHORT = `ghrun-${RUN_ID}-${RUN_ATTEMPT}`; +const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; +type JsonObject = Record; + +function digest(index: number): `sha256:${string}` { + return `sha256:${(index % 15).toString(16).repeat(64)}`; +} + +function cohortContract(): Record { + return { + contractVersion: 2, + cohort: COHORT, + source: { repository: "NVIDIA/NemoClaw", revision: REVISION, release: null }, + run: { id: RUN_ID, attempt: RUN_ATTEMPT }, + platforms: PLATFORMS, + agents: Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, agentIndex) => { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const manifestDigest = digest(agentIndex + 1); + return [ + agent, + { + image, + digest: manifestDigest, + reference: `${image}@${manifestDigest}`, + descriptor: { digest: manifestDigest }, + alias: `${image}:cohort-${COHORT}`, + platforms: Object.fromEntries( + PLATFORMS.map((platform, platformIndex) => { + const platformDigest = digest(agentIndex + platformIndex + 4); + const baseReference = `ghcr.io/nvidia/nemoclaw/base@${digest(agentIndex + platformIndex + 7)}`; + const [os, architecture] = platform.split("/"); + return [ + platform, + { + digest: platformDigest, + reference: `${image}@${platformDigest}`, + baseReference, + publicationEvidence: { + candidateDescriptor: { digest: platformDigest }, + workloadDescriptor: { platform: { os, architecture } }, + attestations: { + slsa: { + statement: { + builderId: `https://github.com/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}`, + bindings: { + agent, + baseReference, + cohort: COHORT, + platform, + revision: REVISION, + source: "https://github.com/NVIDIA/NemoClaw", + }, + }, + }, + }, + }, + }, + ]; + }), + ), + }, + ]; + }), + ), + }; +} + +describe("managed-image cohort publication contract", () => { + it("binds all shipped agents and architectures to the selected publication", () => { + expect( + validateManagedImageCohort(cohortContract(), { + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + }), + ).toEqual({ cohort: COHORT, revision: REVISION, runAttempt: RUN_ATTEMPT, runId: RUN_ID }); + }); + + it("rejects a cohort that omits one image architecture", () => { + const value = cohortContract(); + const agents = value.agents as Record }>; + delete agents.hermes.platforms["linux/arm64"]; + + expect(() => + validateManagedImageCohort(value, { + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + }), + ).toThrow("complete expected set"); + }); + + it("rejects mixed revision provenance within the cohort", () => { + const value = cohortContract(); + const agents = value.agents as Record< + string, + { + platforms: Record< + string, + { + publicationEvidence: { + attestations: { slsa: { statement: { bindings: JsonObject } } }; + }; + } + >; + } + >; + agents.openclaw.platforms[ + "linux/amd64" + ].publicationEvidence.attestations.slsa.statement.bindings.revision = "b".repeat(40); + + expect(() => + validateManagedImageCohort(value, { + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + }), + ).toThrow("revision must be"); + }); +}); diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts new file mode 100644 index 00000000000..2bcb7513a0e --- /dev/null +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, +} from "../../../src/lib/onboard/managed-image/contract.ts"; +import { encodeManagedStartupProfile } from "../../../src/lib/onboard/managed-startup/profile.ts"; +import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; +import { + assertStockManagedImageReceipt, + shouldAssertStockManagedImageReceipt, +} from "../fixtures/managed-image-receipt.ts"; + +const SANDBOX_NAME = "managed-only-stock"; +const REVISION = "d".repeat(40); +const temporaryHomes: string[] = []; + +afterEach(() => { + for (const home of temporaryHomes.splice(0)) { + fs.rmSync(home, { force: true, recursive: true }); + } +}); + +function managedReceipt(sourceRevision = REVISION): Record { + const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); + const reference = `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"a".repeat(64)}`; + return { + schemaVersion: 1, + kind: "managed-image", + reference, + platform: "linux/amd64", + release: "v0.0.100", + sourceRevision, + sourceCohort: "ghrun-32707920950-1", + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }; +} + +function writeRegistry(workload: Record): string { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-only-receipt-")); + temporaryHomes.push(home); + const stateRoot = nemoclawStateRoot(home, 8080); + fs.mkdirSync(stateRoot, { recursive: true }); + fs.writeFileSync( + path.join(stateRoot, "sandboxes.json"), + `${JSON.stringify({ + sandboxes: { + [SANDBOX_NAME]: { + name: SANDBOX_NAME, + agent: "openclaw", + fromDockerfile: null, + imageTag: workload.reference, + workload, + }, + }, + })}\n`, + "utf8", + ); + return home; +} + +describe("stock E2E managed-image receipt assertion", () => { + it("accepts the durable receipt from the selected cohort revision", () => { + const home = writeRegistry(managedReceipt()); + + expect( + assertStockManagedImageReceipt({ + environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toMatchObject({ agent: "openclaw", sourceRevision: REVISION }); + }); + + it("rejects a stock legacy Dockerfile receipt", () => { + const home = writeRegistry({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "stock-legacy:latest", + shared: false, + }); + + expect(() => + assertStockManagedImageReceipt({ + environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, + sandboxName: SANDBOX_NAME, + }), + ).toThrow("must record a managed-image receipt"); + }); + + it("rejects a managed receipt from another cohort revision", () => { + const home = writeRegistry(managedReceipt("b".repeat(40))); + + expect(() => + assertStockManagedImageReceipt({ + environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, + sandboxName: SANDBOX_NAME, + }), + ).toThrow("does not match the selected cohort"); + }); + + it("rejects the stock fallback diagnostic before later probes", () => { + const home = writeRegistry(managedReceipt()); + + expect(() => + assertStockManagedImageReceipt({ + commandOutput: "Managed image unavailable; using the trusted Dockerfile recipe.", + environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, + sandboxName: SANDBOX_NAME, + }), + ).toThrow("fallback diagnostic"); + }); + + it("asserts normal stock onboarding and excludes an explicit custom Dockerfile", () => { + expect( + shouldAssertStockManagedImageReceipt("/workspace/bin/nemoclaw.js", ["onboard"], { + E2E_MANAGED_IMAGE_REVISION: REVISION, + }), + ).toBe(true); + expect( + shouldAssertStockManagedImageReceipt("/workspace/bin/nemoclaw.js", ["onboard"], { + E2E_MANAGED_IMAGE_REVISION: REVISION, + NEMOCLAW_FROM_DOCKERFILE: "/workspace/CustomDockerfile", + }), + ).toBe(false); + expect( + shouldAssertStockManagedImageReceipt( + "/workspace/bin/nemoclaw.js", + ["onboard", "--from", "/workspace/CustomDockerfile"], + { E2E_MANAGED_IMAGE_REVISION: REVISION }, + ), + ).toBe(false); + }); +}); diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index d11ebc60a41..7d39c62f055 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -120,6 +120,15 @@ describe("MCP bridge onboarding environment", () => { ).not.toThrow(); }); + it("accepts the selected cross-release managed-image cohort revision", () => { + expect(() => + assertMcpBridgeManagedImageReceipt({ + environment: { E2E_MANAGED_IMAGE_REVISION: "c".repeat(40) }, + workload: { kind: "managed-image", sourceRevision: "c".repeat(40) }, + }), + ).not.toThrow(); + }); + it("passes only exact-main OpenShell overrides after fixed onboarding values", () => { const env = buildMcpBridgeOnboardEnv({ ...ONBOARD_OPTIONS, diff --git a/test/e2e/support/mcp-workflow-boundary.test.ts b/test/e2e/support/mcp-workflow-boundary.test.ts index cc1bcfc2ae0..30d2b3e135c 100644 --- a/test/e2e/support/mcp-workflow-boundary.test.ts +++ b/test/e2e/support/mcp-workflow-boundary.test.ts @@ -101,7 +101,7 @@ describe("MCP workflow artifact boundary", () => { expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( expect.arrayContaining([ "mcp-bridge must not serialize the independent credential generation-window proof", - "openshell-credential-generation-window must depend only on matrix generation so it can run in parallel", + "openshell-credential-generation-window must depend on publication and matrix generation", ]), ); } finally { diff --git a/test/e2e/support/stock-managed-image-workflow-boundary.test.ts b/test/e2e/support/stock-managed-image-workflow-boundary.test.ts new file mode 100644 index 00000000000..497f7c6000d --- /dev/null +++ b/test/e2e/support/stock-managed-image-workflow-boundary.test.ts @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + type OperationsWorkflow, + validateBaseImagePublicationGate, + validateStockOnboardingPublicationBoundary, +} from "../../../tools/e2e/operations-workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +const STOCK_JOBS = [ + "live", + "mcp-bridge", + "openshell-credential-generation-window", + "mcp-bridge-dev", + "hermes-e2e", + "hermes-gpu-startup", + "cloud-onboard", + "messaging-providers", +] as const; + +const CATALOGUE_JOBS = [ + "catalogue-standard", + "catalogue-nvidia-api", + "catalogue-nvidia-inference", + "catalogue-github-read", + "catalogue-brave-nvidia-inference", +] as const; + +function workflow(): OperationsWorkflow { + return structuredClone(readWorkflow()) as unknown as OperationsWorkflow; +} + +describe("stock onboarding managed-image publication boundary", () => { + it("passes one selected cohort revision to every stock onboarding job", () => { + expect(validateStockOnboardingPublicationBoundary(workflow())).toEqual([]); + }); + + it.each(STOCK_JOBS)("rejects %s without the publication dependency and revision", (jobName) => { + const value = workflow(); + value.jobs[jobName].needs = []; + delete value.jobs[jobName].env?.E2E_MANAGED_IMAGE_REVISION; + + expect(validateStockOnboardingPublicationBoundary(value)).toEqual( + expect.arrayContaining([ + expect.stringContaining(`${jobName} must depend on base-image-publication`), + expect.stringContaining( + `${jobName} must receive the selected managed-image cohort revision`, + ), + ]), + ); + }); + + it.each(CATALOGUE_JOBS)( + "rejects %s without the publication dependency and revision input", + (jobName) => { + const value = workflow(); + value.jobs[jobName].needs = []; + delete value.jobs[jobName].with?.managed_image_revision; + + expect(validateStockOnboardingPublicationBoundary(value)).toEqual( + expect.arrayContaining([ + expect.stringContaining(`${jobName} must depend on base-image-publication`), + expect.stringContaining( + `${jobName} must pass the selected managed-image cohort revision`, + ), + ]), + ); + }, + ); + + it("blocks matrix generation when any publication architecture fails", () => { + const value = workflow(); + value.jobs["generate-matrix"].needs = []; + + expect(validateBaseImagePublicationGate(value)).toContain( + "generate-matrix must wait for complete managed-image publication", + ); + }); +}); diff --git a/test/e2e/support/workload-source-env.test.ts b/test/e2e/support/workload-source-env.test.ts index f148454fa6f..12d0470b527 100644 --- a/test/e2e/support/workload-source-env.test.ts +++ b/test/e2e/support/workload-source-env.test.ts @@ -1,28 +1,32 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import path from "node:path"; - import { describe, expect, it } from "vitest"; -import { REPO_ROOT } from "../fixtures/paths.ts"; import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts"; describe("live E2E workload source environment", () => { - it.each([ - ["openclaw", "Dockerfile"], - ["hermes", "agents/hermes/Dockerfile"], - ["langchain-deepagents-code", "agents/langchain-deepagents-code/Dockerfile"], - ])("honors the explicit legacy-Dockerfile source for %s", (agent, dockerfile) => { - expect( - resolveLiveE2eWorkloadSourceEnv({ - E2E_TARGET_ID: "full-e2e", - E2E_WORKLOAD_SOURCE: "legacy-dockerfile", - NEMOCLAW_AGENT: agent, - }), - ).toMatchObject({ - NEMOCLAW_FROM_DOCKERFILE: path.join(REPO_ROOT, dockerfile), - }); + it.each(["openclaw", "hermes", "langchain-deepagents-code"])( + "rejects automatic legacy-Dockerfile selection for %s", + (agent) => { + expect(() => + resolveLiveE2eWorkloadSourceEnv({ + E2E_TARGET_ID: "full-e2e", + E2E_WORKLOAD_SOURCE: "legacy-dockerfile", + NEMOCLAW_AGENT: agent, + }), + ).toThrow("cannot select a stock legacy Dockerfile"); + }, + ); + + it("preserves an explicit custom Dockerfile", () => { + const input = { + E2E_TARGET_ID: "custom-dockerfile", + E2E_WORKLOAD_SOURCE: "legacy-dockerfile", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_FROM_DOCKERFILE: "/workspace/CustomDockerfile", + }; + expect(resolveLiveE2eWorkloadSourceEnv(input)).toEqual(input); }); it("leaves an unspecified source on the product's default workload path", () => { diff --git a/test/onboard-managed-image-buildless-e2e.test.ts b/test/onboard-managed-image-buildless-e2e.test.ts index 6b16ceeae7e..35280cb0867 100644 --- a/test/onboard-managed-image-buildless-e2e.test.ts +++ b/test/onboard-managed-image-buildless-e2e.test.ts @@ -27,10 +27,10 @@ describe("managed image buildless onboarding orchestration contract", () => { "utf8", ); expect(commands).toContain( - "If registry or catalog availability prevents resolution, the ordinary `prefer-managed` path builds the shipped, reviewed repository Dockerfile instead; it never selects an unpinned `:latest` image.", + "If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile.", ); expect(commands).toContain( - "Available catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent fails closed before sandbox creation.", + "Catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent also fails closed before sandbox creation.", ); progress.phase("validate mocked all-agent buildless orchestration boundaries"); diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index 2c5bff93fda..e26a7cdc9bf 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -103,6 +103,7 @@ export interface PublicationWaitOptions { history: FirstParentHistory; request: (path: string) => Promise; requireWorkflowSuccess?: boolean; + selectNearestSuccessfulRun?: boolean; waitMs: number; pollMs: number; now?: () => number; @@ -284,12 +285,13 @@ export function resolveFirstParentHistory( expectedSha: string, paths: readonly string[], runGit: (args: string[]) => string = defaultGit, + options: { readonly requireCheckedOutCommit?: boolean } = {}, ): FirstParentHistory { sha(expectedSha, "expected SHA"); if (paths.length === 0) throw new Error("at least one base-image path is required"); const checkedOutSha = runGit(["rev-parse", "--verify", "HEAD^{commit}"]); - if (checkedOutSha !== expectedSha) { + if (options.requireCheckedOutCommit !== false && checkedOutSha !== expectedSha) { throw new Error( `checked-out commit ${checkedOutSha || "missing"} does not match ${expectedSha}`, ); @@ -405,6 +407,7 @@ export function selectPublicationRun( payload: unknown, history: FirstParentHistory, workflowId: number, + options: { readonly completedSuccessOnly?: boolean } = {}, ): PublicationSelection { positiveSafeInteger(workflowId, "base-image workflow id"); const response = asRecord(payload); @@ -429,10 +432,13 @@ export function selectPublicationRun( const distance = history.distanceBySha.get(run.headSha); return distance === undefined ? [] : [{ run, distance }]; }); - if (eligible.length === 0) return { state: "missing" }; + const selectable = options.completedSuccessOnly + ? eligible.filter(({ run }) => run.status === "completed" && run.conclusion === "success") + : eligible; + if (selectable.length === 0) return { state: "missing" }; - const nearestDistance = Math.min(...eligible.map(({ distance }) => distance)); - const nearest = eligible.filter(({ distance }) => distance === nearestDistance); + const nearestDistance = Math.min(...selectable.map(({ distance }) => distance)); + const nearest = selectable.filter(({ distance }) => distance === nearestDistance); if (nearest.length !== 1) { throw new Error( `multiple trusted base-image workflow runs match ${nearest[0]?.run.headSha ?? history.relevantSha}`, @@ -631,7 +637,9 @@ export async function waitForBaseImagePublication( const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?branch=${MAIN_BRANCH}&event=push&per_page=100`; while (true) { const runs = await collectPaginated(options.request, runsPath, "workflow_runs"); - const selection = selectPublicationRun(runs, options.history, workflowId); + const selection = selectPublicationRun(runs, options.history, workflowId, { + completedSuccessOnly: options.selectNearestSuccessfulRun === true, + }); if (selection.state === "selected") { const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/attempts/${selection.run.attempt}/jobs?per_page=100`; if (now() > deadline) { @@ -803,6 +811,8 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro const expectedSha = env.EXPECTED_SHA ?? ""; const outputPath = env.GITHUB_OUTPUT ?? ""; const requireManagedImagePublication = env.REQUIRE_MANAGED_IMAGE_PUBLICATION ?? "0"; + const selectNearestSuccessfulRun = env.SELECT_NEAREST_SUCCESSFUL_PUBLICATION ?? "0"; + const allowNonHeadHistory = env.PUBLICATION_HISTORY_ALLOW_NON_HEAD ?? "0"; const workspace = env.GITHUB_WORKSPACE ?? process.cwd(); if (token.length === 0 || token.includes("\r") || token.includes("\n")) { throw new Error("GITHUB_TOKEN must be a non-empty single-line value"); @@ -823,14 +833,26 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro if (requireManagedImagePublication !== "0" && requireManagedImagePublication !== "1") { throw new Error("REQUIRE_MANAGED_IMAGE_PUBLICATION must be 0 or 1"); } + if (selectNearestSuccessfulRun !== "0" && selectNearestSuccessfulRun !== "1") { + throw new Error("SELECT_NEAREST_SUCCESSFUL_PUBLICATION must be 0 or 1"); + } + if (allowNonHeadHistory !== "0" && allowNonHeadHistory !== "1") { + throw new Error("PUBLICATION_HISTORY_ALLOW_NON_HEAD must be 0 or 1"); + } - const workflowSource = readFileSync(resolve(workspace, WORKFLOW_PATH), "utf8"); + const workflowSource = + allowNonHeadHistory === "1" + ? defaultGit(["show", `${expectedSha}:${WORKFLOW_PATH}`]) + : readFileSync(resolve(workspace, WORKFLOW_PATH), "utf8"); const paths = parseBaseImagePushPaths(workflowSource); - const history = resolveFirstParentHistory(expectedSha, paths); + const history = resolveFirstParentHistory(expectedSha, paths, defaultGit, { + requireCheckedOutCommit: allowNonHeadHistory !== "1", + }); const run = await waitForBaseImagePublication({ history, request: (path) => githubRequest(path, token), requireWorkflowSuccess: requireManagedImagePublication === "1", + selectNearestSuccessfulRun: selectNearestSuccessfulRun === "1", waitMs: waitSeconds * 1000, pollMs: pollSeconds * 1000, }); diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index fac576a1ca8..0c504155bc0 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -267,9 +267,6 @@ function validateProducer(errors: string[], producer: WorkflowRecord): void { ".sourceRevision == $candidateSha", "candidate CLI build identity does not match the candidate commit SHA", - ".source.revision == $revision", - ".source.release == $release", - "managed-image catalog source identity does not match the candidate", "--sort=name", "--mtime=@0", "nemoclaw/dist/shared", @@ -359,8 +356,18 @@ function validateConsumer( } let expectedNeeds: string | string[] = CLI_ARTIFACT_PRODUCER_JOB; if (jobName === "mcp-bridge-dev") { - expectedNeeds = [CLI_ARTIFACT_PRODUCER_JOB, "openshell-dev-artifact"]; - } else if (jobName === "live") { + expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB, "openshell-dev-artifact"]; + } else if ( + [ + "live", + "mcp-bridge", + "openshell-credential-generation-window", + "hermes-e2e", + "hermes-gpu-startup", + "cloud-onboard", + "messaging-providers", + ].includes(jobName) + ) { expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; } if (!isDeepStrictEqual(job.needs, expectedNeeds)) { diff --git a/tools/e2e/exact-artifact-download.mts b/tools/e2e/exact-artifact-download.mts index 45a7a5967af..2ff068c58e6 100644 --- a/tools/e2e/exact-artifact-download.mts +++ b/tools/e2e/exact-artifact-download.mts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -15,6 +15,7 @@ import { githubRequest } from "./base-image-publication.mts"; const REPOSITORY = "NVIDIA/NemoClaw"; const API_ROOT = "https://api.github.com"; const CONTRACT_FILE = "contract.json"; +const COHORT_FILE = "cohort.json"; const MAX_ARCHIVE_BYTES = 1024 * 1024; const MAX_CONTRACT_BYTES = 64 * 1024; const MAX_ATTEMPTS = 3; @@ -71,6 +72,11 @@ export function exactArtifactName(expected: ExactArtifactExpectation): string { return `managed-base-${expected.runId}-${expected.runAttempt}-langchain-deepagents-code`; } +/** Derive the immutable all-agent cohort artifact name for one publication attempt. */ +export function exactManagedImageCohortArtifactName(expected: ExactArtifactExpectation): string { + return `managed-image-cohort-${expected.runId}-${expected.runAttempt}`; +} + /** Bind one named artifact and validate every immutable producer attribute. */ export function bindNamedExactArtifact( value: unknown, @@ -274,23 +280,32 @@ export async function downloadBoundArtifact( } /** Extract the sole bounded contract file from a validated artifact ZIP. */ -export function materializeContractArchive(archive: Buffer, outputDirectory: string): string { +export function materializeExactJsonArchive( + archive: Buffer, + outputDirectory: string, + fileName: typeof CONTRACT_FILE | typeof COHORT_FILE, +): string { const entries = listValidatedArtifactZipEntries(archive, { maxEntries: 2 }); - if (JSON.stringify(entries) !== JSON.stringify([CONTRACT_FILE])) { - throw new Error("artifact archive must contain exactly one contract.json regular file"); + if (JSON.stringify(entries) !== JSON.stringify([fileName])) { + throw new Error(`artifact archive must contain exactly one ${fileName} regular file`); } - const contract = readValidatedArtifactZipEntryBytes(archive, CONTRACT_FILE, { + const contract = readValidatedArtifactZipEntryBytes(archive, fileName, { maxBytes: MAX_CONTRACT_BYTES, maxEntries: 2, }); if (!contract) throw new Error("artifact contract archive is malformed"); const resolvedDirectory = path.resolve(outputDirectory); mkdirSync(resolvedDirectory, { mode: 0o700, recursive: true }); - const contractPath = path.join(resolvedDirectory, CONTRACT_FILE); + const contractPath = path.join(resolvedDirectory, fileName); writeFileSync(contractPath, contract, { mode: 0o600 }); return contractPath; } +/** Extract the sole bounded contract file from a validated artifact ZIP. */ +export function materializeContractArchive(archive: Buffer, outputDirectory: string): string { + return materializeExactJsonArchive(archive, outputDirectory, CONTRACT_FILE); +} + /** Read one required positive integer environment value. */ function requiredInteger(value: string | undefined, label: string): number { if (!value || !/^[1-9][0-9]*$/u.test(value)) throw new Error(`${label} is required`); @@ -307,14 +322,33 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro runAttempt: requiredInteger(env.PUBLICATION_RUN_ATTEMPT, "PUBLICATION_RUN_ATTEMPT"), runId: requiredInteger(env.PUBLICATION_RUN_ID, "PUBLICATION_RUN_ID"), }; - const name = exactArtifactName(expected); + const kind = env.PUBLICATION_ARTIFACT_KIND ?? "dcode-base"; + if (kind !== "dcode-base" && kind !== "managed-image-cohort") { + throw new Error("PUBLICATION_ARTIFACT_KIND must be dcode-base or managed-image-cohort"); + } + const name = + kind === "managed-image-cohort" + ? exactManagedImageCohortArtifactName(expected) + : exactArtifactName(expected); + const fileName = kind === "managed-image-cohort" ? COHORT_FILE : CONTRACT_FILE; const response = await githubRequest( `/repos/${REPOSITORY}/actions/runs/${expected.runId}/artifacts?name=${encodeURIComponent(name)}&per_page=100`, token, ); - const identity = bindExactArtifact(response, expected); + const identity = bindNamedExactArtifact(response, expected, name); const archive = await downloadBoundArtifact(identity, token); - materializeContractArchive(archive, argv[0]); + materializeExactJsonArchive(archive, argv[0], fileName); + if (env.GITHUB_OUTPUT) { + const provenance = JSON.stringify({ + artifactDigest: identity.digest, + artifactId: identity.id, + artifactName: identity.name, + revision: identity.headSha, + runAttempt: identity.runAttempt, + runId: identity.runId, + }); + appendFileSync(env.GITHUB_OUTPUT, `provenance=${provenance}\n`, "utf8"); + } } if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { diff --git a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts index cbd2e8d54de..f7767b77148 100644 --- a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts +++ b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts @@ -122,7 +122,11 @@ export function validateHermesGpuStartupWorkflow( if (job["runs-on"] !== "linux-amd64-gpu-rtxpro6000-latest-1") { errors.push(`${JOB_NAME} job must run on the native RTX PRO 6000 GPU runner`); } - if (job.needs !== "generate-matrix" || job.if !== EXPECTED_SELECTOR) { + if ( + JSON.stringify(job.needs) !== + JSON.stringify(["base-image-publication", "generate-matrix"]) || + job.if !== EXPECTED_SELECTOR + ) { errors.push(`${JOB_NAME} job must use the trusted execution plan behind generate-matrix`); } if (job["timeout-minutes"] !== 90) { diff --git a/tools/e2e/managed-image-cohort-contract.mts b/tools/e2e/managed-image-cohort-contract.mts new file mode 100644 index 00000000000..55a308353d0 --- /dev/null +++ b/tools/e2e/managed-image-cohort-contract.mts @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { appendFileSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + MANAGED_IMAGE_REPOSITORIES, + SHIPPED_MANAGED_IMAGE_AGENTS, +} from "../../src/lib/onboard/managed-image/contract.ts"; + +const REPOSITORY = "NVIDIA/NemoClaw"; +const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; +const SHA_PATTERN = /^[0-9a-f]{40}$/u; +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; + +type JsonRecord = Record; + +export interface ManagedImageCohortIdentity { + readonly cohort: string; + readonly revision: string; + readonly runAttempt: number; + readonly runId: number; +} + +function record(value: unknown, label: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be a JSON object`); + } + return value as JsonRecord; +} + +function positiveInteger(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error(`${label} must be a positive integer`); + } + return Number(value); +} + +function exactString(value: unknown, expected: string, label: string): void { + if (value !== expected) throw new Error(`${label} must be ${expected}`); +} + +function digest(value: unknown, label: string): string { + if (typeof value !== "string" || !DIGEST_PATTERN.test(value)) { + throw new Error(`${label} must be an immutable SHA-256 digest`); + } + return value; +} + +function exactKeys(value: JsonRecord, expected: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const sortedExpected = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(sortedExpected)) { + throw new Error(`${label} must contain the complete expected set`); + } +} + +function validatePlatformEvidence( + value: unknown, + expected: { + readonly agent: string; + readonly cohort: string; + readonly platform: (typeof PLATFORMS)[number]; + readonly revision: string; + readonly runAttempt: number; + readonly runId: number; + }, +): void { + const platform = record(value, `${expected.agent} ${expected.platform} publication`); + const platformDigest = digest(platform.digest, `${expected.agent} ${expected.platform} digest`); + const image = + MANAGED_IMAGE_REPOSITORIES[expected.agent as keyof typeof MANAGED_IMAGE_REPOSITORIES]; + exactString( + platform.reference, + `${image}@${platformDigest}`, + `${expected.agent} ${expected.platform} reference`, + ); + const baseReference = platform.baseReference; + if (typeof baseReference !== "string" || !/@sha256:[0-9a-f]{64}$/u.test(baseReference)) { + throw new Error(`${expected.agent} ${expected.platform} base reference must be immutable`); + } + + const publicationEvidence = record( + platform.publicationEvidence, + `${expected.agent} ${expected.platform} publication evidence`, + ); + const candidateDescriptor = record( + publicationEvidence.candidateDescriptor, + `${expected.agent} ${expected.platform} candidate descriptor`, + ); + exactString( + candidateDescriptor.digest, + platformDigest, + `${expected.agent} ${expected.platform} candidate digest`, + ); + const workloadDescriptor = record( + publicationEvidence.workloadDescriptor, + `${expected.agent} ${expected.platform} workload descriptor`, + ); + const workloadPlatform = record( + workloadDescriptor.platform, + `${expected.agent} ${expected.platform} workload platform`, + ); + const [os, architecture] = expected.platform.split("/"); + exactString(workloadPlatform.os, os, `${expected.agent} ${expected.platform} operating system`); + exactString( + workloadPlatform.architecture, + architecture, + `${expected.agent} ${expected.platform} architecture`, + ); + + const attestations = record( + publicationEvidence.attestations, + `${expected.agent} ${expected.platform} attestations`, + ); + const slsa = record(attestations.slsa, `${expected.agent} ${expected.platform} SLSA evidence`); + const statement = record(slsa.statement, `${expected.agent} ${expected.platform} SLSA statement`); + exactString( + statement.builderId, + `https://github.com/${REPOSITORY}/actions/runs/${expected.runId}/attempts/${expected.runAttempt}`, + `${expected.agent} ${expected.platform} builder`, + ); + const bindings = record( + statement.bindings, + `${expected.agent} ${expected.platform} SLSA bindings`, + ); + exactString(bindings.agent, expected.agent, `${expected.agent} ${expected.platform} agent`); + exactString(bindings.cohort, expected.cohort, `${expected.agent} ${expected.platform} cohort`); + exactString( + bindings.platform, + expected.platform, + `${expected.agent} ${expected.platform} binding`, + ); + exactString( + bindings.revision, + expected.revision, + `${expected.agent} ${expected.platform} revision`, + ); + exactString(bindings.source, `https://github.com/${REPOSITORY}`, `${expected.agent} source`); + exactString( + bindings.baseReference, + baseReference, + `${expected.agent} ${expected.platform} base reference binding`, + ); +} + +/** Validate one complete published cohort against its selected workflow attempt. */ +export function validateManagedImageCohort( + value: unknown, + expected: { readonly revision: string; readonly runAttempt: number; readonly runId: number }, +): ManagedImageCohortIdentity { + if (!SHA_PATTERN.test(expected.revision)) throw new Error("expected cohort revision is invalid"); + positiveInteger(expected.runId, "expected cohort run id"); + positiveInteger(expected.runAttempt, "expected cohort run attempt"); + + const cohort = record(value, "managed-image cohort"); + if (cohort.contractVersion !== 2) + throw new Error("managed-image cohort contract version must be 2"); + const expectedCohort = `ghrun-${expected.runId}-${expected.runAttempt}`; + exactString(cohort.cohort, expectedCohort, "managed-image cohort identity"); + const source = record(cohort.source, "managed-image cohort source"); + exactString(source.repository, REPOSITORY, "managed-image cohort source repository"); + exactString(source.revision, expected.revision, "managed-image cohort source revision"); + const run = record(cohort.run, "managed-image cohort run"); + if (run.id !== expected.runId || run.attempt !== expected.runAttempt) { + throw new Error("managed-image cohort run does not match the selected publication"); + } + if (JSON.stringify(cohort.platforms) !== JSON.stringify(PLATFORMS)) { + throw new Error("managed-image cohort must contain linux/amd64 and linux/arm64"); + } + + const agents = record(cohort.agents, "managed-image cohort agents"); + exactKeys(agents, SHIPPED_MANAGED_IMAGE_AGENTS, "managed-image cohort agents"); + for (const agent of SHIPPED_MANAGED_IMAGE_AGENTS) { + const contract = record(agents[agent], `${agent} cohort contract`); + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const manifestDigest = digest(contract.digest, `${agent} cohort digest`); + exactString(contract.image, image, `${agent} cohort image`); + exactString(contract.reference, `${image}@${manifestDigest}`, `${agent} cohort reference`); + exactString(contract.alias, `${image}:cohort-${expectedCohort}`, `${agent} cohort alias`); + exactString( + record(contract.descriptor, `${agent} cohort descriptor`).digest, + manifestDigest, + `${agent} cohort descriptor digest`, + ); + const platforms = record(contract.platforms, `${agent} cohort platforms`); + exactKeys(platforms, PLATFORMS, `${agent} cohort platforms`); + for (const platform of PLATFORMS) { + validatePlatformEvidence(platforms[platform], { + agent, + cohort: expectedCohort, + platform, + revision: expected.revision, + runAttempt: expected.runAttempt, + runId: expected.runId, + }); + } + } + + return { + cohort: expectedCohort, + revision: expected.revision, + runAttempt: expected.runAttempt, + runId: expected.runId, + }; +} + +function requiredInteger(value: string | undefined, label: string): number { + if (!value || !/^[1-9][0-9]*$/u.test(value)) throw new Error(`${label} is required`); + return positiveInteger(Number(value), label); +} + +export function main(argv = process.argv.slice(2), env = process.env): void { + if (argv.length !== 1) throw new Error("expected one managed-image cohort contract path"); + const identity = validateManagedImageCohort( + JSON.parse(readFileSync(argv[0], "utf8")) as unknown, + { + revision: env.PUBLICATION_HEAD_SHA ?? "", + runAttempt: requiredInteger(env.PUBLICATION_RUN_ATTEMPT, "PUBLICATION_RUN_ATTEMPT"), + runId: requiredInteger(env.PUBLICATION_RUN_ID, "PUBLICATION_RUN_ID"), + }, + ); + if (!env.GITHUB_OUTPUT) throw new Error("GITHUB_OUTPUT is required"); + appendFileSync( + env.GITHUB_OUTPUT, + `cohort=${identity.cohort}\nrevision=${identity.revision}\nrun_attempt=${identity.runAttempt}\nrun_id=${identity.runId}\n`, + "utf8", + ); +} + +if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : "unknown managed-image cohort error"); + process.exitCode = 1; + } +} diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index bdaa805a6ee..02c58f55f6c 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -6,7 +6,7 @@ import { createHash } from "node:crypto"; export const MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256 = "052c49d5e8688266dbf38fa911733132d33e4470a29a61deb6e7a11067737559"; export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = - "b9219b0f29da3834499a7c9dcb0acc8287cb3441d01a51bed615d7b2ea4383f9"; + "3384ea63548c78f5c2a592e1d3a4de1b3060dee8167d7b83f0d501e1eaa5e128"; export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index a51f6eeca15..fce9ec3d3dc 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -167,6 +167,12 @@ function validateJobIdentity( jobName, `${jobName} must use its job id as E2E_TARGET_ID`, ); + requireEqual( + errors, + env.E2E_MANAGED_IMAGE_REVISION, + "${{ needs.base-image-publication.outputs.managed_image_revision }}", + `${jobName} must receive the selected managed-image cohort revision`, + ); requireEqual( errors, job["timeout-minutes"], @@ -178,7 +184,9 @@ function validateJobIdentity( errors, JSON.stringify(jobNeeds(job)), JSON.stringify( - jobName === "mcp-bridge-dev" ? ["generate-matrix", DEV_ARTIFACT_JOB] : ["generate-matrix"], + jobName === "mcp-bridge-dev" + ? ["base-image-publication", "generate-matrix", DEV_ARTIFACT_JOB] + : ["base-image-publication", "generate-matrix"], ), `${jobName} must depend on its reviewed artifact producers`, ); @@ -841,8 +849,8 @@ function validateCredentialWindowJob( requireEqual( errors, JSON.stringify(jobNeeds(job)), - JSON.stringify(["generate-matrix"]), - `${CREDENTIAL_WINDOW_JOB} must depend only on matrix generation so it can run in parallel`, + JSON.stringify(["base-image-publication", "generate-matrix"]), + `${CREDENTIAL_WINDOW_JOB} must depend on publication and matrix generation`, ); requireEqual( errors, @@ -865,6 +873,8 @@ function validateCredentialWindowJob( const env = asRecord(job.env); const expectedEnv = { + E2E_MANAGED_IMAGE_REVISION: + "${{ needs.base-image-publication.outputs.managed_image_revision }}", E2E_JOB: "1", E2E_TARGET_ID: CREDENTIAL_WINDOW_JOB, E2E_AGENT_RUNTIME: "openclaw", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index f75bc1d4083..6346502ac4c 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -33,29 +33,32 @@ const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run --test-path const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const COLD_ONBOARD_PERFORMANCE_EVIDENCE_PATH = "e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json"; -const PUBLICATION_REQUIRED_CONDITION = "${{ steps.publication_mode.outputs.required == '1' }}"; -const PUBLICATION_REUSE_CONDITION = "${{ steps.publication_mode.outputs.reuse == '1' }}"; -const PUBLICATION_REQUIRED_OR_REUSE_CONDITION = - "${{ steps.publication_mode.outputs.required == '1' || steps.publication_mode.outputs.reuse == '1' }}"; const PUBLICATION_CLASSIFIER_SCRIPT = [ "set -euo pipefail", - "reuse=0", 'case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in', " NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)", - " required=1", + ' expected_sha="$WORKFLOW_SHA"', + " allow_non_head=0", + " select_nearest_successful=0", " ;;", " NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller)", - " required=0", - " reuse=1", + ' [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {', + ' echo "::error::manual PR publication selection requires an exact base SHA" >&2', + " exit 1", + " }", + ' expected_sha="$BASE_SHA"', + " allow_non_head=1", + " select_nearest_successful=1", " ;;", " *)", ' echo "::error::base-image publication mode is not trusted" >&2', " exit 1", " ;;", "esac", - 'printf \'required=%s\\n\' "${required}" >> "${GITHUB_OUTPUT}"', - 'printf \'reuse=%s\\n\' "${reuse}" >> "${GITHUB_OUTPUT}"', + 'printf \'allow_non_head=%s\\n\' "${allow_non_head}" >> "${GITHUB_OUTPUT}"', + 'printf \'expected_sha=%s\\n\' "${expected_sha}" >> "${GITHUB_OUTPUT}"', + 'printf \'select_nearest_successful=%s\\n\' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}"', ].join("\n") + "\n"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; const ISSUE_MUTATION_BEYOND_COMMENT = @@ -98,6 +101,7 @@ type WorkflowJob = { "runs-on"?: unknown; steps?: WorkflowStep[]; "timeout-minutes"?: unknown; + with?: Record; }; export type OperationsWorkflow = { @@ -484,7 +488,6 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const trustedPublicationCheckout = jobName === "base-image-publication" && step.name === "Check out trusted E2E workflow" && - step.if === PUBLICATION_REQUIRED_OR_REUSE_CONDITION && step.with?.ref === "${{ inputs.checkout_sha || github.sha }}"; const trustedManagedImageRuntimeCheckout = jobName === "managed-image-protected-runtime" && @@ -593,12 +596,15 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): "runs-on": "ubuntu-latest", "timeout-minutes": 55, outputs: { - dcode_base_contract: - "${{ steps.validate_dcode_base.outputs.contract || steps.validate_reused_dcode_base.outputs.contract }}", - dcode_base_ref: - "${{ steps.validate_dcode_base.outputs.base_ref || steps.validate_reused_dcode_base.outputs.base_ref }}", + dcode_base_contract: "${{ steps.validate_dcode_base.outputs.contract }}", + dcode_base_ref: "${{ steps.validate_dcode_base.outputs.base_ref }}", + managed_image_artifact_provenance: + "${{ steps.download_managed_cohort.outputs.provenance }}", + managed_image_cohort: "${{ steps.validate_managed_cohort.outputs.cohort }}", managed_image_revision: - "${{ steps.publication.outputs.head_sha || (steps.publication_mode.outputs.reuse == '1' && 'e38db201413b457614904187377ed9fd002d281d') || inputs.checkout_sha || github.sha }}", + "${{ steps.validate_managed_cohort.outputs.revision }}", + managed_image_run_attempt: "${{ steps.validate_managed_cohort.outputs.run_attempt }}", + managed_image_run_id: "${{ steps.validate_managed_cohort.outputs.run_id }}", }, permissions: { actions: "read", @@ -609,17 +615,18 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): id: "publication_mode", name: "Classify base-image publication requirement", env: { + BASE_SHA: "${{ inputs.base_sha }}", CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EVENT_NAME: "${{ github.event_name }}", REF: "${{ github.ref }}", REPOSITORY: "${{ github.repository }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", }, shell: "bash", run: PUBLICATION_CLASSIFIER_SCRIPT, }, { name: "Check out trusted E2E workflow", - if: PUBLICATION_REQUIRED_OR_REUSE_CONDITION, uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", with: { ref: "${{ inputs.checkout_sha || github.sha }}", @@ -629,7 +636,6 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): }, { name: "Set up Node for publication verification", - if: PUBLICATION_REQUIRED_OR_REUSE_CONDITION, uses: "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", with: { "node-version": 22, @@ -637,25 +643,31 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): }, { id: "publication", - name: "Verify applicable base-image publication", - if: PUBLICATION_REQUIRED_CONDITION, + name: "Select complete base and managed-image publication", env: { - EXPECTED_SHA: "${{ inputs.checkout_sha || github.sha }}", + EXPECTED_SHA: "${{ steps.publication_mode.outputs.expected_sha }}", GITHUB_TOKEN: "${{ github.token }}", + PUBLICATION_HISTORY_ALLOW_NON_HEAD: + "${{ steps.publication_mode.outputs.allow_non_head }}", REQUIRE_MANAGED_IMAGE_PUBLICATION: "1", + SELECT_NEAREST_SUCCESSFUL_PUBLICATION: + "${{ steps.publication_mode.outputs.select_nearest_successful }}", }, shell: "bash", run: [ "set -euo pipefail", "export GITHUB_REF=refs/heads/main", 'export GITHUB_SHA="$EXPECTED_SHA"', - "node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30", + "wait_seconds=3000", + 'if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then', + " wait_seconds=0", + "fi", + 'node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30', "", ].join("\n"), }, { name: "Download immutable Deep Agents Code base contract", - if: PUBLICATION_REQUIRED_CONDITION, env: { GITHUB_TOKEN: "${{ github.token }}", PUBLICATION_HEAD_SHA: "${{ steps.publication.outputs.head_sha }}", @@ -665,37 +677,36 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): run: 'node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract"', }, { - name: "Download reused Deep Agents Code base contract", - if: PUBLICATION_REUSE_CONDITION, + id: "validate_dcode_base", + name: "Validate immutable Deep Agents Code base", env: { - GITHUB_TOKEN: "${{ github.token }}", - PUBLICATION_HEAD_SHA: "e38db201413b457614904187377ed9fd002d281d", - PUBLICATION_RUN_ATTEMPT: "1", - PUBLICATION_RUN_ID: "32544159037", + PUBLICATION_HEAD_SHA: "${{ steps.publication.outputs.head_sha }}", + PUBLICATION_RUN_ATTEMPT: "${{ steps.publication.outputs.run_attempt }}", + PUBLICATION_RUN_ID: "${{ steps.publication.outputs.run_id }}", }, - run: 'node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract-reused"', + run: 'node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json"', }, { - id: "validate_dcode_base", - name: "Validate immutable Deep Agents Code base", - if: PUBLICATION_REQUIRED_CONDITION, + id: "download_managed_cohort", + name: "Download immutable managed-image cohort contract", env: { + GITHUB_TOKEN: "${{ github.token }}", + PUBLICATION_ARTIFACT_KIND: "managed-image-cohort", PUBLICATION_HEAD_SHA: "${{ steps.publication.outputs.head_sha }}", PUBLICATION_RUN_ATTEMPT: "${{ steps.publication.outputs.run_attempt }}", PUBLICATION_RUN_ID: "${{ steps.publication.outputs.run_id }}", }, - run: 'node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json"', + run: 'node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/managed-image-cohort"', }, { - id: "validate_reused_dcode_base", - name: "Validate reused Deep Agents Code base", - if: PUBLICATION_REUSE_CONDITION, + id: "validate_managed_cohort", + name: "Validate immutable managed-image cohort contract", env: { - PUBLICATION_HEAD_SHA: "e38db201413b457614904187377ed9fd002d281d", - PUBLICATION_RUN_ATTEMPT: "1", - PUBLICATION_RUN_ID: "32544159037", + PUBLICATION_HEAD_SHA: "${{ steps.publication.outputs.head_sha }}", + PUBLICATION_RUN_ATTEMPT: "${{ steps.publication.outputs.run_attempt }}", + PUBLICATION_RUN_ID: "${{ steps.publication.outputs.run_id }}", }, - run: 'node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract-reused/contract.json"', + run: 'node --experimental-strip-types --no-warnings tools/e2e/managed-image-cohort-contract.mts "${RUNNER_TEMP}/managed-image-cohort/cohort.json"', }, ], }; @@ -706,8 +717,8 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): ); } const matrix = workflow.jobs["generate-matrix"] ?? {}; - if (needs(matrix).includes("base-image-publication")) { - errors.push("generate-matrix must not wait for base-image-publication"); + if (!sameMembers(needs(matrix), ["base-image-publication"])) { + errors.push("generate-matrix must wait for complete managed-image publication"); } const matrixOutputs = matrix.outputs ?? {}; if ("dcode_base_contract" in matrixOutputs || "dcode_base_ref" in matrixOutputs) { @@ -718,6 +729,8 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): errors.push("live E2E must wait for matrix generation and base-image publication"); } if ( + live.env?.E2E_MANAGED_IMAGE_REVISION !== + "${{ needs.base-image-publication.outputs.managed_image_revision }}" || live.env?.NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF !== "${{ needs.base-image-publication.outputs.dcode_base_ref }}" ) { @@ -756,6 +769,54 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): return errors; } +const STOCK_ONBOARDING_JOBS = [ + "live", + "mcp-bridge", + "openshell-credential-generation-window", + "mcp-bridge-dev", + "hermes-e2e", + "hermes-gpu-startup", + "cloud-onboard", + "messaging-providers", +] as const; + +const STOCK_ONBOARDING_CATALOGUE_JOBS = [ + "catalogue-standard", + "catalogue-nvidia-api", + "catalogue-nvidia-inference", + "catalogue-github-read", + "catalogue-brave-nvidia-inference", +] as const; + +const MANAGED_IMAGE_REVISION_EXPRESSION = + "${{ needs.base-image-publication.outputs.managed_image_revision }}"; + +/** Require publication success and one exact cohort revision for every stock onboarding job. */ +export function validateStockOnboardingPublicationBoundary( + workflow: OperationsWorkflow, +): string[] { + const errors: string[] = []; + for (const jobName of STOCK_ONBOARDING_JOBS) { + const job = workflow.jobs[jobName] ?? {}; + if (!needs(job).includes("base-image-publication")) { + errors.push(`${jobName} must depend on base-image-publication before stock onboarding`); + } + if (job.env?.E2E_MANAGED_IMAGE_REVISION !== MANAGED_IMAGE_REVISION_EXPRESSION) { + errors.push(`${jobName} must receive the selected managed-image cohort revision`); + } + } + for (const jobName of STOCK_ONBOARDING_CATALOGUE_JOBS) { + const job = workflow.jobs[jobName] ?? {}; + if (!needs(job).includes("base-image-publication")) { + errors.push(`${jobName} must depend on base-image-publication before stock onboarding`); + } + if (job.with?.managed_image_revision !== MANAGED_IMAGE_REVISION_EXPRESSION) { + errors.push(`${jobName} must pass the selected managed-image cohort revision`); + } + } + return errors; +} + function validatePrGateEvidenceProducers(errors: string[], workflow: OperationsWorkflow): void { const requiredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); for (const jobId of requiredJobs) { @@ -1290,6 +1351,7 @@ export function validateE2eOperationsWorkflow( workflow as unknown as Record, ); errors.push(...validateBaseImagePublicationGate(workflow)); + errors.push(...validateStockOnboardingPublicationBoundary(workflow)); validateManualPrDispatch(errors, workflow); validatePrGateEvidenceProducers(errors, workflow); validateAggregation(errors, workflow); diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index 55da34c0179..7e5156c5723 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -129,8 +129,13 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi errors.push(`workflow is missing ${contract.job}`); continue; } - if (job.needs !== "generate-matrix" || job.uses !== PROFILE_WORKFLOW) { - errors.push(`${contract.job} must call the standard E2E profile after matrix generation`); + if ( + !isDeepStrictEqual(job.needs, ["base-image-publication", "generate-matrix"]) || + job.uses !== PROFILE_WORKFLOW + ) { + errors.push( + `${contract.job} must call the standard E2E profile after publication and matrix generation`, + ); } if (job.name !== "${{ matrix.display_name }}") { errors.push(`${contract.job} must use the planned outcome-first display name`); @@ -158,7 +163,8 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi risk_signal_correlation_id: "${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}", cli_artifact_provenance: "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}", - managed_image_catalog: "${{ needs.generate-matrix.outputs.managed_image_catalog }}", + managed_image_revision: + "${{ needs.base-image-publication.outputs.managed_image_revision }}", credential_boundary: contract.credentialBoundary, catalogue_id: "${{ matrix.id }}", target_id: "${{ matrix.target_id }}", @@ -206,7 +212,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi risk_signal_expected_sha: "string", risk_signal_correlation_id: "string", cli_artifact_provenance: "string", - managed_image_catalog: "string", + managed_image_revision: "string", credential_boundary: "string", catalogue_id: "string", target_id: "string", @@ -275,6 +281,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi const expectedJobEnv = { E2E_JOB: "1", E2E_TARGET_ID: "${{ inputs.target_id }}", + E2E_MANAGED_IMAGE_REVISION: "${{ inputs.managed_image_revision }}", NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.candidate_sha }}", NEMOCLAW_E2E_CORRELATION_ID: "${{ inputs.risk_signal_correlation_id }}", @@ -297,7 +304,6 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi "Install target host dependencies", "Prepare E2E workspace", "Restore exact-commit CLI artifact", - "Materialize temporary managed-image catalog", "Install reviewed cloudflared", "Add swap for Hermes image rebuild", "Initialize runner comparison telemetry", @@ -448,40 +454,6 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi ) { errors.push("standard E2E profile must restore the planned exact-commit CLI artifact"); } - const managedCatalog = requireStep( - errors, - workflowSteps, - "Materialize temporary managed-image catalog", - ); - const managedCatalogRun = String(managedCatalog?.run ?? ""); - if ( - managedCatalog?.if !== "${{ inputs.managed_image_catalog != '' }}" || - managedCatalog.shell !== EXECUTION_PLAN_SHELL || - !isDeepStrictEqual(record(managedCatalog.env), { - CANDIDATE_SHA: "${{ inputs.candidate_sha }}", - MANAGED_IMAGE_CATALOG: "${{ inputs.managed_image_catalog }}", - RESTORE_CLI: "${{ inputs.restore_cli && 'true' || 'false' }}", - }) || - !managedCatalogRun.includes(".source.revision == $revision") || - !managedCatalogRun.includes("[.[].source.release] | unique | length") || - !managedCatalogRun.includes("[.[].source.cohort] | unique | length") || - !managedCatalogRun.includes('[[ "$RESTORE_CLI" == "true" ]]') || - !managedCatalogRun.includes(".source.release == $release") || - !managedCatalogRun.includes( - "managed-image catalog source identity does not match the candidate", - ) || - !managedCatalogRun.includes( - "managed-image catalog release does not match the restored CLI", - ) || - !managedCatalogRun.includes("NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG") || - managedCatalogRun.includes("NEMOCLAW_E2E_EXACT_RELEASE") || - managedCatalogRun.includes(".source.release = $release") || - workflowSteps.indexOf(managedCatalog ?? {}) !== workflowSteps.indexOf(restore ?? {}) + 1 - ) { - errors.push( - "standard E2E profile must materialize only the exact-candidate managed-image catalog", - ); - } const cloudflared = requireStep(errors, workflowSteps, "Install reviewed cloudflared"); const cloudflaredRun = String(cloudflared?.run ?? ""); if ( @@ -499,7 +471,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi !cloudflaredRun.includes('dpkg-deb -f "${cloudflared_deb}" Package') || !cloudflaredRun.includes('"${architecture}" != "amd64"') || cloudflaredRun.includes("command -v cloudflared") || - workflowSteps.indexOf(cloudflared ?? {}) !== workflowSteps.indexOf(managedCatalog ?? {}) + 1 + workflowSteps.indexOf(cloudflared ?? {}) !== workflowSteps.indexOf(restore ?? {}) + 1 ) { errors.push("standard E2E profile must install only the reviewed cloudflared package"); } diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index ca841803cc5..af271502c92 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -258,7 +258,7 @@ export function validateTrustedHermesSwapWorkflow(workflowValue: unknown): strin continue; } - if (job.needs !== "generate-matrix") { + if (!isDeepStrictEqual(job.needs, ["base-image-publication", "generate-matrix"])) { errors.push(`${jobName} trusted Hermes swap job must depend on controller validation`); } if (provisionSteps.length !== 1) { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 570c026a9bf..997fac624ca 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1176,8 +1176,15 @@ function validateFreeStandingJobSelector( const job = asRecord(jobs[jobName]); const expectedNeeds = jobName === "mcp-bridge-dev" - ? ["generate-matrix", "openshell-dev-artifact"] - : "generate-matrix"; + ? ["base-image-publication", "generate-matrix", "openshell-dev-artifact"] + : [ + "mcp-bridge", + "openshell-credential-generation-window", + "cloud-onboard", + "messaging-providers", + ].includes(jobName) + ? ["base-image-publication", "generate-matrix"] + : "generate-matrix"; if (!isDeepStrictEqual(job.needs, expectedNeeds)) { errors.push(`${jobName} job must depend on generate-matrix`); } @@ -1630,8 +1637,8 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { return; } - if (job.needs !== "generate-matrix") { - errors.push("hermes-e2e job must depend on generate-matrix validation"); + if (!isDeepStrictEqual(job.needs, ["base-image-publication", "generate-matrix"])) { + errors.push("hermes-e2e job must depend on publication and generate-matrix validation"); } if (job.if !== "${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}") { errors.push("hermes-e2e job must use validated hermes_selected output"); @@ -1743,7 +1750,7 @@ function validateJetsonControllerBoundary(errors: string[], jobs: WorkflowRecord const publication = asRecord(jobs["base-image-publication"]); if ( asRecord(publication.outputs).managed_image_revision !== - "${{ steps.publication.outputs.head_sha || (steps.publication_mode.outputs.reuse == '1' && 'e38db201413b457614904187377ed9fd002d281d') || inputs.checkout_sha || github.sha }}" + "${{ steps.validate_managed_cohort.outputs.revision }}" ) { errors.push("base-image-publication must expose the managed-image revision to Jetson dispatch"); } @@ -2575,43 +2582,6 @@ function validateTrustedE2ePlannerBoundary( } } -function validateExactPrManagedImageCatalogBoundary( - errors: string[], - generateSteps: WorkflowRecord[], - generate: WorkflowRecord | undefined, - generateCheckout: WorkflowRecord | undefined, -): void { - const managedCatalog = requireStep( - errors, - generateSteps, - "Resolve exact PR managed-image catalog", - ); - if ( - managedCatalog?.if !== - "${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}" || - !isDeepStrictEqual(asRecord(managedCatalog?.env), { - BASE_SHA: "${{ inputs.base_sha }}", - CANDIDATE_REPOSITORY: "${{ inputs.checkout_repository }}", - CANDIDATE_SHA: "${{ inputs.checkout_sha }}", - GITHUB_TOKEN: "${{ github.token }}", - PR_NUMBER: "${{ inputs.pr_number }}", - }) || - managedCatalog?.run !== - 'node --experimental-strip-types --no-warnings tools/e2e/pr-managed-image-publication.mts "${RUNNER_TEMP}/pr-managed-image-catalog.json"' - ) { - errors.push("manual PR E2E must resolve the exact candidate managed-image publication"); - } - if ( - generate && - managedCatalog && - generateCheckout && - (generateSteps.indexOf(managedCatalog) <= generateSteps.indexOf(generate) || - generateSteps.indexOf(managedCatalog) >= generateSteps.indexOf(generateCheckout)) - ) { - errors.push("exact managed-image publication must resolve before candidate checkout"); - } -} - export function validateE2eWorkflow(workflowValue: unknown): string[] { const workflow = asRecord(workflowValue); const errors: string[] = []; @@ -2836,12 +2806,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); validateTrustedE2ePlannerBoundary(errors, generateSteps, generate, generateCheckout); - validateExactPrManagedImageCatalogBoundary( - errors, - generateSteps, - generate, - generateCheckout, - ); const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") { errors.push("matrix generation step must bind controller checkout through CHECKOUT_SHA env"); From 81d1ff9e34b6faa6addf9d69e47aad33109492b5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 09:57:25 -0500 Subject: [PATCH 02/37] fix(e2e): allow publication evidence validation --- .github/workflows/e2e.yaml | 2 +- tools/e2e/operations-workflow-boundary.mts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fefe21a5efe..58baf8697d8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -173,7 +173,7 @@ jobs: export GITHUB_SHA="$EXPECTED_SHA" wait_seconds=3000 if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then - wait_seconds=0 + wait_seconds=300 fi node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30 diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 6346502ac4c..a3cf1ef0b19 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -660,7 +660,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): 'export GITHUB_SHA="$EXPECTED_SHA"', "wait_seconds=3000", 'if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then', - " wait_seconds=0", + " wait_seconds=300", "fi", 'node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30', "", From e7b1c4187d7165f7579de1e493390197e77eae8d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 10:18:04 -0500 Subject: [PATCH 03/37] fix(e2e): bind public NVIDIA qualification key --- test/e2e/fixtures/inference-adapter.ts | 12 ++++++++---- test/e2e/support/inference-adapter.test.ts | 13 +++++++++++-- tools/e2e/target-catalogue.mts | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index ae8e9a02dbe..29393ae4d1e 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -27,9 +27,10 @@ import type { TestProgress, TestProgressCapability } from "./progress.ts"; * `mock` exposes an authenticated local compatible endpoint and stages only * `COMPATIBLE_API_KEY`; `internal-nvidia` stages the internal NVIDIA endpoint * as compatible inference and rejects endpoint overrides outside its static - * allowlist; `public-nvidia` uses the public NVIDIA provider and stages only - * `NVIDIA_INFERENCE_API_KEY`. Every mode registers its credential for artifact - * redaction and removes credentials owned by the other modes. + * allowlist; `public-nvidia` reads the public `NVIDIA_API_KEY` credential and + * stages it only under the runtime's historical `NVIDIA_INFERENCE_API_KEY` + * alias. Every mode registers its credential for artifact redaction and + * removes credentials owned by the other modes. * * Tests normally consume the `inference` fixture from `e2e-test.ts`, pass * `inference.env()` to install/onboard commands, use its model and provider @@ -77,6 +78,7 @@ export interface E2EInferenceAdapterOptions { const DEFAULT_MOCK_MODEL = "nvidia/nvidia/nemotron-3-ultra"; const DEFAULT_PUBLIC_NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; const DEFAULT_PUBLIC_NVIDIA_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const PUBLIC_NVIDIA_CREDENTIAL_ENV = "NVIDIA_API_KEY"; const DIRECT_CHAT_TIMEOUT_MS = 120_000; const INTERNAL_NVIDIA_ALLOWED_HOSTS = ["inference-api.nvidia.com"] as const; const MODEL_PROBE_TIMEOUT_MS = 30_000; @@ -434,7 +436,9 @@ export async function createE2EInferenceAdapter( artifacts: options.artifacts, }); } - const apiKey = requirePublicNvidiaInferenceKey(options.secrets.required(HOSTED_INFERENCE_SECRET)); + const apiKey = requirePublicNvidiaInferenceKey( + options.secrets.required(PUBLIC_NVIDIA_CREDENTIAL_ENV), + ); const model = env.NEMOCLAW_MODEL || DEFAULT_PUBLIC_NVIDIA_MODEL; return new PublicNvidiaInferenceAdapter({ apiKey, diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index cfd3133c5e1..588bd6bd109 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -331,7 +331,7 @@ describe("E2E inference adapter", () => { artifacts: artifactSink, env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" }, provider: provider((request) => requests.push(request)), - secrets: { NVIDIA_INFERENCE_API_KEY: apiKey }, + secrets: { NVIDIA_API_KEY: apiKey }, }); const env = adapter.env({ COMPATIBLE_API_KEY: "ambient-compatible-key", @@ -370,11 +370,20 @@ describe("E2E inference adapter", () => { await expect( createAdapter({ env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" }, - secrets: { NVIDIA_INFERENCE_API_KEY: "sk-compatible-key" }, + secrets: { NVIDIA_API_KEY: "sk-compatible-key" }, }), ).rejects.toThrow(/must start with nvapi-/); }); + it("does not treat the internal NVIDIA inference credential as public authority", async () => { + await expect( + createAdapter({ + env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" }, + secrets: { NVIDIA_INFERENCE_API_KEY: "nvapi-wrong-source" }, + }), + ).rejects.toThrow(/missing NVIDIA_API_KEY/); + }); + it("rejects unknown explicit modes instead of silently falling back", async () => { await expect( createAdapter({ env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvida" } }), diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index e6eb13360c6..4e10e72693a 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -1155,7 +1155,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ displayName: "Pi: qualifies managed runtime on Linux AMD64", agentRuntime: "pi", environmentOrInferenceEndpoint: "Linux AMD64 Docker; NVIDIA hosted inference", - profile: "nvidia-inference", + profile: "nvidia-api", testFile: "test/e2e/live/pi-agent-qualification.test.ts", timeoutMinutes: 100, installMode: "authenticated", @@ -1189,7 +1189,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ displayName: "Pi: qualifies managed runtime on Linux ARM64", agentRuntime: "pi", environmentOrInferenceEndpoint: "Linux ARM64 Docker; NVIDIA hosted inference", - profile: "nvidia-inference", + profile: "nvidia-api", testFile: "test/e2e/live/pi-agent-qualification.test.ts", timeoutMinutes: 100, installMode: "authenticated", From dd3bead5abbc1f4c62cbae048a20e8ccac87e21f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 10:30:18 -0500 Subject: [PATCH 04/37] fix(e2e): bind trusted cohort receipt --- .github/workflows/e2e.yaml | 4 +- test/e2e/live/mcp-bridge-onboard-env.ts | 107 ++++++++++++- test/e2e/live/mcp-bridge.test.ts | 5 +- .../managed-image-cohort-contract.test.ts | 25 ++- .../support/mcp-bridge-onboard-env.test.ts | 147 +++++++++++++++--- test/e2e/support/workload-source-env.test.ts | 7 +- tools/e2e/managed-image-cohort-contract.mts | 43 ++++- tools/e2e/mcp-workflow-boundary.mts | 8 + tools/e2e/operations-workflow-boundary.mts | 5 +- 9 files changed, 311 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 58baf8697d8..3d087c41dd5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -103,6 +103,7 @@ jobs: dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }} managed_image_artifact_provenance: ${{ steps.download_managed_cohort.outputs.provenance }} managed_image_cohort: ${{ steps.validate_managed_cohort.outputs.cohort }} + managed_image_receipt: ${{ steps.validate_managed_cohort.outputs.receipt }} managed_image_revision: ${{ steps.validate_managed_cohort.outputs.revision }} managed_image_run_attempt: ${{ steps.validate_managed_cohort.outputs.run_attempt }} managed_image_run_id: ${{ steps.validate_managed_cohort.outputs.run_id }} @@ -149,7 +150,7 @@ jobs: - name: Check out trusted E2E workflow uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.checkout_sha || github.sha }} + ref: ${{ github.workflow_sha }} fetch-depth: 0 persist-credentials: false @@ -3280,6 +3281,7 @@ jobs: coverage_variant: deepagents env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge" E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index a91152c8221..a07cb103f0b 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -1,6 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; + +import { + MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_REPOSITORIES, + parseManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ManagedImagePlatform, + type ShippedManagedImageAgent, +} from "../../../src/lib/onboard/managed-image/contract.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; const EXACT_MAIN_OVERLAY_KEYS = new Set([ @@ -12,6 +22,7 @@ const EXACT_MAIN_OVERLAY_KEYS = new Set([ const MCP_BRIDGE_QUALIFICATION_ENV_KEYS = [ "E2E_MANAGED_IMAGE_REVISION", + "E2E_MANAGED_IMAGE_COHORT_RECEIPT", "NEMOCLAW_E2E_EXPECTED_SHA", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG", "NEMOCLAW_RUN_LIVE_E2E", @@ -25,9 +36,7 @@ const MCP_BRIDGE_ONBOARD_ARGS = [ "--yes-i-accept-third-party-software", ] as const; -export function buildMcpBridgeOnboardArgs( - environment: NodeJS.ProcessEnv = process.env, -): string[] { +export function buildMcpBridgeOnboardArgs(environment: NodeJS.ProcessEnv = process.env): string[] { const catalogPath = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG?.trim(); return catalogPath ? [ @@ -42,24 +51,108 @@ export function buildMcpBridgeOnboardArgs( export function assertMcpBridgeManagedImageReceipt(options: { environment?: NodeJS.ProcessEnv; + expectedAgent: ShippedManagedImageAgent; workload?: Record; }): void { const environment = options.environment ?? process.env; const selectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim(); + const selectedReceipt = environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT?.trim(); const exactCandidateCatalog = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG?.trim(); if (!selectedRevision && !exactCandidateCatalog) return; - const expectedRevision = - selectedRevision ?? environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; + const expectedRevision = selectedRevision ?? environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { throw new Error("managed-image MCP qualification requires an exact cohort revision"); } + + const workloadPlatform = options.workload?.platform; + if ( + typeof workloadPlatform !== "string" || + !(MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(workloadPlatform) + ) { + throw new Error("managed-image MCP qualification requires an exact workload platform"); + } + const expectedPlatform = workloadPlatform as ManagedImagePlatform; + + let expectedReference: string; + let expectedCohort: string; + if (selectedRevision) { + if (!selectedReceipt || Buffer.byteLength(selectedReceipt, "utf8") > 8 * 1024) { + throw new Error("managed-image MCP qualification requires the selected cohort receipt"); + } + let receipt: Record; + try { + const parsed = JSON.parse(selectedReceipt) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); + receipt = parsed as Record; + } catch { + throw new Error("managed-image MCP qualification cohort receipt is invalid"); + } + const runId = receipt.runId; + const runAttempt = receipt.runAttempt; + const images = receipt.images; + const cohort = receipt.cohort; + if ( + JSON.stringify(Object.keys(receipt).sort()) !== + JSON.stringify(["cohort", "images", "kind", "revision", "runAttempt", "runId"]) || + receipt.kind !== "nemoclaw-managed-image-cohort-receipt-v1" || + receipt.revision !== expectedRevision || + !Number.isSafeInteger(runId) || + Number(runId) < 1 || + !Number.isSafeInteger(runAttempt) || + Number(runAttempt) < 1 || + cohort !== `ghrun-${String(runId)}-${String(runAttempt)}` || + !images || + typeof images !== "object" || + Array.isArray(images) || + JSON.stringify(Object.keys(images).sort()) !== + JSON.stringify([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()) + ) { + throw new Error("managed-image MCP qualification cohort receipt is invalid"); + } + const agentImages = (images as Record)[options.expectedAgent]; + if ( + !agentImages || + typeof agentImages !== "object" || + Array.isArray(agentImages) || + JSON.stringify(Object.keys(agentImages).sort()) !== + JSON.stringify([...MANAGED_IMAGE_PLATFORMS].sort()) + ) { + throw new Error("managed-image MCP qualification cohort receipt is invalid"); + } + expectedReference = (agentImages as Record)[expectedPlatform] as string; + expectedCohort = cohort; + } else { + let catalog: Record; + try { + const parsed = JSON.parse(fs.readFileSync(exactCandidateCatalog!, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); + catalog = parsed as Record; + } catch { + throw new Error("managed-image MCP qualification catalog is invalid"); + } + const contract = parseManagedImageContractV1( + catalog[options.expectedAgent], + options.expectedAgent, + expectedPlatform, + ); + if (contract.source.revision !== expectedRevision) { + throw new Error("managed-image MCP qualification catalog revision is invalid"); + } + expectedReference = contract.reference; + expectedCohort = contract.source.cohort; + } + if ( + typeof expectedReference !== "string" || + !expectedReference.startsWith(`${MANAGED_IMAGE_REPOSITORIES[options.expectedAgent]}@sha256:`) || options.workload?.kind !== "managed-image" || - options.workload.sourceRevision !== expectedRevision + options.workload.sourceRevision !== expectedRevision || + options.workload.sourceCohort !== expectedCohort || + options.workload.reference !== expectedReference ) { throw new Error( - "MCP qualification must use the exact managed image instead of a Dockerfile build", + "MCP qualification must use the exact agent image from the selected cohort receipt", ); } } diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 70e117f3dd8..d931dced453 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -102,11 +102,12 @@ function mcpBridgeShardTest(shard: McpBridgeShard) { const test = mcpBridgeShardTest("openclaw"); type McpAgent = "openclaw" | "hermes" | "langchain-deepagents-code"; -function expectManagedImageQualificationReceipt(sandboxName: string): void { +function expectManagedImageQualificationReceipt(sandboxName: string, agent: McpAgent): void { const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { sandboxes?: Record }>; }; assertMcpBridgeManagedImageReceipt({ + expectedAgent: agent, workload: registry.sandboxes?.[sandboxName]?.workload, }); } @@ -149,7 +150,7 @@ async function onboardAgent( }, ); expectExitZero(result, `onboard ${options.agent} sandbox for MCP bridge`); - expectManagedImageQualificationReceipt(options.sandboxName); + expectManagedImageQualificationReceipt(options.sandboxName, options.agent); } async function assertSecretAbsentFromSandbox( sandbox: SandboxClient, diff --git a/test/e2e/support/managed-image-cohort-contract.test.ts b/test/e2e/support/managed-image-cohort-contract.test.ts index 80efc634e8d..e6a78199714 100644 --- a/test/e2e/support/managed-image-cohort-contract.test.ts +++ b/test/e2e/support/managed-image-cohort-contract.test.ts @@ -88,7 +88,30 @@ describe("managed-image cohort publication contract", () => { runAttempt: RUN_ATTEMPT, runId: RUN_ID, }), - ).toEqual({ cohort: COHORT, revision: REVISION, runAttempt: RUN_ATTEMPT, runId: RUN_ID }); + ).toEqual({ + cohort: COHORT, + receipt: { + kind: "nemoclaw-managed-image-cohort-receipt-v1", + cohort: COHORT, + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + images: Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, agentIndex) => [ + agent, + Object.fromEntries( + PLATFORMS.map((platform, platformIndex) => [ + platform, + `${MANAGED_IMAGE_REPOSITORIES[agent]}@${digest(agentIndex + platformIndex + 4)}`, + ]), + ), + ]), + ), + }, + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + }); }); it("rejects a cohort that omits one image architecture", () => { diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index 7d39c62f055..3312020bcb3 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -1,8 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; +import { + MANAGED_IMAGE_REPOSITORIES, + SHIPPED_MANAGED_IMAGE_AGENTS, +} from "../../../src/lib/onboard/managed-image/contract.ts"; + import { assertMcpBridgeManagedImageReceipt, buildMcpBridgeExactMainEnv, @@ -19,6 +28,48 @@ const ONBOARD_OPTIONS = { endpointUrl: "https://inference.example.test/v1", sandboxName: "e2e-mcp-dcode", }; +const SELECTED_REVISION = "c".repeat(40); +const SELECTED_COHORT = "ghrun-123-4"; +const PLATFORM = "linux/amd64"; +const selectedReferences = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, agentIndex) => [ + agent, + Object.fromEntries( + ["linux/amd64", "linux/arm64"].map((platform, platformIndex) => [ + platform, + `${MANAGED_IMAGE_REPOSITORIES[agent]}@sha256:${String(agentIndex + platformIndex + 1).repeat(64)}`, + ]), + ), + ]), +) as Record>; + +function selectedEnvironment(): NodeJS.ProcessEnv { + return { + E2E_MANAGED_IMAGE_REVISION: SELECTED_REVISION, + E2E_MANAGED_IMAGE_COHORT_RECEIPT: JSON.stringify({ + kind: "nemoclaw-managed-image-cohort-receipt-v1", + cohort: SELECTED_COHORT, + revision: SELECTED_REVISION, + runAttempt: 4, + runId: 123, + images: selectedReferences, + }), + }; +} + +function selectedWorkload( + agent: keyof typeof MANAGED_IMAGE_REPOSITORIES, + overrides: Record = {}, +): Record { + return { + kind: "managed-image", + platform: PLATFORM, + reference: selectedReferences[agent][PLATFORM], + sourceCohort: SELECTED_COHORT, + sourceRevision: SELECTED_REVISION, + ...overrides, + }; +} describe("MCP bridge onboarding environment", () => { it("restores exact-main OpenShell overrides after child environment sanitization", () => { @@ -71,25 +122,23 @@ describe("MCP bridge onboarding environment", () => { it("rejects a Dockerfile workload in managed-image MCP qualification", () => { expect(() => assertMcpBridgeManagedImageReceipt({ - environment: { - NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), - NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: "/tmp/managed-pr-catalog.json", - }, - workload: { kind: "dockerfile" }, + environment: selectedEnvironment(), + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("langchain-deepagents-code", { kind: "dockerfile" }), }), - ).toThrow("must use the exact managed image instead of a Dockerfile build"); + ).toThrow("must use the exact agent image from the selected cohort receipt"); }); it("rejects a managed image from a different candidate revision", () => { expect(() => assertMcpBridgeManagedImageReceipt({ - environment: { - NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), - NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: "/tmp/managed-pr-catalog.json", - }, - workload: { kind: "managed-image", sourceRevision: "b".repeat(40) }, + environment: selectedEnvironment(), + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("langchain-deepagents-code", { + sourceRevision: "b".repeat(40), + }), }), - ).toThrow("must use the exact managed image instead of a Dockerfile build"); + ).toThrow("must use the exact agent image from the selected cohort receipt"); }); it("activates the exact managed runtime when the qualification catalog is present", () => { @@ -109,26 +158,82 @@ describe("MCP bridge onboarding environment", () => { }); it("accepts the exact managed image candidate revision", () => { - expect(() => - assertMcpBridgeManagedImageReceipt({ - environment: { - NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), - NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: "/tmp/managed-pr-catalog.json", + const revision = "a".repeat(40); + const cohort = "ghrun-77-2"; + const reference = `${MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"]}@sha256:${"d".repeat(64)}`; + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-catalog-")); + const catalogPath = path.join(fixtureRoot, "catalog.json"); + fs.writeFileSync( + catalogPath, + JSON.stringify({ + "langchain-deepagents-code": { + contractVersion: 1, + agent: "langchain-deepagents-code", + platform: PLATFORM, + image: MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"], + digest: `sha256:${"d".repeat(64)}`, + reference, + source: { repository: "NVIDIA/NemoClaw", revision, release: "v0.0.114", cohort }, + startupProfileContractVersion: 1, + capabilityContractVersion: 1, }, - workload: { kind: "managed-image", sourceRevision: "a".repeat(40) }, }), - ).not.toThrow(); + { mode: 0o600 }, + ); + try { + expect(() => + assertMcpBridgeManagedImageReceipt({ + environment: { + NEMOCLAW_E2E_EXPECTED_SHA: revision, + NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, + }, + expectedAgent: "langchain-deepagents-code", + workload: { + kind: "managed-image", + platform: PLATFORM, + reference, + sourceCohort: cohort, + sourceRevision: revision, + }, + }), + ).not.toThrow(); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } }); it("accepts the selected cross-release managed-image cohort revision", () => { expect(() => assertMcpBridgeManagedImageReceipt({ - environment: { E2E_MANAGED_IMAGE_REVISION: "c".repeat(40) }, - workload: { kind: "managed-image", sourceRevision: "c".repeat(40) }, + environment: selectedEnvironment(), + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("langchain-deepagents-code"), }), ).not.toThrow(); }); + it("rejects a different agent image from the selected cohort", () => { + expect(() => + assertMcpBridgeManagedImageReceipt({ + environment: selectedEnvironment(), + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("openclaw"), + }), + ).toThrow("must use the exact agent image from the selected cohort receipt"); + }); + + it("rejects a different publication cohort with the selected revision", () => { + expect(() => + assertMcpBridgeManagedImageReceipt({ + environment: selectedEnvironment(), + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("langchain-deepagents-code", { + sourceCohort: "ghrun-999-1", + }), + }), + ).toThrow("must use the exact agent image from the selected cohort receipt"); + }); + it("passes only exact-main OpenShell overrides after fixed onboarding values", () => { const env = buildMcpBridgeOnboardEnv({ ...ONBOARD_OPTIONS, diff --git a/test/e2e/support/workload-source-env.test.ts b/test/e2e/support/workload-source-env.test.ts index 12d0470b527..7b0c9b09e21 100644 --- a/test/e2e/support/workload-source-env.test.ts +++ b/test/e2e/support/workload-source-env.test.ts @@ -34,11 +34,8 @@ describe("live E2E workload source environment", () => { expect(resolveLiveE2eWorkloadSourceEnv(input)).toEqual(input); }); - it.each([ - "managed-image-protected-runtime", - "podman-native-cpu", - "mxc-runtime-proof", - ])("honors the provider-neutral managed-image source for %s", (targetId) => { + it("honors the provider-neutral managed-image source", () => { + const targetId = "managed-image-protected-runtime"; expect( resolveLiveE2eWorkloadSourceEnv({ E2E_TARGET_ID: targetId, diff --git a/tools/e2e/managed-image-cohort-contract.mts b/tools/e2e/managed-image-cohort-contract.mts index 55a308353d0..109603bb61e 100644 --- a/tools/e2e/managed-image-cohort-contract.mts +++ b/tools/e2e/managed-image-cohort-contract.mts @@ -8,6 +8,8 @@ import { fileURLToPath } from "node:url"; import { MANAGED_IMAGE_REPOSITORIES, SHIPPED_MANAGED_IMAGE_AGENTS, + type ManagedImagePlatform, + type ShippedManagedImageAgent, } from "../../src/lib/onboard/managed-image/contract.ts"; const REPOSITORY = "NVIDIA/NemoClaw"; @@ -19,11 +21,23 @@ type JsonRecord = Record; export interface ManagedImageCohortIdentity { readonly cohort: string; + readonly receipt: ManagedImageCohortReceipt; readonly revision: string; readonly runAttempt: number; readonly runId: number; } +export interface ManagedImageCohortReceipt { + readonly kind: "nemoclaw-managed-image-cohort-receipt-v1"; + readonly cohort: string; + readonly revision: string; + readonly runAttempt: number; + readonly runId: number; + readonly images: Readonly< + Record>> + >; +} + function record(value: unknown, label: string): JsonRecord { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`${label} must be a JSON object`); @@ -199,8 +213,35 @@ export function validateManagedImageCohort( } } + const images = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent) => { + const platforms = record( + record(agents[agent], `${agent} cohort contract`).platforms, + `${agent} cohort platforms`, + ); + return [ + agent, + Object.fromEntries( + PLATFORMS.map((platform) => [ + platform, + record(platforms[platform], `${agent} ${platform} publication`).reference, + ]), + ), + ]; + }), + ) as ManagedImageCohortReceipt["images"]; + const receipt: ManagedImageCohortReceipt = { + kind: "nemoclaw-managed-image-cohort-receipt-v1", + cohort: expectedCohort, + revision: expected.revision, + runAttempt: expected.runAttempt, + runId: expected.runId, + images, + }; + return { cohort: expectedCohort, + receipt, revision: expected.revision, runAttempt: expected.runAttempt, runId: expected.runId, @@ -225,7 +266,7 @@ export function main(argv = process.argv.slice(2), env = process.env): void { if (!env.GITHUB_OUTPUT) throw new Error("GITHUB_OUTPUT is required"); appendFileSync( env.GITHUB_OUTPUT, - `cohort=${identity.cohort}\nrevision=${identity.revision}\nrun_attempt=${identity.runAttempt}\nrun_id=${identity.runId}\n`, + `cohort=${identity.cohort}\nreceipt=${JSON.stringify(identity.receipt)}\nrevision=${identity.revision}\nrun_attempt=${identity.runAttempt}\nrun_id=${identity.runId}\n`, "utf8", ); } diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index fce9ec3d3dc..c09784cbdfe 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -173,6 +173,14 @@ function validateJobIdentity( "${{ needs.base-image-publication.outputs.managed_image_revision }}", `${jobName} must receive the selected managed-image cohort revision`, ); + if (jobName === "mcp-bridge") { + requireEqual( + errors, + env.E2E_MANAGED_IMAGE_COHORT_RECEIPT, + "${{ needs.base-image-publication.outputs.managed_image_receipt }}", + "mcp-bridge must receive the complete selected managed-image cohort receipt", + ); + } requireEqual( errors, job["timeout-minutes"], diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a3cf1ef0b19..a78ed18aea1 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -488,7 +488,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const trustedPublicationCheckout = jobName === "base-image-publication" && step.name === "Check out trusted E2E workflow" && - step.with?.ref === "${{ inputs.checkout_sha || github.sha }}"; + step.with?.ref === "${{ github.workflow_sha }}"; const trustedManagedImageRuntimeCheckout = jobName === "managed-image-protected-runtime" && step.name === "Checkout trusted protected runtime qualification" && @@ -601,6 +601,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): managed_image_artifact_provenance: "${{ steps.download_managed_cohort.outputs.provenance }}", managed_image_cohort: "${{ steps.validate_managed_cohort.outputs.cohort }}", + managed_image_receipt: "${{ steps.validate_managed_cohort.outputs.receipt }}", managed_image_revision: "${{ steps.validate_managed_cohort.outputs.revision }}", managed_image_run_attempt: "${{ steps.validate_managed_cohort.outputs.run_attempt }}", @@ -629,7 +630,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): name: "Check out trusted E2E workflow", uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", with: { - ref: "${{ inputs.checkout_sha || github.sha }}", + ref: "${{ github.workflow_sha }}", "fetch-depth": 0, "persist-credentials": false, }, From 7aed52b3f7d483ad7e9d5fc16e506c3a477967d8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 10:53:34 -0500 Subject: [PATCH 05/37] fix(e2e): repair exact runtime lifecycle probes --- .../onboard/machine/finalization-deps.test.ts | 15 ++--- src/lib/onboard/machine/finalization-deps.ts | 44 ++++++++++---- src/lib/onboard/messaging-prep.test.ts | 25 ++++++++ src/lib/onboard/messaging-prep.ts | 12 ++-- test/e2e/live/messaging-providers-helpers.ts | 57 ++++++++++++++++++- test/e2e/live/messaging-providers.test.ts | 35 ++++++++++-- test/e2e/live/openclaw-pairing-helpers.ts | 26 +++++---- test/e2e/live/openclaw-slack-pairing.test.ts | 12 +++- ...shell-credential-generation-window.test.ts | 49 ++++++++++------ .../openclaw-discord-pairing-helpers.test.ts | 18 +++--- 10 files changed, 226 insertions(+), 67 deletions(-) diff --git a/src/lib/onboard/machine/finalization-deps.test.ts b/src/lib/onboard/machine/finalization-deps.test.ts index 9cadcb507ea..babf925c4d7 100644 --- a/src/lib/onboard/machine/finalization-deps.test.ts +++ b/src/lib/onboard/machine/finalization-deps.test.ts @@ -113,7 +113,7 @@ describe("ordinary OpenClaw pairing settlement", () => { expect(scope.deps.runApproval).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); }); - it("runs the canonical request probe before waiting for fresh pairing (#10014)", async () => { + it("waits for canonical pairing before running the one request producer (#10014)", async () => { const observePairing = vi .fn(() => SCOPE_UPGRADE_PENDING) .mockImplementationOnce(() => { @@ -135,7 +135,7 @@ describe("ordinary OpenClaw pairing settlement", () => { kind: "settled", }); - expect(scope.calls).toEqual(["warmup", "sleep", "approval"]); + expect(scope.calls).toEqual(["warmup", "approval"]); expect(scope.deps.runWarmup).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); expect(scope.deps.runApproval).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); }); @@ -416,7 +416,7 @@ describe("ordinary OpenClaw pairing settlement", () => { reason: "scope-upgrade-incomplete", }); expect(scope.deps.sleep).toHaveBeenCalledTimes(39); - expect(scope.deps.runWarmup).toHaveBeenCalledOnce(); + expect(scope.deps.runWarmup).not.toHaveBeenCalled(); expect(scope.deps.runApproval).toHaveBeenCalledOnce(); }); @@ -436,6 +436,7 @@ describe("ordinary OpenClaw pairing settlement", () => { .mockImplementationOnce(() => { throw new Error("not pending"); }) + .mockReturnValueOnce(PAIRING_ONLY) .mockReturnValueOnce(SCOPE_UPGRADE_PENDING) .mockReturnValue(SETTLED), runWarmup: vi.fn(() => { @@ -452,7 +453,7 @@ describe("ordinary OpenClaw pairing settlement", () => { expect(scope.deps.runWarmup).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); expect(scope.deps.runApproval).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); - expect(scope.deps.observePairing).toHaveBeenCalledTimes(4); + expect(scope.deps.observePairing).toHaveBeenCalledTimes(5); expect(now).toBe( OPENCLAW_PAIRING_OBSERVATION_TIMEOUT_MS + OPENCLAW_ONBOARDING_PAIRING_TIMEOUT_MS - @@ -489,11 +490,11 @@ describe("ordinary OpenClaw pairing settlement", () => { reason: "pairing-unavailable", }); expect(scope.deps.sleep).not.toHaveBeenCalled(); - expect(scope.deps.runWarmup).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); + expect(scope.deps.runWarmup).not.toHaveBeenCalled(); expect(scope.deps.runApproval).not.toHaveBeenCalled(); }); - it("performs one request-producer write when a canonical CLI pairing never appears (#9844)", async () => { + it("does not run a request producer without a canonical CLI pairing (#9844)", async () => { const scope = ordinaryPairingDeps({ observePairing: vi.fn(() => { throw new Error("not published"); @@ -505,7 +506,7 @@ describe("ordinary OpenClaw pairing settlement", () => { reason: "pairing-unavailable", }); - expect(scope.deps.runWarmup).toHaveBeenCalledExactlyOnceWith("alpha", "nemoclaw"); + expect(scope.deps.runWarmup).not.toHaveBeenCalled(); expect(scope.deps.runApproval).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/machine/finalization-deps.ts b/src/lib/onboard/machine/finalization-deps.ts index 55d323f53ea..c2f382c618a 100644 --- a/src/lib/onboard/machine/finalization-deps.ts +++ b/src/lib/onboard/machine/finalization-deps.ts @@ -222,7 +222,7 @@ export async function settleOrdinaryOpenClawPairing( ); sawCanonicalPairing = true; } catch { - // Pairing may not have appeared yet; the producer handles that path. + // Pairing may not have appeared yet; the bounded observer handles that path. } if (!samePairingTarget(target, deps.getTarget(name))) { return { kind: "incomplete", reason: "runtime-identity-invalid" }; @@ -232,14 +232,36 @@ export async function settleOrdinaryOpenClawPairing( } if (initial?.state === "settled") return { kind: "settled" }; - let baseline: PairingWaitResult; - if (initial?.state === "scope-upgrade-pending") { - baseline = { kind: "observed", value: initial }; - } else { - // A valid non-interactive path can reach finalization before the - // startup watcher publishes its first CLI request. Run the bounded - // direct producer once, then require canonical evidence that its - // exact write upgrade is pending before the approval pass (#10014). + const pairingAppearanceDeadline = Math.min( + settlementDeadline, + deps.now() + OPENCLAW_ONBOARDING_PAIRING_TIMEOUT_MS, + ); + let baseline: PairingWaitResult = initial + ? { kind: "observed", value: initial } + : await waitForPairingObservation( + name, + target, + pairingAppearanceDeadline, + (value) => { + sawCanonicalPairing = true; + return true; + }, + deps, + ); + if (baseline.kind === "target-changed") { + return { kind: "incomplete", reason: "runtime-identity-invalid" }; + } + if (baseline.kind === "timeout") { + return { kind: "incomplete", reason: "pairing-unavailable" }; + } + if (baseline.value.state === "settled") return { kind: "settled" }; + + if (baseline.value.state === "pairing-only") { + // Wait for the startup watcher to publish the canonical CLI + // pairing before issuing the one bounded write-scope producer. + // Running the producer before that identity exists can create no + // upgrade and leaves an otherwise healthy fresh onboard stuck at + // pairing-only (#10014). try { await deps.runWarmup(name, target.gatewayName); } catch { @@ -248,14 +270,14 @@ export async function settleOrdinaryOpenClawPairing( if (!samePairingTarget(target, deps.getTarget(name))) { return { kind: "incomplete", reason: "runtime-identity-invalid" }; } - const pairingAppearanceDeadline = Math.min( + const scopeUpgradeDeadline = Math.min( settlementDeadline, deps.now() + OPENCLAW_ONBOARDING_PAIRING_TIMEOUT_MS, ); baseline = await waitForPairingObservation( name, target, - pairingAppearanceDeadline, + scopeUpgradeDeadline, (value) => { sawCanonicalPairing = true; return value.state !== "pairing-only"; diff --git a/src/lib/onboard/messaging-prep.test.ts b/src/lib/onboard/messaging-prep.test.ts index 015fda9710d..875e5303072 100644 --- a/src/lib/onboard/messaging-prep.test.ts +++ b/src/lib/onboard/messaging-prep.test.ts @@ -83,6 +83,31 @@ describe("prepareCreateSandboxMessaging", () => { ); }); + it("reattaches an exact durable provider when rebuild resumes without channel prompts", () => { + const providerMatchesGatewayCredential = vi.fn( + (name: string, type: string, credentialKey: string) => + name === "demo-discord-bridge" && + type === "nemoclaw-mcp-v1" && + credentialKey === "DISCORD_BOT_TOKEN", + ); + + const result = prepareCreateSandboxMessaging( + createInput({ + enabledChannels: null, + requireExactProviderBinding: true, + providerMatchesGatewayCredential, + }), + ); + + expect(result.reusableMessagingProviders).toContain("demo-discord-bridge"); + expect(result.reusableMessagingChannels).toContain("discord"); + expect(providerMatchesGatewayCredential).toHaveBeenCalledWith( + "demo-discord-bridge", + "nemoclaw-mcp-v1", + "DISCORD_BOT_TOKEN", + ); + }); + it("reuses an existing gateway bridge provider when the bridge secret is not resolvable", () => { // Deferred rebuild in a fresh process: the pasted secret is env-only and // gone, so no bridge token def exists — but the gateway still durably diff --git a/src/lib/onboard/messaging-prep.ts b/src/lib/onboard/messaging-prep.ts index 2e52abe84cd..cdf40a51b00 100644 --- a/src/lib/onboard/messaging-prep.ts +++ b/src/lib/onboard/messaging-prep.ts @@ -192,7 +192,7 @@ export function prepareCreateSandboxMessaging( : []; const reusableMessagingChannels: string[] = []; - if (input.enabledChannels != null) { + if (input.enabledChannels != null || input.requireExactProviderBinding === true) { for (const { name, envKey, @@ -203,7 +203,11 @@ export function prepareCreateSandboxMessaging( const channel = input.getMessagingChannelForEnvKey(envKey); if (!channel) continue; const channelDisabled = disabledChannelNames.has(channel); - if (!input.enabledChannels.includes(channel) && !(channelDisabled && retainWhileDisabled)) { + if ( + input.enabledChannels != null && + !input.enabledChannels.includes(channel) && + !(channelDisabled && retainWhileDisabled) + ) { continue; } if (channelDisabled && !retainWhileDisabled) continue; @@ -231,10 +235,10 @@ export function prepareCreateSandboxMessaging( // The name carries the channel but not the agent, and onboard can recreate a // sandbox name under a different agent, so match the gateway binding against // the selected profile rather than accepting any provider with that name. - if (input.enabledChannels != null) { + if (input.enabledChannels != null || input.requireExactProviderBinding === true) { for (const profile of bridgeProfiles) { const channel = profile.channelId; - if (!input.enabledChannels.includes(channel)) continue; + if (input.enabledChannels != null && !input.enabledChannels.includes(channel)) continue; if (disabledChannelNames.has(channel)) continue; for (const name of bridgeProviderNamesForChannel(input.sandboxName, channel, [profile])) { if (messagingTokenDefs.some((def) => def.name === name && def.token)) continue; diff --git a/test/e2e/live/messaging-providers-helpers.ts b/test/e2e/live/messaging-providers-helpers.ts index 5cfdfc289d3..ee40dc418d4 100644 --- a/test/e2e/live/messaging-providers-helpers.ts +++ b/test/e2e/live/messaging-providers-helpers.ts @@ -92,6 +92,7 @@ export { shellQuote }; export type FakeDockerApi = { kind: string; port: string; + alternatePort?: string; dir: string; captureFile: string; container: string; @@ -319,11 +320,13 @@ export async function runHost( artifactName: string; env: NodeJS.ProcessEnv; redactionValues: string[]; + cwd?: string; timeoutMs?: number; }, ): Promise { return host.command(command, args, { artifactName: options.artifactName, + cwd: options.cwd, env: options.env, redactionValues: options.redactionValues, timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS, @@ -602,6 +605,9 @@ export async function startFakeDockerApi( "-e", `${options.captureFileEnv}=/tmp/fake/capture.jsonl`, ]; + if (options.kind === "slack") { + dockerArgs.splice(7, 0, "-p", "0:8080"); + } for (const [key, value] of Object.entries(options.expectedEnv)) { dockerArgs.push("-e", `${key}=${value}`); } @@ -647,9 +653,24 @@ export async function startFakeDockerApi( redactionValues: options.redactionValues, timeoutMs: 30_000, }); - const published = port.stdout.trim().split(":").at(-1)?.trim(); - if (published) { - return { kind: options.kind, port: published, dir, captureFile, container }; + const published = [ + ...new Set( + port.stdout + .trim() + .split(/\r?\n/u) + .map((line) => line.split(":").at(-1)?.trim() ?? "") + .filter(Boolean), + ), + ]; + if (published.length >= (options.kind === "slack" ? 2 : 1)) { + return { + kind: options.kind, + port: published[0], + ...(options.kind === "slack" ? { alternatePort: published[1] } : {}), + dir, + captureFile, + container, + }; } } await sleep(100); @@ -663,6 +684,7 @@ export async function applyRestRewritePolicy( api: FakeDockerApi, env: NodeJS.ProcessEnv, redactionValues: string[], + providerName?: string, ): Promise { const result = await runHost( host, @@ -691,6 +713,35 @@ export async function applyRestRewritePolicy( }, ); expectExitZero(result, `apply ${api.kind} fake REST policy`); + if (!providerName) return; + + const binding = await runHost( + host, + "bash", + [ + "-lc", + String.raw`set -eu +policy_file="$(mktemp)" +trap 'rm -f "$policy_file"' EXIT +"$1" policy get --base "$2" >"$policy_file" +node --import tsx "$5" "$policy_file" "$3" host.openshell.internal "$4" rest +"$1" policy set --policy "$policy_file" --wait "$2"`, + `bind-fake-${api.kind}-rest-policy`, + host.openshellCommandPath, + SANDBOX_NAME, + providerName, + api.port, + path.join(REPO_ROOT, "test/e2e/fixtures/hermes-discord-policy-binding.ts"), + ], + { + artifactName: `apply-${api.kind}-rest-policy-credential-binding`, + cwd: REPO_ROOT, + env, + redactionValues, + timeoutMs: 120_000, + }, + ); + expectExitZero(binding, `bind ${api.kind} fake REST policy credential`); } export async function applyWebSocketRewritePolicy( diff --git a/test/e2e/live/messaging-providers.test.ts b/test/e2e/live/messaging-providers.test.ts index 2ed5a1a4690..eadbae996e5 100644 --- a/test/e2e/live/messaging-providers.test.ts +++ b/test/e2e/live/messaging-providers.test.ts @@ -12,7 +12,7 @@ import fs from "node:fs"; import { testTimeoutOptions } from "../../helpers/timeouts"; -import { test } from "../fixtures/e2e-test.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { accountBool, @@ -837,7 +837,28 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); env: state.env, redactionValues, }); - await applyRestRewritePolicy(host, fakeSlack, state.env, redactionValues); + await applyRestRewritePolicy( + host, + fakeSlack, + state.env, + redactionValues, + `${SANDBOX_NAME}-slack-bridge`, + ); + expect( + fakeSlack.alternatePort, + "fake Slack API must publish an independent app-token port", + ).toMatch(/^[1-9][0-9]*$/u); + const fakeSlackApp = { + ...fakeSlack, + port: fakeSlack.alternatePort!, + }; + await applyRestRewritePolicy( + host, + fakeSlackApp, + state.env, + redactionValues, + `${SANDBOX_NAME}-slack-app`, + ); const slackAuth = await runSlackApiRequest( sandbox, @@ -889,7 +910,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); const slackApp = await runSlackApiRequest( sandbox, - fakeSlack.port, + fakeSlackApp.port, "/api/apps.connections.open", "Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", redactionValues, @@ -963,7 +984,13 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); env: state.env, redactionValues, }); - await applyRestRewritePolicy(host, fakeTelegram, state.env, redactionValues); + await applyRestRewritePolicy( + host, + fakeTelegram, + state.env, + redactionValues, + `${SANDBOX_NAME}-telegram-bridge`, + ); const telegramMockTarget = "42424242"; const telegramMockText = "NemoClaw OpenClaw Telegram plugin mock E2E"; const installedTelegramProof = await runInstalledTelegramRuntimeProof( diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index 9a6307bcd8c..d1c21d0d424 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -382,10 +382,10 @@ NODE // Slack probe delegates Socket Mode/REST traffic to a shared fake-provider client // instead of hand-rolled sockets. export const SLACK_PROBE_INPUT_VALIDATION_SOURCE = String.raw` -function parseFakeSlackPort() { - const raw = process.env.FAKE_SLACK_API_PORT || ""; +function parseFakeSlackPort(name) { + const raw = process.env[name] || ""; const port = Number(raw); - if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("FAKE_SLACK_API_PORT must be an integer in 1..65535"); + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(name + " must be an integer in 1..65535"); return port; } function parseProxyTarget() { @@ -410,13 +410,14 @@ set -eu set -a [ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh set +a -fake_slack_api_port="$1" -slack_pairing_user="$2" +fake_slack_rest_port="$1" +fake_slack_websocket_port="$2" +slack_pairing_user="$3" : "${"$"}{OPENCLAW_HOME:?OPENCLAW_HOME missing}" : "${"$"}{OPENCLAW_STATE_DIR:?OPENCLAW_STATE_DIR missing}" : "${"$"}{OPENCLAW_CONFIG_PATH:?OPENCLAW_CONFIG_PATH missing}" : "${"$"}{OPENCLAW_OAUTH_DIR:?OPENCLAW_OAUTH_DIR missing}" -exec env HOME=/sandbox PATH="/usr/local/bin:/usr/bin:/bin:${"$"}{PATH:-}" OPENCLAW_HOME="$OPENCLAW_HOME" OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" HTTP_PROXY="${"$"}{HTTP_PROXY:-}" HTTPS_PROXY="${"$"}{HTTPS_PROXY:-}" http_proxy="${"$"}{http_proxy:-}" https_proxy="${"$"}{https_proxy:-}" NO_PROXY="${"$"}{NO_PROXY:-}" no_proxy="${"$"}{no_proxy:-}" NODE_OPTIONS="${"$"}{NODE_OPTIONS:-}" FAKE_SLACK_API_HOST="host.openshell.internal" FAKE_SLACK_API_PORT="$fake_slack_api_port" SLACK_PAIRING_USER="$slack_pairing_user" node --input-type=module <<'NODE' +exec env HOME=/sandbox PATH="/usr/local/bin:/usr/bin:/bin:${"$"}{PATH:-}" OPENCLAW_HOME="$OPENCLAW_HOME" OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" HTTP_PROXY="${"$"}{HTTP_PROXY:-}" HTTPS_PROXY="${"$"}{HTTPS_PROXY:-}" http_proxy="${"$"}{http_proxy:-}" https_proxy="${"$"}{https_proxy:-}" NO_PROXY="${"$"}{NO_PROXY:-}" no_proxy="${"$"}{no_proxy:-}" NODE_OPTIONS="${"$"}{NODE_OPTIONS:-}" FAKE_SLACK_API_HOST="host.openshell.internal" FAKE_SLACK_REST_PORT="$fake_slack_rest_port" FAKE_SLACK_WEBSOCKET_PORT="$fake_slack_websocket_port" SLACK_PAIRING_USER="$slack_pairing_user" node --input-type=module <<'NODE' __LOAD_CONVERSATION_RUNTIME_SOURCE__ import crypto from "node:crypto"; import http from "node:http"; @@ -454,7 +455,7 @@ function decodeServerFrame(buffer) { } function receiveSlackSocketEvent() { const host = "host.openshell.internal"; - const port = parseFakeSlackPort(); + const port = parseFakeSlackPort("FAKE_SLACK_WEBSOCKET_PORT"); const proxy = parseProxyTarget(); return new Promise((resolve, reject) => { const socket = proxy ? net.createConnection({ host: proxy.host, port: proxy.port }) : net.createConnection({ host, port }); @@ -512,7 +513,7 @@ function receiveSlackSocketEvent() { } function postPairingReply(text, channel) { const host = "host.openshell.internal"; - const port = parseFakeSlackPort(); + const port = parseFakeSlackPort("FAKE_SLACK_REST_PORT"); const token = "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"; const data = new URLSearchParams({ token, channel, text }).toString(); return new Promise((resolve, reject) => { @@ -597,12 +598,17 @@ export async function issuePairingRequest(options: { sandboxName: string; channel: PairingChannel; redactions: string[]; - fakeSlackPort?: string; + fakeSlackRestPort?: string; + fakeSlackWebsocketPort?: string; }): Promise { const script = options.channel === "slack" ? SLACK_PAIRING_SCRIPT : DISCORD_PAIRING_SCRIPT; const args = options.channel === "slack" - ? [options.fakeSlackPort ?? "", PAIRING_USER.slack] + ? [ + options.fakeSlackRestPort ?? "", + options.fakeSlackWebsocketPort ?? "", + PAIRING_USER.slack, + ] : [PAIRING_USER.discord, DISCORD_DM_CHANNEL]; return sandboxShWithArgs(options.sandbox, options.sandboxName, script, args, { artifactName: `${options.channel}-issue-pairing-request`, diff --git a/test/e2e/live/openclaw-slack-pairing.test.ts b/test/e2e/live/openclaw-slack-pairing.test.ts index 195cea5b89f..2431d682ed7 100644 --- a/test/e2e/live/openclaw-slack-pairing.test.ts +++ b/test/e2e/live/openclaw-slack-pairing.test.ts @@ -173,10 +173,17 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap redactions, artifactName: "apply-slack-rest-policy", }); + const websocketPort = fakeSlack.alternatePort ?? ""; + expect(websocketPort, "fake Slack API must publish an independent websocket port").toMatch( + /^[1-9][0-9]*$/u, + ); await applyFakePolicy({ host, sandboxName: SANDBOX_NAME, - api: fakeSlack, + api: { + ...fakeSlack, + port: websocketPort, + }, protocol: "websocket", rewrite: "websocket-credential-rewrite", providerName: `${SANDBOX_NAME}-slack-app`, @@ -191,7 +198,8 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap sandboxName: SANDBOX_NAME, channel: "slack", redactions, - fakeSlackPort: fakeSlack.port, + fakeSlackRestPort: fakeSlack.port, + fakeSlackWebsocketPort: websocketPort, }); expectExitZero(issue, "Slack pairing request creation"); const code = extractPairingCode(resultText(issue), "PAIRING_E2E_RESULT"); diff --git a/test/e2e/live/openshell-credential-generation-window.test.ts b/test/e2e/live/openshell-credential-generation-window.test.ts index bbc9b811aae..abb3720373a 100644 --- a/test/e2e/live/openshell-credential-generation-window.test.ts +++ b/test/e2e/live/openshell-credential-generation-window.test.ts @@ -329,8 +329,8 @@ test("openshell-credential-generation-window", { "attach the MCP provider and observe its initial generation", "prove a retained credential generation expires", "rotate beyond the retained generation window", - "prove key removal and provider detach revoke access", - "restart the bridge and confirm old-process fallback", + "prove key and bridge removal revoke access", + "re-add the bridge and confirm old-process fallback", "rebuild the sandbox and confirm credential reuse", "remove the MCP bridge and audit denied requests", ], @@ -677,7 +677,7 @@ test("openshell-credential-generation-window", { placeholderAbsent: true, }); - progress.phase("prove key removal and provider detach revoke access"); + progress.phase("prove key and bridge removal revoke access"); await updateProviderCredential( sandbox, providerName, @@ -736,16 +736,15 @@ test("openshell-credential-generation-window", { placeholderAbsent: true, }); - const detach = await sandbox.openshell( - ["sandbox", "provider", "detach", SANDBOX_NAME, providerName], + const removeBeforeReadd = await host.nemoclaw( + [SANDBOX_NAME, "mcp", "remove", SERVER_NAME], { - artifactName: "credential-window-direct-provider-detach", - env: openshellEnv(), - timeoutMs: 90_000, + artifactName: "credential-window-remove-before-readd", + env: buildAvailabilityProbeEnv(), + timeoutMs: 4 * 60_000, }, ); - expectExitZero(detach, "detach credential-window provider"); - expect(resultText(detach)).toMatch(/Detached provider/iu); + expectExitZero(removeBeforeReadd, "remove credential-window bridge before re-add"); await expectFreshCredentialAbsent( sandbox, "credential-window-fresh-credential-absent-after-detach", @@ -765,14 +764,30 @@ test("openshell-credential-generation-window", { ).seen, ).toBe(false); - progress.phase("restart the bridge and confirm old-process fallback"); - await rotateCredential( - host, - fakeMcp, - restartSecret, - CREDENTIAL_WINDOW_ROTATION_COUNT + 1, - allSecrets, + progress.phase("re-add the bridge and confirm old-process fallback"); + fakeMcp.setSecret(restartSecret); + const readd = await host.nemoclaw( + [ + SANDBOX_NAME, + "mcp", + "add", + SERVER_NAME, + "--url", + tunnel.url, + "--env", + CREDENTIAL_WINDOW_ENV_NAME, + ], + { + artifactName: "credential-window-readd-after-removal", + env: { + ...buildAvailabilityProbeEnv(), + [CREDENTIAL_WINDOW_ENV_NAME]: restartSecret, + }, + redactionValues: [...allSecrets], + timeoutMs: 4 * 60_000, + }, ); + expectExitZero(readd, "re-add credential-window bridge"); restartedRevision = await observeFreshRevision( sandbox, "credential-window-fresh-revision-after-restart", diff --git a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts index 7fdc8ed5c3a..1b874ce560d 100644 --- a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts +++ b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts @@ -220,37 +220,37 @@ describe("OpenClaw Discord pairing helper contracts", () => { it.each([ { name: "missing fake port", - env: { FAKE_SLACK_API_PORT: "", HTTP_PROXY: "", http_proxy: "" }, - error: "FAKE_SLACK_API_PORT must be an integer in 1..65535", + env: { FAKE_SLACK_REST_PORT: "", HTTP_PROXY: "", http_proxy: "" }, + error: "FAKE_SLACK_REST_PORT must be an integer in 1..65535", }, { name: "out-of-range fake port", - env: { FAKE_SLACK_API_PORT: "70000", HTTP_PROXY: "", http_proxy: "" }, - error: "FAKE_SLACK_API_PORT must be an integer in 1..65535", + env: { FAKE_SLACK_REST_PORT: "70000", HTTP_PROXY: "", http_proxy: "" }, + error: "FAKE_SLACK_REST_PORT must be an integer in 1..65535", }, { name: "malformed proxy", - env: { FAKE_SLACK_API_PORT: "12345", HTTP_PROXY: "http://[", http_proxy: "" }, + env: { FAKE_SLACK_REST_PORT: "12345", HTTP_PROXY: "http://[", http_proxy: "" }, error: "HTTP proxy for Slack pairing probe is malformed", }, { name: "non-HTTP proxy", - env: { FAKE_SLACK_API_PORT: "12345", HTTP_PROXY: "socks5://127.0.0.1:1080", http_proxy: "" }, + env: { FAKE_SLACK_REST_PORT: "12345", HTTP_PROXY: "socks5://127.0.0.1:1080", http_proxy: "" }, error: "Slack pairing probe only supports HTTP proxies", }, { name: "invalid proxy port", - env: { FAKE_SLACK_API_PORT: "12345", HTTP_PROXY: "http://127.0.0.1:70000", http_proxy: "" }, + env: { FAKE_SLACK_REST_PORT: "12345", HTTP_PROXY: "http://127.0.0.1:70000", http_proxy: "" }, error: "HTTP proxy for Slack pairing probe is malformed", }, { name: "unexpected valid proxy host", - env: { FAKE_SLACK_API_PORT: "12345", HTTP_PROXY: "http://127.0.0.1:3128", http_proxy: "" }, + env: { FAKE_SLACK_REST_PORT: "12345", HTTP_PROXY: "http://127.0.0.1:3128", http_proxy: "" }, error: "unexpected HTTP proxy for Slack pairing probe", }, ])("fails closed on invalid Slack probe input before network access: $name", ({ env, error }) => { const result = spawnSync(process.execPath, ["--input-type=module"], { - input: `${SLACK_PROBE_INPUT_VALIDATION_SOURCE}\nlet networkAttempted = false;\ntry { parseFakeSlackPort(); parseProxyTarget(); networkAttempted = true; } catch (error) { console.error(error.message); console.error("NETWORK_ATTEMPTED=" + networkAttempted); process.exit(1); }\n`, + input: `${SLACK_PROBE_INPUT_VALIDATION_SOURCE}\nlet networkAttempted = false;\ntry { parseFakeSlackPort("FAKE_SLACK_REST_PORT"); parseProxyTarget(); networkAttempted = true; } catch (error) { console.error(error.message); console.error("NETWORK_ATTEMPTED=" + networkAttempted); process.exit(1); }\n`, encoding: "utf8", env: { ...process.env, ...env }, }); From 202c292c9d3e1f1c9a8ac55a29f35771dfc7dfe0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 11:09:42 -0500 Subject: [PATCH 06/37] fix(e2e): enforce complete cohort receipts --- .github/workflows/e2e-standard-profile.yaml | 4 + .github/workflows/e2e.yaml | 12 +++ docs/deployment/sandbox-hardening.mdx | 2 +- docs/get-started/quickstart-hermes.mdx | 3 +- .../quickstart-langchain-deepagents-code.mdx | 3 +- docs/get-started/quickstart.mdx | 3 +- docs/reference/architecture.mdx | 2 +- test/e2e/fixtures/availability-env.ts | 1 + test/e2e/fixtures/inference-adapter.ts | 4 +- test/e2e/fixtures/managed-image-receipt.ts | 81 ++++++++++++++++ test/e2e/live/mcp-bridge-onboard-env.ts | 54 ++--------- test/e2e/support/inference-adapter.test.ts | 2 +- .../e2e/support/managed-image-receipt.test.ts | 95 ++++++++++++++++++- .../support/mcp-bridge-onboard-env.test.ts | 8 +- ...ck-managed-image-workflow-boundary.test.ts | 14 ++- ...nboard-managed-image-buildless-e2e.test.ts | 15 +++ .../e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/mcp-workflow-boundary.mts | 2 + tools/e2e/operations-workflow-boundary.mts | 8 ++ .../standard-profile-workflow-boundary.mts | 4 + 20 files changed, 253 insertions(+), 66 deletions(-) diff --git a/.github/workflows/e2e-standard-profile.yaml b/.github/workflows/e2e-standard-profile.yaml index 5a17e9c2464..53f1fcf64a5 100644 --- a/.github/workflows/e2e-standard-profile.yaml +++ b/.github/workflows/e2e-standard-profile.yaml @@ -24,6 +24,9 @@ on: managed_image_revision: required: true type: string + managed_image_receipt: + required: true + type: string credential_boundary: required: true type: string @@ -110,6 +113,7 @@ jobs: NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.candidate_sha }} E2E_MANAGED_IMAGE_REVISION: ${{ inputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ inputs.managed_image_receipt }} NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.risk_signal_correlation_id }} NEMOCLAW_E2E_RISK_SIGNAL_EXPECTED_SHA: ${{ inputs.risk_signal_expected_sha }} NEMOCLAW_LLAMA_CPP_QUALIFICATION_HEAD_SHA: ${{ inputs.candidate_sha }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3d087c41dd5..bbe36f680c5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2698,6 +2698,7 @@ jobs: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.base-image-publication.outputs.dcode_base_ref }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -2989,6 +2990,7 @@ jobs: risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} + managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} credential_boundary: no provider credential target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3029,6 +3031,7 @@ jobs: risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} + managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} credential_boundary: NVIDIA API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3070,6 +3073,7 @@ jobs: risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} + managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} credential_boundary: NVIDIA inference API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3111,6 +3115,7 @@ jobs: risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} + managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} credential_boundary: GitHub read token target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3152,6 +3157,7 @@ jobs: risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} + managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} credential_boundary: Brave and NVIDIA inference API keys target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} @@ -3615,6 +3621,7 @@ jobs: timeout-minutes: 90 env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "openshell-credential-generation-window" E2E_AGENT_RUNTIME: "openclaw" @@ -3788,6 +3795,7 @@ jobs: coverage_variant: deepagents env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge-dev" E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" @@ -4958,6 +4966,7 @@ jobs: timeout-minutes: 85 env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "hermes-e2e" E2E_AGENT_RUNTIME: "hermes" @@ -5066,6 +5075,7 @@ jobs: coverage_variant: compatibility-only env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "hermes-gpu-startup" E2E_AGENT_RUNTIME: "hermes" @@ -5371,6 +5381,7 @@ jobs: timeout-minutes: 70 env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "cloud-onboard" E2E_AGENT_RUNTIME: "openclaw" @@ -5496,6 +5507,7 @@ jobs: timeout-minutes: 90 env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} + E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_JOB: "1" E2E_TARGET_ID: "messaging-providers" E2E_AGENT_RUNTIME: "openclaw" diff --git a/docs/deployment/sandbox-hardening.mdx b/docs/deployment/sandbox-hardening.mdx index b3a2fe9bd5a..2640c721333 100644 --- a/docs/deployment/sandbox-hardening.mdx +++ b/docs/deployment/sandbox-hardening.mdx @@ -15,7 +15,7 @@ The NemoClaw sandbox image applies several security measures to reduce the attac ## Immutable Managed Image Selection Stock onboarding through the OpenShell Docker driver selects an exact managed-image digest and validates the complete OpenClaw, Hermes, and LangChain Deep Agents Code publication cohort before sandbox creation. -If registry or catalog availability prevents resolution, the ordinary `prefer-managed` path builds the shipped, reviewed repository Dockerfile instead; it never selects an unpinned `:latest` image. +If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. An available but incomplete, mixed, mutable, wrong-platform, or identity-inconsistent cohort fails closed before sandbox creation. Passing `--from ` remains a separate explicit opt-in whose complete custom image must be reviewed independently. diff --git a/docs/get-started/quickstart-hermes.mdx b/docs/get-started/quickstart-hermes.mdx index 61293de4490..fe953dd66f5 100644 --- a/docs/get-started/quickstart-hermes.mdx +++ b/docs/get-started/quickstart-hermes.mdx @@ -62,8 +62,9 @@ Review the [Prerequisites](prerequisites) before you begin. If the installer does not offer Express setup, or if you enter `n` at the Express prompt on a supported non-N1x host, choose an inference provider and model, then provide its credential when prompted. For that interactive path, skip optional web search and messaging setup on a first run, then accept the suggested network policy tier. With the OpenShell Docker driver, stock Hermes onboarding normally uses the release's exact managed-image digest. - If registry or catalog availability prevents resolution, it builds the shipped repository Dockerfile instead; it never selects an unpinned `:latest` image. + If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Invalid or inconsistent catalog evidence fails closed before sandbox creation. + An explicit `nemoclaw onboard --from ` remains a separate custom-image path. diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 45167aa3a80..0301fee482f 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -65,8 +65,9 @@ Review the [Prerequisites](prerequisites) before you begin. If the installer does not offer Express setup, or if you enter `n` at the Express prompt on a supported non-N1x host, choose an inference provider and model, then provide its credential when prompted. For that interactive path, accept the suggested network policy tier on a first run. With the OpenShell Docker driver, stock Deep Agents Code onboarding normally uses the release's exact managed-image digest. - If registry or catalog availability prevents resolution, it builds the shipped repository Dockerfile instead; it never selects an unpinned `:latest` image. + If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Invalid or inconsistent catalog evidence fails closed before sandbox creation. + An explicit `nemoclaw onboard --from ` remains a separate custom-image path. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 8636a36378f..db61cf48071 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -51,8 +51,9 @@ Review the [Prerequisites](prerequisites) before you begin. Press Enter to accept the suggested `my-assistant` sandbox name. For a first run, skip optional web search and messaging setup, then accept the suggested network policy tier. With the OpenShell Docker driver, stock OpenClaw onboarding normally uses the release's exact managed-image digest. - If registry or catalog availability prevents resolution, it builds the shipped repository Dockerfile instead; it never selects an unpinned `:latest` image. + If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Invalid or inconsistent catalog evidence fails closed before sandbox creation. + An explicit `nemoclaw onboard --from ` remains a separate custom-image path. The installer can display `Run express install with these settings? [Y/n]:` before the agent-selection prompt on DGX Spark, qualifying DGX Station, or Windows Subsystem for Linux (WSL) hosts. diff --git a/docs/reference/architecture.mdx b/docs/reference/architecture.mdx index beb1443949d..a7d0755f22e 100644 --- a/docs/reference/architecture.mdx +++ b/docs/reference/architecture.mdx @@ -312,7 +312,7 @@ The maintained onboarding path for this agent does not consume the component. Stock onboarding through the OpenShell Docker driver for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. Before selecting one agent image, NemoClaw validates a complete three-agent cohort with one release, source revision, publication cohort, and compatible startup and capability contracts. -If registry or catalog availability prevents resolution, the ordinary `prefer-managed` path builds the shipped, reviewed repository Dockerfile instead and never selects an unpinned tag. +If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Available catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent fails closed before sandbox creation. An explicit `--from ` remains a separate complete custom-image path. The portable experimental profile retains its existing workload path, and native Podman remains disabled. diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index 213b4e189ca..627449cf23d 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -11,6 +11,7 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "DOCKER_CERT_PATH", "DOCKER_API_VERSION", "E2E_MANAGED_IMAGE_REVISION", + "E2E_MANAGED_IMAGE_COHORT_RECEIPT", "GITHUB_WORKSPACE", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index 29393ae4d1e..5e4ca42209c 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -96,7 +96,9 @@ export function normalizeMode(env: NodeJS.ProcessEnv): E2EInferenceMode { export function requirePublicNvidiaInferenceKey(value: string): string { if (!value.startsWith("nvapi-")) { - throw new Error(`${HOSTED_INFERENCE_SECRET} must start with nvapi- for public NVIDIA mode`); + throw new Error( + `${PUBLIC_NVIDIA_CREDENTIAL_ENV} must start with nvapi- for public NVIDIA mode`, + ); } return value; } diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts index ee625d4424c..bfd09c27baa 100644 --- a/test/e2e/fixtures/managed-image-receipt.ts +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -5,6 +5,13 @@ import os from "node:os"; import path from "node:path"; import { DEFAULT_GATEWAY_PORT } from "../../../src/lib/core/ports.ts"; +import { + isShippedManagedImageAgent, + MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_REPOSITORIES, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "../../../src/lib/onboard/managed-image/contract.ts"; import { readManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts"; import { readConfigFile } from "../../../src/lib/state/config-io.ts"; import { parseSandboxRegistryEntries } from "../../../src/lib/state/registry-normalization.ts"; @@ -14,6 +21,73 @@ import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; const REVISION_PATTERN = /^[0-9a-f]{40}$/u; const FALLBACK_DIAGNOSTIC = "Managed image unavailable; using the trusted Dockerfile recipe."; +export function assertManagedImageReceiptMatchesSelectedCohort(options: { + readonly environment: NodeJS.ProcessEnv; + readonly expectedAgent: ShippedManagedImageAgent; + readonly workload?: Record; +}): void { + const revision = options.environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; + const rawReceipt = options.environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT?.trim() ?? ""; + if (!REVISION_PATTERN.test(revision) || !rawReceipt || Buffer.byteLength(rawReceipt) > 8 * 1024) { + throw new Error("stock onboarding requires the complete selected managed-image cohort receipt"); + } + let receipt: Record; + try { + const parsed = JSON.parse(rawReceipt) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); + receipt = parsed as Record; + } catch { + throw new Error("stock onboarding selected managed-image cohort receipt is invalid"); + } + const runId = receipt.runId; + const runAttempt = receipt.runAttempt; + const images = receipt.images; + const cohort = receipt.cohort; + if ( + JSON.stringify(Object.keys(receipt).sort()) !== + JSON.stringify(["cohort", "images", "kind", "revision", "runAttempt", "runId"]) || + receipt.kind !== "nemoclaw-managed-image-cohort-receipt-v1" || + receipt.revision !== revision || + !Number.isSafeInteger(runId) || + Number(runId) < 1 || + !Number.isSafeInteger(runAttempt) || + Number(runAttempt) < 1 || + cohort !== `ghrun-${String(runId)}-${String(runAttempt)}` || + !images || + typeof images !== "object" || + Array.isArray(images) || + JSON.stringify(Object.keys(images).sort()) !== + JSON.stringify([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()) + ) { + throw new Error("stock onboarding selected managed-image cohort receipt is invalid"); + } + + const platform = options.workload?.platform; + const agentImages = (images as Record)[options.expectedAgent]; + if ( + typeof platform !== "string" || + !(MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) || + !agentImages || + typeof agentImages !== "object" || + Array.isArray(agentImages) || + JSON.stringify(Object.keys(agentImages).sort()) !== + JSON.stringify([...MANAGED_IMAGE_PLATFORMS].sort()) + ) { + throw new Error("stock onboarding selected managed-image cohort receipt is invalid"); + } + const expectedReference = (agentImages as Record)[platform]; + if ( + typeof expectedReference !== "string" || + !expectedReference.startsWith(`${MANAGED_IMAGE_REPOSITORIES[options.expectedAgent]}@sha256:`) || + options.workload?.kind !== "managed-image" || + options.workload.reference !== expectedReference || + options.workload.sourceRevision !== revision || + options.workload.sourceCohort !== cohort + ) { + throw new Error("stock onboarding must use the exact agent image from the selected cohort"); + } +} + export interface StockManagedImageReceiptEvidence { readonly agent: string; readonly reference: string; @@ -81,6 +155,13 @@ export function assertStockManagedImageReceipt(options: { if (options.expectedAgent && authority.agent !== options.expectedAgent) { throw new Error(`stock sandbox '${options.sandboxName}' managed-image agent does not match`); } + if (isShippedManagedImageAgent(authority.agent)) { + assertManagedImageReceiptMatchesSelectedCohort({ + environment, + expectedAgent: authority.agent, + workload: authority.receipt as unknown as Record, + }); + } return { agent: authority.agent, reference: authority.receipt.reference, diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index a07cb103f0b..c41c9b2783c 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -7,11 +7,11 @@ import { MANAGED_IMAGE_PLATFORMS, MANAGED_IMAGE_REPOSITORIES, parseManagedImageContractV1, - SHIPPED_MANAGED_IMAGE_AGENTS, type ManagedImagePlatform, type ShippedManagedImageAgent, } from "../../../src/lib/onboard/managed-image/contract.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertManagedImageReceiptMatchesSelectedCohort } from "../fixtures/managed-image-receipt.ts"; const EXACT_MAIN_OVERLAY_KEYS = new Set([ "PATH", @@ -56,7 +56,6 @@ export function assertMcpBridgeManagedImageReceipt(options: { }): void { const environment = options.environment ?? process.env; const selectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim(); - const selectedReceipt = environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT?.trim(); const exactCandidateCatalog = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG?.trim(); if (!selectedRevision && !exactCandidateCatalog) return; @@ -77,51 +76,12 @@ export function assertMcpBridgeManagedImageReceipt(options: { let expectedReference: string; let expectedCohort: string; if (selectedRevision) { - if (!selectedReceipt || Buffer.byteLength(selectedReceipt, "utf8") > 8 * 1024) { - throw new Error("managed-image MCP qualification requires the selected cohort receipt"); - } - let receipt: Record; - try { - const parsed = JSON.parse(selectedReceipt) as unknown; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); - receipt = parsed as Record; - } catch { - throw new Error("managed-image MCP qualification cohort receipt is invalid"); - } - const runId = receipt.runId; - const runAttempt = receipt.runAttempt; - const images = receipt.images; - const cohort = receipt.cohort; - if ( - JSON.stringify(Object.keys(receipt).sort()) !== - JSON.stringify(["cohort", "images", "kind", "revision", "runAttempt", "runId"]) || - receipt.kind !== "nemoclaw-managed-image-cohort-receipt-v1" || - receipt.revision !== expectedRevision || - !Number.isSafeInteger(runId) || - Number(runId) < 1 || - !Number.isSafeInteger(runAttempt) || - Number(runAttempt) < 1 || - cohort !== `ghrun-${String(runId)}-${String(runAttempt)}` || - !images || - typeof images !== "object" || - Array.isArray(images) || - JSON.stringify(Object.keys(images).sort()) !== - JSON.stringify([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()) - ) { - throw new Error("managed-image MCP qualification cohort receipt is invalid"); - } - const agentImages = (images as Record)[options.expectedAgent]; - if ( - !agentImages || - typeof agentImages !== "object" || - Array.isArray(agentImages) || - JSON.stringify(Object.keys(agentImages).sort()) !== - JSON.stringify([...MANAGED_IMAGE_PLATFORMS].sort()) - ) { - throw new Error("managed-image MCP qualification cohort receipt is invalid"); - } - expectedReference = (agentImages as Record)[expectedPlatform] as string; - expectedCohort = cohort; + assertManagedImageReceiptMatchesSelectedCohort({ + environment, + expectedAgent: options.expectedAgent, + workload: options.workload, + }); + return; } else { let catalog: Record; try { diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index 588bd6bd109..8dec4d09cdc 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -372,7 +372,7 @@ describe("E2E inference adapter", () => { env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" }, secrets: { NVIDIA_API_KEY: "sk-compatible-key" }, }), - ).rejects.toThrow(/must start with nvapi-/); + ).rejects.toThrow(/NVIDIA_API_KEY must start with nvapi-/); }); it("does not treat the internal NVIDIA inference credential as public authority", async () => { diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts index 2bcb7513a0e..18dc87cd37f 100644 --- a/test/e2e/support/managed-image-receipt.test.ts +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -23,6 +23,8 @@ import { const SANDBOX_NAME = "managed-only-stock"; const REVISION = "d".repeat(40); +const COHORT = "ghrun-32707920950-1"; +const REFERENCE = `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"a".repeat(64)}`; const temporaryHomes: string[] = []; afterEach(() => { @@ -33,15 +35,14 @@ afterEach(() => { function managedReceipt(sourceRevision = REVISION): Record { const encodedProfile = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); - const reference = `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"a".repeat(64)}`; return { schemaVersion: 1, kind: "managed-image", - reference, + reference: REFERENCE, platform: "linux/amd64", release: "v0.0.100", sourceRevision, - sourceCohort: "ghrun-32707920950-1", + sourceCohort: COHORT, capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, encodedProfile, @@ -51,6 +52,34 @@ function managedReceipt(sourceRevision = REVISION): Record { }; } +function selectedEnvironment(home: string): NodeJS.ProcessEnv { + return { + E2E_MANAGED_IMAGE_REVISION: REVISION, + E2E_MANAGED_IMAGE_COHORT_RECEIPT: JSON.stringify({ + kind: "nemoclaw-managed-image-cohort-receipt-v1", + cohort: COHORT, + revision: REVISION, + runAttempt: 1, + runId: 32707920950, + images: { + openclaw: { + "linux/amd64": REFERENCE, + "linux/arm64": `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"b".repeat(64)}`, + }, + hermes: { + "linux/amd64": `${MANAGED_IMAGE_REPOSITORIES.hermes}@sha256:${"c".repeat(64)}`, + "linux/arm64": `${MANAGED_IMAGE_REPOSITORIES.hermes}@sha256:${"d".repeat(64)}`, + }, + "langchain-deepagents-code": { + "linux/amd64": `${MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"]}@sha256:${"e".repeat(64)}`, + "linux/arm64": `${MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"]}@sha256:${"f".repeat(64)}`, + }, + }, + }), + HOME: home, + }; +} + function writeRegistry(workload: Record): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-only-receipt-")); temporaryHomes.push(home); @@ -80,7 +109,7 @@ describe("stock E2E managed-image receipt assertion", () => { expect( assertStockManagedImageReceipt({ - environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, + environment: selectedEnvironment(home), expectedAgent: "openclaw", sandboxName: SANDBOX_NAME, }), @@ -114,6 +143,64 @@ describe("stock E2E managed-image receipt assertion", () => { ).toThrow("does not match the selected cohort"); }); + it("rejects the selected revision with another publication cohort", () => { + const home = writeRegistry({ ...managedReceipt(), sourceCohort: "ghrun-999-1" }); + + expect(() => + assertStockManagedImageReceipt({ + environment: selectedEnvironment(home), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("exact agent image from the selected cohort"); + }); + + it("rejects the selected revision with another immutable image reference", () => { + const home = writeRegistry({ + ...managedReceipt(), + reference: `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"9".repeat(64)}`, + }); + + expect(() => + assertStockManagedImageReceipt({ + environment: selectedEnvironment(home), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("exact agent image from the selected cohort"); + }); + + it("rejects a cohort receipt that maps the stock agent to another repository", () => { + const home = writeRegistry(managedReceipt()); + const environment = selectedEnvironment(home); + const receipt = JSON.parse(environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT!) as { + images: Record>; + }; + receipt.images.openclaw["linux/amd64"] = + `${MANAGED_IMAGE_REPOSITORIES.hermes}@sha256:${"c".repeat(64)}`; + environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT = JSON.stringify(receipt); + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("exact agent image from the selected cohort"); + }); + + it("rejects a platform reference that differs from the durable workload", () => { + const home = writeRegistry({ ...managedReceipt(), platform: "linux/arm64" }); + + expect(() => + assertStockManagedImageReceipt({ + environment: selectedEnvironment(home), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("exact agent image from the selected cohort"); + }); + it("rejects the stock fallback diagnostic before later probes", () => { const home = writeRegistry(managedReceipt()); diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index 3312020bcb3..d50a9f337b1 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -126,7 +126,7 @@ describe("MCP bridge onboarding environment", () => { expectedAgent: "langchain-deepagents-code", workload: selectedWorkload("langchain-deepagents-code", { kind: "dockerfile" }), }), - ).toThrow("must use the exact agent image from the selected cohort receipt"); + ).toThrow("must use the exact agent image from the selected cohort"); }); it("rejects a managed image from a different candidate revision", () => { @@ -138,7 +138,7 @@ describe("MCP bridge onboarding environment", () => { sourceRevision: "b".repeat(40), }), }), - ).toThrow("must use the exact agent image from the selected cohort receipt"); + ).toThrow("must use the exact agent image from the selected cohort"); }); it("activates the exact managed runtime when the qualification catalog is present", () => { @@ -219,7 +219,7 @@ describe("MCP bridge onboarding environment", () => { expectedAgent: "langchain-deepagents-code", workload: selectedWorkload("openclaw"), }), - ).toThrow("must use the exact agent image from the selected cohort receipt"); + ).toThrow("must use the exact agent image from the selected cohort"); }); it("rejects a different publication cohort with the selected revision", () => { @@ -231,7 +231,7 @@ describe("MCP bridge onboarding environment", () => { sourceCohort: "ghrun-999-1", }), }), - ).toThrow("must use the exact agent image from the selected cohort receipt"); + ).toThrow("must use the exact agent image from the selected cohort"); }); it("passes only exact-main OpenShell overrides after fixed onboarding values", () => { diff --git a/test/e2e/support/stock-managed-image-workflow-boundary.test.ts b/test/e2e/support/stock-managed-image-workflow-boundary.test.ts index 497f7c6000d..aa0537423d6 100644 --- a/test/e2e/support/stock-managed-image-workflow-boundary.test.ts +++ b/test/e2e/support/stock-managed-image-workflow-boundary.test.ts @@ -34,14 +34,15 @@ function workflow(): OperationsWorkflow { } describe("stock onboarding managed-image publication boundary", () => { - it("passes one selected cohort revision to every stock onboarding job", () => { + it("passes one selected cohort receipt to every stock onboarding job", () => { expect(validateStockOnboardingPublicationBoundary(workflow())).toEqual([]); }); - it.each(STOCK_JOBS)("rejects %s without the publication dependency and revision", (jobName) => { + it.each(STOCK_JOBS)("rejects %s without the publication dependency and receipt", (jobName) => { const value = workflow(); value.jobs[jobName].needs = []; delete value.jobs[jobName].env?.E2E_MANAGED_IMAGE_REVISION; + delete value.jobs[jobName].env?.E2E_MANAGED_IMAGE_COHORT_RECEIPT; expect(validateStockOnboardingPublicationBoundary(value)).toEqual( expect.arrayContaining([ @@ -49,16 +50,20 @@ describe("stock onboarding managed-image publication boundary", () => { expect.stringContaining( `${jobName} must receive the selected managed-image cohort revision`, ), + expect.stringContaining( + `${jobName} must receive the complete selected managed-image cohort receipt`, + ), ]), ); }); it.each(CATALOGUE_JOBS)( - "rejects %s without the publication dependency and revision input", + "rejects %s without the publication dependency and receipt inputs", (jobName) => { const value = workflow(); value.jobs[jobName].needs = []; delete value.jobs[jobName].with?.managed_image_revision; + delete value.jobs[jobName].with?.managed_image_receipt; expect(validateStockOnboardingPublicationBoundary(value)).toEqual( expect.arrayContaining([ @@ -66,6 +71,9 @@ describe("stock onboarding managed-image publication boundary", () => { expect.stringContaining( `${jobName} must pass the selected managed-image cohort revision`, ), + expect.stringContaining( + `${jobName} must pass the complete selected managed-image cohort receipt`, + ), ]), ); }, diff --git a/test/onboard-managed-image-buildless-e2e.test.ts b/test/onboard-managed-image-buildless-e2e.test.ts index 35280cb0867..3e3426a157b 100644 --- a/test/onboard-managed-image-buildless-e2e.test.ts +++ b/test/onboard-managed-image-buildless-e2e.test.ts @@ -10,6 +10,16 @@ import { describe, expect } from "vitest"; import { test } from "./e2e/fixtures/workflow-e2e-test.ts"; import { runManagedImageBuildlessE2e } from "./helpers/managed-image-buildless-e2e"; +function expectManagedOnlyGuide(relativePath: string): void { + const guide = readFileSync(path.join(import.meta.dirname, "..", "docs", relativePath), "utf8"); + expect(guide).toContain( + "stock onboarding stops before sandbox creation and does not build a shipped Dockerfile", + ); + expect(guide).toContain("--from "); + expect(guide).not.toContain("builds the shipped repository Dockerfile instead"); + expect(guide).not.toContain("ordinary `prefer-managed` path"); +} + describe("managed image buildless onboarding orchestration contract", () => { test("renders every shipped agent's immutable launch without entering Dockerfile orchestration (#7744)", { timeout: 240_000, @@ -32,6 +42,11 @@ describe("managed image buildless onboarding orchestration contract", () => { expect(commands).toContain( "Catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent also fails closed before sandbox creation.", ); + expectManagedOnlyGuide("deployment/sandbox-hardening.mdx"); + expectManagedOnlyGuide("reference/architecture.mdx"); + expectManagedOnlyGuide("get-started/quickstart.mdx"); + expectManagedOnlyGuide("get-started/quickstart-hermes.mdx"); + expectManagedOnlyGuide("get-started/quickstart-langchain-deepagents-code.mdx"); progress.phase("validate mocked all-agent buildless orchestration boundaries"); runManagedImageBuildlessE2e(); diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 02c58f55f6c..371a20fc95c 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -6,7 +6,7 @@ import { createHash } from "node:crypto"; export const MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256 = "052c49d5e8688266dbf38fa911733132d33e4470a29a61deb6e7a11067737559"; export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = - "3384ea63548c78f5c2a592e1d3a4de1b3060dee8167d7b83f0d501e1eaa5e128"; + "8f136a82c39b727db2b1da43253ff660e2ab44f2d82144900b26b344400311a4"; export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index c09784cbdfe..4a32291f4f3 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -883,6 +883,8 @@ function validateCredentialWindowJob( const expectedEnv = { E2E_MANAGED_IMAGE_REVISION: "${{ needs.base-image-publication.outputs.managed_image_revision }}", + E2E_MANAGED_IMAGE_COHORT_RECEIPT: + "${{ needs.base-image-publication.outputs.managed_image_receipt }}", E2E_JOB: "1", E2E_TARGET_ID: CREDENTIAL_WINDOW_JOB, E2E_AGENT_RUNTIME: "openclaw", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a78ed18aea1..0ada8658a5e 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -791,6 +791,8 @@ const STOCK_ONBOARDING_CATALOGUE_JOBS = [ const MANAGED_IMAGE_REVISION_EXPRESSION = "${{ needs.base-image-publication.outputs.managed_image_revision }}"; +const MANAGED_IMAGE_RECEIPT_EXPRESSION = + "${{ needs.base-image-publication.outputs.managed_image_receipt }}"; /** Require publication success and one exact cohort revision for every stock onboarding job. */ export function validateStockOnboardingPublicationBoundary( @@ -805,6 +807,9 @@ export function validateStockOnboardingPublicationBoundary( if (job.env?.E2E_MANAGED_IMAGE_REVISION !== MANAGED_IMAGE_REVISION_EXPRESSION) { errors.push(`${jobName} must receive the selected managed-image cohort revision`); } + if (job.env?.E2E_MANAGED_IMAGE_COHORT_RECEIPT !== MANAGED_IMAGE_RECEIPT_EXPRESSION) { + errors.push(`${jobName} must receive the complete selected managed-image cohort receipt`); + } } for (const jobName of STOCK_ONBOARDING_CATALOGUE_JOBS) { const job = workflow.jobs[jobName] ?? {}; @@ -814,6 +819,9 @@ export function validateStockOnboardingPublicationBoundary( if (job.with?.managed_image_revision !== MANAGED_IMAGE_REVISION_EXPRESSION) { errors.push(`${jobName} must pass the selected managed-image cohort revision`); } + if (job.with?.managed_image_receipt !== MANAGED_IMAGE_RECEIPT_EXPRESSION) { + errors.push(`${jobName} must pass the complete selected managed-image cohort receipt`); + } } return errors; } diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index 7e5156c5723..8e89535e670 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -165,6 +165,8 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi cli_artifact_provenance: "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}", managed_image_revision: "${{ needs.base-image-publication.outputs.managed_image_revision }}", + managed_image_receipt: + "${{ needs.base-image-publication.outputs.managed_image_receipt }}", credential_boundary: contract.credentialBoundary, catalogue_id: "${{ matrix.id }}", target_id: "${{ matrix.target_id }}", @@ -213,6 +215,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi risk_signal_correlation_id: "string", cli_artifact_provenance: "string", managed_image_revision: "string", + managed_image_receipt: "string", credential_boundary: "string", catalogue_id: "string", target_id: "string", @@ -282,6 +285,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi E2E_JOB: "1", E2E_TARGET_ID: "${{ inputs.target_id }}", E2E_MANAGED_IMAGE_REVISION: "${{ inputs.managed_image_revision }}", + E2E_MANAGED_IMAGE_COHORT_RECEIPT: "${{ inputs.managed_image_receipt }}", NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.candidate_sha }}", NEMOCLAW_E2E_CORRELATION_ID: "${{ inputs.risk_signal_correlation_id }}", From fbedb28343756697a30def2ca097cb09bf4f08a8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 11:38:28 -0500 Subject: [PATCH 07/37] test(onboard): remove stock fallback fixtures --- ci/test-file-size-budget.json | 2 +- test/helpers/onboard-script-mocks.cjs | 43 +++++++++++----- ...oard-extra-provider-reconciliation.test.ts | 11 ++-- test/onboard-installer-restore-intent.test.ts | 17 +++++-- ...onboard-mcp-observability-redirect.test.ts | 4 +- test/onboard-messaging.test.ts | 29 ++++------- test/onboard-prepared-build-context.test.ts | 1 - test/onboard-reservation-recreate.test.ts | 6 ++- test/onboard-sandbox-build.test.ts | 23 ++++++--- test/onboard-sandbox-recreation.test.ts | 51 +++++++++++++++---- test/onboard-terminal-dashboard.test.ts | 6 ++- test/onboard.test.ts | 1 - test/shellquote-sandbox.test.ts | 6 ++- 13 files changed, 130 insertions(+), 70 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 19c1200f60c..eb17f80b9e9 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -8,7 +8,7 @@ "test/generate-openclaw-config.test.ts": 1907, "test/install-preflight.test.ts": 3025, "test/nemoclaw-start.test.ts": 4671, - "test/onboard-messaging.test.ts": 2023, + "test/onboard-messaging.test.ts": 2012, "test/onboard-selection.test.ts": 4177 } } diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index c05209df217..e0760de64b2 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -252,6 +252,7 @@ function mockSandboxExecCurl(command, options = {}) { } function mockOnboardRunCapture(command, options = {}) { + mockCustomDockerfilePluginDiscovery(); const normalized = normalizeCommand(command); if ( normalized.startsWith("docker ps -a --no-trunc ") && @@ -279,6 +280,33 @@ function mockOnboardRunCapture(command, options = {}) { return mockSandboxExecCurl(command, options); } +let customDockerfilePluginDiscoveryMocked = false; +function mockCustomDockerfilePluginDiscovery() { + if ( + customDockerfilePluginDiscoveryMocked || + (process.argv[1] || "").toLowerCase().includes("/node_modules/vitest/") + ) { + return; + } + customDockerfilePluginDiscoveryMocked = true; + const childProcess = require("node:child_process"); + const originalSpawnSync = childProcess.spawnSync; + childProcess.spawnSync = (command, args, options) => { + const normalized = normalizeCommand([command, ...(Array.isArray(args) ? args : [])]); + if (command === "ssh" && normalized.includes("installed_plugin_index")) { + return { + status: 0, + signal: null, + stdout: Buffer.from( + JSON.stringify({ version: 1, installRecords: {}, loadPaths: [] }), + ), + stderr: Buffer.alloc(0), + }; + } + return originalSpawnSync(command, args, options); + }; +} + function mockStructuredOpenShellCaptureFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); const client = require( @@ -348,24 +376,11 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { }; } -function mockManagedImageFallback() { - const catalog = require( - path.resolve(__dirname, "../../src/lib/onboard/managed-image/catalog.ts"), - ); - catalog.resolveManagedImageCatalogFromGhcr = async () => { - throw new catalog.ManagedImageCatalogUnavailableError( - "integration fixture intentionally exercises the trusted Dockerfile fallback", - ); - }; -} - -process.env.NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK === "1" && mockManagedImageFallback(); - module.exports = { createStatefulMessagingProviderRunner, isOpenClawSecurityInventoryProbe, mockDockerSandboxLifecycleReleaseFromRunner, - mockManagedImageFallback, + mockCustomDockerfilePluginDiscovery, mockOnboardRunCapture, mockSandboxExecCurl, mockStandaloneGatewayTeardownAuthority, diff --git a/test/onboard-extra-provider-reconciliation.test.ts b/test/onboard-extra-provider-reconciliation.test.ts index eb8e3e87697..45540425690 100644 --- a/test/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboard-extra-provider-reconciliation.test.ts @@ -125,8 +125,14 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxNames = [ - await createSandbox(null, "gpt-5.4"), - await createSandbox(null, "gpt-5.4"), + await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ), + await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ), ]; console.log(JSON.stringify({ sandboxNames, @@ -149,7 +155,6 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", NEMOCLAW_SANDBOX_PREBUILD: "1", OPENSHELL_GATEWAY_ENDPOINT: undefined, }, diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index b2fc0d69ea2..7f2eded1f38 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -153,7 +153,10 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.NEMOCLAW_RECREATE_SANDBOX; process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); // Prove the recreated + restored sandbox is reachable through the real // "nemoclaw exec" boundary and can read a preserved workspace marker. @@ -190,7 +193,6 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", NEMOCLAW_SANDBOX_PREBUILD: "1", }; delete env["NEMOCLAW_RECREATE_SANDBOX"]; @@ -340,7 +342,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; try { - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ error: null, mutations })); } catch (caught) { const message = caught instanceof Error ? caught.message : String(caught); @@ -437,7 +442,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.NEMOCLAW_RECREATE_SANDBOX; delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { console.error(error); @@ -451,7 +459,6 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", }; delete env["NEMOCLAW_RECREATE_SANDBOX"]; delete env["NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE"]; diff --git a/test/onboard-mcp-observability-redirect.test.ts b/test/onboard-mcp-observability-redirect.test.ts index 5bd9da7c4b6..cbdd6485898 100644 --- a/test/onboard-mcp-observability-redirect.test.ts +++ b/test/onboard-mcp-observability-redirect.test.ts @@ -69,7 +69,8 @@ registry.getSandbox = () => ({ registry.getDefault = () => null; const { createSandbox } = require(${onboardPath}); createSandbox( - null, "model", "provider", "openai-completions", "alpha", null, null, null, + null, "model", "provider", "openai-completions", "alpha", null, null, + ${JSON.stringify(path.join(repoRoot, "agents", "langchain-deepagents-code", "Dockerfile"))}, { name: "langchain-deepagents-code", policyAdditionsPath: ${dcodePolicyPath} }, null, null, null, [], null, null, { @@ -91,7 +92,6 @@ createSandbox( HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", }, }); diff --git a/test/onboard-messaging.test.ts b/test/onboard-messaging.test.ts index 41420a29256..a27f207960c 100644 --- a/test/onboard-messaging.test.ts +++ b/test/onboard-messaging.test.ts @@ -46,7 +46,6 @@ const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), ); beforeEach(() => { - vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK", "1"); vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); describe("onboard messaging", () => { @@ -145,7 +144,7 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); process.env.KUBECONFIG = "/tmp/host-kubeconfig"; process.env.SSH_AUTH_SOCK = "/tmp/host-ssh-agent.sock"; await setupMessagingChannels(null, null, "my-assistant"); - const sandboxName = await createSandbox(null, "gpt-5.4"); + const sandboxName = await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, @@ -562,7 +561,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; - const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"]); + const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registered })); })().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); `; @@ -756,9 +755,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.TELEGRAM_BOT_TOKEN; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["telegram"], ["telegram"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], - ); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -912,9 +909,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], - ); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -1073,9 +1068,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], - ); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -1179,7 +1172,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - await createSandbox(null, "gpt-5.4"); + await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); // Should not reach here console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { @@ -1258,7 +1251,7 @@ const { createSandbox } = require(${onboardPath}); process.env.DISCORD_BOT_TOKEN = "test-discord-token"; process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; process.env.SLACK_APP_TOKEN = "xapp-test-slack-token"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1387,9 +1380,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], - ); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1522,9 +1513,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Empty array — user deselected all channels - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], - ); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index bf7ad7c39cf..05ee4fa906a 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -268,7 +268,6 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, NEMOCLAW_HOME: path.join(tmpDir, ".nemoclaw"), NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", PATH: `${fakeBin}:${process.env.PATH ?? ""}`, }, }); diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index 9751bb3e1f7..8010e763467 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -133,7 +133,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); @@ -145,7 +148,6 @@ const { createSandbox } = require(${onboardPath}); const result = runOnboardProcess([scriptPath], { env: workspaceEnv(workspace, { NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", NEMOCLAW_SANDBOX_PREBUILD: "1", }), timeoutMs: 30_000, diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index 65a294a17b7..b9d7fd67c97 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -17,7 +17,6 @@ import { } from "./helpers/onboard-split-context"; beforeEach(() => { - vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK", "1"); vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); @@ -109,7 +108,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4"); + const sandboxName = await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands, registerCalls, updateCalls, defaultCalls })); })().catch((error) => { console.error(error); @@ -346,7 +348,7 @@ const { createSandbox } = require(${onboardPath}); "hermes-sandbox", null, [], - null, + ${JSON.stringify(path.join(repoRoot, "agents", "hermes", "Dockerfile"))}, agent, null, null, @@ -532,7 +534,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ commands, logs, baseResolutionCalls })); })().catch((error) => { console.error(error); @@ -640,7 +645,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.com"; - await createSandbox(null, "gpt-5.4"); + await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify(commands)); })().catch((error) => { console.error(error); @@ -748,7 +756,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4"); + const sandboxName = await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 7db31a2922d..a9c379e5e12 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -13,7 +13,6 @@ import { writeOkOpenshell } from "./helpers/onboard-openshell-fixture"; import { type CommandEntry, onboardScriptMocksPath } from "./helpers/onboard-split-context"; beforeEach(() => { - vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK", "1"); vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); @@ -65,7 +64,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { console.error(error); @@ -192,7 +194,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands, registeredSandbox })); })().catch((error) => { console.error(error); @@ -350,7 +355,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); @@ -495,7 +503,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); @@ -643,7 +654,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); @@ -780,7 +794,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); const session = onboardSession.loadSession(); console.log(JSON.stringify({ policyPresets: session && session.policyPresets })); })().catch((error) => { @@ -909,7 +926,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1061,7 +1081,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1208,7 +1231,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1391,7 +1417,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4"); + const sandboxName = await createSandbox( + null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ sandboxName, diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index 347b2d0be71..33e55b19815 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -173,7 +173,10 @@ const agent = agentDefs.loadAgent("langchain-deepagents-code"); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.test:19000"; process.env.NEMOCLAW_DASHBOARD_PORT = "19000"; - const resultName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, sandboxName, null, null, null, agent); + const resultName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, sandboxName, null, null, + ${JSON.stringify(path.join(repoRoot, "agents", "langchain-deepagents-code", "Dockerfile"))}, agent, + ); console.log(JSON.stringify({ resultName, commands, registerCalls, updateCalls })); clearInterval(keepAlive); })().catch((error) => { @@ -192,7 +195,6 @@ const agent = agentDefs.loadAgent("langchain-deepagents-code"); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", OPENSHELL_DRIVERS: scenario === "create" ? "vm" : "docker", }, timeout: 15000, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 017d3675a05..e94f39a0cd4 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -1106,7 +1106,6 @@ const { createSandboxWithTemporaryManagedRuntime } = require(${onboardPath}); NEMOCLAW_RECREATE_SANDBOX: "1", }; delete env.NEMOCLAW_RECREATE_WITHOUT_BACKUP; - delete env.NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK; const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index b1f0c605de0..41b36a5c26c 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -134,7 +134,10 @@ try { process.env.NEMOCLAW_HEALTH_POLL_COUNT = "1"; Object.defineProperty(process, "platform", { value: "darwin" }); Object.defineProperty(process, "arch", { value: "x64" }); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ); console.log(JSON.stringify({ sandboxName, commands })); } catch (error) { console.error(error && error.stack ? error.stack : String(error)); @@ -158,7 +161,6 @@ try { env: { HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_TEST_MANAGED_IMAGE_FALLBACK: "1", }, timeout: 30_000, }, From 603b5eba8c5b85f51490942908a5cf2064a132b6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 11:51:45 -0500 Subject: [PATCH 08/37] test(onboard): remove stock fallback fixtures --- test/fixtures/explicit-custom.Dockerfile | 8 ++++++++ ...oard-extra-provider-reconciliation.test.ts | 4 ++-- test/onboard-installer-restore-intent.test.ts | 6 +++--- test/onboard-messaging.test.ts | 18 ++++++++--------- test/onboard-reservation-recreate.test.ts | 2 +- test/onboard-sandbox-build.test.ts | 8 ++++---- test/onboard-sandbox-recreation.test.ts | 20 +++++++++---------- test/shellquote-sandbox.test.ts | 2 +- 8 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 test/fixtures/explicit-custom.Dockerfile diff --git a/test/fixtures/explicit-custom.Dockerfile b/test/fixtures/explicit-custom.Dockerfile new file mode 100644 index 00000000000..9664e47d9b7 --- /dev/null +++ b/test/fixtures/explicit-custom.Dockerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +FROM scratch +ARG NEMOCLAW_MESSAGING_PLAN_B64= +ARG NEMOCLAW_TOOL_DISCLOSURE=progressive +ENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE} +CMD ["/bin/true"] diff --git a/test/onboard-extra-provider-reconciliation.test.ts b/test/onboard-extra-provider-reconciliation.test.ts index 45540425690..4bf5db88eb1 100644 --- a/test/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboard-extra-provider-reconciliation.test.ts @@ -127,11 +127,11 @@ const { createSandbox } = require(${onboardPath}); const sandboxNames = [ await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ), await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ), ]; console.log(JSON.stringify({ diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index 7f2eded1f38..0b6b31ef672 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -155,7 +155,7 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); // Prove the recreated + restored sandbox is reachable through the real @@ -344,7 +344,7 @@ const { createSandbox } = require(${onboardPath}); try { await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ error: null, mutations })); } catch (caught) { @@ -444,7 +444,7 @@ const { createSandbox } = require(${onboardPath}); delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { diff --git a/test/onboard-messaging.test.ts b/test/onboard-messaging.test.ts index a27f207960c..f56a0f4b6a9 100644 --- a/test/onboard-messaging.test.ts +++ b/test/onboard-messaging.test.ts @@ -144,7 +144,7 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); process.env.KUBECONFIG = "/tmp/host-kubeconfig"; process.env.SSH_AUTH_SOCK = "/tmp/host-ssh-agent.sock"; await setupMessagingChannels(null, null, "my-assistant"); - const sandboxName = await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, @@ -561,7 +561,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; - const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registered })); })().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); `; @@ -755,7 +755,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.TELEGRAM_BOT_TOKEN; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["telegram"], ["telegram"])})).toString("base64"); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -909,7 +909,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -1068,7 +1068,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -1172,7 +1172,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.DISCORD_BOT_TOKEN = "test-discord-token-value"; - await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + await createSandbox(null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); // Should not reach here console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { @@ -1251,7 +1251,7 @@ const { createSandbox } = require(${onboardPath}); process.env.DISCORD_BOT_TOKEN = "test-discord-token"; process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; process.env.SLACK_APP_TOKEN = "xapp-test-slack-token"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1380,7 +1380,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1513,7 +1513,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Empty array — user deselected all channels - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index 8010e763467..e4a14a35791 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -135,7 +135,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index b9d7fd67c97..d27487eef24 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -110,7 +110,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands, registerCalls, updateCalls, defaultCalls })); })().catch((error) => { @@ -536,7 +536,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ commands, logs, baseResolutionCalls })); })().catch((error) => { @@ -647,7 +647,7 @@ const { createSandbox } = require(${onboardPath}); process.env.CHAT_UI_URL = "https://chat.example.com"; await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify(commands)); })().catch((error) => { @@ -758,7 +758,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index a9c379e5e12..6da43665046 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -66,7 +66,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { @@ -196,7 +196,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands, registeredSandbox })); })().catch((error) => { @@ -357,7 +357,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -505,7 +505,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -656,7 +656,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -796,7 +796,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); const session = onboardSession.loadSession(); console.log(JSON.stringify({ policyPresets: session && session.policyPresets })); @@ -928,7 +928,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1083,7 +1083,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1233,7 +1233,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1419,7 +1419,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( null, "gpt-5.4", undefined, undefined, undefined, undefined, undefined, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index 41b36a5c26c..236b18f9155 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -136,7 +136,7 @@ try { Object.defineProperty(process, "arch", { value: "x64" }); const sandboxName = await createSandbox( null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "Dockerfile"))}, + ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, ); console.log(JSON.stringify({ sandboxName, commands })); } catch (error) { From 1432c79de262293b1531f54a8b1489d5b3a2d70d Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 24 Aug 2026 10:04:53 -0700 Subject: [PATCH 09/37] fix(e2e): bind self-hosted catalog revision Signed-off-by: Senthil Ravichandran --- .github/workflows/pr-self-hosted.yaml | 46 ++++++++++++++++++- .../pr-self-hosted-llama-selector.test.ts | 30 +++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index 63ad377faae..8d04cafaace 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -30,8 +30,13 @@ concurrency: jobs: select-llama-cpp-generic-gpu: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 55 + permissions: + actions: read + contents: read outputs: + base_sha: ${{ steps.changed.outputs.base_sha }} + managed_image_revision: ${{ steps.publication.outputs.head_sha }} selected: ${{ steps.changed.outputs.selected }} steps: - id: changed @@ -48,6 +53,7 @@ jobs: pr_number="${BASH_REMATCH[1]}" pr_json="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$pr_number")" head_sha="$(jq -er '.head.sha | select(test("^[a-f0-9]{40}$"))' <<<"$pr_json")" + base_sha="$(jq -er '.base.sha | select(test("^[a-f0-9]{40}$"))' <<<"$pr_json")" [[ "$head_sha" == "$GITHUB_SHA" ]] || { echo "::error::Copied PR branch SHA does not match the current PR head" >&2 exit 1 @@ -79,7 +85,42 @@ jobs: else selected=false fi - printf 'selected=%s\n' "$selected" >>"$GITHUB_OUTPUT" + { + printf 'base_sha=%s\n' "$base_sha" + printf 'selected=%s\n' "$selected" + } >>"$GITHUB_OUTPUT" + + - name: Check out trusted publication gate + if: ${{ steps.changed.outputs.selected == 'true' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ steps.changed.outputs.base_sha }} + + - name: Set up Node for publication verification + if: ${{ steps.changed.outputs.selected == 'true' }} + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Install trusted publication gate dependencies + if: ${{ steps.changed.outputs.selected == 'true' }} + run: npm ci --ignore-scripts --no-audit --no-fund + + - id: publication + name: Verify applicable base-image publication + if: ${{ steps.changed.outputs.selected == 'true' }} + env: + EXPECTED_SHA: ${{ steps.changed.outputs.base_sha }} + GITHUB_TOKEN: ${{ github.token }} + REQUIRE_MANAGED_IMAGE_PUBLICATION: "1" + shell: bash + run: | + set -euo pipefail + export GITHUB_REF=refs/heads/main + export GITHUB_SHA="$EXPECTED_SHA" + node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30 llama-cpp-generic-gpu: name: llama.cpp on generic NVIDIA GPU @@ -90,6 +131,7 @@ jobs: env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-generic-gpu E2E_JOB: "1" + E2E_MANAGED_IMAGE_REVISION: ${{ needs.select-llama-cpp-generic-gpu.outputs.managed_image_revision }} E2E_TARGET_ID: llama-cpp-generic-gpu NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js diff --git a/test/e2e/support/pr-self-hosted-llama-selector.test.ts b/test/e2e/support/pr-self-hosted-llama-selector.test.ts index 23e4d1b557d..ec201b410d1 100644 --- a/test/e2e/support/pr-self-hosted-llama-selector.test.ts +++ b/test/e2e/support/pr-self-hosted-llama-selector.test.ts @@ -15,8 +15,13 @@ type Workflow = { const WORKFLOW_PATH = ".github/workflows/pr-self-hosted.yaml"; const CANDIDATE_SHA = "a".repeat(40); +const BASE_SHA = "b".repeat(40); -function selectGenericGpuLane(changedFiles: readonly string[], copiedSha = CANDIDATE_SHA) { +function selectGenericGpuLane( + changedFiles: readonly string[], + copiedSha = CANDIDATE_SHA, + baseSha = BASE_SHA, +) { const workflow = YAML.parse(readFileSync(WORKFLOW_PATH, "utf8")) as Workflow; const script = workflow.jobs["select-llama-cpp-generic-gpu"]?.steps?.find( (step) => step.name === "Select llama.cpp generic GPU E2E from PR files", @@ -57,7 +62,11 @@ fi GITHUB_SHA: copiedSha, PATH: `${binDirectory}:${process.env.PATH ?? ""}`, PR_FILES_JSON: JSON.stringify([changedFiles.map((filename) => ({ filename }))]), - PR_JSON: JSON.stringify({ number: 8748, head: { sha: CANDIDATE_SHA } }), + PR_JSON: JSON.stringify({ + number: 8748, + base: { sha: baseSha }, + head: { sha: CANDIDATE_SHA }, + }), }, }, ); @@ -76,12 +85,17 @@ describe("generic NVIDIA GPU PR selection", () => { "src/lib/onboard/fatal-runtime-preflight.ts", "src/lib/onboard/overlayfs-auto-fix.ts", "src/lib/onboard/preflight.ts", - ])("selects the generic NVIDIA GPU E2E job when %s can change installer readiness", (changedFile) => { - expect(selectGenericGpuLane([changedFile])).toBe("selected=true"); - }); + ])( + "selects the generic NVIDIA GPU E2E job when %s can change installer readiness", + (changedFile) => { + expect(selectGenericGpuLane([changedFile])).toBe(`base_sha=${BASE_SHA}\nselected=true`); + }, + ); it("does not select the generic NVIDIA GPU E2E job for unrelated documentation", () => { - expect(selectGenericGpuLane(["docs/get-started/quickstart.mdx"])).toBe("selected=false"); + expect(selectGenericGpuLane(["docs/get-started/quickstart.mdx"])).toBe( + `base_sha=${BASE_SHA}\nselected=false`, + ); }); it("rejects a copied branch whose commit does not match the current PR head", () => { @@ -89,4 +103,8 @@ describe("generic NVIDIA GPU PR selection", () => { "Copied PR branch SHA does not match the current PR head", ); }); + + it("rejects a PR base that cannot identify an exact managed-image publication", () => { + expect(() => selectGenericGpuLane(["scripts/install.sh"], CANDIDATE_SHA, "main")).toThrow(); + }); }); From dac3a860b50d2cd808f55444f43a616efec894d4 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:15:26 -0700 Subject: [PATCH 10/37] test(e2e): bind publication workflow contract Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/pr-self-hosted.yaml | 6 +- ci/source-shape-test-budget.json | 5 ++ .../pr-self-hosted-llama-selector.test.ts | 73 ++++++++++++++++++- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index 8d04cafaace..c21c7488d75 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -90,7 +90,7 @@ jobs: printf 'selected=%s\n' "$selected" } >>"$GITHUB_OUTPUT" - - name: Check out trusted publication gate + - name: Check out PR base SHA for publication verification if: ${{ steps.changed.outputs.selected == 'true' }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -104,12 +104,12 @@ jobs: with: node-version: 22 - - name: Install trusted publication gate dependencies + - name: Install publication verifier dependencies if: ${{ steps.changed.outputs.selected == 'true' }} run: npm ci --ignore-scripts --no-audit --no-fund - id: publication - name: Verify applicable base-image publication + name: Verify complete base and managed-image publication if: ${{ steps.changed.outputs.selected == 'true' }} env: EXPECTED_SHA: ${{ steps.changed.outputs.base_sha }} diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index bff42eae90d..8f1f2546493 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -101,6 +101,11 @@ "test": "selects exact-commit rootless evidence for Portable recovery changes (#9707)", "category": "security" }, + { + "file": "test/e2e/support/pr-self-hosted-llama-selector.test.ts", + "test": "binds trusted base publication to the generic NVIDIA GPU job", + "category": "security" + }, { "file": "test/growth-guardrails-workflow-boundary.test.ts", "test": "runs the trusted Vitest guardrails against pull request data", diff --git a/test/e2e/support/pr-self-hosted-llama-selector.test.ts b/test/e2e/support/pr-self-hosted-llama-selector.test.ts index ec201b410d1..39c1126e473 100644 --- a/test/e2e/support/pr-self-hosted-llama-selector.test.ts +++ b/test/e2e/support/pr-self-hosted-llama-selector.test.ts @@ -9,21 +9,42 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import YAML from "yaml"; +type WorkflowStep = { + env?: Record; + id?: string; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + outputs?: Record; + permissions?: Record; + steps?: WorkflowStep[]; +}; + type Workflow = { - jobs: Record }>; + jobs: Record; }; const WORKFLOW_PATH = ".github/workflows/pr-self-hosted.yaml"; const CANDIDATE_SHA = "a".repeat(40); const BASE_SHA = "b".repeat(40); +function workflow(): Workflow { + return YAML.parse(readFileSync(WORKFLOW_PATH, "utf8")) as Workflow; +} + function selectGenericGpuLane( changedFiles: readonly string[], copiedSha = CANDIDATE_SHA, baseSha = BASE_SHA, ) { - const workflow = YAML.parse(readFileSync(WORKFLOW_PATH, "utf8")) as Workflow; - const script = workflow.jobs["select-llama-cpp-generic-gpu"]?.steps?.find( + const value = workflow(); + const script = value.jobs["select-llama-cpp-generic-gpu"]?.steps?.find( (step) => step.name === "Select llama.cpp generic GPU E2E from PR files", )?.run; expect(script).toEqual(expect.any(String)); @@ -104,7 +125,51 @@ describe("generic NVIDIA GPU PR selection", () => { ); }); - it("rejects a PR base that cannot identify an exact managed-image publication", () => { + it("rejects a PR whose base SHA is not a lowercase 40-character SHA", () => { expect(() => selectGenericGpuLane(["scripts/install.sh"], CANDIDATE_SHA, "main")).toThrow(); }); + + // source-shape-contract: security -- The copied PR workflow must run the publication verifier from the validated PR base before the generic GPU job receives its managed-image revision + it("binds trusted base publication to the generic NVIDIA GPU job", () => { + const value = workflow(); + const selector = value.jobs["select-llama-cpp-generic-gpu"]; + + expect(selector?.permissions).toEqual({ actions: "read", contents: "read" }); + expect(selector?.outputs).toMatchObject({ + base_sha: "${{ steps.changed.outputs.base_sha }}", + managed_image_revision: "${{ steps.publication.outputs.head_sha }}", + }); + + const checkout = selector?.steps?.find( + (step) => step.name === "Check out PR base SHA for publication verification", + ); + expect(checkout).toMatchObject({ + if: "${{ steps.changed.outputs.selected == 'true' }}", + uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + with: { + "fetch-depth": 0, + "persist-credentials": false, + ref: "${{ steps.changed.outputs.base_sha }}", + }, + }); + + const publication = selector?.steps?.find((step) => step.id === "publication"); + expect(publication).toMatchObject({ + env: { + EXPECTED_SHA: "${{ steps.changed.outputs.base_sha }}", + GITHUB_TOKEN: "${{ github.token }}", + REQUIRE_MANAGED_IMAGE_PUBLICATION: "1", + }, + if: "${{ steps.changed.outputs.selected == 'true' }}", + }); + expect(publication?.run).toContain("export GITHUB_REF=refs/heads/main"); + expect(publication?.run).toContain('export GITHUB_SHA="$EXPECTED_SHA"'); + expect(publication?.run).toContain( + "node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30", + ); + + expect(value.jobs["llama-cpp-generic-gpu"]?.env?.E2E_MANAGED_IMAGE_REVISION).toBe( + "${{ needs.select-llama-cpp-generic-gpu.outputs.managed_image_revision }}", + ); + }); }); From f1f33922df5998282ade249fbd5884dd1b7dd1ff Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 12:24:28 -0500 Subject: [PATCH 11/37] fix(e2e): bind runtime cohort descriptors --- ...vider-inference-host-local-startup.test.ts | 19 +++++++++++++++++++ .../machine/handlers/provider-inference.ts | 1 + test/e2e/fixtures/managed-image-receipt.ts | 2 ++ test/e2e/live/gpu-e2e.test.ts | 2 ++ .../managed-image-cohort-contract.test.ts | 8 ++++++-- .../e2e/support/managed-image-receipt.test.ts | 6 ++++++ tools/e2e/managed-image-cohort-contract.mts | 17 +++++++++++++---- 7 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/machine/handlers/provider-inference-host-local-startup.test.ts b/src/lib/onboard/machine/handlers/provider-inference-host-local-startup.test.ts index 6a177709960..d62a296bfc5 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-host-local-startup.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-host-local-startup.test.ts @@ -235,6 +235,25 @@ function llamaCppLifecycleSelection( } describe("provider inference host-local startup selection", () => { + it("does not require host-local application support for a hosted candidate provider", () => { + const resolver = vi.fn(); + const resolve = createCachedHostLocalInferenceSetupResolver({ + resolver, + application: "pi", + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + acceleration: "cpu", + requireToolCalling: null, + freshRequireToolCalling: true, + allowPublishedResume: false, + recover: false, + recordToolCallingRequirement: vi.fn(), + }); + + expect(resolve("pi-sandbox")).toEqual({}); + expect(resolver).not.toHaveBeenCalled(); + }); + it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( "dispatches a published %s llama.cpp route through the common lifecycle exactly once", async (application) => { diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index bc5a8362804..9f2700e6c4b 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -525,6 +525,7 @@ function hostLocalInferenceSetupOptions( (candidate): candidate is HostLocalInferenceApplication => candidate === input.application, ); if (!application) { + if (!isHostLocalInferenceProvider(input.provider)) return {}; throw new Error(`Unsupported host-local inference application '${input.application}'.`); } const selected = resolver({ diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts index bfd09c27baa..ec2d44f8f3e 100644 --- a/test/e2e/fixtures/managed-image-receipt.ts +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -176,6 +176,8 @@ export function shouldAssertStockManagedImageReceipt( environment: NodeJS.ProcessEnv, ): boolean { if (!environment.E2E_MANAGED_IMAGE_REVISION?.trim()) return false; + const selectedAgent = environment.NEMOCLAW_AGENT?.trim(); + if (selectedAgent && !isShippedManagedImageAgent(selectedAgent)) return false; if (environment.NEMOCLAW_FROM_DOCKERFILE?.trim()) return false; const executable = path.basename(command); let onboardArgumentIndex = -1; diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 3ff9b1c5f68..ed73157f46a 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -357,6 +357,8 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], // Keep this assertion about routed inference, not the model's reasoning-token budget. reasoning_effort: "none", + seed: 0, + temperature: 0, max_tokens: 32, }, )}'`, diff --git a/test/e2e/support/managed-image-cohort-contract.test.ts b/test/e2e/support/managed-image-cohort-contract.test.ts index e6a78199714..f07a495b56e 100644 --- a/test/e2e/support/managed-image-cohort-contract.test.ts +++ b/test/e2e/support/managed-image-cohort-contract.test.ts @@ -42,6 +42,7 @@ function cohortContract(): Record { platforms: Object.fromEntries( PLATFORMS.map((platform, platformIndex) => { const platformDigest = digest(agentIndex + platformIndex + 4); + const workloadDigest = digest(agentIndex + platformIndex + 10); const baseReference = `ghcr.io/nvidia/nemoclaw/base@${digest(agentIndex + platformIndex + 7)}`; const [os, architecture] = platform.split("/"); return [ @@ -52,7 +53,10 @@ function cohortContract(): Record { baseReference, publicationEvidence: { candidateDescriptor: { digest: platformDigest }, - workloadDescriptor: { platform: { os, architecture } }, + workloadDescriptor: { + digest: workloadDigest, + platform: { os, architecture }, + }, attestations: { slsa: { statement: { @@ -102,7 +106,7 @@ describe("managed-image cohort publication contract", () => { Object.fromEntries( PLATFORMS.map((platform, platformIndex) => [ platform, - `${MANAGED_IMAGE_REPOSITORIES[agent]}@${digest(agentIndex + platformIndex + 4)}`, + `${MANAGED_IMAGE_REPOSITORIES[agent]}@${digest(agentIndex + platformIndex + 10)}`, ]), ), ]), diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts index 18dc87cd37f..d6515638e3b 100644 --- a/test/e2e/support/managed-image-receipt.test.ts +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -232,5 +232,11 @@ describe("stock E2E managed-image receipt assertion", () => { { E2E_MANAGED_IMAGE_REVISION: REVISION }, ), ).toBe(false); + expect( + shouldAssertStockManagedImageReceipt("/workspace/bin/nemoclaw.js", ["onboard"], { + E2E_MANAGED_IMAGE_REVISION: REVISION, + NEMOCLAW_AGENT: "pi", + }), + ).toBe(false); }); }); diff --git a/tools/e2e/managed-image-cohort-contract.mts b/tools/e2e/managed-image-cohort-contract.mts index 109603bb61e..a1d7c3127a3 100644 --- a/tools/e2e/managed-image-cohort-contract.mts +++ b/tools/e2e/managed-image-cohort-contract.mts @@ -222,10 +222,19 @@ export function validateManagedImageCohort( return [ agent, Object.fromEntries( - PLATFORMS.map((platform) => [ - platform, - record(platforms[platform], `${agent} ${platform} publication`).reference, - ]), + PLATFORMS.map((platform) => { + const publication = record(platforms[platform], `${agent} ${platform} publication`); + const workloadDescriptor = record( + record(publication.publicationEvidence, `${agent} ${platform} publication evidence`) + .workloadDescriptor, + `${agent} ${platform} workload descriptor`, + ); + const workloadDigest = digest( + workloadDescriptor.digest, + `${agent} ${platform} workload digest`, + ); + return [platform, `${MANAGED_IMAGE_REPOSITORIES[agent]}@${workloadDigest}`]; + }), ), ]; }), From eaecfba599b97a9b2b8e5e4649f1f6003491d79f Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:02:33 -0700 Subject: [PATCH 12/37] fix(onboard): preserve trusted agent build context --- .../onboard-orchestration.test.ts | 15 +++++++++++-- .../managed-workload/onboard-orchestration.ts | 6 +++++- test/onboard-installer-restore-intent.test.ts | 9 +++++++- test/onboard-sandbox-build.test.ts | 1 + test/onboard-sandbox-recreation.test.ts | 21 +++++++++++++++---- ...shell-credential-generation-window.test.ts | 4 +++- 6 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index d0bedccdc40..6422033b52e 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -311,12 +311,22 @@ describe("managed workload onboard orchestration", () => { it("resolves final-image patch metadata after managed build-context staging", async () => { const resolutionMetadata = { key: "published-dcode-base" }; + const trustedDockerfile = path.join( + process.cwd(), + "agents", + "langchain-deepagents-code", + "Dockerfile", + ); let staged = false; const resolvePatchInput = vi.fn(() => { expect(staged).toBe(true); - return { preResolvedBaseImageMetadata: resolutionMetadata } as never; + return { + fromDockerfile: trustedDockerfile, + preResolvedBaseImageMetadata: resolutionMetadata, + } as never; }); const resolveSandboxBuildPatch = vi.fn(async (input: Record) => { + expect(input.fromDockerfile).toBeNull(); expect(input.preResolvedBaseImageMetadata).toBe(resolutionMetadata); expect(input.stagedDockerfile).toBe("/tmp/nemoclaw-staged-context/Dockerfile"); return { buildId: "dcode-build", dashboardRemoteBindPrepared: false }; @@ -362,8 +372,9 @@ describe("managed workload onboard orchestration", () => { agent: { name: "langchain-deepagents-code", displayName: "LangChain Deep Agents Code", + dockerfilePath: trustedDockerfile, }, - fromDockerfile: null, + fromDockerfile: trustedDockerfile, createAgentSandbox: () => { staged = true; return { diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 9b6830624eb..aaae9c94077 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -473,11 +473,15 @@ export async function prepareOnboardSandboxWorkloadLaunch( } else { const buildContext = requireLegacyBuildContext(legacyBuildContext); input.dependencies.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); + const patchInput = input.legacy.resolvePatchInput(); const patch = await (input.dependencies.resolveSandboxBuildPatch ?? resolveSandboxBuildPatch)({ // Build-context staging resolves managed-agent base-image provenance. // Read the patch input only after that boundary so the final image gets // the exact metadata produced by the same staging operation. - ...input.legacy.resolvePatchInput(), + ...patchInput, + // An explicit path to the checked-in agent Dockerfile is staged through + // the trusted agent builder. Preserve that classification at patch time. + fromDockerfile: buildContext.origin === "generated" ? null : patchInput.fromDockerfile, selectedGpuRoute: initialGpuRoute, stagedDockerfile: buildContext.stagedDockerfile, }); diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index 0b6b31ef672..369ba7dc920 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -101,7 +101,14 @@ registry.removeSandbox = () => true; sandboxState.getLatestBackup = (name) => { events.push({ kind: "getLatestBackup", name }); - return { backupPath: PRE_UPGRADE_BACKUP, timestamp: "2026-05-25T00:00:00Z" }; + return { + agentType: "openclaw", + dir: "/sandbox/.openclaw", + openclawImagePluginInstalls: [], + reconcileOpenClawImagePluginProvenance: true, + backupPath: PRE_UPGRADE_BACKUP, + timestamp: "2026-05-25T00:00:00Z", + }; }; sandboxState.backupSandboxState = (name) => { events.push({ kind: "backup", name }); diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index d27487eef24..2b1915bc0ff 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -333,6 +333,7 @@ const { createSandbox } = require(${onboardPath}); const agent = { name: "hermes", displayName: "Hermes Agent", + dockerfilePath: ${JSON.stringify(path.join(repoRoot, "agents", "hermes", "Dockerfile"))}, forwardPort: 18789, forward_ports: [18789, 8642], healthProbe: { url: "http://127.0.0.1:8642/health", port: 8642, timeout_seconds: 90 }, diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 6da43665046..ba02a7a182e 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -53,9 +53,14 @@ runner.runCapture = (command) => { // Existing sandbox that is NOT ready if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive" }); +registry.getSandbox = () => ({ + name: "my-assistant", + toolDisclosure: "progressive", + fromDockerfile: ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, +}); childProcess.spawn = () => { throw new Error("unexpected sandbox create"); }; @@ -318,7 +323,11 @@ sandboxState.backupSandboxState = (name) => { failedDirs: [], backedUpFiles: ["UPGRADE_MARKER.md"], failedFiles: [], - manifest: { backupPath: "/tmp/fake-backup-path", timestamp: "2026-05-25T00:00:00Z" }, + manifest: { + agentType: "openclaw", dir: "/sandbox/.openclaw", + openclawImagePluginInstalls: [], reconcileOpenClawImagePluginProvenance: true, + backupPath: "/tmp/fake-backup-path", timestamp: "2026-05-25T00:00:00Z", + }, }; }; sandboxState.restoreRecreatedSandboxState = (name, backupPath, options) => { @@ -409,7 +418,7 @@ const { createSandbox } = require(${onboardPath}); const restoreEvent = events[restoreIndex]; assert.equal(restoreEvent?.backupPath, "/tmp/fake-backup-path", "restore must use backup path"); assert.equal(restoreEvent?.options?.targetAgentType, "openclaw"); - assert.equal(restoreEvent?.options?.freshOpenClawImagePluginInstalls, undefined); + assert.deepEqual(restoreEvent?.options?.freshOpenClawImagePluginInstalls, []); }); it("recreate-sandbox with NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 skips backup", { @@ -617,7 +626,11 @@ sandboxState.backupSandboxState = (name) => { failedDirs: [], backedUpFiles: ["UPGRADE_MARKER.md"], failedFiles: [], - manifest: { backupPath: "/tmp/fake-backup-notready", timestamp: "2026-05-25T00:00:00Z" }, + manifest: { + agentType: "openclaw", dir: "/sandbox/.openclaw", + openclawImagePluginInstalls: [], reconcileOpenClawImagePluginProvenance: true, + backupPath: "/tmp/fake-backup-notready", timestamp: "2026-05-25T00:00:00Z", + }, }; }; sandboxState.restoreRecreatedSandboxState = (name, backupPath) => { diff --git a/test/openshell-credential-generation-window.test.ts b/test/openshell-credential-generation-window.test.ts index 849cbdddd75..2d9006681c2 100644 --- a/test/openshell-credential-generation-window.test.ts +++ b/test/openshell-credential-generation-window.test.ts @@ -104,7 +104,9 @@ describe("OpenShell exact-main credential generation-window proof", () => { expect(liveTarget).toContain('["nemoclaw-start", "node", "-e"'); expect(liveTarget).toContain("CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry"); expect(liveTarget).toContain("CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval"); - expect(liveTarget).toContain('["sandbox", "provider", "detach"'); + expect(liveTarget).toContain('[SANDBOX_NAME, "mcp", "remove", SERVER_NAME]'); + expect(liveTarget).toMatch(/\[\s*SANDBOX_NAME,\s*"mcp",\s*"add",\s*SERVER_NAME,/u); + expect(liveTarget).not.toContain('["sandbox", "provider", "detach"'); expect(liveTarget).toContain('[SANDBOX_NAME, "mcp", "restart", SERVER_NAME]'); expect(liveTarget).toContain('[SANDBOX_NAME, "rebuild", "--yes"]'); expect(liveTarget).toContain('!request.auth.includes("openshell:resolve:env")'); From 916c6abfbefb02af5ad096a4a10a61d149310a2e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:50:19 -0700 Subject: [PATCH 13/37] ci(e2e): trigger exact managed image publication From b8990bd8b0e58e25100864ba5a2a74660272b902 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:52:24 -0700 Subject: [PATCH 14/37] docs(onboard): clarify generated build context --- src/lib/onboard/managed-workload/onboard-orchestration.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index aaae9c94077..437fabdd7df 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -479,8 +479,8 @@ export async function prepareOnboardSandboxWorkloadLaunch( // Read the patch input only after that boundary so the final image gets // the exact metadata produced by the same staging operation. ...patchInput, - // An explicit path to the checked-in agent Dockerfile is staged through - // the trusted agent builder. Preserve that classification at patch time. + // An explicit path to the checked-in agent Dockerfile is staged as a + // generated build context. Preserve that origin at patch time. fromDockerfile: buildContext.origin === "generated" ? null : patchInput.fromDockerfile, selectedGpuRoute: initialGpuRoute, stagedDockerfile: buildContext.stagedDockerfile, From 947364541452505ab9860f1ea6faaae9202fad47 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:48:46 -0700 Subject: [PATCH 15/37] ci(test): rebalance E2E support shards --- test/cli-coverage-sequencer.test.ts | 2 +- test/helpers/cli-coverage-sequencer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli-coverage-sequencer.test.ts b/test/cli-coverage-sequencer.test.ts index 18f13705d62..7e1f27b7877 100644 --- a/test/cli-coverage-sequencer.test.ts +++ b/test/cli-coverage-sequencer.test.ts @@ -116,7 +116,7 @@ describe("stable CLI coverage sharding", () => { expect(Object.fromEntries(owners)).toEqual({ "cli:src/lib/example.test.ts": 6, - "e2e-support:test/e2e/support/example.test.ts": 8, + "e2e-support:test/e2e/support/example.test.ts": 2, "integration:test/hermes-restart-config-seal-write-lock.test.ts": 4, "integration:test/local-credential-helper-fields.test.ts": 7, "integration:test/regular-0.test.ts": 4, diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index 835d9e97a95..e34ff9578ee 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -43,7 +43,7 @@ const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); // of relying on combined weight from the parallel CLI and E2E-support lanes. const stableShardSalt = "7257"; const integrationShardSalt = "12432"; -const e2eSupportShardSalt = "13930"; +const e2eSupportShardSalt = "15448"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); From 36446227c0e59e63d82faf54dace9a6b6fa47e40 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:12:32 -0700 Subject: [PATCH 16/37] test(e2e): correct credential and Slack fixtures --- test/e2e/lib/fake-slack-api.cjs | 54 +++++++++++++---- test/e2e/live/messaging-providers-helpers.ts | 30 +++++----- ...shell-credential-generation-window.test.ts | 60 +++++++++++-------- .../openshell-credential-generation-window.ts | 6 +- ...messaging-providers-runtime-proofs.test.ts | 54 +++++++++++++++++ ...shell-credential-generation-window.test.ts | 2 +- 6 files changed, 152 insertions(+), 54 deletions(-) diff --git a/test/e2e/lib/fake-slack-api.cjs b/test/e2e/lib/fake-slack-api.cjs index 30b72a19eff..0935ff8f0d7 100755 --- a/test/e2e/lib/fake-slack-api.cjs +++ b/test/e2e/lib/fake-slack-api.cjs @@ -11,6 +11,8 @@ const http = require("http"); const host = process.env.FAKE_SLACK_API_HOST || "0.0.0.0"; const rawPort = process.env.FAKE_SLACK_API_PORT || "0"; const port = Number(rawPort); +const rawWebsocketPort = process.env.FAKE_SLACK_API_WEBSOCKET_PORT || "0"; +const websocketPort = Number(rawWebsocketPort); const portFile = process.env.FAKE_SLACK_API_PORT_FILE || ""; const captureFile = process.env.FAKE_SLACK_API_CAPTURE_FILE || ""; const expectedBotToken = process.env.FAKE_SLACK_API_EXPECTED_BOT_TOKEN || ""; @@ -26,6 +28,18 @@ if (!Number.isInteger(port) || port < 0 || port > 65535) { process.exit(2); } +if (!Number.isInteger(websocketPort) || websocketPort < 0 || websocketPort > 65535) { + console.error( + `FAKE_SLACK_API_WEBSOCKET_PORT must be an integer between 0 and 65535 (received: ${rawWebsocketPort})`, + ); + process.exit(2); +} + +if (port !== 0 && websocketPort === port) { + console.error("FAKE_SLACK_API_PORT and FAKE_SLACK_API_WEBSOCKET_PORT must be distinct"); + process.exit(2); +} + if (!expectedBotToken || !expectedAppToken) { console.error("FAKE_SLACK_API_EXPECTED_BOT_TOKEN and FAKE_SLACK_API_EXPECTED_APP_TOKEN are required"); process.exit(2); @@ -156,7 +170,7 @@ function sendSocketModeEvent(socket) { record({ event: "websocket-event-sent", path: "/socket-mode", envelopeId: envelope.envelope_id }); } -const server = http.createServer((req, res) => { +function handleRequest(req, res) { const chunks = []; let bodyBytes = 0; let bodyTooLarge = false; @@ -223,9 +237,9 @@ const server = http.createServer((req, res) => { }); res.end(JSON.stringify(response.body)); }); -}); +} -server.on("upgrade", (req, socket) => { +function handleUpgrade(req, socket) { const pathname = new URL(req.url || "/", "http://fake-slack.local").pathname; if (pathname !== "/socket-mode") { socket.destroy(); @@ -292,19 +306,39 @@ server.on("upgrade", (req, socket) => { } } }); -}); +} + +function createSlackServer() { + const slackServer = http.createServer(handleRequest); + slackServer.on("upgrade", handleUpgrade); + return slackServer; +} + +const restServer = createSlackServer(); +const websocketServer = createSlackServer(); +const listeningPorts = {}; -server.listen(port, host, () => { +function recordListening(kind, server) { const address = server.address(); - if (portFile) { - fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + listeningPorts[kind] = address.port; + record({ event: "listening", host, kind, port: address.port }); + if (portFile && listeningPorts.rest && listeningPorts.websocket) { + fs.writeFileSync(portFile, `${listeningPorts.rest}\n`, { mode: 0o600 }); } - record({ event: "listening", host, port: address.port }); -}); +} + +restServer.listen(port, host, () => recordListening("rest", restServer)); +websocketServer.listen(websocketPort, host, () => recordListening("websocket", websocketServer)); for (const signal of ["SIGTERM", "SIGINT"]) { process.on(signal, () => { - server.close(() => process.exit(0)); + let openServers = 2; + const finish = () => { + openServers -= 1; + if (openServers === 0) process.exit(0); + }; + restServer.close(finish); + websocketServer.close(finish); setTimeout(() => process.exit(0), 1000).unref(); }); } diff --git a/test/e2e/live/messaging-providers-helpers.ts b/test/e2e/live/messaging-providers-helpers.ts index ee40dc418d4..ac1f49442a9 100644 --- a/test/e2e/live/messaging-providers-helpers.ts +++ b/test/e2e/live/messaging-providers-helpers.ts @@ -606,7 +606,7 @@ export async function startFakeDockerApi( `${options.captureFileEnv}=/tmp/fake/capture.jsonl`, ]; if (options.kind === "slack") { - dockerArgs.splice(7, 0, "-p", "0:8080"); + dockerArgs.splice(7, 0, "-p", "0:8081", "-e", "FAKE_SLACK_API_WEBSOCKET_PORT=8081"); } for (const [key, value] of Object.entries(options.expectedEnv)) { dockerArgs.push("-e", `${key}=${value}`); @@ -647,26 +647,28 @@ export async function startFakeDockerApi( for (let attempt = 0; attempt < 100; attempt += 1) { if (fs.existsSync(portFile) && fs.statSync(portFile).size > 0) { - const port = await runHost(host, "docker", ["port", container, "8080/tcp"], { + const restPort = await runHost(host, "docker", ["port", container, "8080/tcp"], { artifactName: `port-fake-${options.kind}-api`, env: options.env, redactionValues: options.redactionValues, timeoutMs: 30_000, }); - const published = [ - ...new Set( - port.stdout - .trim() - .split(/\r?\n/u) - .map((line) => line.split(":").at(-1)?.trim() ?? "") - .filter(Boolean), - ), - ]; - if (published.length >= (options.kind === "slack" ? 2 : 1)) { + const publishedRestPort = restPort.stdout.trim().split(":").at(-1)?.trim() ?? ""; + let publishedWebsocketPort = ""; + if (options.kind === "slack") { + const websocketPort = await runHost(host, "docker", ["port", container, "8081/tcp"], { + artifactName: "port-fake-slack-websocket-api", + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 30_000, + }); + publishedWebsocketPort = websocketPort.stdout.trim().split(":").at(-1)?.trim() ?? ""; + } + if (publishedRestPort && (options.kind !== "slack" || publishedWebsocketPort)) { return { kind: options.kind, - port: published[0], - ...(options.kind === "slack" ? { alternatePort: published[1] } : {}), + port: publishedRestPort, + ...(options.kind === "slack" ? { alternatePort: publishedWebsocketPort } : {}), dir, captureFile, container, diff --git a/test/e2e/live/openshell-credential-generation-window.test.ts b/test/e2e/live/openshell-credential-generation-window.test.ts index abb3720373a..5ffdefccb74 100644 --- a/test/e2e/live/openshell-credential-generation-window.test.ts +++ b/test/e2e/live/openshell-credential-generation-window.test.ts @@ -330,7 +330,7 @@ test("openshell-credential-generation-window", { "prove a retained credential generation expires", "rotate beyond the retained generation window", "prove key and bridge removal revoke access", - "re-add the bridge and confirm old-process fallback", + "re-add the bridge and keep the old process revoked", "rebuild the sandbox and confirm credential reuse", "remove the MCP bridge and audit denied requests", ], @@ -764,7 +764,7 @@ test("openshell-credential-generation-window", { ).seen, ).toBe(false); - progress.phase("re-add the bridge and confirm old-process fallback"); + progress.phase("re-add the bridge and keep the old process revoked"); fakeMcp.setSecret(restartSecret); const readd = await host.nemoclaw( [ @@ -794,31 +794,36 @@ test("openshell-credential-generation-window", { ); expect(restartedRevision).not.toBe(currentRevision); expect(restartedRevision).not.toBe(restoredKeyRevision); + const freshAfterReaddId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-readd`; + const freshAfterReadd = await runFreshRequest( + sandbox, + tunnel.url, + freshAfterReaddId, + allSecrets, + "credential-window-fresh-request-after-readd", + ); + expect(freshAfterReadd).toEqual({ + revision: restartedRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, freshAfterReaddId, restartSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); await writeControl( sandbox, - CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, - "credential-window-signal-fallback-after-restart", + CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + "credential-window-signal-denied-after-readd", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, "allowed"); - await expect - .poll( - () => - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart), - restartSecret, - ), - { - interval: 500, - timeout: 30_000, - message: "old revision fallback after restart", - }, - ) - .toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, "denied"); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + restartSecret, + ).seen, + ).toBe(false); } finally { await writeControl( sandbox, @@ -846,8 +851,8 @@ test("openshell-credential-generation-window", { { step: CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, outcome: "denied" }, { step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, outcome: "denied" }, { - step: CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart, - outcome: "allowed", + step: CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + outcome: "denied", }, ], }); @@ -921,6 +926,9 @@ test("openshell-credential-generation-window", { expect(upstreamRequestIds).not.toContain( credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), ); + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + ); expect( fakeMcp.requests.every( (request: CredentialWindowRequest) => !request.auth.includes("openshell:resolve:env"), diff --git a/test/e2e/live/openshell-credential-generation-window.ts b/test/e2e/live/openshell-credential-generation-window.ts index 5d56d44993c..db5065cf0be 100644 --- a/test/e2e/live/openshell-credential-generation-window.ts +++ b/test/e2e/live/openshell-credential-generation-window.ts @@ -24,7 +24,7 @@ export const CREDENTIAL_WINDOW_STEPS = { fallbackAfterEviction: "fallback-after-eviction", deniedAfterKeyRemoval: "denied-after-key-removal", deniedAfterDetach: "denied-after-detach", - fallbackAfterRestart: "fallback-after-restart", + deniedAfterReadd: "denied-after-readd", stop: "stop", } as const; @@ -34,7 +34,7 @@ export type CredentialWindowRequestStep = | (typeof CREDENTIAL_WINDOW_STEPS)["fallbackAfterEviction"] | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterKeyRemoval"] | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterDetach"] - | (typeof CREDENTIAL_WINDOW_STEPS)["fallbackAfterRestart"]; + | (typeof CREDENTIAL_WINDOW_STEPS)["deniedAfterReadd"]; export function credentialWindowSecret(generation: number): string { return `${MCP_BRIDGE_TEST_CREDENTIALS.generationWindow}${String(generation).padStart(2, "0")}`; @@ -108,7 +108,7 @@ const requestSteps = new Set([ config.steps.fallbackAfterEviction, config.steps.deniedAfterKeyRemoval, config.steps.deniedAfterDetach, - config.steps.fallbackAfterRestart, + config.steps.deniedAfterReadd, ]); const seen = new Set(); const outcomes = []; diff --git a/test/e2e/support/messaging-providers-runtime-proofs.test.ts b/test/e2e/support/messaging-providers-runtime-proofs.test.ts index 5528f3c6c2b..9a5952c445c 100644 --- a/test/e2e/support/messaging-providers-runtime-proofs.test.ts +++ b/test/e2e/support/messaging-providers-runtime-proofs.test.ts @@ -24,6 +24,11 @@ import { import { TELEGRAM_INSTALLED_RUNTIME_PROOF_SOURCE } from "../live/messaging-providers-telegram-runtime-proof.ts"; const FAKE_TELEGRAM_API = path.resolve(import.meta.dirname, "../lib/fake-telegram-api.cjs"); +const FAKE_SLACK_API = path.resolve(import.meta.dirname, "../lib/fake-slack-api.cjs"); +const MESSAGING_PROVIDERS_HELPERS_SOURCE = fs.readFileSync( + path.resolve(import.meta.dirname, "../live/messaging-providers-helpers.ts"), + "utf8", +); const LIVE_MESSAGING_PROVIDERS_SOURCE = fs.readFileSync( path.resolve(import.meta.dirname, "../live/messaging-providers.test.ts"), "utf8", @@ -48,6 +53,55 @@ async function waitFor(predicate: () => boolean, message: string): Promise } describe("messaging provider installed-runtime proofs", () => { + it("publishes independent fake Slack REST and websocket ports", async () => { + expect(MESSAGING_PROVIDERS_HELPERS_SOURCE.match(/"0:8080"/gu)).toHaveLength(1); + expect(MESSAGING_PROVIDERS_HELPERS_SOURCE).toContain('"0:8081"'); + expect(MESSAGING_PROVIDERS_HELPERS_SOURCE).toContain('"FAKE_SLACK_API_WEBSOCKET_PORT=8081"'); + expect(MESSAGING_PROVIDERS_HELPERS_SOURCE).toContain('["port", container, "8081/tcp"]'); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fake-slack-ports-")); + const portFile = path.join(dir, "port"); + const captureFile = path.join(dir, "capture.jsonl"); + const child = spawn(process.execPath, [FAKE_SLACK_API], { + env: { + ...process.env, + FAKE_SLACK_API_HOST: "127.0.0.1", + FAKE_SLACK_API_PORT: "0", + FAKE_SLACK_API_WEBSOCKET_PORT: "0", + FAKE_SLACK_API_PORT_FILE: portFile, + FAKE_SLACK_API_CAPTURE_FILE: captureFile, + FAKE_SLACK_API_EXPECTED_BOT_TOKEN: "xoxb-fake-slack-port-test", + FAKE_SLACK_API_EXPECTED_APP_TOKEN: "xapp-fake-slack-port-test", + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderr += chunk; + }); + + try { + await waitFor(() => fs.existsSync(portFile), `fake Slack listeners did not start: ${stderr}`); + const listening = fs + .readFileSync(captureFile, "utf8") + .trim() + .split(/\r?\n/u) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((entry) => entry.event === "listening"); + expect(listening).toHaveLength(2); + expect(listening.map((entry) => entry.kind).sort()).toEqual(["rest", "websocket"]); + expect(new Set(listening.map((entry) => entry.port)).size).toBe(2); + } finally { + child.kill("SIGTERM"); + await new Promise((resolve) => + child.exitCode !== null ? resolve() : child.once("exit", () => resolve()), + ); + fs.rmSync(dir, { recursive: true, force: true }); + } + }, 10_000); + it("keeps raw process-probe tokens out of argv and fails closed", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-process-token-probe-")); const token = `xoxb-nemoclaw-process-probe-secret-${process.pid}`; diff --git a/test/openshell-credential-generation-window.test.ts b/test/openshell-credential-generation-window.test.ts index 2d9006681c2..03efed4029d 100644 --- a/test/openshell-credential-generation-window.test.ts +++ b/test/openshell-credential-generation-window.test.ts @@ -52,7 +52,7 @@ describe("OpenShell exact-main credential generation-window proof", () => { expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach)); - expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.fallbackAfterRestart)); + expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd)); expect(script).toContain(JSON.stringify(CREDENTIAL_WINDOW_STEPS.stop)); expect(script).not.toContain(MCP_BRIDGE_TEST_CREDENTIALS.generationWindow); }); From 871edc304c388e8a0a4888d63f3ceca83c64a932 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:34:39 -0700 Subject: [PATCH 17/37] docs(e2e): correct workflow and CLI guidance --- docs/get-started/quickstart-hermes.mdx | 2 +- .../quickstart-langchain-deepagents-code.mdx | 2 +- test/e2e/README.md | 12 ++++++------ test/e2e/fixtures/availability-env.ts | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/get-started/quickstart-hermes.mdx b/docs/get-started/quickstart-hermes.mdx index fe953dd66f5..eb1361c6538 100644 --- a/docs/get-started/quickstart-hermes.mdx +++ b/docs/get-started/quickstart-hermes.mdx @@ -64,7 +64,7 @@ Review the [Prerequisites](prerequisites) before you begin. With the OpenShell Docker driver, stock Hermes onboarding normally uses the release's exact managed-image digest. If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Invalid or inconsistent catalog evidence fails closed before sandbox creation. - An explicit `nemoclaw onboard --from ` remains a separate custom-image path. + An explicit `nemohermes onboard --from ` remains a separate custom-image path. diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 0301fee482f..c553707e0f5 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -67,7 +67,7 @@ Review the [Prerequisites](prerequisites) before you begin. With the OpenShell Docker driver, stock Deep Agents Code onboarding normally uses the release's exact managed-image digest. If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Invalid or inconsistent catalog evidence fails closed before sandbox creation. - An explicit `nemoclaw onboard --from ` remains a separate custom-image path. + An explicit `nemo-deepagents onboard --from ` remains a separate custom-image path. diff --git a/test/e2e/README.md b/test/e2e/README.md index bdce05810f2..0ee33524f1b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -122,7 +122,8 @@ This baseline measures only the replaced build step. Artifact upload, download, validation, and the dependency on `generate-matrix` add runtime and can affect the workflow critical path. Do not use the build-step median to claim savings in runner time or workflow elapsed time. -A same-repository manual PR E2E run tests candidate code and executes `.github/workflows/e2e.yaml` from the PR branch at the exact latest PR commit. +A manual PR E2E run tests candidate code but executes `.github/workflows/e2e.yaml` from trusted `main`. +The PR run cannot measure this workflow change before merge. After merge, use a passing `main` run and complete these steps: 1. Match the job selection, runner labels, and first attempt to the baseline. @@ -561,7 +562,7 @@ to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map before checking out test code, and it consumes the variable only when the workflow repository is `NVIDIA/NemoClaw`, the ref is `refs/heads/main`, and no alternate checkout SHA is requested. Manual PR E2E dispatches therefore remain on -standard runners because an alternate candidate cannot select the administrator-managed label. +standard runners even though they use the trusted workflow definition from `main`. Manual PR E2E dispatches and direct push or manual `main` runs use a bounded swap fallback for eligible hosted Hermes image-building lanes. The @@ -1258,8 +1259,7 @@ The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. Jetson and Launchable dispatch additionally require the PR branch to be in `NVIDIA/NemoClaw`; their operator and image-producer backends do not accept a sibling-repository candidate. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -For an NVIDIA-owned PR, the same-repository PR branch supplies the workflow definition and `workflow_sha` must match the latest PR commit. -The workflow binds the latest PR commit to the current PR base SHA. +The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. It does not run GitHub's synthetic merge commit. Before candidate execution, the workflow uploads a `nemoclaw-e2e-dispatch-v2` receipt for the trusted manual run. The full-main `Release qualification` aggregate does not use this receipt. @@ -1342,7 +1342,7 @@ For a manual PR run, provide these inputs: - The lowercase 40-character SHA of the latest PR commit. - The PR source repository. - The lowercase 40-character PR base SHA. -- The exact SHA of the workflow commit on the same-repository PR branch, equal to the latest PR commit. +- The exact SHA of the trusted workflow commit on `main`. For the default NVIDIA-owned PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. @@ -1356,7 +1356,7 @@ To select native runtime qualification evidence production, set `jobs=native-run Leave `targets` empty and keep `include_staging_brev_launchable=false`. For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -A trusted controller pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. +A trusted `main` workflow pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. That ownership authorizes the full ordinary plan and credential profiles; external sources retain the bounded controller plan. A second validation after checkout rejects a changed candidate commit, base commit, PR source repository, or NVIDIA ownership before preparation. Candidate runs cannot publish release qualification. diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index 627449cf23d..2d11753fcdc 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -26,10 +26,10 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ export function buildAvailabilityProbeEnv( base: NodeJS.ProcessEnv = process.env, ): NodeJS.ProcessEnv { - // Availability probes run outside live target phases, but they need - // the same child-env and PATH policy. Add Docker discovery knobs and the - // workflow-owned local-model pull budget and exact PR catalog authority on - // top of the shared boundary. + // Availability probes run outside live target phases but need the shared + // child environment and PATH policy. Add Docker discovery settings, the + // workflow-owned local-model pull budget, and the selected managed-image + // cohort revision and receipt to that boundary. return buildChildEnv(base, { additionalAllowedEnv: AVAILABILITY_PROBE_EXTRA_ENV_KEYS, fixtureOverlay: {}, From 55c4d2c0e812a4677824303385f233fb8453ca9b Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:25:13 -0700 Subject: [PATCH 18/37] test(e2e): tunnel Slack socket probe through proxy --- test/e2e/live/openclaw-pairing-helpers.ts | 41 +++++-- .../support/fixtures/slack-connect-proxy.ts | 107 ++++++++++++++++++ .../openclaw-discord-pairing-helpers.test.ts | 71 +++++++++--- 3 files changed, 197 insertions(+), 22 deletions(-) create mode 100644 test/e2e/support/fixtures/slack-connect-proxy.ts diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index d1c21d0d424..5bdfe61edd4 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -460,14 +460,15 @@ function receiveSlackSocketEvent() { return new Promise((resolve, reject) => { const socket = proxy ? net.createConnection({ host: proxy.host, port: proxy.port }) : net.createConnection({ host, port }); const timer = setTimeout(() => { socket.destroy(); reject(new Error("timed out waiting for fake Slack Socket Mode event")); }, 30000); + let proxyHandshake = Buffer.alloc(0); let handshake = Buffer.alloc(0); let framed = Buffer.alloc(0); + let tunnelEstablished = !proxy; let upgraded = false; - socket.on("connect", () => { + function sendUpgradeRequest() { const key = crypto.randomBytes(16).toString("base64"); - const requestTarget = proxy ? "http://" + host + ":" + port + "/socket-mode" : "/socket-mode"; socket.write([ - "GET " + requestTarget + " HTTP/1.1", + "GET /socket-mode HTTP/1.1", "Host: " + host + ":" + port, "Upgrade: websocket", "Connection: Upgrade", @@ -475,8 +476,36 @@ function receiveSlackSocketEvent() { "Sec-WebSocket-Version: 13", "\r\n", ].join("\r\n")); + } + socket.on("connect", () => { + if (!proxy) { + sendUpgradeRequest(); + return; + } + socket.write([ + "CONNECT " + host + ":" + port + " HTTP/1.1", + "Host: " + host + ":" + port, + "\r\n", + ].join("\r\n")); }); socket.on("data", (chunk) => { + if (!tunnelEstablished) { + proxyHandshake = Buffer.concat([proxyHandshake, chunk]); + const end = proxyHandshake.indexOf("\r\n\r\n"); + if (end === -1) return; + const statusLine = proxyHandshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; + if (!/^HTTP\/1\.[01] 200(?: |$)/.test(statusLine)) { + clearTimeout(timer); + socket.destroy(); + reject(new Error("OpenShell proxy CONNECT failed: " + statusLine)); + return; + } + tunnelEstablished = true; + chunk = proxyHandshake.slice(end + 4); + proxyHandshake = Buffer.alloc(0); + sendUpgradeRequest(); + if (chunk.length === 0) return; + } if (!upgraded) { handshake = Buffer.concat([handshake, chunk]); const end = handshake.indexOf("\r\n\r\n"); @@ -604,11 +633,7 @@ export async function issuePairingRequest(options: { const script = options.channel === "slack" ? SLACK_PAIRING_SCRIPT : DISCORD_PAIRING_SCRIPT; const args = options.channel === "slack" - ? [ - options.fakeSlackRestPort ?? "", - options.fakeSlackWebsocketPort ?? "", - PAIRING_USER.slack, - ] + ? [options.fakeSlackRestPort ?? "", options.fakeSlackWebsocketPort ?? "", PAIRING_USER.slack] : [PAIRING_USER.discord, DISCORD_DM_CHANNEL]; return sandboxShWithArgs(options.sandbox, options.sandboxName, script, args, { artifactName: `${options.channel}-issue-pairing-request`, diff --git a/test/e2e/support/fixtures/slack-connect-proxy.ts b/test/e2e/support/fixtures/slack-connect-proxy.ts new file mode 100644 index 00000000000..b1b049b1d4a --- /dev/null +++ b/test/e2e/support/fixtures/slack-connect-proxy.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import net from "node:net"; +import vm from "node:vm"; + +import { SLACK_PAIRING_SCRIPT } from "../../live/openclaw-pairing-helpers.ts"; + +export function createSlackSocketClient(proxyPort: number, targetPort: number) { + const sourceStart = SLACK_PAIRING_SCRIPT.indexOf("function parseFakeSlackPort"); + const sourceEnd = SLACK_PAIRING_SCRIPT.indexOf("function postPairingReply"); + if (sourceStart < 0 || sourceEnd <= sourceStart) { + throw new Error("Slack Socket Mode client source is missing"); + } + const source = SLACK_PAIRING_SCRIPT.slice(sourceStart, sourceEnd); + return vm.runInNewContext(`${source}\nreceiveSlackSocketEvent`, { + Buffer, + URL, + clearTimeout, + crypto, + net: { + createConnection: () => net.createConnection({ host: "127.0.0.1", port: proxyPort }), + }, + process: { + env: { + FAKE_SLACK_WEBSOCKET_PORT: String(targetPort), + HTTP_PROXY: "http://10.200.0.1:3128", + http_proxy: "", + }, + }, + setTimeout, + }) as () => Promise>; +} + +function encodeServerText(payload: Record): Buffer { + const body = Buffer.from(JSON.stringify(payload), "utf8"); + if (body.length > 125) throw new Error("test WebSocket payload is too large"); + return Buffer.concat([Buffer.from([0x81, body.length]), body]); +} + +export async function listenOnLoopback(server: net.Server): Promise { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test proxy has no TCP port"); + return address.port; +} + +export async function closeServer(server: net.Server): Promise { + await new Promise((resolve) => server.close(() => resolve())); +} + +export function createSuccessfulSlackConnectProxy(envelope: Record): { + server: net.Server; + requests: string[]; + websocketBytes: () => number; +} { + const requests: string[] = []; + let receivedWebsocketBytes = 0; + const server = net.createServer((socket) => { + let buffer = Buffer.alloc(0); + let phase: "connect" | "upgrade" | "websocket" = "connect"; + socket.on("data", (chunk) => { + if (phase === "websocket") { + receivedWebsocketBytes += chunk.length; + return; + } + buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]); + const end = buffer.indexOf("\r\n\r\n"); + if (end === -1) return; + requests.push(buffer.slice(0, end).toString("latin1")); + buffer = buffer.slice(end + 4); + if (phase === "connect") { + phase = "upgrade"; + socket.write("HTTP/1.1 200 Connection Established\r\n"); + setImmediate(() => socket.write("\r\n")); + return; + } + phase = "websocket"; + socket.write( + Buffer.concat([ + Buffer.from( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n", + "latin1", + ), + encodeServerText(envelope), + ]), + ); + }); + }); + return { + server, + requests, + websocketBytes: () => receivedWebsocketBytes, + }; +} + +export function createRejectedSlackConnectProxy(): net.Server { + return net.createServer((socket) => { + socket.once("data", () => { + socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); + }); + }); +} diff --git a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts index 1b874ce560d..cf826a8d143 100644 --- a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts +++ b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts @@ -9,6 +9,13 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + closeServer, + createRejectedSlackConnectProxy, + createSlackSocketClient, + createSuccessfulSlackConnectProxy, + listenOnLoopback, +} from "./fixtures/slack-connect-proxy.ts"; import { buildPairingApproveCommand, buildPairingPendingCommand, @@ -124,7 +131,43 @@ async function sendDiscordIdentify(port: number, token: string): Promise { }); } -describe("OpenClaw Discord pairing helper contracts", () => { +describe("OpenClaw pairing helper contracts", () => { + it("establishes an HTTP CONNECT tunnel before the fake Slack WebSocket upgrade", async () => { + const targetPort = 4443; + const envelope = { payload: { event: { type: "message" } } }; + const proxy = createSuccessfulSlackConnectProxy(envelope); + const proxyPort = await listenOnLoopback(proxy.server); + + try { + await expect(createSlackSocketClient(proxyPort, targetPort)()).resolves.toEqual(envelope); + await vi.waitFor(() => expect(proxy.websocketBytes()).toBeGreaterThan(0), { + interval: 10, + timeout: 1_000, + }); + expect(proxy.requests).toHaveLength(2); + expect(proxy.requests[0]).toMatch( + new RegExp(`^CONNECT host\\.openshell\\.internal:${targetPort} HTTP/1\\.1`, "u"), + ); + expect(proxy.requests[1]).toMatch(/^GET \/socket-mode HTTP\/1\.1/u); + expect(proxy.requests[1]).not.toContain("http://"); + } finally { + await closeServer(proxy.server); + } + }); + + it("rejects a non-200 OpenShell proxy CONNECT response", async () => { + const proxy = createRejectedSlackConnectProxy(); + const proxyPort = await listenOnLoopback(proxy); + + try { + await expect(createSlackSocketClient(proxyPort, 4443)()).rejects.toThrow( + "OpenShell proxy CONNECT failed: HTTP/1.1 502 Bad Gateway", + ); + } finally { + await closeServer(proxy); + } + }); + it("shell-quotes pairing code and user without command substitution", () => { const code = "abc$(touch /tmp/e2e-should-not-run)"; const user = "user`touch /tmp/e2e-should-not-run`"; @@ -292,20 +335,20 @@ describe("OpenClaw Discord pairing helper contracts", () => { env: { HTTP_PROXY: "http://127.0.0.1:3128", http_proxy: "" }, error: "unexpected HTTP proxy for Discord Gateway proof", }, - ])("fails closed on invalid Discord Gateway proxy input before network access: $name", ({ - env, - error, - }) => { - const result = spawnSync(process.execPath, ["--input-type=module"], { - input: `${DISCORD_GATEWAY_PROOF_SOURCE}\n`, - encoding: "utf8", - env: { ...process.env, FAKE_DISCORD_GATEWAY_PORT: "12345", ...env }, - }); + ])( + "fails closed on invalid Discord Gateway proxy input before network access: $name", + ({ env, error }) => { + const result = spawnSync(process.execPath, ["--input-type=module"], { + input: `${DISCORD_GATEWAY_PROOF_SOURCE}\n`, + encoding: "utf8", + env: { ...process.env, FAKE_DISCORD_GATEWAY_PORT: "12345", ...env }, + }); - expect(result.status).not.toBe(0); - expect(result.stderr).toEqual(expect.stringContaining(error)); - expect(result.stderr).not.toContain("ECONNREFUSED"); - }); + expect(result.status).not.toBe(0); + expect(result.stderr).toEqual(expect.stringContaining(error)); + expect(result.stderr).not.toContain("ECONNREFUSED"); + }, + ); it("rejects malformed sandboxNode env keys before sandbox execution", async () => { const execShell = vi.fn(async () => { From 7b3b33db4470d2c53675bf11b6b58ab3c4213267 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 20:38:19 -0500 Subject: [PATCH 19/37] fix(test): repair grouped imports after main merge --- .../langchain-deepagents-code-image.test.ts | 2 +- .../openclaw-2026-7-startup-compat.test.ts | 2 +- .../openclaw/openclaw-lifecycle-policy.test.ts | 2 +- ...penclaw-security-revision-container-e2e.test.ts | 4 ++-- ...claw-start-extra-placeholder-breadcrumb.test.ts | 2 +- .../runtime/nemoclaw-start-gateway-health.test.ts | 2 +- .../nemoclaw-start-gateway-token-env.test.ts | 2 +- test/automation/e2e/e2e-private-file.test.ts | 2 +- test/automation/e2e/e2e-recommendations.test.ts | 4 ++-- .../e2e/e2e-risk-signal-reporter.test.ts | 4 ++-- .../pr-review-advisor-writing-guide.test.ts | 14 +++++++------- .../managed-image-activation-command.test.ts | 2 +- ...lama-auth-proxy-handler-startup-cleanup.test.ts | 2 +- .../ollama/ollama-auth-proxy-handler.test.ts | 2 +- test/mcp/mcp-agent-matrix-artifact-proof.test.ts | 2 +- test/mcp/mcp-artifact-secret-scan.test.ts | 4 ++-- test/mcp/mcp-bridge-servers.test.ts | 4 ++-- test/mcp/mcp-openshell-workflow.test.ts | 2 +- test/repository/source-shape-scanner.test.ts | 8 ++++---- test/repository/vitest-coverage-thresholds.test.ts | 2 +- .../gateway/gateway-serving-watchdog.test.ts | 2 +- .../messaging-build-applier-integrity.test.ts | 2 +- .../messaging/messaging-build-applier.test.ts | 2 +- .../sandbox/sandbox-download-upload-cli.test.ts | 2 +- .../sandbox-sessions-admin-agent-cli.test.ts | 2 +- .../sandbox/sandbox-sessions-export-cli.test.ts | 2 +- 26 files changed, 40 insertions(+), 40 deletions(-) diff --git a/test/agents/deepagents/langchain-deepagents-code-image.test.ts b/test/agents/deepagents/langchain-deepagents-code-image.test.ts index df57960dd71..2f18ec065f2 100644 --- a/test/agents/deepagents/langchain-deepagents-code-image.test.ts +++ b/test/agents/deepagents/langchain-deepagents-code-image.test.ts @@ -13,7 +13,7 @@ import YAML from "yaml"; import { loadAgent } from "../../../src/lib/agent/defs.ts"; import { prepareInitialSandboxCreatePolicy } from "../../../src/lib/onboard/initial-policy.ts"; import { TOKEN_PREFIX_PATTERNS } from "../../../src/lib/security/secret-patterns.ts"; -import { cloudExperimentalChecksForOnboarding } from "./e2e/live/cloud-experimental-check-list.ts"; +import { cloudExperimentalChecksForOnboarding } from "../../e2e/live/cloud-experimental-check-list.ts"; import { ANALYTICS_DISABLE_ENV_NAMES, DCODE_CANONICAL_PATH, diff --git a/test/agents/openclaw/openclaw-2026-7-startup-compat.test.ts b/test/agents/openclaw/openclaw-2026-7-startup-compat.test.ts index 2f8d09e11b3..8ba0ad3f7fa 100644 --- a/test/agents/openclaw/openclaw-2026-7-startup-compat.test.ts +++ b/test/agents/openclaw/openclaw-2026-7-startup-compat.test.ts @@ -8,7 +8,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { safeTmpHelpers } from "./nemoclaw-start-gateway.test-helpers"; +import { safeTmpHelpers } from "../../nemoclaw-start-gateway.test-helpers"; const ROOT = path.resolve(import.meta.dirname, "../../.."); const NORMALIZER = path.join(ROOT, "scripts", "lib", "normalize_mutable_config_perms.py"); diff --git a/test/agents/openclaw/openclaw-lifecycle-policy.test.ts b/test/agents/openclaw/openclaw-lifecycle-policy.test.ts index ca9ff25df92..c0f2374628e 100644 --- a/test/agents/openclaw/openclaw-lifecycle-policy.test.ts +++ b/test/agents/openclaw/openclaw-lifecycle-policy.test.ts @@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import policy from "../ci/reviewed-npm-lifecycle-allowlist.json"; +import policy from "../../../ci/reviewed-npm-lifecycle-allowlist.json"; import { reviewedOpenClawPluginIntegrityByPackageSpec } from "../../../src/lib/messaging/applier/build/messaging-build-applier.mts"; const REPO_ROOT = path.join(import.meta.dirname, "../../.."); diff --git a/test/agents/openclaw/openclaw-security-revision-container-e2e.test.ts b/test/agents/openclaw/openclaw-security-revision-container-e2e.test.ts index 93406a17945..fce37cc7adf 100644 --- a/test/agents/openclaw/openclaw-security-revision-container-e2e.test.ts +++ b/test/agents/openclaw/openclaw-security-revision-container-e2e.test.ts @@ -5,8 +5,8 @@ import { randomUUID } from "node:crypto"; import { describe } from "vitest"; -import { type DockerCommandResult, DockerProbe, resultText } from "./e2e/fixtures/docker-probe.ts"; -import { expect, test } from "./e2e/fixtures/e2e-test.ts"; +import { type DockerCommandResult, DockerProbe, resultText } from "../../e2e/fixtures/docker-probe.ts"; +import { expect, test } from "../../e2e/fixtures/e2e-test.ts"; const TARGET_ID = "openclaw-security-revision-container-e2e"; const RUN_ENV = "NEMOCLAW_RUN_OPENCLAW_SECURITY_REVISION_CONTAINER_E2E"; diff --git a/test/agents/openclaw/runtime/nemoclaw-start-extra-placeholder-breadcrumb.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-extra-placeholder-breadcrumb.test.ts index 84180ffd2e9..60b57676db5 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-extra-placeholder-breadcrumb.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-extra-placeholder-breadcrumb.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from "vitest"; import { placeholderPlan, runRefresh, -} from "./nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts"; +} from "../../../nemoclaw-start-extra-placeholder-breadcrumb-helpers.ts"; // The extra-placeholder canonicalization + accepted-keys breadcrumb contract is // asserted end-to-end only in the live messaging-providers E2E (cases X4a/X4b diff --git a/test/agents/openclaw/runtime/nemoclaw-start-gateway-health.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-gateway-health.test.ts index 8e8ff533767..6acf705f1dd 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-gateway-health.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-gateway-health.test.ts @@ -22,7 +22,7 @@ import { START_SCRIPT, safeTmpHelpers, writeProcStatFunction, -} from "./nemoclaw-start-gateway.test-helpers"; +} from "../../../nemoclaw-start-gateway.test-helpers"; function gatewayMarkerFunction(src: string, name: string, markerPath: string): string { return extractShellFunction(src, name).replaceAll("/tmp/nemoclaw-gateway-local", markerPath); diff --git a/test/agents/openclaw/runtime/nemoclaw-start-gateway-token-env.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-gateway-token-env.test.ts index 4fdd7f91ac5..81e7bba1bb2 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-gateway-token-env.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-gateway-token-env.test.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { safeTmpHelpers } from "./nemoclaw-start-gateway.test-helpers"; +import { safeTmpHelpers } from "../../../nemoclaw-start-gateway.test-helpers"; import { extractShellFunctionFromSource } from "../../../support/shell-function-extractor"; const START_SCRIPT = path.resolve(import.meta.dirname, "../../../../scripts/nemoclaw-start.sh"); diff --git a/test/automation/e2e/e2e-private-file.test.ts b/test/automation/e2e/e2e-private-file.test.ts index ecfa8ba1175..cac51166570 100644 --- a/test/automation/e2e/e2e-private-file.test.ts +++ b/test/automation/e2e/e2e-private-file.test.ts @@ -12,7 +12,7 @@ import { appendPrivateRegularFile, readPrivateRegularFile, writePrivateRegularFile, -} from "../tools/e2e/private-file.mts"; +} from "../../../tools/e2e/private-file.mts"; describe("private E2E controller files", () => { it("writes private regular files without following links or truncating hardlink targets", () => { diff --git a/test/automation/e2e/e2e-recommendations.test.ts b/test/automation/e2e/e2e-recommendations.test.ts index 7ad1aaaf4ac..87eaf1019c3 100644 --- a/test/automation/e2e/e2e-recommendations.test.ts +++ b/test/automation/e2e/e2e-recommendations.test.ts @@ -14,8 +14,8 @@ import { normalizeE2eCoverageResult, normalizeE2eTargetAdvisorResult, trustedE2eRecommendationInventory, -} from "../tools/advisors/e2e-recommendations.mts"; -import { isCommandShapedE2eText } from "../tools/advisors/e2e-text.mts"; +} from "../../../tools/advisors/e2e-recommendations.mts"; +import { isCommandShapedE2eText } from "../../../tools/advisors/e2e-text.mts"; // Tests target the session-free recommendation normalizer shared by the // unified PR Review Advisor. Model prompt and comment rendering are covered by diff --git a/test/automation/e2e/e2e-risk-signal-reporter.test.ts b/test/automation/e2e/e2e-risk-signal-reporter.test.ts index 360465399cf..168320e3211 100644 --- a/test/automation/e2e/e2e-risk-signal-reporter.test.ts +++ b/test/automation/e2e/e2e-risk-signal-reporter.test.ts @@ -15,7 +15,7 @@ import { parseLiveTestOutcome, readLiveTestOutcome, writeLiveTestOutcome, -} from "../tools/e2e/live-test-outcome.mts"; +} from "../../../tools/e2e/live-test-outcome.mts"; import { configuredEnvironment, default as E2eRiskSignalReporter, @@ -23,7 +23,7 @@ import { RISK_SIGNAL_FILE, type RiskSignalEnvironment, writeRiskSignal, -} from "./e2e/risk-signal-reporter.ts"; +} from "../../e2e/risk-signal-reporter.ts"; vi.mock("node:child_process", () => ({ execFileSync: vi.fn(() => "a".repeat(40)), diff --git a/test/automation/pull-requests/pr-review-advisor-writing-guide.test.ts b/test/automation/pull-requests/pr-review-advisor-writing-guide.test.ts index 3bb3e2630aa..cce6640f523 100644 --- a/test/automation/pull-requests/pr-review-advisor-writing-guide.test.ts +++ b/test/automation/pull-requests/pr-review-advisor-writing-guide.test.ts @@ -20,7 +20,7 @@ describe("PR Review Advisor writing guide", () => { try { process.chdir(prWorktree); const { readTrustedWritingGuide } = - await import("../tools/pr-review-advisor/trusted-guidance.mts"); + await import("../../../tools/pr-review-advisor/trusted-guidance.mts"); const writingGuide = readTrustedWritingGuide(); expect(writingGuide).toContain("# NemoClaw Writing Guide"); @@ -33,7 +33,7 @@ describe("PR Review Advisor writing guide", () => { it("stops when the trusted guide is unavailable", async () => { const { readTrustedWritingGuide } = - await import("../tools/pr-review-advisor/trusted-guidance.mts"); + await import("../../../tools/pr-review-advisor/trusted-guidance.mts"); vi.spyOn(fs, "readFileSync").mockImplementationOnce(() => { throw new Error("missing guide fixture"); }); @@ -42,8 +42,8 @@ describe("PR Review Advisor writing guide", () => { }); it("writes failure artifacts when the trusted security rubric is unavailable", async () => { - const { preparePromptArtifacts } = await import("../tools/pr-review-advisor/analyze.mts"); - const { artifactPaths } = await import("../tools/pr-review-advisor/artifacts.mts"); + const { preparePromptArtifacts } = await import("../../../tools/pr-review-advisor/analyze.mts"); + const { artifactPaths } = await import("../../../tools/pr-review-advisor/artifacts.mts"); const outDir = fs.mkdtempSync(path.join(tmpdir(), "advisor-rubric-failure-")); const headSha = "b".repeat(40); const realReadFileSync = fs.readFileSync.bind(fs); @@ -111,10 +111,10 @@ describe("PR Review Advisor writing guide", () => { }); it("writes failure artifacts when trusted prompt inputs are unavailable", async () => { - const { preparePromptArtifacts } = await import("../tools/pr-review-advisor/analyze.mts"); - const { artifactPaths } = await import("../tools/pr-review-advisor/artifacts.mts"); + const { preparePromptArtifacts } = await import("../../../tools/pr-review-advisor/analyze.mts"); + const { artifactPaths } = await import("../../../tools/pr-review-advisor/artifacts.mts"); const { readTrustedSecurityRubric } = - await import("../tools/pr-review-advisor/trusted-guidance.mts"); + await import("../../../tools/pr-review-advisor/trusted-guidance.mts"); const outDir = fs.mkdtempSync(path.join(tmpdir(), "advisor-prompt-failure-")); const headSha = "a".repeat(40); const securityRubric = readTrustedSecurityRubric(); diff --git a/test/inference/managed/managed-image-activation-command.test.ts b/test/inference/managed/managed-image-activation-command.test.ts index 79dad95b2d7..80d737baf9e 100644 --- a/test/inference/managed/managed-image-activation-command.test.ts +++ b/test/inference/managed/managed-image-activation-command.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; -import { managedActivationOnboardArgs } from "./e2e/live/managed-image-activation-e2e-helpers"; +import { managedActivationOnboardArgs } from "../../e2e/live/managed-image-activation-e2e-helpers"; describe("managed image activation command", () => { it("uses an exact stock catalog without enabling candidate activation", () => { diff --git a/test/inference/ollama/ollama-auth-proxy-handler-startup-cleanup.test.ts b/test/inference/ollama/ollama-auth-proxy-handler-startup-cleanup.test.ts index 21896a8a7c8..88ea0a51db1 100644 --- a/test/inference/ollama/ollama-auth-proxy-handler-startup-cleanup.test.ts +++ b/test/inference/ollama/ollama-auth-proxy-handler-startup-cleanup.test.ts @@ -17,7 +17,7 @@ vi.mock("../../helpers/child-process-lifecycle.ts", () => ({ ownChildProcess: ownerMocks.ownChildProcess, })); -import { forceKill, freePort, startProxy, terminate } from "./ollama-auth-proxy-handler-helpers.ts"; +import { forceKill, freePort, startProxy, terminate } from "../../ollama-auth-proxy-handler-helpers.ts"; const TOKEN = "unit-test-secret-token"; diff --git a/test/inference/ollama/ollama-auth-proxy-handler.test.ts b/test/inference/ollama/ollama-auth-proxy-handler.test.ts index e55b979f394..4cb03fc3fdd 100644 --- a/test/inference/ollama/ollama-auth-proxy-handler.test.ts +++ b/test/inference/ollama/ollama-auth-proxy-handler.test.ts @@ -32,7 +32,7 @@ import { startProxy, terminate, waitForProxyReadiness, -} from "./ollama-auth-proxy-handler-helpers.ts"; +} from "../../ollama-auth-proxy-handler-helpers.ts"; const TOKEN = "unit-test-secret-token"; diff --git a/test/mcp/mcp-agent-matrix-artifact-proof.test.ts b/test/mcp/mcp-agent-matrix-artifact-proof.test.ts index aceb03a3756..6d202271c89 100644 --- a/test/mcp/mcp-agent-matrix-artifact-proof.test.ts +++ b/test/mcp/mcp-agent-matrix-artifact-proof.test.ts @@ -11,7 +11,7 @@ import { assertMcpAgentMatrixArtifacts, REQUIRED_MCP_AGENT_TEST_IDS, writeMcpAgentMatrixProof, -} from "../tools/e2e/assert-mcp-agent-matrix-artifacts.mts"; +} from "../../tools/e2e/assert-mcp-agent-matrix-artifacts.mts"; const directories: string[] = []; diff --git a/test/mcp/mcp-artifact-secret-scan.test.ts b/test/mcp/mcp-artifact-secret-scan.test.ts index e9371b1ffef..d56629faf5f 100644 --- a/test/mcp/mcp-artifact-secret-scan.test.ts +++ b/test/mcp/mcp-artifact-secret-scan.test.ts @@ -7,8 +7,8 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { scanMcpArtifactSecrets } from "../tools/e2e/assert-mcp-artifact-secrets-absent.mts"; -import { MCP_BRIDGE_TEST_CREDENTIALS } from "./e2e/fixtures/mcp-bridge-credentials.ts"; +import { scanMcpArtifactSecrets } from "../../tools/e2e/assert-mcp-artifact-secrets-absent.mts"; +import { MCP_BRIDGE_TEST_CREDENTIALS } from "../e2e/fixtures/mcp-bridge-credentials.ts"; const roots: string[] = []; diff --git a/test/mcp/mcp-bridge-servers.test.ts b/test/mcp/mcp-bridge-servers.test.ts index d7aa9d8a479..45216c3b9ed 100644 --- a/test/mcp/mcp-bridge-servers.test.ts +++ b/test/mcp/mcp-bridge-servers.test.ts @@ -12,7 +12,7 @@ import path from "node:path"; import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { MCP_BRIDGE_ALLOWED_METHODS } from "../../src/lib/actions/sandbox/mcp-bridge-policy"; -import { startTestProgress } from "./e2e/fixtures/progress.ts"; +import { startTestProgress } from "../e2e/fixtures/progress.ts"; import { buildCloudflaredQuickTunnelArgs, HERMES_DEFERRED_TOOL_SEARCH_MISS, @@ -21,7 +21,7 @@ import { startCompatibleMock, startFakeMcpHttpsServer, startPublicMcpHttpsTunnel, -} from "./e2e/live/mcp-bridge-servers"; +} from "../e2e/live/mcp-bridge-servers"; const servers: StartedHttpServer[] = []; function progressProbe() { diff --git a/test/mcp/mcp-openshell-workflow.test.ts b/test/mcp/mcp-openshell-workflow.test.ts index 00ae8cf610c..46433429b61 100644 --- a/test/mcp/mcp-openshell-workflow.test.ts +++ b/test/mcp/mcp-openshell-workflow.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; -import { validateMcpOpenShellWorkflowBoundary } from "../tools/e2e/mcp-workflow-boundary.mts"; +import { validateMcpOpenShellWorkflowBoundary } from "../../tools/e2e/mcp-workflow-boundary.mts"; describe("MCP OpenShell workflow boundary", () => { it("validates the unified stable and explicit-dev MCP workflow contract", () => { diff --git a/test/repository/source-shape-scanner.test.ts b/test/repository/source-shape-scanner.test.ts index d4e3017bec2..be8f64bf4b0 100644 --- a/test/repository/source-shape-scanner.test.ts +++ b/test/repository/source-shape-scanner.test.ts @@ -434,7 +434,7 @@ describe("source-shape scanner", () => { const cases = detectedCaseNames(` import { expect, it } from "vitest"; import { readWorkflow } from "../helpers/e2e-workflow-contract"; - import { listTargets } from "./e2e/registry/registry"; + import { listTargets } from "../e2e/registry/registry"; it("mirrors workflow jobs through a selector", () => { const workflow = readWorkflow(); @@ -492,9 +492,9 @@ describe("source-shape scanner", () => { const cases = detectedCaseNames(` import { expect, it } from "vitest"; import * as workflows from "../helpers/e2e-workflow-contract"; - import * as registry from "./e2e/registry/registry"; - import { probesForState } from "./e2e/registry/expected-states"; - import { loadManifest, loadManifestsFromDir } from "./e2e/registry/manifests"; + import * as registry from "../e2e/registry/registry"; + import { probesForState } from "../e2e/registry/expected-states"; + import { loadManifest, loadManifestsFromDir } from "../e2e/registry/manifests"; it("mirrors a namespace-loaded workflow", () => { expect(Object.keys(workflows.readWorkflow().jobs)).toEqual(["test", "build"]); diff --git a/test/repository/vitest-coverage-thresholds.test.ts b/test/repository/vitest-coverage-thresholds.test.ts index 2a4a8e2bcbd..bfb4c259f3d 100644 --- a/test/repository/vitest-coverage-thresholds.test.ts +++ b/test/repository/vitest-coverage-thresholds.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; -import rootVitestConfig from "../vitest.config"; +import rootVitestConfig from "../../vitest.config"; import { resolveVitestCoverageThresholds, securityCoverageThresholds, diff --git a/test/runtime/gateway/gateway-serving-watchdog.test.ts b/test/runtime/gateway/gateway-serving-watchdog.test.ts index 9bcfc1dba9a..d702d9e8b4d 100644 --- a/test/runtime/gateway/gateway-serving-watchdog.test.ts +++ b/test/runtime/gateway/gateway-serving-watchdog.test.ts @@ -27,7 +27,7 @@ import { START_SCRIPT, safeTmpHelpers, writeProcStatFunction, -} from "./nemoclaw-start-gateway.test-helpers"; +} from "../../nemoclaw-start-gateway.test-helpers"; function watchdogFunctions(gatewayLog: string): string { const src = fs.readFileSync(START_SCRIPT, "utf-8"); diff --git a/test/runtime/messaging/messaging-build-applier-integrity.test.ts b/test/runtime/messaging/messaging-build-applier-integrity.test.ts index cd19a58c66a..2332c4cfc4d 100644 --- a/test/runtime/messaging/messaging-build-applier-integrity.test.ts +++ b/test/runtime/messaging/messaging-build-applier-integrity.test.ts @@ -14,7 +14,7 @@ import { reviewedOpenClawPluginTarballUrlByPackageSpec, } from "../../../src/lib/messaging/applier/build/messaging-build-applier.mts"; import { testTimeout } from "../../helpers/timeouts"; -import { withLegacyMessagingPlanEnvDirect } from "./messaging-plan-test-helper"; +import { withLegacyMessagingPlanEnvDirect } from "../../messaging-plan-test-helper"; vi.mock("../../../scripts/lib/openclaw-npm-remediation.mts", async (importOriginal) => { const original = diff --git a/test/runtime/messaging/messaging-build-applier.test.ts b/test/runtime/messaging/messaging-build-applier.test.ts index 0034ee0a042..450128b47d2 100644 --- a/test/runtime/messaging/messaging-build-applier.test.ts +++ b/test/runtime/messaging/messaging-build-applier.test.ts @@ -15,7 +15,7 @@ import { readMessagingBuildPlanFromEnv, } from "../../../src/lib/messaging/applier/build/messaging-build-applier.mts"; import { execTimeout, testTimeout } from "../../helpers/timeouts"; -import { withLegacyMessagingPlanEnvDirect } from "./messaging-plan-test-helper"; +import { withLegacyMessagingPlanEnvDirect } from "../../messaging-plan-test-helper"; const { remediateReviewedArchive } = vi.hoisted(() => ({ remediateReviewedArchive: vi.fn(({ archivePath }: { archivePath: string }) => ({ diff --git a/test/runtime/sandbox/sandbox-download-upload-cli.test.ts b/test/runtime/sandbox/sandbox-download-upload-cli.test.ts index f8cced6e0e0..a2515dd2e6f 100644 --- a/test/runtime/sandbox/sandbox-download-upload-cli.test.ts +++ b/test/runtime/sandbox/sandbox-download-upload-cli.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { runWithEnv, writeSandboxRegistry } from "./cli/helpers"; +import { runWithEnv, writeSandboxRegistry } from "../../cli/helpers"; function buildStubOpenshell(home: string, logFile: string): string { const localBin = path.join(home, "bin"); diff --git a/test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts b/test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts index 70adc9ff2b3..57600b27197 100644 --- a/test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts +++ b/test/runtime/sandbox/sandbox-sessions-admin-agent-cli.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { runWithEnv, writeSandboxRegistry } from "./cli/helpers"; +import { runWithEnv, writeSandboxRegistry } from "../../cli/helpers"; function buildStubOpenshell(home: string, logFile: string, nativeDeleteExit = 0): string { const localBin = path.join(home, "bin"); diff --git a/test/runtime/sandbox/sandbox-sessions-export-cli.test.ts b/test/runtime/sandbox/sandbox-sessions-export-cli.test.ts index f6f1edf47ee..85ee938a318 100644 --- a/test/runtime/sandbox/sandbox-sessions-export-cli.test.ts +++ b/test/runtime/sandbox/sandbox-sessions-export-cli.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { runWithEnv, writeSandboxRegistry } from "./cli/helpers"; +import { runWithEnv, writeSandboxRegistry } from "../../cli/helpers"; function buildStubOpenshell( home: string, From b96c25f3b4f95d96a52440c4176571a87109d323 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 20:55:33 -0500 Subject: [PATCH 20/37] fix(hermes): refresh managed wrapper integrity pin --- agents/hermes/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index bfc78c7d28b..4db2cd3d407 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -701,7 +701,7 @@ RUN node --experimental-strip-types \ # silent supply-chain tampering of the build context (an attacker rewriting a # file has to also rewrite the Dockerfile-committed hash, which reviewers gate). # Regenerate with `sha256sum agents/hermes/{hermes-wrapper.py,hermes-cli-adapter-v1.json,validate-cli-adapter.py,validate-env-secret-boundary.py,finalize-tirith-marker.py,cron-restore-control.py}`. -ARG NEMOCLAW_HERMES_WRAPPER_SHA256=f4276e9833638b7a620176c88bd329d6b6d4948538a3227b727a1397146a0e0e +ARG NEMOCLAW_HERMES_WRAPPER_SHA256=4db45043f45d8296dd39228315b721ee19b0a4e0591579ec0ceeec2777bbb40d ARG NEMOCLAW_HERMES_CLI_ADAPTER_SHA256=989edf54a8c09c6efb348600a8aa2f264c0b71408eb9d7bcd579b92cbeccf9b1 ARG NEMOCLAW_HERMES_CLI_ADAPTER_VALIDATOR_SHA256=db4046e79e513eab67b069a8eda20167b8b65529cf26842531d2ad673c670330 ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=b355d1365fb1d15475e327f312ceb854ae96f9ebed28cf96bc8817f550df2688 From adc42ec8c711837815375f6b5c13e9ceb54f7d0c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 21:06:51 -0500 Subject: [PATCH 21/37] ci(e2e): bind PR qualification to branch image cohort --- .github/workflows/e2e.yaml | 20 +++-- ...mage-publication-workflow-boundary.test.ts | 51 +++++++++---- .../support/base-image-publication.test.ts | 68 +++++++++++++++++ tools/e2e/base-image-publication.mts | 75 ++++++++++++++++--- tools/e2e/operations-workflow-boundary.mts | 30 ++++---- 5 files changed, 199 insertions(+), 45 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6ddb995439e..7420cdc3874 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -127,16 +127,20 @@ jobs: NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) expected_sha="$WORKFLOW_SHA" allow_non_head=0 + publication_branch=main + publication_event=push select_nearest_successful=0 ;; NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller) - [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { - echo "::error::manual PR publication selection requires an exact base SHA" >&2 + [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::manual PR publication selection requires an exact candidate SHA" >&2 exit 1 } - expected_sha="$BASE_SHA" - allow_non_head=1 - select_nearest_successful=1 + expected_sha="$CHECKOUT_SHA" + allow_non_head=0 + publication_branch="${REF#refs/heads/}" + publication_event=workflow_dispatch + select_nearest_successful=0 ;; *) echo "::error::base-image publication mode is not trusted" >&2 @@ -145,6 +149,8 @@ jobs: esac printf 'allow_non_head=%s\n' "${allow_non_head}" >> "${GITHUB_OUTPUT}" printf 'expected_sha=%s\n' "${expected_sha}" >> "${GITHUB_OUTPUT}" + printf 'publication_branch=%s\n' "${publication_branch}" >> "${GITHUB_OUTPUT}" + printf 'publication_event=%s\n' "${publication_event}" >> "${GITHUB_OUTPUT}" printf 'select_nearest_successful=%s\n' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}" - name: Check out trusted E2E workflow @@ -164,13 +170,15 @@ jobs: env: EXPECTED_SHA: ${{ steps.publication_mode.outputs.expected_sha }} GITHUB_TOKEN: ${{ github.token }} + PUBLICATION_BRANCH: ${{ steps.publication_mode.outputs.publication_branch }} + PUBLICATION_EVENT: ${{ steps.publication_mode.outputs.publication_event }} PUBLICATION_HISTORY_ALLOW_NON_HEAD: ${{ steps.publication_mode.outputs.allow_non_head }} REQUIRE_MANAGED_IMAGE_PUBLICATION: "1" SELECT_NEAREST_SUCCESSFUL_PUBLICATION: ${{ steps.publication_mode.outputs.select_nearest_successful }} shell: bash run: | set -euo pipefail - export GITHUB_REF=refs/heads/main + export GITHUB_REF="refs/heads/$PUBLICATION_BRANCH" export GITHUB_SHA="$EXPECTED_SHA" wait_seconds=3000 if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then diff --git a/test/e2e/support/base-image-publication-workflow-boundary.test.ts b/test/e2e/support/base-image-publication-workflow-boundary.test.ts index 6243eb0a8a5..6e60072a0fe 100644 --- a/test/e2e/support/base-image-publication-workflow-boundary.test.ts +++ b/test/e2e/support/base-image-publication-workflow-boundary.test.ts @@ -111,29 +111,53 @@ describe("base-image publication workflow boundary (#7372)", () => { }); it.each([ - ["push to main", "push", "", "refs/heads/main", "0", "c".repeat(40), "0"], - ["manual main", "workflow_dispatch", "", "refs/heads/main", "0", "c".repeat(40), "0"], + ["push to main", "push", "", "refs/heads/main", "0", "c".repeat(40), "main", "push", "0"], [ - "controller-selected PR", + "manual main", + "workflow_dispatch", + "", + "refs/heads/main", + "0", + "c".repeat(40), + "main", + "push", + "0", + ], + [ + "exact PR branch publication", "workflow_dispatch", "a".repeat(40), "refs/heads/candidate", - "1", - "b".repeat(40), - "1", + "0", + "a".repeat(40), + "candidate", + "workflow_dispatch", + "0", ], [ "pinned a4f9b59 diagnostic", "workflow_dispatch", "a4f9b59aa64f88532a3e64e949dd1b4068aa1f1e", "refs/heads/candidate", - "1", - "b".repeat(40), - "1", + "0", + "a4f9b59aa64f88532a3e64e949dd1b4068aa1f1e", + "candidate", + "workflow_dispatch", + "0", ], ])( "classifies %s without executing untrusted code (#7372)", - (_case, eventName, checkoutSha, ref, allowNonHead, expectedSha, selectNearest) => { + ( + _case, + eventName, + checkoutSha, + ref, + allowNonHead, + expectedSha, + publicationBranch, + publicationEvent, + selectNearest, + ) => { expect( runClassifier({ checkoutSha, @@ -142,7 +166,7 @@ describe("base-image publication workflow boundary (#7372)", () => { repository: "NVIDIA/NemoClaw", }), ).toEqual({ - output: `allow_non_head=${allowNonHead}\nexpected_sha=${expectedSha}\nselect_nearest_successful=${selectNearest}\n`, + output: `allow_non_head=${allowNonHead}\nexpected_sha=${expectedSha}\npublication_branch=${publicationBranch}\npublication_event=${publicationEvent}\nselect_nearest_successful=${selectNearest}\n`, status: 0, }); }, @@ -241,10 +265,7 @@ describe("base-image publication workflow boundary (#7372)", () => { "node tools/e2e/dcode-base-image-contract.mts contract.json"), ], ["step count", (value) => gateSteps(value).push({ name: "Unreviewed step", run: "true" })], - [ - "matrix publication dependency", - (value) => (value.jobs["generate-matrix"].needs = []), - ], + ["matrix publication dependency", (value) => (value.jobs["generate-matrix"].needs = [])], ["live publication dependency", (value) => (value.jobs.live.needs = ["generate-matrix"])], [ "live managed-image revision", diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 69285ba7e7f..10db170327d 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -120,6 +120,8 @@ function selectedRun(overrides: Partial = {}): PublicationRun { id: RUN_ID, attempt: 1, workflowId: WORKFLOW_ID, + event: "push", + headBranch: "main", headSha: RELEVANT_SHA, status: "completed", conclusion: "success", @@ -503,6 +505,31 @@ describe("base-image publication evidence", () => { }); }); + it("binds an exact branch workflow-dispatch publication", () => { + const branch = "fix/managed-only"; + const selection = selectPublicationRun( + runsPayload([ + workflowRun({ + event: "workflow_dispatch", + head_branch: branch, + head_sha: EXPECTED_SHA, + }), + ]), + history(), + WORKFLOW_ID, + { publicationBranch: branch, publicationEvent: "workflow_dispatch" }, + ); + + expect(selection).toMatchObject({ + state: "selected", + run: { + event: "workflow_dispatch", + headBranch: branch, + headSha: EXPECTED_SHA, + }, + }); + }); + it("does not select an incomplete or failed publication for branch reuse", () => { expect( selectPublicationRun( @@ -748,6 +775,47 @@ describe("base-image publication evidence", () => { expect(notices).toHaveLength(2); }); + it("queries and verifies an exact branch workflow-dispatch publication", async () => { + const branch = "fix/managed-only"; + const branchRun = workflowRun({ + event: "workflow_dispatch", + head_branch: branch, + head_sha: EXPECTED_SHA, + }); + const responses = [ + workflowMetadata(), + runsPayload([branchRun]), + { + total_count: 3, + jobs: successfulJobs().map((job) => ({ ...job, head_sha: EXPECTED_SHA })), + }, + branchRun, + ]; + const requests: string[] = []; + + await expect( + waitForBaseImagePublication({ + history: history(), + publicationBranch: branch, + publicationEvent: "workflow_dispatch", + request: async (requestPath) => { + requests.push(requestPath); + return responses.shift(); + }, + requireWorkflowSuccess: true, + waitMs: 100, + pollMs: 10, + }), + ).resolves.toMatchObject({ + event: "workflow_dispatch", + headBranch: branch, + headSha: EXPECTED_SHA, + }); + expect(requests[1]).toBe( + "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=fix%2Fmanaged-only&event=workflow_dispatch&per_page=100&page=1", + ); + }); + it("accepts required publishers while managed-image jobs remain in progress (#9549)", async () => { const inProgressRun = workflowRun({ status: "in_progress", conclusion: null }); const jobs = [ diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index e26a7cdc9bf..973933a8c3e 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -89,6 +89,8 @@ export interface PublicationRun { id: number; attempt: number; workflowId: number; + event: "push" | "workflow_dispatch"; + headBranch: string; headSha: string; status: string; conclusion: string | null; @@ -101,6 +103,8 @@ export type PublicationSelection = export interface PublicationWaitOptions { history: FirstParentHistory; + publicationBranch?: string; + publicationEvent?: "push" | "workflow_dispatch"; request: (path: string) => Promise; requireWorkflowSuccess?: boolean; selectNearestSuccessfulRun?: boolean; @@ -144,7 +148,11 @@ function positiveSafeInteger(value: unknown, label: string): number { return Number(value); } -function exactString(value: unknown, expected: string, label: string): string { +function exactString( + value: unknown, + expected: Expected, + label: string, +): Expected { if (value !== expected) { throw new Error(`${label} must be ${expected}`); } @@ -356,7 +364,13 @@ export function validateWorkflow(payload: unknown): number { return workflowId; } -function validateRun(value: unknown, index: number, expectedWorkflowId: number): PublicationRun { +function validateRun( + value: unknown, + index: number, + expectedWorkflowId: number, + expectedEvent: "push" | "workflow_dispatch", + expectedBranch: string, +): PublicationRun { const run = asRecord(value); const id = positiveSafeInteger(run.id, `workflow run ${index} id`); const attempt = positiveSafeInteger(run.run_attempt, `workflow run ${index} attempt`); @@ -366,8 +380,8 @@ function validateRun(value: unknown, index: number, expectedWorkflowId: number): throw new Error(`workflow run ${index} workflow id does not match the base-image workflow`); } const headSha = sha(run.head_sha, `workflow run ${index} head SHA`); - exactString(run.event, "push", `workflow run ${index} event`); - exactString(run.head_branch, MAIN_BRANCH, `workflow run ${index} branch`); + const event = exactString(run.event, expectedEvent, `workflow run ${index} event`); + const headBranch = exactString(run.head_branch, expectedBranch, `workflow run ${index} branch`); exactString(run.path, WORKFLOW_PATH, `workflow run ${index} path`); exactString(run.name, WORKFLOW_NAME, `workflow run ${index} name`); exactString(asRecord(run.repository).full_name, REPOSITORY, `workflow run ${index} repository`); @@ -396,6 +410,8 @@ function validateRun(value: unknown, index: number, expectedWorkflowId: number): id, attempt, workflowId: expectedWorkflowId, + event, + headBranch, headSha, status, conclusion, @@ -407,7 +423,11 @@ export function selectPublicationRun( payload: unknown, history: FirstParentHistory, workflowId: number, - options: { readonly completedSuccessOnly?: boolean } = {}, + options: { + readonly completedSuccessOnly?: boolean; + readonly publicationBranch?: string; + readonly publicationEvent?: "push" | "workflow_dispatch"; + } = {}, ): PublicationSelection { positiveSafeInteger(workflowId, "base-image workflow id"); const response = asRecord(payload); @@ -419,10 +439,12 @@ export function selectPublicationRun( throw new Error("workflow run listing is incomplete"); } + const publicationBranch = options.publicationBranch ?? MAIN_BRANCH; + const publicationEvent = options.publicationEvent ?? "push"; const runs = response.workflow_runs.flatMap((value, index) => { const run = asRecord(value); return typeof run.head_sha === "string" && history.distanceBySha.has(run.head_sha) - ? [validateRun(run, index, workflowId)] + ? [validateRun(run, index, workflowId, publicationEvent, publicationBranch)] : []; }); if (new Set(runs.map((run) => run.id)).size !== runs.length) { @@ -523,7 +545,7 @@ export function validatePublisherJobs(payload: unknown, run: PublicationRun): "p } export function validateBoundRun(payload: unknown, expected: PublicationRun): PublicationRun { - const actual = validateRun(payload, 0, expected.workflowId); + const actual = validateRun(payload, 0, expected.workflowId, expected.event, expected.headBranch); if ( actual.id !== expected.id || actual.attempt !== expected.attempt || @@ -629,16 +651,38 @@ export async function waitForBaseImagePublication( if (!Number.isSafeInteger(options.pollMs) || options.pollMs < 1) { throw new Error("pollMs must be a positive integer"); } + const publicationBranch = options.publicationBranch ?? MAIN_BRANCH; + const publicationEvent = options.publicationEvent ?? "push"; + if ( + publicationBranch.length === 0 || + !SAFE_PATH_PATTERN.test(publicationBranch) || + publicationBranch.startsWith("/") || + publicationBranch.endsWith("/") || + publicationBranch.includes("//") || + publicationBranch.split("/").some((segment) => segment === "." || segment === "..") + ) { + throw new Error("publication branch is invalid"); + } + if (publicationEvent === "push" && publicationBranch !== MAIN_BRANCH) { + throw new Error("non-main publication must use workflow_dispatch"); + } const deadline = now() + options.waitMs; const workflowId = validateWorkflow( await options.request(`/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}`), ); - const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?branch=${MAIN_BRANCH}&event=push&per_page=100`; + const query = new URLSearchParams({ + branch: publicationBranch, + event: publicationEvent, + per_page: String(PAGE_SIZE), + }); + const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?${query.toString()}`; while (true) { const runs = await collectPaginated(options.request, runsPath, "workflow_runs"); const selection = selectPublicationRun(runs, options.history, workflowId, { completedSuccessOnly: options.selectNearestSuccessfulRun === true, + publicationBranch, + publicationEvent, }); if (selection.state === "selected") { const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/attempts/${selection.run.attempt}/jobs?per_page=100`; @@ -690,7 +734,7 @@ export async function waitForBaseImagePublication( notice( selection.state === "selected" ? `Required base image publishers are not complete for ${selection.run.headSha}; selected workflow run status ${selection.run.status}; ${selection.run.url}` - : `Waiting for a trusted base-image push run covering ${options.history.relevantSha}`, + : `Waiting for a trusted ${publicationBranch} base-image ${publicationEvent} run covering ${options.history.relevantSha}`, ); await sleep(Math.min(options.pollMs, Math.max(1, deadline - now()))); } @@ -813,6 +857,8 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro const requireManagedImagePublication = env.REQUIRE_MANAGED_IMAGE_PUBLICATION ?? "0"; const selectNearestSuccessfulRun = env.SELECT_NEAREST_SUCCESSFUL_PUBLICATION ?? "0"; const allowNonHeadHistory = env.PUBLICATION_HISTORY_ALLOW_NON_HEAD ?? "0"; + const publicationBranch = env.PUBLICATION_BRANCH ?? MAIN_BRANCH; + const publicationEvent = env.PUBLICATION_EVENT ?? "push"; const workspace = env.GITHUB_WORKSPACE ?? process.cwd(); if (token.length === 0 || token.includes("\r") || token.includes("\n")) { throw new Error("GITHUB_TOKEN must be a non-empty single-line value"); @@ -821,8 +867,8 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro if (env.GITHUB_REPOSITORY !== REPOSITORY) { throw new Error(`GITHUB_REPOSITORY must be ${REPOSITORY}`); } - if (env.GITHUB_REF !== "refs/heads/main") { - throw new Error("GITHUB_REF must be refs/heads/main"); + if (env.GITHUB_REF !== `refs/heads/${publicationBranch}`) { + throw new Error("GITHUB_REF must match PUBLICATION_BRANCH"); } if (!isBaseImagePublicationEvent(env.GITHUB_EVENT_NAME)) { throw new Error("GITHUB_EVENT_NAME must be push or workflow_dispatch"); @@ -850,6 +896,13 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro }); const run = await waitForBaseImagePublication({ history, + publicationBranch, + publicationEvent: + publicationEvent === "push" || publicationEvent === "workflow_dispatch" + ? publicationEvent + : (() => { + throw new Error("PUBLICATION_EVENT must be push or workflow_dispatch"); + })(), request: (path) => githubRequest(path, token), requireWorkflowSuccess: requireManagedImagePublication === "1", selectNearestSuccessfulRun: selectNearestSuccessfulRun === "1", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index dc3cab501e6..a246b4477ad 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -40,16 +40,20 @@ const PUBLICATION_CLASSIFIER_SCRIPT = " NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)", ' expected_sha="$WORKFLOW_SHA"', " allow_non_head=0", + " publication_branch=main", + " publication_event=push", " select_nearest_successful=0", " ;;", " NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller)", - ' [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {', - ' echo "::error::manual PR publication selection requires an exact base SHA" >&2', + ' [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || {', + ' echo "::error::manual PR publication selection requires an exact candidate SHA" >&2', " exit 1", " }", - ' expected_sha="$BASE_SHA"', - " allow_non_head=1", - " select_nearest_successful=1", + ' expected_sha="$CHECKOUT_SHA"', + " allow_non_head=0", + ' publication_branch="${REF#refs/heads/}"', + " publication_event=workflow_dispatch", + " select_nearest_successful=0", " ;;", " *)", ' echo "::error::base-image publication mode is not trusted" >&2', @@ -58,6 +62,8 @@ const PUBLICATION_CLASSIFIER_SCRIPT = "esac", 'printf \'allow_non_head=%s\\n\' "${allow_non_head}" >> "${GITHUB_OUTPUT}"', 'printf \'expected_sha=%s\\n\' "${expected_sha}" >> "${GITHUB_OUTPUT}"', + 'printf \'publication_branch=%s\\n\' "${publication_branch}" >> "${GITHUB_OUTPUT}"', + 'printf \'publication_event=%s\\n\' "${publication_event}" >> "${GITHUB_OUTPUT}"', 'printf \'select_nearest_successful=%s\\n\' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}"', ].join("\n") + "\n"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; @@ -598,12 +604,10 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): outputs: { dcode_base_contract: "${{ steps.validate_dcode_base.outputs.contract }}", dcode_base_ref: "${{ steps.validate_dcode_base.outputs.base_ref }}", - managed_image_artifact_provenance: - "${{ steps.download_managed_cohort.outputs.provenance }}", + managed_image_artifact_provenance: "${{ steps.download_managed_cohort.outputs.provenance }}", managed_image_cohort: "${{ steps.validate_managed_cohort.outputs.cohort }}", managed_image_receipt: "${{ steps.validate_managed_cohort.outputs.receipt }}", - managed_image_revision: - "${{ steps.validate_managed_cohort.outputs.revision }}", + managed_image_revision: "${{ steps.validate_managed_cohort.outputs.revision }}", managed_image_run_attempt: "${{ steps.validate_managed_cohort.outputs.run_attempt }}", managed_image_run_id: "${{ steps.validate_managed_cohort.outputs.run_id }}", }, @@ -648,6 +652,8 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): env: { EXPECTED_SHA: "${{ steps.publication_mode.outputs.expected_sha }}", GITHUB_TOKEN: "${{ github.token }}", + PUBLICATION_BRANCH: "${{ steps.publication_mode.outputs.publication_branch }}", + PUBLICATION_EVENT: "${{ steps.publication_mode.outputs.publication_event }}", PUBLICATION_HISTORY_ALLOW_NON_HEAD: "${{ steps.publication_mode.outputs.allow_non_head }}", REQUIRE_MANAGED_IMAGE_PUBLICATION: "1", @@ -657,7 +663,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): shell: "bash", run: [ "set -euo pipefail", - "export GITHUB_REF=refs/heads/main", + 'export GITHUB_REF="refs/heads/$PUBLICATION_BRANCH"', 'export GITHUB_SHA="$EXPECTED_SHA"', "wait_seconds=3000", 'if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then', @@ -805,9 +811,7 @@ const MANAGED_IMAGE_RECEIPT_EXPRESSION = "${{ needs.base-image-publication.outputs.managed_image_receipt }}"; /** Require publication success and one exact cohort revision for every stock onboarding job. */ -export function validateStockOnboardingPublicationBoundary( - workflow: OperationsWorkflow, -): string[] { +export function validateStockOnboardingPublicationBoundary(workflow: OperationsWorkflow): string[] { const errors: string[] = []; for (const jobName of STOCK_ONBOARDING_JOBS) { const job = workflow.jobs[jobName] ?? {}; From 16b7e5f74e6a4716471fa691be9eac9be9043f22 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 24 Aug 2026 21:09:02 -0500 Subject: [PATCH 22/37] Revert "ci(e2e): bind PR qualification to branch image cohort" This reverts commit adc42ec8c711837815375f6b5c13e9ceb54f7d0c. --- .github/workflows/e2e.yaml | 20 ++--- ...mage-publication-workflow-boundary.test.ts | 51 ++++--------- .../support/base-image-publication.test.ts | 68 ----------------- tools/e2e/base-image-publication.mts | 75 +++---------------- tools/e2e/operations-workflow-boundary.mts | 30 ++++---- 5 files changed, 45 insertions(+), 199 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7420cdc3874..6ddb995439e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -127,20 +127,16 @@ jobs: NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) expected_sha="$WORKFLOW_SHA" allow_non_head=0 - publication_branch=main - publication_event=push select_nearest_successful=0 ;; NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller) - [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { - echo "::error::manual PR publication selection requires an exact candidate SHA" >&2 + [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::manual PR publication selection requires an exact base SHA" >&2 exit 1 } - expected_sha="$CHECKOUT_SHA" - allow_non_head=0 - publication_branch="${REF#refs/heads/}" - publication_event=workflow_dispatch - select_nearest_successful=0 + expected_sha="$BASE_SHA" + allow_non_head=1 + select_nearest_successful=1 ;; *) echo "::error::base-image publication mode is not trusted" >&2 @@ -149,8 +145,6 @@ jobs: esac printf 'allow_non_head=%s\n' "${allow_non_head}" >> "${GITHUB_OUTPUT}" printf 'expected_sha=%s\n' "${expected_sha}" >> "${GITHUB_OUTPUT}" - printf 'publication_branch=%s\n' "${publication_branch}" >> "${GITHUB_OUTPUT}" - printf 'publication_event=%s\n' "${publication_event}" >> "${GITHUB_OUTPUT}" printf 'select_nearest_successful=%s\n' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}" - name: Check out trusted E2E workflow @@ -170,15 +164,13 @@ jobs: env: EXPECTED_SHA: ${{ steps.publication_mode.outputs.expected_sha }} GITHUB_TOKEN: ${{ github.token }} - PUBLICATION_BRANCH: ${{ steps.publication_mode.outputs.publication_branch }} - PUBLICATION_EVENT: ${{ steps.publication_mode.outputs.publication_event }} PUBLICATION_HISTORY_ALLOW_NON_HEAD: ${{ steps.publication_mode.outputs.allow_non_head }} REQUIRE_MANAGED_IMAGE_PUBLICATION: "1" SELECT_NEAREST_SUCCESSFUL_PUBLICATION: ${{ steps.publication_mode.outputs.select_nearest_successful }} shell: bash run: | set -euo pipefail - export GITHUB_REF="refs/heads/$PUBLICATION_BRANCH" + export GITHUB_REF=refs/heads/main export GITHUB_SHA="$EXPECTED_SHA" wait_seconds=3000 if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then diff --git a/test/e2e/support/base-image-publication-workflow-boundary.test.ts b/test/e2e/support/base-image-publication-workflow-boundary.test.ts index 6e60072a0fe..6243eb0a8a5 100644 --- a/test/e2e/support/base-image-publication-workflow-boundary.test.ts +++ b/test/e2e/support/base-image-publication-workflow-boundary.test.ts @@ -111,53 +111,29 @@ describe("base-image publication workflow boundary (#7372)", () => { }); it.each([ - ["push to main", "push", "", "refs/heads/main", "0", "c".repeat(40), "main", "push", "0"], + ["push to main", "push", "", "refs/heads/main", "0", "c".repeat(40), "0"], + ["manual main", "workflow_dispatch", "", "refs/heads/main", "0", "c".repeat(40), "0"], [ - "manual main", - "workflow_dispatch", - "", - "refs/heads/main", - "0", - "c".repeat(40), - "main", - "push", - "0", - ], - [ - "exact PR branch publication", + "controller-selected PR", "workflow_dispatch", "a".repeat(40), "refs/heads/candidate", - "0", - "a".repeat(40), - "candidate", - "workflow_dispatch", - "0", + "1", + "b".repeat(40), + "1", ], [ "pinned a4f9b59 diagnostic", "workflow_dispatch", "a4f9b59aa64f88532a3e64e949dd1b4068aa1f1e", "refs/heads/candidate", - "0", - "a4f9b59aa64f88532a3e64e949dd1b4068aa1f1e", - "candidate", - "workflow_dispatch", - "0", + "1", + "b".repeat(40), + "1", ], ])( "classifies %s without executing untrusted code (#7372)", - ( - _case, - eventName, - checkoutSha, - ref, - allowNonHead, - expectedSha, - publicationBranch, - publicationEvent, - selectNearest, - ) => { + (_case, eventName, checkoutSha, ref, allowNonHead, expectedSha, selectNearest) => { expect( runClassifier({ checkoutSha, @@ -166,7 +142,7 @@ describe("base-image publication workflow boundary (#7372)", () => { repository: "NVIDIA/NemoClaw", }), ).toEqual({ - output: `allow_non_head=${allowNonHead}\nexpected_sha=${expectedSha}\npublication_branch=${publicationBranch}\npublication_event=${publicationEvent}\nselect_nearest_successful=${selectNearest}\n`, + output: `allow_non_head=${allowNonHead}\nexpected_sha=${expectedSha}\nselect_nearest_successful=${selectNearest}\n`, status: 0, }); }, @@ -265,7 +241,10 @@ describe("base-image publication workflow boundary (#7372)", () => { "node tools/e2e/dcode-base-image-contract.mts contract.json"), ], ["step count", (value) => gateSteps(value).push({ name: "Unreviewed step", run: "true" })], - ["matrix publication dependency", (value) => (value.jobs["generate-matrix"].needs = [])], + [ + "matrix publication dependency", + (value) => (value.jobs["generate-matrix"].needs = []), + ], ["live publication dependency", (value) => (value.jobs.live.needs = ["generate-matrix"])], [ "live managed-image revision", diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 10db170327d..69285ba7e7f 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -120,8 +120,6 @@ function selectedRun(overrides: Partial = {}): PublicationRun { id: RUN_ID, attempt: 1, workflowId: WORKFLOW_ID, - event: "push", - headBranch: "main", headSha: RELEVANT_SHA, status: "completed", conclusion: "success", @@ -505,31 +503,6 @@ describe("base-image publication evidence", () => { }); }); - it("binds an exact branch workflow-dispatch publication", () => { - const branch = "fix/managed-only"; - const selection = selectPublicationRun( - runsPayload([ - workflowRun({ - event: "workflow_dispatch", - head_branch: branch, - head_sha: EXPECTED_SHA, - }), - ]), - history(), - WORKFLOW_ID, - { publicationBranch: branch, publicationEvent: "workflow_dispatch" }, - ); - - expect(selection).toMatchObject({ - state: "selected", - run: { - event: "workflow_dispatch", - headBranch: branch, - headSha: EXPECTED_SHA, - }, - }); - }); - it("does not select an incomplete or failed publication for branch reuse", () => { expect( selectPublicationRun( @@ -775,47 +748,6 @@ describe("base-image publication evidence", () => { expect(notices).toHaveLength(2); }); - it("queries and verifies an exact branch workflow-dispatch publication", async () => { - const branch = "fix/managed-only"; - const branchRun = workflowRun({ - event: "workflow_dispatch", - head_branch: branch, - head_sha: EXPECTED_SHA, - }); - const responses = [ - workflowMetadata(), - runsPayload([branchRun]), - { - total_count: 3, - jobs: successfulJobs().map((job) => ({ ...job, head_sha: EXPECTED_SHA })), - }, - branchRun, - ]; - const requests: string[] = []; - - await expect( - waitForBaseImagePublication({ - history: history(), - publicationBranch: branch, - publicationEvent: "workflow_dispatch", - request: async (requestPath) => { - requests.push(requestPath); - return responses.shift(); - }, - requireWorkflowSuccess: true, - waitMs: 100, - pollMs: 10, - }), - ).resolves.toMatchObject({ - event: "workflow_dispatch", - headBranch: branch, - headSha: EXPECTED_SHA, - }); - expect(requests[1]).toBe( - "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=fix%2Fmanaged-only&event=workflow_dispatch&per_page=100&page=1", - ); - }); - it("accepts required publishers while managed-image jobs remain in progress (#9549)", async () => { const inProgressRun = workflowRun({ status: "in_progress", conclusion: null }); const jobs = [ diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index 973933a8c3e..e26a7cdc9bf 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -89,8 +89,6 @@ export interface PublicationRun { id: number; attempt: number; workflowId: number; - event: "push" | "workflow_dispatch"; - headBranch: string; headSha: string; status: string; conclusion: string | null; @@ -103,8 +101,6 @@ export type PublicationSelection = export interface PublicationWaitOptions { history: FirstParentHistory; - publicationBranch?: string; - publicationEvent?: "push" | "workflow_dispatch"; request: (path: string) => Promise; requireWorkflowSuccess?: boolean; selectNearestSuccessfulRun?: boolean; @@ -148,11 +144,7 @@ function positiveSafeInteger(value: unknown, label: string): number { return Number(value); } -function exactString( - value: unknown, - expected: Expected, - label: string, -): Expected { +function exactString(value: unknown, expected: string, label: string): string { if (value !== expected) { throw new Error(`${label} must be ${expected}`); } @@ -364,13 +356,7 @@ export function validateWorkflow(payload: unknown): number { return workflowId; } -function validateRun( - value: unknown, - index: number, - expectedWorkflowId: number, - expectedEvent: "push" | "workflow_dispatch", - expectedBranch: string, -): PublicationRun { +function validateRun(value: unknown, index: number, expectedWorkflowId: number): PublicationRun { const run = asRecord(value); const id = positiveSafeInteger(run.id, `workflow run ${index} id`); const attempt = positiveSafeInteger(run.run_attempt, `workflow run ${index} attempt`); @@ -380,8 +366,8 @@ function validateRun( throw new Error(`workflow run ${index} workflow id does not match the base-image workflow`); } const headSha = sha(run.head_sha, `workflow run ${index} head SHA`); - const event = exactString(run.event, expectedEvent, `workflow run ${index} event`); - const headBranch = exactString(run.head_branch, expectedBranch, `workflow run ${index} branch`); + exactString(run.event, "push", `workflow run ${index} event`); + exactString(run.head_branch, MAIN_BRANCH, `workflow run ${index} branch`); exactString(run.path, WORKFLOW_PATH, `workflow run ${index} path`); exactString(run.name, WORKFLOW_NAME, `workflow run ${index} name`); exactString(asRecord(run.repository).full_name, REPOSITORY, `workflow run ${index} repository`); @@ -410,8 +396,6 @@ function validateRun( id, attempt, workflowId: expectedWorkflowId, - event, - headBranch, headSha, status, conclusion, @@ -423,11 +407,7 @@ export function selectPublicationRun( payload: unknown, history: FirstParentHistory, workflowId: number, - options: { - readonly completedSuccessOnly?: boolean; - readonly publicationBranch?: string; - readonly publicationEvent?: "push" | "workflow_dispatch"; - } = {}, + options: { readonly completedSuccessOnly?: boolean } = {}, ): PublicationSelection { positiveSafeInteger(workflowId, "base-image workflow id"); const response = asRecord(payload); @@ -439,12 +419,10 @@ export function selectPublicationRun( throw new Error("workflow run listing is incomplete"); } - const publicationBranch = options.publicationBranch ?? MAIN_BRANCH; - const publicationEvent = options.publicationEvent ?? "push"; const runs = response.workflow_runs.flatMap((value, index) => { const run = asRecord(value); return typeof run.head_sha === "string" && history.distanceBySha.has(run.head_sha) - ? [validateRun(run, index, workflowId, publicationEvent, publicationBranch)] + ? [validateRun(run, index, workflowId)] : []; }); if (new Set(runs.map((run) => run.id)).size !== runs.length) { @@ -545,7 +523,7 @@ export function validatePublisherJobs(payload: unknown, run: PublicationRun): "p } export function validateBoundRun(payload: unknown, expected: PublicationRun): PublicationRun { - const actual = validateRun(payload, 0, expected.workflowId, expected.event, expected.headBranch); + const actual = validateRun(payload, 0, expected.workflowId); if ( actual.id !== expected.id || actual.attempt !== expected.attempt || @@ -651,38 +629,16 @@ export async function waitForBaseImagePublication( if (!Number.isSafeInteger(options.pollMs) || options.pollMs < 1) { throw new Error("pollMs must be a positive integer"); } - const publicationBranch = options.publicationBranch ?? MAIN_BRANCH; - const publicationEvent = options.publicationEvent ?? "push"; - if ( - publicationBranch.length === 0 || - !SAFE_PATH_PATTERN.test(publicationBranch) || - publicationBranch.startsWith("/") || - publicationBranch.endsWith("/") || - publicationBranch.includes("//") || - publicationBranch.split("/").some((segment) => segment === "." || segment === "..") - ) { - throw new Error("publication branch is invalid"); - } - if (publicationEvent === "push" && publicationBranch !== MAIN_BRANCH) { - throw new Error("non-main publication must use workflow_dispatch"); - } const deadline = now() + options.waitMs; const workflowId = validateWorkflow( await options.request(`/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}`), ); - const query = new URLSearchParams({ - branch: publicationBranch, - event: publicationEvent, - per_page: String(PAGE_SIZE), - }); - const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?${query.toString()}`; + const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?branch=${MAIN_BRANCH}&event=push&per_page=100`; while (true) { const runs = await collectPaginated(options.request, runsPath, "workflow_runs"); const selection = selectPublicationRun(runs, options.history, workflowId, { completedSuccessOnly: options.selectNearestSuccessfulRun === true, - publicationBranch, - publicationEvent, }); if (selection.state === "selected") { const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/attempts/${selection.run.attempt}/jobs?per_page=100`; @@ -734,7 +690,7 @@ export async function waitForBaseImagePublication( notice( selection.state === "selected" ? `Required base image publishers are not complete for ${selection.run.headSha}; selected workflow run status ${selection.run.status}; ${selection.run.url}` - : `Waiting for a trusted ${publicationBranch} base-image ${publicationEvent} run covering ${options.history.relevantSha}`, + : `Waiting for a trusted base-image push run covering ${options.history.relevantSha}`, ); await sleep(Math.min(options.pollMs, Math.max(1, deadline - now()))); } @@ -857,8 +813,6 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro const requireManagedImagePublication = env.REQUIRE_MANAGED_IMAGE_PUBLICATION ?? "0"; const selectNearestSuccessfulRun = env.SELECT_NEAREST_SUCCESSFUL_PUBLICATION ?? "0"; const allowNonHeadHistory = env.PUBLICATION_HISTORY_ALLOW_NON_HEAD ?? "0"; - const publicationBranch = env.PUBLICATION_BRANCH ?? MAIN_BRANCH; - const publicationEvent = env.PUBLICATION_EVENT ?? "push"; const workspace = env.GITHUB_WORKSPACE ?? process.cwd(); if (token.length === 0 || token.includes("\r") || token.includes("\n")) { throw new Error("GITHUB_TOKEN must be a non-empty single-line value"); @@ -867,8 +821,8 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro if (env.GITHUB_REPOSITORY !== REPOSITORY) { throw new Error(`GITHUB_REPOSITORY must be ${REPOSITORY}`); } - if (env.GITHUB_REF !== `refs/heads/${publicationBranch}`) { - throw new Error("GITHUB_REF must match PUBLICATION_BRANCH"); + if (env.GITHUB_REF !== "refs/heads/main") { + throw new Error("GITHUB_REF must be refs/heads/main"); } if (!isBaseImagePublicationEvent(env.GITHUB_EVENT_NAME)) { throw new Error("GITHUB_EVENT_NAME must be push or workflow_dispatch"); @@ -896,13 +850,6 @@ export async function main(argv = process.argv.slice(2), env = process.env): Pro }); const run = await waitForBaseImagePublication({ history, - publicationBranch, - publicationEvent: - publicationEvent === "push" || publicationEvent === "workflow_dispatch" - ? publicationEvent - : (() => { - throw new Error("PUBLICATION_EVENT must be push or workflow_dispatch"); - })(), request: (path) => githubRequest(path, token), requireWorkflowSuccess: requireManagedImagePublication === "1", selectNearestSuccessfulRun: selectNearestSuccessfulRun === "1", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a246b4477ad..dc3cab501e6 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -40,20 +40,16 @@ const PUBLICATION_CLASSIFIER_SCRIPT = " NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)", ' expected_sha="$WORKFLOW_SHA"', " allow_non_head=0", - " publication_branch=main", - " publication_event=push", " select_nearest_successful=0", " ;;", " NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller)", - ' [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || {', - ' echo "::error::manual PR publication selection requires an exact candidate SHA" >&2', + ' [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {', + ' echo "::error::manual PR publication selection requires an exact base SHA" >&2', " exit 1", " }", - ' expected_sha="$CHECKOUT_SHA"', - " allow_non_head=0", - ' publication_branch="${REF#refs/heads/}"', - " publication_event=workflow_dispatch", - " select_nearest_successful=0", + ' expected_sha="$BASE_SHA"', + " allow_non_head=1", + " select_nearest_successful=1", " ;;", " *)", ' echo "::error::base-image publication mode is not trusted" >&2', @@ -62,8 +58,6 @@ const PUBLICATION_CLASSIFIER_SCRIPT = "esac", 'printf \'allow_non_head=%s\\n\' "${allow_non_head}" >> "${GITHUB_OUTPUT}"', 'printf \'expected_sha=%s\\n\' "${expected_sha}" >> "${GITHUB_OUTPUT}"', - 'printf \'publication_branch=%s\\n\' "${publication_branch}" >> "${GITHUB_OUTPUT}"', - 'printf \'publication_event=%s\\n\' "${publication_event}" >> "${GITHUB_OUTPUT}"', 'printf \'select_nearest_successful=%s\\n\' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}"', ].join("\n") + "\n"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; @@ -604,10 +598,12 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): outputs: { dcode_base_contract: "${{ steps.validate_dcode_base.outputs.contract }}", dcode_base_ref: "${{ steps.validate_dcode_base.outputs.base_ref }}", - managed_image_artifact_provenance: "${{ steps.download_managed_cohort.outputs.provenance }}", + managed_image_artifact_provenance: + "${{ steps.download_managed_cohort.outputs.provenance }}", managed_image_cohort: "${{ steps.validate_managed_cohort.outputs.cohort }}", managed_image_receipt: "${{ steps.validate_managed_cohort.outputs.receipt }}", - managed_image_revision: "${{ steps.validate_managed_cohort.outputs.revision }}", + managed_image_revision: + "${{ steps.validate_managed_cohort.outputs.revision }}", managed_image_run_attempt: "${{ steps.validate_managed_cohort.outputs.run_attempt }}", managed_image_run_id: "${{ steps.validate_managed_cohort.outputs.run_id }}", }, @@ -652,8 +648,6 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): env: { EXPECTED_SHA: "${{ steps.publication_mode.outputs.expected_sha }}", GITHUB_TOKEN: "${{ github.token }}", - PUBLICATION_BRANCH: "${{ steps.publication_mode.outputs.publication_branch }}", - PUBLICATION_EVENT: "${{ steps.publication_mode.outputs.publication_event }}", PUBLICATION_HISTORY_ALLOW_NON_HEAD: "${{ steps.publication_mode.outputs.allow_non_head }}", REQUIRE_MANAGED_IMAGE_PUBLICATION: "1", @@ -663,7 +657,7 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): shell: "bash", run: [ "set -euo pipefail", - 'export GITHUB_REF="refs/heads/$PUBLICATION_BRANCH"', + "export GITHUB_REF=refs/heads/main", 'export GITHUB_SHA="$EXPECTED_SHA"', "wait_seconds=3000", 'if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then', @@ -811,7 +805,9 @@ const MANAGED_IMAGE_RECEIPT_EXPRESSION = "${{ needs.base-image-publication.outputs.managed_image_receipt }}"; /** Require publication success and one exact cohort revision for every stock onboarding job. */ -export function validateStockOnboardingPublicationBoundary(workflow: OperationsWorkflow): string[] { +export function validateStockOnboardingPublicationBoundary( + workflow: OperationsWorkflow, +): string[] { const errors: string[] = []; for (const jobName of STOCK_ONBOARDING_JOBS) { const job = workflow.jobs[jobName] ?? {}; From 6fe1c0b7474a4db569123f91f66e077fe5481f7d Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:22:39 -0700 Subject: [PATCH 23/37] fix(hermes): sync profile policy integrity pin Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- agents/hermes/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 4db2cd3d407..ebcacc4995f 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -909,7 +909,7 @@ RUN install -o root -g root -m 0444 \ # Fresh named profiles do not receive config.yaml. Patch the pinned Hermes # fallback readers from the generated manifest, then validate a real profile. -ARG NEMOCLAW_HERMES_PROFILE_POLICY_PATCHER_SHA256=7468555c7596b3b95732fb98aec6152537778d8519a4409c3da8aa6a76c9a3f7 +ARG NEMOCLAW_HERMES_PROFILE_POLICY_PATCHER_SHA256=424336d2ee3a12b4fb979ed84401ef105bf9c70e36dc3aa27a70f2a46b46def9 # hadolint ignore=DL4006 RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_PROFILE_POLICY_PATCHER_SHA256" /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py \ From ee88761d9d81729927f7527ffce660918da74c09 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:48:44 -0700 Subject: [PATCH 24/37] test: restore virtual source-shape imports Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/cli-coverage-sequencer.test.ts | 21 ++++++++++++-------- test/repository/source-shape-scanner.test.ts | 20 +++++++++---------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/test/cli-coverage-sequencer.test.ts b/test/cli-coverage-sequencer.test.ts index db5cde3d487..faff05bf5af 100644 --- a/test/cli-coverage-sequencer.test.ts +++ b/test/cli-coverage-sequencer.test.ts @@ -95,10 +95,14 @@ describe("stable CLI coverage sharding", () => { ]); const withRemoval = assignmentOwners(entries.slice(1)); - expect(entries.every((entry) => - Object.is(withAddition.get(entry.key), baseline.get(entry.key)))).toBe(true); - expect(entries.slice(1).every((entry) => - Object.is(withRemoval.get(entry.key), baseline.get(entry.key)))).toBe(true); + expect( + entries.every((entry) => Object.is(withAddition.get(entry.key), baseline.get(entry.key))), + ).toBe(true); + expect( + entries + .slice(1) + .every((entry) => Object.is(withRemoval.get(entry.key), baseline.get(entry.key))), + ).toBe(true); }); it("keeps recorded project and path keys on their stable shards", () => { @@ -137,9 +141,7 @@ describe("stable CLI coverage sharding", () => { ); expect(integrationEntries.length).toBeGreaterThan(0); - const weights = assignStableShards(integrationEntries, 12).map( - (shard) => shard.totalWeightMs, - ); + const weights = assignStableShards(integrationEntries, 12).map((shard) => shard.totalWeightMs); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.1); @@ -156,7 +158,10 @@ describe("stable CLI coverage sharding", () => { it("wires stable project and path ownership into the Vitest sequencer", async () => { const specifications = [ testSpecification("test/local-credential-helper-fields.test.ts", "local-credentials"), - testSpecification("test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts", "hermes-config"), + testSpecification( + "test/agents/hermes/hermes-restart-config-seal-write-lock.test.ts", + "hermes-config", + ), ...Array.from({ length: 8 }, (_, index) => testSpecification(`test/regular-${index}.test.ts`, `regular-${index}`), ), diff --git a/test/repository/source-shape-scanner.test.ts b/test/repository/source-shape-scanner.test.ts index be8f64bf4b0..e3cf307fc68 100644 --- a/test/repository/source-shape-scanner.test.ts +++ b/test/repository/source-shape-scanner.test.ts @@ -165,7 +165,7 @@ describe("source-shape scanner", () => { import { readFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { it } from "vitest"; - import { validateBlueprint } from "../../src/lib/config-validator"; + import { validateBlueprint } from "../src/lib/config-validator"; const cjsSame = require("node:assert").deepEqual; const esmSame = nodeAssert.deepStrictEqual; @@ -411,7 +411,7 @@ describe("source-shape scanner", () => { import { readFileSync } from "node:fs"; import YAML from "yaml"; import { expect, it } from "vitest"; - import { validateBlueprint } from "../../src/lib/config-validator"; + import { validateBlueprint } from "../src/lib/config-validator"; it("asserts validator behavior", () => { const raw = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); @@ -433,8 +433,8 @@ describe("source-shape scanner", () => { it("detects explicit raw-config accessors and local selectors", () => { const cases = detectedCaseNames(` import { expect, it } from "vitest"; - import { readWorkflow } from "../helpers/e2e-workflow-contract"; - import { listTargets } from "../e2e/registry/registry"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; + import { listTargets } from "./e2e/registry/registry"; it("mirrors workflow jobs through a selector", () => { const workflow = readWorkflow(); @@ -491,10 +491,10 @@ describe("source-shape scanner", () => { it("tracks namespace accessors but not derived registry and manifest helpers", () => { const cases = detectedCaseNames(` import { expect, it } from "vitest"; - import * as workflows from "../helpers/e2e-workflow-contract"; - import * as registry from "../e2e/registry/registry"; - import { probesForState } from "../e2e/registry/expected-states"; - import { loadManifest, loadManifestsFromDir } from "../e2e/registry/manifests"; + import * as workflows from "./helpers/e2e-workflow-contract"; + import * as registry from "./e2e/registry/registry"; + import { probesForState } from "./e2e/registry/expected-states"; + import { loadManifest, loadManifestsFromDir } from "./e2e/registry/manifests"; it("mirrors a namespace-loaded workflow", () => { expect(Object.keys(workflows.readWorkflow().jobs)).toEqual(["test", "build"]); @@ -516,7 +516,7 @@ describe("source-shape scanner", () => { const cases = detectedCaseNames(` import { spawnSync } from "node:child_process"; import { expect, it } from "vitest"; - import { readWorkflow } from "../helpers/e2e-workflow-contract"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; it("asserts executed behavior", () => { const workflow = readWorkflow(); @@ -531,7 +531,7 @@ describe("source-shape scanner", () => { it("does not taint execution of a program extracted from config", () => { const cases = detectedCaseNames(` import { expect, it } from "vitest"; - import { readWorkflow } from "../helpers/e2e-workflow-contract"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; const DynamicFunction = Object.getPrototypeOf(async () => undefined).constructor; const workflow = readWorkflow(); From 9464631ab7a7e9262e891866ea2972054c119f8c Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:05:08 -0700 Subject: [PATCH 25/37] test: repair grouped test paths Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../nemoclaw-start-gateway-ws-host.test.ts | 4 +-- .../pull-public-exact-digest.test.ts | 2 +- .../generate-openclaw-config.test.ts | 2 +- ...nboard-managed-image-buildless-e2e.test.ts | 2 +- test/repository/source-require-loader.test.ts | 4 +-- test/repository/type-safety-hotspots.test.ts | 26 +++++++++---------- .../gateway-watchdog-validation.test.ts | 2 +- test/security/shellquote-sandbox.test.ts | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test/agents/openclaw/runtime/nemoclaw-start-gateway-ws-host.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-gateway-ws-host.test.ts index 6c6435c1f3d..5db277fb223 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-gateway-ws-host.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-gateway-ws-host.test.ts @@ -12,7 +12,7 @@ import { describe, expect, it } from "vitest"; const requireForTest = createRequire(import.meta.url); const YAML = requireForTest("yaml"); -const START_SCRIPT = path.join(import.meta.dirname, "../../..", "scripts", "nemoclaw-start.sh"); +const START_SCRIPT = path.join(import.meta.dirname, "../../../..", "scripts", "nemoclaw-start.sh"); const startScriptSource = fs.readFileSync(START_SCRIPT, "utf-8"); @@ -558,7 +558,7 @@ describe("gateway websocket url host derivation", () => { describe("gateway dial-back base policy", () => { function loadYaml(relativePath: string): Record { return YAML.parse( - fs.readFileSync(path.join(import.meta.dirname, "../../..", relativePath), "utf-8"), + fs.readFileSync(path.join(import.meta.dirname, "../../../..", relativePath), "utf-8"), ) as Record; } diff --git a/test/e2e-runtime/pull-public-exact-digest.test.ts b/test/e2e-runtime/pull-public-exact-digest.test.ts index 8d49f2cdc59..4349b72ddbf 100644 --- a/test/e2e-runtime/pull-public-exact-digest.test.ts +++ b/test/e2e-runtime/pull-public-exact-digest.test.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -const repoRoot = path.resolve(import.meta.dirname, ".."); +const repoRoot = path.resolve(import.meta.dirname, "../.."); const puller = path.join(repoRoot, "scripts/checks/pull-public-exact-digest.sh"); const reference = `ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:${"a".repeat(64)}`; diff --git a/test/generation/generate-openclaw-config.test.ts b/test/generation/generate-openclaw-config.test.ts index fc25a21fc43..3de923d1c03 100644 --- a/test/generation/generate-openclaw-config.test.ts +++ b/test/generation/generate-openclaw-config.test.ts @@ -1063,7 +1063,7 @@ describe("generate-openclaw-config.mts: config generation", () => { NEMOCLAW_EXTRA_AGENTS_JSON_B64: extraAgentsB64([ makeExtra({ workspace: "/sandbox/.openclaw/foo/../workspace-research", - agentDir: "/sandbox/.openclaw/bar/../../agents/research", + agentDir: "/sandbox/.openclaw/bar/../agents/research", }), ]), }); diff --git a/test/onboarding/onboard-managed-image-buildless-e2e.test.ts b/test/onboarding/onboard-managed-image-buildless-e2e.test.ts index cc9e6b1c4e8..c5111ac32a8 100644 --- a/test/onboarding/onboard-managed-image-buildless-e2e.test.ts +++ b/test/onboarding/onboard-managed-image-buildless-e2e.test.ts @@ -11,7 +11,7 @@ import { test } from "../e2e/fixtures/workflow-e2e-test.ts"; import { runManagedImageBuildlessE2e } from "../helpers/managed-image-buildless-e2e"; function expectManagedOnlyGuide(relativePath: string): void { - const guide = readFileSync(path.join(import.meta.dirname, "..", "docs", relativePath), "utf8"); + const guide = readFileSync(path.join(import.meta.dirname, "../..", "docs", relativePath), "utf8"); expect(guide).toContain( "stock onboarding stops before sandbox creation and does not build a shipped Dockerfile", ); diff --git a/test/repository/source-require-loader.test.ts b/test/repository/source-require-loader.test.ts index b20ec2774e9..b2ae52b24ed 100644 --- a/test/repository/source-require-loader.test.ts +++ b/test/repository/source-require-loader.test.ts @@ -433,8 +433,8 @@ fs.linkSync = function replaceSourceRequireLock(existingPath, claimPath) { const Module = require("node:module"); const path = require("node:path"); const expected = new Set([ - path.resolve(${JSON.stringify(path.join(import.meta.dirname, "helpers", "register-source-require.ts"))}), - path.resolve(${JSON.stringify(path.join(import.meta.dirname, "helpers", "source-require-cache.ts"))}), + path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "register-source-require.ts"))}), + path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-require-cache.ts"))}), ]); const compiled = []; const originalCompile = Module.prototype._compile; diff --git a/test/repository/type-safety-hotspots.test.ts b/test/repository/type-safety-hotspots.test.ts index 1a05e8d9066..ac9be35ee14 100644 --- a/test/repository/type-safety-hotspots.test.ts +++ b/test/repository/type-safety-hotspots.test.ts @@ -136,11 +136,11 @@ export const value = 1; }; } `, - "src/use-a.ts": `import { normalizeConfig } from ".././config"; + "src/use-a.ts": `import { normalizeConfig } from "./config"; export const configA = normalizeConfig("{}"); `, - "src/use-b.ts": `import { normalizeConfig } from ".././config"; + "src/use-b.ts": `import { normalizeConfig } from "./config"; export const configB = normalizeConfig("{}"); `, @@ -196,17 +196,17 @@ export function normalizeConfig(raw: UserConfig | null): string | null { return raw?.owner ?? null; } `, - "src/use-a.ts": `import type { UserConfig } from ".././config"; + "src/use-a.ts": `import type { UserConfig } from "./config"; export const configA = null as UserConfig | null; `, - "src/use-b.ts": `import type { UserConfig } from ".././config"; + "src/use-b.ts": `import type { UserConfig } from "./config"; export function readOwner(value: UserConfig | null): string | null { return value?.owner ?? null; } `, - "src/use-c.ts": `import type { MaybeConfig } from ".././config"; + "src/use-c.ts": `import type { MaybeConfig } from "./config"; interface UserConfig { shadow: string | null; @@ -215,31 +215,31 @@ interface UserConfig { export const maybe = null as MaybeConfig; export const shadow = null as UserConfig | null; `, - "src/use-d.ts": `import type { UserConfig as ImportedConfig } from ".././config"; + "src/use-d.ts": `import type { UserConfig as ImportedConfig } from "./config"; export const aliased = null as ImportedConfig | null; `, - "src/use-e.ts": `import type * as Config from ".././config"; + "src/use-e.ts": `import type * as Config from "./config"; export const namespaced = null as Config.UserConfig | null; `, - "src/barrel.ts": `export type { UserConfig } from ".././config"; + "src/barrel.ts": `export type { UserConfig } from "./config"; `, - "src/star-barrel.ts": `export * from ".././config"; + "src/star-barrel.ts": `export * from "./config"; `, - "src/use-f.ts": `import type { UserConfig as BarrelConfig } from ".././barrel"; + "src/use-f.ts": `import type { UserConfig as BarrelConfig } from "./barrel"; export const barreled = null as BarrelConfig | null; `, - "src/use-g.ts": `import type * as Barrel from ".././barrel"; + "src/use-g.ts": `import type * as Barrel from "./barrel"; export const namespacedBarrel = null as Barrel.UserConfig | null; `, - "src/use-h.ts": `import type { UserConfig as StarConfig } from ".././star-barrel"; + "src/use-h.ts": `import type { UserConfig as StarConfig } from "./star-barrel"; export const starBarreled = null as StarConfig | null; `, - "src/use-i.ts": `import type { HiddenConfig } from ".././config"; + "src/use-i.ts": `import type { HiddenConfig } from "./config"; export const hidden = null as HiddenConfig | null; `, diff --git a/test/runtime/gateway/gateway-watchdog-validation.test.ts b/test/runtime/gateway/gateway-watchdog-validation.test.ts index 240bea3dfa9..4f8b026541a 100644 --- a/test/runtime/gateway/gateway-watchdog-validation.test.ts +++ b/test/runtime/gateway/gateway-watchdog-validation.test.ts @@ -9,7 +9,7 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; const HERE = path.dirname(fileURLToPath(import.meta.url)); -const START_SCRIPT = path.resolve(HERE, "..", "scripts", "nemoclaw-start.sh"); +const START_SCRIPT = path.resolve(HERE, "../../..", "scripts", "nemoclaw-start.sh"); function requireNonNegative(value: number, message: string): number { return value >= 0 diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index ebe789e7f39..75355882ef7 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -54,7 +54,7 @@ describe("sandboxName command hardening in onboard.js", () => { }); it("runs setup-dns-proxy.sh through the argv helper instead of bash -c interpolation", () => { - const repoRoot = path.join(import.meta.dirname, ".."); + const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dns-argv-")); const fakeBin = path.join(tmpDir, "bin"); const scriptPath = path.join(tmpDir, "create-sandbox-dns-argv.cjs"); From 69848aafb6b0ad6f9c826c6f8ac468ece954402d Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 25 Aug 2026 14:09:11 +0700 Subject: [PATCH 26/37] fix(onboard): clean stale Hermes state on recreation Signed-off-by: San Dang --- .../managed-workload/onboard-orchestration.ts | 9 +- .../sandbox-create/orchestration.test.ts | 133 ++++++++++++++++++ .../onboard/sandbox-create/orchestration.ts | 96 ++++++++++++- 3 files changed, 230 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 437fabdd7df..4e34bcdfdea 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -20,10 +20,7 @@ import { } from "../docker-gpu-route"; import type { HermesDashboardOnboardState } from "../hermes-dashboard"; import type { InitialSandboxPolicy } from "../initial-policy"; -import { - isShippedManagedImageAgent, - managedImageRuntimeIdentity, -} from "../managed-image/contract"; +import { isShippedManagedImageAgent, managedImageRuntimeIdentity } from "../managed-image/contract"; import { type BuiltManagedStartupOnboardProfile, buildManagedStartupOnboardProfile, @@ -35,6 +32,7 @@ import type { MessagingTokenDef } from "../messaging-prep"; import { resolveSandboxBuildContext, resolveSandboxBuildPatch } from "../prepared-dcode-rebuild"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, + normalizeRuntimeProviderIdentity, type RuntimeProviderBundle, resolveRuntimeProviderBundle, } from "../runtime-provider/access"; @@ -69,6 +67,7 @@ import { import { resolveSandboxWorkloadRuntimeCapabilities } from "../workload/runtime"; import { prepareManagedHermesStateVolume, + removeManagedHermesStateVolume, type ManagedHermesStateVolumeContext, type ManagedHermesStateVolumeDeps, } from "./hermes-state-volume"; @@ -82,6 +81,8 @@ type SandboxInferenceConfig = import("../../inference/config").SandboxInferenceC type SupportedBootstrap = Extract; type BootstrapProvider = RuntimeProviderBundle & { readonly bootstrap: SupportedBootstrap }; +export { normalizeRuntimeProviderIdentity, removeManagedHermesStateVolume }; + export type ManagedHermesStateVolumeOnboardLifecycle = { materializeSandboxCreatePlan( input: MaterializeSandboxCreatePlanInput, diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 1b2bcdf5aa2..497dccb0c70 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -6,12 +6,145 @@ import { describe, expect, it, vi } from "vitest"; import type { SandboxEntry } from "../../state/registry"; import { applyAbsentSandboxRebuildPolicyCarryForward, + cleanupRecreatedSourceHermesStateVolume, completeHermesPortableSandboxRegistration, + finalizeRecreatedSourceHermesStateVolume, hasManagedMcpRebuildHandoff, proveRecreateSourceBeforePolicyCarryForward, readManagedDcodeCreateSelectionDrift, } from "./orchestration"; +function managedHermesSource(): SandboxEntry { + return { + name: "alpha", + agent: "hermes", + openshellDriver: "docker-linux", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/hermes@sha256:${"a".repeat(64)}`, + release: "v0.0.100", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-100-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: "fixture-profile", + startupProfileSha256: "c".repeat(64), + credentialProxyReplayRequired: false, + shared: true, + }, + }; +} + +type HermesVolumeCleanupDeps = Parameters[1]; + +function hermesVolumeCleanupDeps( + removeManagedHermesStateVolume: HermesVolumeCleanupDeps["removeManagedHermesStateVolume"], +): HermesVolumeCleanupDeps { + return { + normalizeRuntimeProviderIdentity: vi.fn(() => "docker"), + removeManagedHermesStateVolume, + note: vi.fn(), + warn: vi.fn(), + redact: vi.fn((message: string) => message.replace("secret", "[REDACTED]")), + }; +} + +describe("recreated managed Hermes state volume", () => { + it("preserves the volume when the replacement keeps the managed Docker Hermes lifecycle", () => { + const removeManagedHermesStateVolume = + vi.fn(); + const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); + + cleanupRecreatedSourceHermesStateVolume( + { + sandboxName: "alpha", + sourceEntry: managedHermesSource(), + targetKeepsManagedHermesStateVolume: true, + }, + deps, + ); + + expect(removeManagedHermesStateVolume).not.toHaveBeenCalled(); + }); + + it("removes the owned volume when the replacement changes its lifecycle", () => { + const removeManagedHermesStateVolume = vi.fn< + HermesVolumeCleanupDeps["removeManagedHermesStateVolume"] + >(() => ({ status: "removed" as const })); + const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); + + cleanupRecreatedSourceHermesStateVolume( + { + sandboxName: "alpha", + sourceEntry: managedHermesSource(), + targetKeepsManagedHermesStateVolume: false, + }, + deps, + ); + + expect(removeManagedHermesStateVolume).toHaveBeenCalledExactlyOnceWith({ + agentName: "hermes", + runtimeProviderId: "docker", + sandboxName: "alpha", + workloadKind: "managed-image", + }); + expect(deps.note).toHaveBeenCalledWith(" Removed managed Hermes state volume for 'alpha'."); + }); + + it("leaves a foreign same-name volume untouched", () => { + const removeManagedHermesStateVolume = vi.fn< + HermesVolumeCleanupDeps["removeManagedHermesStateVolume"] + >(() => ({ + status: "not-owned" as const, + detail: "the exact NemoClaw ownership labels are absent or changed", + volumeName: "nemoclaw-hermes-state-v1-alpha", + })); + const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); + + cleanupRecreatedSourceHermesStateVolume( + { + sandboxName: "alpha", + sourceEntry: managedHermesSource(), + targetKeepsManagedHermesStateVolume: false, + }, + deps, + ); + + expect(deps.warn).toHaveBeenCalledWith( + " Left Docker volume 'nemoclaw-hermes-state-v1-alpha' untouched because the exact NemoClaw ownership labels are absent or changed.", + ); + }); + + it("fails with redacted recovery evidence and allows an exact retry", () => { + const removeManagedHermesStateVolume = vi + .fn() + .mockReturnValueOnce({ + status: "failed" as const, + detail: "secret Docker failure", + volumeName: "nemoclaw-hermes-state-v1-alpha", + }) + .mockReturnValueOnce({ status: "removed" as const }); + const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); + const removeSourceRegistryEntry = vi.fn(); + const finalizationDeps = { ...deps, removeSourceRegistryEntry }; + const input = { + sandboxName: "alpha", + sourceEntry: managedHermesSource(), + sourceConfirmedAbsent: true, + targetKeepsManagedHermesStateVolume: false, + }; + + expect(() => finalizeRecreatedSourceHermesStateVolume(input, finalizationDeps)).toThrow( + "[REDACTED] Docker failure", + ); + expect(removeSourceRegistryEntry).not.toHaveBeenCalled(); + expect(() => finalizeRecreatedSourceHermesStateVolume(input, finalizationDeps)).not.toThrow(); + expect(removeManagedHermesStateVolume).toHaveBeenCalledTimes(2); + expect(removeSourceRegistryEntry).toHaveBeenCalledExactlyOnceWith(input.sourceEntry, "alpha"); + }); +}); + describe("managed MCP rebuild handoff", () => { const targetIntentFingerprint = "a".repeat(64); const recreateTransaction = { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index ecf38209b92..0db5df2313e 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -10,6 +10,10 @@ import type { SandboxEntry } from "../../state/registry"; import type { HermesAuthMethod } from "../hermes-auth"; import type { PreparedSandboxBuildContext } from "../build-context-stage"; import type { DcodeSelectionDriftReader } from "../dcode-selection-drift"; +import type { + ManagedHermesStateVolumeCleanupResult, + ManagedHermesStateVolumeContext, +} from "../managed-workload/hermes-state-volume"; import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import type { PortableOnboardRuntimeContext } from "../session-bootstrap"; @@ -37,6 +41,66 @@ type SandboxRecreateReasonInput = { existingSandboxState: string; }; +type RecreatedSourceHermesStateVolumeCleanupInput = { + readonly sandboxName: string; + readonly sourceEntry: SandboxEntry | null; + readonly targetKeepsManagedHermesStateVolume: boolean; +}; + +type RecreatedSourceHermesStateVolumeCleanupDeps = { + readonly normalizeRuntimeProviderIdentity: (driverName: string | null | undefined) => string; + readonly removeManagedHermesStateVolume: ( + context: ManagedHermesStateVolumeContext, + ) => ManagedHermesStateVolumeCleanupResult; + readonly note: (message: string) => void; + readonly warn: (message: string) => void; + readonly redact: (message: string) => string; +}; + +type RecreatedSourceHermesStateVolumeFinalizationInput = + RecreatedSourceHermesStateVolumeCleanupInput & { + readonly sourceConfirmedAbsent: boolean; + }; + +type RecreatedSourceHermesStateVolumeFinalizationDeps = + RecreatedSourceHermesStateVolumeCleanupDeps & { + readonly removeSourceRegistryEntry: (entry: SandboxEntry, sandboxName: string) => void; + }; + +export function cleanupRecreatedSourceHermesStateVolume( + input: RecreatedSourceHermesStateVolumeCleanupInput, + deps: RecreatedSourceHermesStateVolumeCleanupDeps, +): void { + if (!input.sourceEntry || input.targetKeepsManagedHermesStateVolume) return; + + const cleanup = deps.removeManagedHermesStateVolume({ + agentName: input.sourceEntry.agent, + runtimeProviderId: deps.normalizeRuntimeProviderIdentity(input.sourceEntry.openshellDriver), + sandboxName: input.sandboxName, + workloadKind: input.sourceEntry.workload?.kind ?? "", + }); + if (cleanup.status === "failed") { + throw new Error( + `Sandbox '${input.sandboxName}' is gone, but its managed Hermes state volume '${cleanup.volumeName}' could not be removed: ${deps.redact(cleanup.detail)}. The sandbox registry entry was preserved so exact cleanup can be retried.`, + ); + } + if (cleanup.status === "not-owned") { + deps.warn(` Left Docker volume '${cleanup.volumeName}' untouched because ${cleanup.detail}.`); + } else if (cleanup.status === "removed") { + deps.note(` Removed managed Hermes state volume for '${input.sandboxName}'.`); + } +} + +export function finalizeRecreatedSourceHermesStateVolume( + input: RecreatedSourceHermesStateVolumeFinalizationInput, + deps: RecreatedSourceHermesStateVolumeFinalizationDeps, +): void { + if (!input.sourceConfirmedAbsent || !input.sourceEntry) return; + + cleanupRecreatedSourceHermesStateVolume(input, deps); + deps.removeSourceRegistryEntry(input.sourceEntry, input.sandboxName); +} + export function readManagedDcodeCreateSelectionDrift( input: { sandboxName: string; @@ -117,9 +181,7 @@ export function hasManagedMcpRebuildHandoff( createIntent: SandboxCreateIntent | null | undefined, ): boolean { const handoff = createIntent?.recreateJournalTargetIntentFingerprint; - return Boolean( - handoff && createIntent?.recreateTransaction?.targetIntentFingerprint === handoff, - ); + return Boolean(handoff && createIntent?.recreateTransaction?.targetIntentFingerprint === handoff); } function shouldRefuseManagedMcpRecreate( @@ -544,6 +606,27 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sandboxName, workloadKind: workload.source.kind, }); + const finalizeRecreatedSourceHermesVolume = ( + sourceConfirmedAbsent: boolean, + sourceEntry: SandboxEntry | null, + targetKeepsManagedHermesStateVolume: boolean, + ) => + finalizeRecreatedSourceHermesStateVolume( + { + sandboxName, + sourceConfirmedAbsent, + sourceEntry, + targetKeepsManagedHermesStateVolume, + }, + { + normalizeRuntimeProviderIdentity: managedWorkloadOnboard.normalizeRuntimeProviderIdentity, + removeManagedHermesStateVolume: managedWorkloadOnboard.removeManagedHermesStateVolume, + removeSourceRegistryEntry: sandboxLifecycle.removeSandboxUnlessSessionReservation, + note, + warn: (message) => console.warn(message), + redact, + }, + ); // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox. const sandboxWasLiveDefault = liveExists && wasSandboxDefault(registry.getDefault(), sandboxName); @@ -894,7 +977,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); } recreateRuntime.confirmDeleted(); - sandboxLifecycle.removeSandboxUnlessSessionReservation(previousEntry, sandboxName); + finalizeRecreatedSourceHermesVolume(true, previousEntry, hermesStateVolumeLifecycle !== null); await hermesApiPortReservationScope.rebindAfterOwnedForwardDelete( hermesApiPortReservationInput, ); @@ -905,6 +988,11 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche } preparedSandboxWorkload = await ensurePreparedSandboxWorkload(); hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload); + finalizeRecreatedSourceHermesVolume( + !liveExists, + existingEntry, + hermesStateVolumeLifecycle !== null, + ); } sandboxCreatePlanMaterialization.applyOrdinaryExtraProviderReconciliation( agentCreateInput.hermesPortableLifecycle, From e91f8b19c2e4afe2c1f17f72c7e0007356e24f83 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 25 Aug 2026 14:59:56 +0700 Subject: [PATCH 27/37] fix(onboard): bind managed image provenance and cleanup --- .github/workflows/e2e.yaml | 1 - .../onboard-orchestration.test.ts | 2 +- .../managed-workload/onboard-orchestration.ts | 7 +- .../sandbox-create/orchestration.test.ts | 93 +++++++++++---- .../onboard/sandbox-create/orchestration.ts | 12 +- test/e2e/fixtures/clients/host.ts | 1 - test/e2e/fixtures/managed-image-receipt.ts | 6 - test/e2e/fixtures/shell-probe.ts | 3 +- test/e2e/fixtures/workload-source-env.ts | 14 --- ...untime-compatible-anthropic-raw-command.ts | 3 +- test/e2e/live/cloud-onboard.test.ts | 1 - test/e2e/live/dashboard-connect-handoff.ts | 3 +- test/e2e/live/hermes-e2e.test.ts | 1 - test/e2e/live/hermes-gpu-startup.test.ts | 1 - test/e2e/live/jetson-nvmap-gpu.test.ts | 1 - test/e2e/live/messaging-providers.test.ts | 1 - ...time-compatible-anthropic-progress.test.ts | 21 ---- .../managed-image-cohort-contract.test.ts | 109 +++++++++++++++++- .../e2e/support/managed-image-receipt.test.ts | 12 -- test/e2e/support/workload-source-env.test.ts | 49 -------- test/fixtures/explicit-custom.Dockerfile | 8 -- test/helpers/onboard-script-mocks.cjs | 29 ----- .../onboard-installer-restore-intent.test.ts | 20 ++-- test/onboarding/onboard-messaging.test.ts | 14 +-- .../onboard-reservation-recreate.test.ts | 5 +- .../onboard-sandbox-recreation.test.ts | 48 +++----- test/security/shellquote-sandbox.test.ts | 8 +- tools/e2e/managed-image-cohort-contract.mts | 40 +++++++ ...ge-protected-runtime-workflow-boundary.mts | 1 - 29 files changed, 276 insertions(+), 238 deletions(-) delete mode 100644 test/e2e/fixtures/workload-source-env.ts delete mode 100644 test/e2e/support/workload-source-env.test.ts delete mode 100644 test/fixtures/explicit-custom.Dockerfile diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a1789b9b759..3cfdbcb3d74 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4676,7 +4676,6 @@ jobs: E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" - E2E_WORKLOAD_SOURCE: "managed-image" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index 029f20f323b..9b7e6ef3c4f 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -237,7 +237,7 @@ describe("managed workload onboard orchestration", () => { }, ); - lifecycle.materializeSandboxCreatePlan({} as never, (input) => { + lifecycle!.materializeSandboxCreatePlan({} as never, (input) => { expect(input.managedStateMount).toMatchObject({ target: "/sandbox/.hermes" }); return {} as never; }); diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 4e34bcdfdea..71ade27bcf4 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -96,7 +96,7 @@ export function createManagedHermesStateVolumeOnboardLifecycle( readonly runtimeProvider: RuntimeProviderBundle | null; }, deps: ManagedHermesStateVolumeDeps = {}, -): ManagedHermesStateVolumeOnboardLifecycle { +): ManagedHermesStateVolumeOnboardLifecycle | null { const scope = prepareManagedHermesStateVolume( { agentName: input.agentName, @@ -106,12 +106,13 @@ export function createManagedHermesStateVolumeOnboardLifecycle( }, deps, ); + if (!scope) return null; return { materializeSandboxCreatePlan(input, materialize) { - return materialize({ ...input, managedStateMount: scope?.mount }); + return materialize({ ...input, managedStateMount: scope.mount }); }, commit() { - scope?.commit(); + scope.commit(); }, }; } diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 497dccb0c70..6c7c91d9a35 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import type { SandboxEntry } from "../../state/registry"; +import { createHermesStateVolumeDockerHarness } from "../__test-helpers__/hermes-state-volume"; +import { createManagedHermesStateVolumeOnboardLifecycle } from "../managed-workload/onboard-orchestration"; import { applyAbsentSandboxRebuildPolicyCarryForward, cleanupRecreatedSourceHermesStateVolume, @@ -51,7 +53,28 @@ function hermesVolumeCleanupDeps( } describe("recreated managed Hermes state volume", () => { - it("preserves the volume when the replacement keeps the managed Docker Hermes lifecycle", () => { + it("preserves the volume when managed Docker Hermes replaces managed Docker Hermes", () => { + const docker = createHermesStateVolumeDockerHarness({ + name: "nemoclaw-hermes-state-v1-alpha", + labels: { + "io.nvidia.nemoclaw.hermes-state.managed": "true", + "io.nvidia.nemoclaw.hermes-state.schema": "1", + "io.nvidia.nemoclaw.hermes-state.sandbox": "alpha", + "io.nvidia.nemoclaw.hermes-state.target": "/sandbox/.hermes", + }, + }); + const targetLifecycle = createManagedHermesStateVolumeOnboardLifecycle( + { + agentName: "hermes", + runtimeProvider: { identity: { id: "docker" } } as never, + sandboxName: "alpha", + workloadKind: "managed-image", + }, + { + runDocker: docker.runDocker as never, + registerExitCleanup: () => vi.fn(), + }, + ); const removeManagedHermesStateVolume = vi.fn(); const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); @@ -60,37 +83,61 @@ describe("recreated managed Hermes state volume", () => { { sandboxName: "alpha", sourceEntry: managedHermesSource(), - targetKeepsManagedHermesStateVolume: true, + targetKeepsManagedHermesStateVolume: targetLifecycle !== null, }, deps, ); + expect(targetLifecycle).not.toBeNull(); + expect(docker.calls.some((args) => args[0] === "create")).toBe(false); expect(removeManagedHermesStateVolume).not.toHaveBeenCalled(); + targetLifecycle?.commit(); }); - it("removes the owned volume when the replacement changes its lifecycle", () => { - const removeManagedHermesStateVolume = vi.fn< - HermesVolumeCleanupDeps["removeManagedHermesStateVolume"] - >(() => ({ status: "removed" as const })); - const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); + it.each([ + ["OpenClaw", "openclaw", "docker", "managed-image"], + ["custom Dockerfile Hermes", "hermes", "docker", "legacy-dockerfile"], + ["managed-image Hermes on a non-Docker runtime", "hermes", "kubernetes", "managed-image"], + ])( + "removes the owned volume when managed Docker Hermes changes to %s", + (_replacement, agentName, runtimeProviderId, workloadKind) => { + const runDocker = vi.fn(() => { + throw new Error("a replacement that does not own the volume must not access Docker"); + }); + const targetLifecycle = createManagedHermesStateVolumeOnboardLifecycle( + { + agentName, + runtimeProvider: { identity: { id: runtimeProviderId } } as never, + sandboxName: "alpha", + workloadKind, + }, + { runDocker: runDocker as never }, + ); + const removeManagedHermesStateVolume = vi.fn< + HermesVolumeCleanupDeps["removeManagedHermesStateVolume"] + >(() => ({ status: "removed" as const })); + const deps = hermesVolumeCleanupDeps(removeManagedHermesStateVolume); - cleanupRecreatedSourceHermesStateVolume( - { - sandboxName: "alpha", - sourceEntry: managedHermesSource(), - targetKeepsManagedHermesStateVolume: false, - }, - deps, - ); + cleanupRecreatedSourceHermesStateVolume( + { + sandboxName: "alpha", + sourceEntry: managedHermesSource(), + targetKeepsManagedHermesStateVolume: targetLifecycle !== null, + }, + deps, + ); - expect(removeManagedHermesStateVolume).toHaveBeenCalledExactlyOnceWith({ - agentName: "hermes", - runtimeProviderId: "docker", - sandboxName: "alpha", - workloadKind: "managed-image", - }); - expect(deps.note).toHaveBeenCalledWith(" Removed managed Hermes state volume for 'alpha'."); - }); + expect(targetLifecycle).toBeNull(); + expect(runDocker).not.toHaveBeenCalled(); + expect(removeManagedHermesStateVolume).toHaveBeenCalledExactlyOnceWith({ + agentName: "hermes", + runtimeProviderId: "docker", + sandboxName: "alpha", + workloadKind: "managed-image", + }); + expect(deps.note).toHaveBeenCalledWith(" Removed managed Hermes state volume for 'alpha'."); + }, + ); it("leaves a foreign same-name volume untouched", () => { const removeManagedHermesStateVolume = vi.fn< diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 0db5df2313e..23a58b0abf3 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1119,10 +1119,12 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, dependencies: { materializeSandboxCreatePlan: (input) => - hermesStateVolumeLifecycle.materializeSandboxCreatePlan( - input, - sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, - ), + hermesStateVolumeLifecycle + ? hermesStateVolumeLifecycle.materializeSandboxCreatePlan( + input, + sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, + ) + : sandboxCreatePlanMaterialization.materializeSandboxCreatePlan(input), prepareSandboxBuildPatchConfig: sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig, }, @@ -1390,7 +1392,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche cleanupInitialCreateSource(); await completeCreatedSandboxRegistration(created, null); } - hermesStateVolumeLifecycle.commit(); + hermesStateVolumeLifecycle?.commit(); if ("complete" in recreateRuntime) recreateRuntime.complete(); if (agentCreateInput.hermesPortableLifecycle) return sandboxName; return completeOrdinaryOnboardSandboxCreation( diff --git a/test/e2e/fixtures/clients/host.ts b/test/e2e/fixtures/clients/host.ts index eefec95c3b3..4991582e2c3 100644 --- a/test/e2e/fixtures/clients/host.ts +++ b/test/e2e/fixtures/clients/host.ts @@ -79,7 +79,6 @@ export class HostCliClient { throw new Error("stock managed-image receipt assertion requires a sandbox name"); } assertStockManagedImageReceipt({ - commandOutput: resultText(result), environment, expectedAgent: environment.NEMOCLAW_AGENT?.trim(), sandboxName, diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts index ec2d44f8f3e..ff89d9b7ef9 100644 --- a/test/e2e/fixtures/managed-image-receipt.ts +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -19,7 +19,6 @@ import { cloneSandboxWorkloadReceipt } from "../../../src/lib/state/registry/wor import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; const REVISION_PATTERN = /^[0-9a-f]{40}$/u; -const FALLBACK_DIAGNOSTIC = "Managed image unavailable; using the trusted Dockerfile recipe."; export function assertManagedImageReceiptMatchesSelectedCohort(options: { readonly environment: NodeJS.ProcessEnv; @@ -110,7 +109,6 @@ function gatewayPort(environment: NodeJS.ProcessEnv): number { /** Assert the durable receipt before an E2E test begins post-onboarding probes. */ export function assertStockManagedImageReceipt(options: { - readonly commandOutput?: string; readonly environment?: NodeJS.ProcessEnv; readonly expectedAgent?: string; readonly sandboxName: string; @@ -120,10 +118,6 @@ export function assertStockManagedImageReceipt(options: { if (!REVISION_PATTERN.test(revision)) { throw new Error("stock onboarding requires one exact managed-image cohort revision"); } - if (options.commandOutput?.includes(FALLBACK_DIAGNOSTIC)) { - throw new Error("stock onboarding emitted a legacy Dockerfile fallback diagnostic"); - } - const home = environment.HOME?.trim() || os.homedir(); const registryPath = path.join( nemoclawStateRoot(home, gatewayPort(environment)), diff --git a/test/e2e/fixtures/shell-probe.ts b/test/e2e/fixtures/shell-probe.ts index 929e30fd08d..0a66f223d35 100644 --- a/test/e2e/fixtures/shell-probe.ts +++ b/test/e2e/fixtures/shell-probe.ts @@ -5,7 +5,6 @@ import type { ArtifactSink } from "./artifacts.ts"; import { type ChildProcessProgress, spawnObservedChild } from "./observed-child-process.ts"; import { superviseChild } from "./shell/supervisor.ts"; import type { TrustedShellCommand } from "./shell/trusted-command.ts"; -import { resolveLiveE2eWorkloadSourceEnv } from "./workload-source-env.ts"; /** * Fixture-flavoured host shell probe. @@ -223,7 +222,7 @@ export class ShellProbe { spawn: { cwd: options.cwd, detached: true, - env: resolveLiveE2eWorkloadSourceEnv({ ...(options.env ?? {}) }), + env: { ...(options.env ?? {}) }, stdio: ["ignore", "pipe", "pipe"], }, }); diff --git a/test/e2e/fixtures/workload-source-env.ts b/test/e2e/fixtures/workload-source-env.ts deleted file mode 100644 index 52ca5f2b1e6..00000000000 --- a/test/e2e/fixtures/workload-source-env.ts +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Reject automatic legacy source selection at the final E2E spawn boundary. - * An explicit custom Dockerfile remains a separate user-supplied input. - */ -export function resolveLiveE2eWorkloadSourceEnv(input: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - const targetId = input.E2E_TARGET_ID; - const source = input.E2E_WORKLOAD_SOURCE; - if (!targetId || source !== "legacy-dockerfile") return input; - if (input.NEMOCLAW_FROM_DOCKERFILE) return input; - throw new Error(`live E2E target '${targetId}' cannot select a stock legacy Dockerfile`); -} diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts index df248747f03..a98e1a76221 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts @@ -6,7 +6,6 @@ import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { TestProgress, TestProgressCapability } from "../fixtures/progress.ts"; import { redactString } from "../fixtures/redaction.ts"; -import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts"; import { projectRawOutputForArtifact, type RawArtifactOutputMode, @@ -96,7 +95,7 @@ export async function runRawCommand( spawn: { cwd: options.cwd ?? REPO_ROOT, detached: true, - env: resolveLiveE2eWorkloadSourceEnv({ ...(options.env ?? {}) }), + env: { ...(options.env ?? {}) }, stdio: ["ignore", "pipe", "pipe"], }, }); diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index 29298776e77..5ea4a7808cf 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -223,7 +223,6 @@ test("cloud onboard: public installer creates healthy sandbox with security chec ); expect(install.exitCode, resultText(install)).toBe(0); assertStockManagedImageReceipt({ - commandOutput: resultText(install), environment: testEnv(), expectedAgent: "openclaw", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/dashboard-connect-handoff.ts b/test/e2e/live/dashboard-connect-handoff.ts index e05dafd0cc1..23d07f60b1e 100644 --- a/test/e2e/live/dashboard-connect-handoff.ts +++ b/test/e2e/live/dashboard-connect-handoff.ts @@ -9,7 +9,6 @@ import { spawnObservedChild, } from "../fixtures/observed-child-process.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; -import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts"; import { dashboardRemoteBindConnectStarted } from "./dashboard-remote-bind-env.ts"; const CONNECT_CAPTURE_LIMIT_BYTES = 1024 * 1024; @@ -88,7 +87,7 @@ export async function runDashboardConnectUntilForwardHandoff( spawn: { cwd: REPO_ROOT, detached: true, - env: resolveLiveE2eWorkloadSourceEnv({ ...options.env }), + env: { ...options.env }, stdio: ["ignore", "pipe", "pipe"], }, }); diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 8dffd809adf..28f34d0895d 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -338,7 +338,6 @@ test( )); expect(install.exitCode, resultText(install)).toBe(0); assertStockManagedImageReceipt({ - commandOutput: resultText(install), environment: env, expectedAgent: "hermes", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index 1b79ef8df6a..eba237e1ae6 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -459,7 +459,6 @@ test( : Promise.resolve()); expect(install.exitCode, resultText(install)).toBe(0); assertStockManagedImageReceipt({ - commandOutput: resultText(install), environment: env, expectedAgent: "hermes", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index a10acda3b0e..3790e04d28d 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -287,7 +287,6 @@ fi`, await artifacts.writeText("install-jetson-nvmap.log", resultText(install)); expect(install.exitCode, resultText(install)).toBe(0); assertStockManagedImageReceipt({ - commandOutput: resultText(install), environment: env(inferenceEnv), expectedAgent: "openclaw", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/messaging-providers.test.ts b/test/e2e/live/messaging-providers.test.ts index eadbae996e5..8026309fbfb 100644 --- a/test/e2e/live/messaging-providers.test.ts +++ b/test/e2e/live/messaging-providers.test.ts @@ -168,7 +168,6 @@ test( } expectExitZero(install, "M0: install.sh completed"); assertStockManagedImageReceipt({ - commandOutput: outputText(install), environment: state.env, expectedAgent: "openclaw", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts index c8abc433dc8..228d6faad16 100644 --- a/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts +++ b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts @@ -54,27 +54,6 @@ afterEach(async () => { }); describe("Bedrock raw-command progress", () => { - it("rejects a stock legacy Dockerfile at the raw spawn boundary", async () => { - const artifacts = await artifactSink("bedrock-workload-source"); - const observation = progressProbe(); - await expect( - runRawCommand( - process.execPath, - ["-e", "process.stdout.write(process.env.NEMOCLAW_FROM_DOCKERFILE ?? '')"], - { - artifactName: "bedrock-workload-source", - artifacts, - env: { - E2E_TARGET_ID: "inference-routing", - E2E_WORKLOAD_SOURCE: "legacy-dockerfile", - NEMOCLAW_AGENT: "langchain-deepagents-code", - }, - progress: observation.progress, - }, - ), - ).rejects.toThrow("cannot select a stock legacy Dockerfile"); - }); - it("reports timestamp-only output activity without forwarding child payloads", async () => { const secret = "opaque-bedrock-progress-secret"; const artifacts = await artifactSink("bedrock-progress-output"); diff --git a/test/e2e/support/managed-image-cohort-contract.test.ts b/test/e2e/support/managed-image-cohort-contract.test.ts index f07a495b56e..92599eab8a1 100644 --- a/test/e2e/support/managed-image-cohort-contract.test.ts +++ b/test/e2e/support/managed-image-cohort-contract.test.ts @@ -15,6 +15,16 @@ const RUN_ATTEMPT = 1; const COHORT = `ghrun-${RUN_ID}-${RUN_ATTEMPT}`; const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; type JsonObject = Record; +type PlatformPublication = { + publicationEvidence: { + workloadDescriptor: JsonObject; + attestations: { + manifestDescriptor: { annotations: JsonObject }; + slsa: { statement: { bindings: JsonObject; subject: JsonObject } }; + spdx: { statement: { subject: JsonObject } }; + }; + }; +}; function digest(index: number): `sha256:${string}` { return `sha256:${(index % 15).toString(16).repeat(64)}`; @@ -52,15 +62,44 @@ function cohortContract(): Record { reference: `${image}@${platformDigest}`, baseReference, publicationEvidence: { - candidateDescriptor: { digest: platformDigest }, + candidateDescriptor: { + digest: platformDigest, + mediaType: "application/vnd.oci.image.index.v1+json", + size: 100, + }, workloadDescriptor: { digest: workloadDigest, + mediaType: "application/vnd.oci.image.manifest.v1+json", platform: { os, architecture }, + size: 200, }, attestations: { + manifestDescriptor: { + annotations: { + "vnd.docker.reference.digest": workloadDigest, + "vnd.docker.reference.type": "attestation-manifest", + }, + digest: digest(agentIndex + platformIndex + 12), + mediaType: "application/vnd.oci.image.manifest.v1+json", + platform: { os: "unknown", architecture: "unknown" }, + size: 300, + }, slsa: { + descriptor: { + annotations: { + "in-toto.io/predicate-type": "https://slsa.dev/provenance/v1", + }, + digest: digest(agentIndex + platformIndex + 13), + mediaType: "application/vnd.in-toto+json", + size: 400, + }, statement: { + type: "https://in-toto.io/Statement/v1", + predicateType: "https://slsa.dev/provenance/v1", + buildType: + "https://github.com/moby/buildkit/blob/master/docs/attestations/slsa-definitions.md", builderId: `https://github.com/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}`, + subject: { name: image, digest: workloadDigest }, bindings: { agent, baseReference, @@ -71,6 +110,21 @@ function cohortContract(): Record { }, }, }, + spdx: { + descriptor: { + annotations: { + "in-toto.io/predicate-type": "https://spdx.dev/Document", + }, + digest: digest(agentIndex + platformIndex + 14), + mediaType: "application/vnd.in-toto+json", + size: 500, + }, + statement: { + type: "https://in-toto.io/Statement/v1", + predicateType: "https://spdx.dev/Document", + subject: { name: image, digest: workloadDigest }, + }, + }, }, }, }, @@ -84,6 +138,11 @@ function cohortContract(): Record { }; } +function platformPublication(value: Record): PlatformPublication { + const agents = value.agents as Record }>; + return agents.openclaw.platforms["linux/amd64"]; +} + describe("managed-image cohort publication contract", () => { it("binds all shipped agents and architectures to the selected publication", () => { expect( @@ -159,4 +218,52 @@ describe("managed-image cohort publication contract", () => { }), ).toThrow("revision must be"); }); + + it.each([ + [ + "a missing attestation manifest workload digest binding", + (publication: PlatformPublication) => + delete publication.publicationEvidence.attestations.manifestDescriptor.annotations[ + "vnd.docker.reference.digest" + ], + ], + [ + "a changed attestation manifest workload digest binding", + (publication: PlatformPublication) => + (publication.publicationEvidence.attestations.manifestDescriptor.annotations[ + "vnd.docker.reference.digest" + ] = digest(9)), + ], + [ + "a missing SLSA subject workload digest binding", + (publication: PlatformPublication) => + delete publication.publicationEvidence.attestations.slsa.statement.subject.digest, + ], + [ + "a changed SLSA subject workload digest binding", + (publication: PlatformPublication) => + (publication.publicationEvidence.attestations.slsa.statement.subject.digest = digest(9)), + ], + [ + "a missing SPDX subject workload digest binding", + (publication: PlatformPublication) => + delete publication.publicationEvidence.attestations.spdx.statement.subject.digest, + ], + [ + "a changed SPDX subject workload digest binding", + (publication: PlatformPublication) => + (publication.publicationEvidence.attestations.spdx.statement.subject.digest = digest(9)), + ], + ])("rejects %s", (_label, corruptBinding) => { + const value = cohortContract(); + corruptBinding(platformPublication(value)); + + expect(() => + validateManagedImageCohort(value, { + revision: REVISION, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + }), + ).toThrow("workload digest must be"); + }); }); diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts index d6515638e3b..c78e4d198d6 100644 --- a/test/e2e/support/managed-image-receipt.test.ts +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -201,18 +201,6 @@ describe("stock E2E managed-image receipt assertion", () => { ).toThrow("exact agent image from the selected cohort"); }); - it("rejects the stock fallback diagnostic before later probes", () => { - const home = writeRegistry(managedReceipt()); - - expect(() => - assertStockManagedImageReceipt({ - commandOutput: "Managed image unavailable; using the trusted Dockerfile recipe.", - environment: { E2E_MANAGED_IMAGE_REVISION: REVISION, HOME: home }, - sandboxName: SANDBOX_NAME, - }), - ).toThrow("fallback diagnostic"); - }); - it("asserts normal stock onboarding and excludes an explicit custom Dockerfile", () => { expect( shouldAssertStockManagedImageReceipt("/workspace/bin/nemoclaw.js", ["onboard"], { diff --git a/test/e2e/support/workload-source-env.test.ts b/test/e2e/support/workload-source-env.test.ts deleted file mode 100644 index 7b0c9b09e21..00000000000 --- a/test/e2e/support/workload-source-env.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { resolveLiveE2eWorkloadSourceEnv } from "../fixtures/workload-source-env.ts"; - -describe("live E2E workload source environment", () => { - it.each(["openclaw", "hermes", "langchain-deepagents-code"])( - "rejects automatic legacy-Dockerfile selection for %s", - (agent) => { - expect(() => - resolveLiveE2eWorkloadSourceEnv({ - E2E_TARGET_ID: "full-e2e", - E2E_WORKLOAD_SOURCE: "legacy-dockerfile", - NEMOCLAW_AGENT: agent, - }), - ).toThrow("cannot select a stock legacy Dockerfile"); - }, - ); - - it("preserves an explicit custom Dockerfile", () => { - const input = { - E2E_TARGET_ID: "custom-dockerfile", - E2E_WORKLOAD_SOURCE: "legacy-dockerfile", - NEMOCLAW_AGENT: "openclaw", - NEMOCLAW_FROM_DOCKERFILE: "/workspace/CustomDockerfile", - }; - expect(resolveLiveE2eWorkloadSourceEnv(input)).toEqual(input); - }); - - it("leaves an unspecified source on the product's default workload path", () => { - const input = { E2E_TARGET_ID: "full-e2e", NEMOCLAW_AGENT: "openclaw" }; - expect(resolveLiveE2eWorkloadSourceEnv(input)).toEqual(input); - }); - - it("honors the provider-neutral managed-image source", () => { - const targetId = "managed-image-protected-runtime"; - expect( - resolveLiveE2eWorkloadSourceEnv({ - E2E_TARGET_ID: targetId, - E2E_WORKLOAD_SOURCE: "managed-image", - }), - ).toEqual({ - E2E_TARGET_ID: targetId, - E2E_WORKLOAD_SOURCE: "managed-image", - }); - }); -}); diff --git a/test/fixtures/explicit-custom.Dockerfile b/test/fixtures/explicit-custom.Dockerfile deleted file mode 100644 index 9664e47d9b7..00000000000 --- a/test/fixtures/explicit-custom.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -FROM scratch -ARG NEMOCLAW_MESSAGING_PLAN_B64= -ARG NEMOCLAW_TOOL_DISCLOSURE=progressive -ENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE} -CMD ["/bin/true"] diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index f26d3fa8340..aa1ede5e1fc 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -252,7 +252,6 @@ function mockSandboxExecCurl(command, options = {}) { } function mockOnboardRunCapture(command, options = {}) { - mockCustomDockerfilePluginDiscovery(); const normalized = normalizeCommand(command); if ( normalized.startsWith("docker ps -a --no-trunc ") && @@ -280,33 +279,6 @@ function mockOnboardRunCapture(command, options = {}) { return mockSandboxExecCurl(command, options); } -let customDockerfilePluginDiscoveryMocked = false; -function mockCustomDockerfilePluginDiscovery() { - if ( - customDockerfilePluginDiscoveryMocked || - (process.argv[1] || "").toLowerCase().includes("/node_modules/vitest/") - ) { - return; - } - customDockerfilePluginDiscoveryMocked = true; - const childProcess = require("node:child_process"); - const originalSpawnSync = childProcess.spawnSync; - childProcess.spawnSync = (command, args, options) => { - const normalized = normalizeCommand([command, ...(Array.isArray(args) ? args : [])]); - if (command === "ssh" && normalized.includes("installed_plugin_index")) { - return { - status: 0, - signal: null, - stdout: Buffer.from( - JSON.stringify({ version: 1, installRecords: {}, loadPaths: [] }), - ), - stderr: Buffer.alloc(0), - }; - } - return originalSpawnSync(command, args, options); - }; -} - function mockStructuredOpenShellCaptureFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); const client = require( @@ -593,7 +565,6 @@ module.exports = { createStatefulMessagingProviderRunner, isOpenClawSecurityInventoryProbe, mockDockerSandboxLifecycleReleaseFromRunner, - mockCustomDockerfilePluginDiscovery, mockFreshOpenClawPluginDiscovery, mockOnboardRunCapture, mockStandaloneGatewayTeardownAuthority, diff --git a/test/onboarding/onboard-installer-restore-intent.test.ts b/test/onboarding/onboard-installer-restore-intent.test.ts index beb11ca4dbc..27e6cf4ad71 100644 --- a/test/onboarding/onboard-installer-restore-intent.test.ts +++ b/test/onboarding/onboard-installer-restore-intent.test.ts @@ -80,7 +80,7 @@ runner.runCapture = (command) => { if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } - if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -161,8 +161,7 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 delete process.env.NEMOCLAW_RECREATE_SANDBOX; process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); // Prove the recreated + restored sandbox is reachable through the real @@ -200,6 +199,7 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", NEMOCLAW_SANDBOX_PREBUILD: "1", }; delete env["NEMOCLAW_RECREATE_SANDBOX"]; @@ -323,7 +323,7 @@ runner.runCapture = (command) => { if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; if (normalized.includes("sandbox list")) return ""; if (normalized.includes("forward list")) { - return "my-assistant 127.0.0.1 18789 12345 running"; + return "my-assistant 127.0.0.1 28789 12345 running"; } const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -350,8 +350,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; try { await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ error: null, mutations })); } catch (caught) { @@ -370,6 +369,8 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_PREBUILD: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", NEMOCLAW_TEST_REGISTRY_RACE: race, }, @@ -427,7 +428,7 @@ runner.runCapture = (command) => { // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. if (_n(command).includes("forward list")) { - return "my-assistant 127.0.0.1 18789 12345 running"; + return "my-assistant 127.0.0.1 28789 12345 running"; } return ""; }; @@ -450,8 +451,7 @@ const { createSandbox } = require(${onboardPath}); delete process.env.NEMOCLAW_RECREATE_SANDBOX; delete process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE; await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { @@ -466,6 +466,8 @@ const { createSandbox } = require(${onboardPath}); HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_PREBUILD: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", }; delete env["NEMOCLAW_RECREATE_SANDBOX"]; delete env["NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE"]; diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index b8a91f24ced..c648cd08da7 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -539,7 +539,7 @@ runner.runCapture = (command) => { if (_n(command).includes("sandbox list")) return "my-assistant Ready"; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; @@ -556,7 +556,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; - const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); + const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], null, null, 28789); console.log(JSON.stringify({ sandboxName, commands, registered })); })().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); `; @@ -1251,7 +1251,7 @@ const { createSandbox } = require(${onboardPath}); process.env.DISCORD_BOT_TOKEN = "test-discord-token"; process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token"; process.env.SLACK_APP_TOKEN = "xapp-test-slack-token"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1348,7 +1348,7 @@ runner.runCapture = (command) => { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; registry.registerSandbox = () => true; @@ -1380,7 +1380,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, 28789); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1481,7 +1481,7 @@ runner.runCapture = (command) => { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; registry.registerSandbox = () => true; @@ -1513,7 +1513,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Empty array — user deselected all channels - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}); + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, [], null, null, 28789); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index c63a9c1f472..8756e6a39f5 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -82,7 +82,7 @@ runner.runCapture = (command) => { if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } - if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -134,8 +134,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { diff --git a/test/onboarding/onboard-sandbox-recreation.test.ts b/test/onboarding/onboard-sandbox-recreation.test.ts index 359b1b2f8d1..efde0440467 100644 --- a/test/onboarding/onboard-sandbox-recreation.test.ts +++ b/test/onboarding/onboard-sandbox-recreation.test.ts @@ -60,7 +60,6 @@ runner.runCapture = (command) => { registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive", - fromDockerfile: ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, }); childProcess.spawn = () => { throw new Error("unexpected sandbox create"); @@ -70,10 +69,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, - ); + await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); console.log("ERROR_DID_NOT_EXIT"); })().catch((error) => { console.error(error); @@ -160,7 +156,7 @@ runner.run = (command, opts = {}) => { runner.runCapture = (command) => { if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -201,8 +197,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, commands, registeredSandbox })); })().catch((error) => { @@ -301,7 +296,7 @@ runner.runCapture = (command) => { const cmd = _n(command); if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -366,8 +361,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -419,7 +413,7 @@ const { createSandbox } = require(${onboardPath}); const restoreEvent = events[restoreIndex]; assert.equal(restoreEvent?.backupPath, "/tmp/fake-backup-path", "restore must use backup path"); assert.equal(restoreEvent?.options?.targetAgentType, "openclaw"); - assert.deepEqual(restoreEvent?.options?.freshOpenClawImagePluginInstalls, []); + assert.equal(restoreEvent?.options?.freshOpenClawImagePluginInstalls, undefined); }); it("recreate-sandbox with NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 skips backup", { @@ -465,7 +459,7 @@ runner.runCapture = (command) => { const cmd = _n(command); if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -514,8 +508,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -604,7 +597,7 @@ runner.runCapture = (command) => { if (cmd.includes("sandbox list")) { return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } - if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -669,8 +662,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { @@ -760,7 +752,7 @@ runner.run = (command, opts = {}) => { runner.runCapture = (command) => { if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -809,8 +801,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); const session = onboardSession.loadSession(); console.log(JSON.stringify({ policyPresets: session && session.policyPresets })); @@ -913,7 +904,7 @@ runner.runFile = (file, args = [], opts = {}) => { runner.runCapture = (command) => { if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; return ""; }; registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive" }); @@ -941,8 +932,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1055,7 +1045,7 @@ runner.runFile = (file, args = [], opts = {}) => { runner.runCapture = (command) => { if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -1096,8 +1086,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1194,7 +1183,7 @@ runner.runCapture = (command) => { if (_n(command).includes("sandbox list")) { return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -1246,8 +1235,7 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index 75355882ef7..4eef8608b6f 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -82,6 +82,7 @@ for (const key of Object.keys(process.env)) { delete process.env[key]; } } +process.env.NEMOCLAW_SANDBOX_PREBUILD = "1"; process.env.NEMOCLAW_OPENSHELL_BIN = ${JSON.stringify(path.join(fakeBin, "openshell"))}; const commands = []; const asText = (command) => Array.isArray(command) ? command.join(" ") : String(command); @@ -105,7 +106,7 @@ runner.runCapture = (command) => { const text = asText(command); if (text.includes("sandbox get") && text.includes("my-assistant")) return ""; if (text.includes("sandbox list")) return "my-assistant Ready"; - if (text.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (text.includes("forward list")) return "my-assistant 127.0.0.1 28789 12345 running"; if (text.includes("sandbox exec") && text.includes("http://localhost:") && text.includes("/health")) return "200"; if (text === "uname -r") return "6.8.0"; const mockedCapture = require(${JSON.stringify( @@ -135,8 +136,7 @@ try { Object.defineProperty(process, "platform", { value: "darwin" }); Object.defineProperty(process, "arch", { value: "x64" }); const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, - ${JSON.stringify(path.join(repoRoot, "test", "fixtures", "explicit-custom.Dockerfile"))}, + null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, 28789, ); console.log(JSON.stringify({ sandboxName, commands })); } catch (error) { @@ -161,6 +161,8 @@ try { env: { HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_SANDBOX_PREBUILD: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", }, timeout: 30_000, }, diff --git a/tools/e2e/managed-image-cohort-contract.mts b/tools/e2e/managed-image-cohort-contract.mts index a1d7c3127a3..16da770a46a 100644 --- a/tools/e2e/managed-image-cohort-contract.mts +++ b/tools/e2e/managed-image-cohort-contract.mts @@ -113,6 +113,10 @@ function validatePlatformEvidence( publicationEvidence.workloadDescriptor, `${expected.agent} ${expected.platform} workload descriptor`, ); + const workloadDigest = digest( + workloadDescriptor.digest, + `${expected.agent} ${expected.platform} workload digest`, + ); const workloadPlatform = record( workloadDescriptor.platform, `${expected.agent} ${expected.platform} workload platform`, @@ -129,8 +133,30 @@ function validatePlatformEvidence( publicationEvidence.attestations, `${expected.agent} ${expected.platform} attestations`, ); + const manifestDescriptor = record( + attestations.manifestDescriptor, + `${expected.agent} ${expected.platform} attestation manifest descriptor`, + ); + const manifestAnnotations = record( + manifestDescriptor.annotations, + `${expected.agent} ${expected.platform} attestation manifest annotations`, + ); + exactString( + manifestAnnotations["vnd.docker.reference.digest"], + workloadDigest, + `${expected.agent} ${expected.platform} attestation manifest workload digest`, + ); const slsa = record(attestations.slsa, `${expected.agent} ${expected.platform} SLSA evidence`); const statement = record(slsa.statement, `${expected.agent} ${expected.platform} SLSA statement`); + const slsaSubject = record( + statement.subject, + `${expected.agent} ${expected.platform} SLSA subject`, + ); + exactString( + slsaSubject.digest, + workloadDigest, + `${expected.agent} ${expected.platform} SLSA subject workload digest`, + ); exactString( statement.builderId, `https://github.com/${REPOSITORY}/actions/runs/${expected.runId}/attempts/${expected.runAttempt}`, @@ -158,6 +184,20 @@ function validatePlatformEvidence( baseReference, `${expected.agent} ${expected.platform} base reference binding`, ); + const spdx = record(attestations.spdx, `${expected.agent} ${expected.platform} SPDX evidence`); + const spdxStatement = record( + spdx.statement, + `${expected.agent} ${expected.platform} SPDX statement`, + ); + const spdxSubject = record( + spdxStatement.subject, + `${expected.agent} ${expected.platform} SPDX subject`, + ); + exactString( + spdxSubject.digest, + workloadDigest, + `${expected.agent} ${expected.platform} SPDX subject workload digest`, + ); } /** Validate one complete published cohort against its selected workflow attempt. */ diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index e7031afd020..25fe4abd7b2 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -122,7 +122,6 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime", E2E_JOB: "1", E2E_TARGET_ID: JOB_ID, - E2E_WORKLOAD_SOURCE: "managed-image", RELEASE_E2E_ACTIVATION_PATH: ACTIVATION_PATH, NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", NEMOCLAW_E2E_SHARD: "linux-amd64-gpu", From c021ad24550d77e2b79ee96030af485ca6b50858 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 25 Aug 2026 15:12:02 +0700 Subject: [PATCH 28/37] test(e2e): stabilize PTY identity fixture --- test/e2e/support/launch-agent-turn.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 53cb57400d8..01a3e5c9fdf 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -421,6 +421,7 @@ if (process.argv[2] !== "tui") { }) + "\n"; client.end(body); }); + process.umask(0o177); await new Promise((resolve, reject) => { replacement.once("error", reject); replacement.listen(socketPath, resolve); From da42148cd267c8b042c6f59032a1529d28cb3087 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:40:53 -0700 Subject: [PATCH 29/37] test(e2e): preserve trusted PR controller compatibility Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/fixtures/hosted-inference.ts | 1 + test/e2e/fixtures/inference-adapter.ts | 43 +++- test/e2e/fixtures/managed-image-receipt.ts | 116 +++++++++- test/e2e/live/full-e2e-workload-evidence.ts | 32 +-- test/e2e/live/openclaw-pairing-helpers.ts | 35 +-- ...onnect-proxy.ts => slack-forward-proxy.ts} | 17 +- test/e2e/support/inference-adapter.test.ts | 41 ++++ .../e2e/support/managed-image-receipt.test.ts | 200 ++++++++++++++++++ .../openclaw-discord-pairing-helpers.test.ts | 25 +-- 9 files changed, 422 insertions(+), 88 deletions(-) rename test/e2e/support/fixtures/{slack-connect-proxy.ts => slack-forward-proxy.ts} (86%) diff --git a/test/e2e/fixtures/hosted-inference.ts b/test/e2e/fixtures/hosted-inference.ts index 1c594ab74e0..a32d9b3a5e1 100644 --- a/test/e2e/fixtures/hosted-inference.ts +++ b/test/e2e/fixtures/hosted-inference.ts @@ -21,6 +21,7 @@ export const DEFAULT_HOSTED_INFERENCE_MODEL = "nvidia/nvidia/nemotron-3-ultra"; const PORTABLE_DESCRIPTOR_VALIDITY_MS = 60 * 60_000; export interface HostedInferenceSecrets { + optional?(name: string): string | undefined; required(name: string): string; } diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index 5e4ca42209c..c7b1fcb6d8e 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -29,8 +29,10 @@ import type { TestProgress, TestProgressCapability } from "./progress.ts"; * as compatible inference and rejects endpoint overrides outside its static * allowlist; `public-nvidia` reads the public `NVIDIA_API_KEY` credential and * stages it only under the runtime's historical `NVIDIA_INFERENCE_API_KEY` - * alias. Every mode registers its credential for artifact redaction and - * removes credentials owned by the other modes. + * alias. A trusted-main manual PR controller can supply the public credential + * through the historical alias until it supplies `NVIDIA_API_KEY`. Every mode + * registers its credential for artifact redaction and removes credentials owned + * by the other modes. * * Tests normally consume the `inference` fixture from `e2e-test.ts`, pass * `inference.env()` to install/onboard commands, use its model and provider @@ -84,6 +86,9 @@ const INTERNAL_NVIDIA_ALLOWED_HOSTS = ["inference-api.nvidia.com"] as const; const MODEL_PROBE_TIMEOUT_MS = 30_000; const PUBLIC_NVIDIA_ALLOWED_HOSTS = ["integrate.api.nvidia.com"] as const; const SANDBOX_HOST_ALIAS = "host.openshell.internal"; +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const CORRELATION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; export function normalizeMode(env: NodeJS.ProcessEnv): E2EInferenceMode { const raw = env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase(); @@ -103,6 +108,36 @@ export function requirePublicNvidiaInferenceKey(value: string): string { return value; } +function usesTrustedMainManualPrController(environment: NodeJS.ProcessEnv): boolean { + const candidateRevision = environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; + const workflowRevision = environment.GITHUB_SHA?.trim() ?? ""; + return ( + environment.GITHUB_ACTIONS === "true" && + environment.GITHUB_EVENT_NAME === "workflow_dispatch" && + REVISION_PATTERN.test(candidateRevision) && + REVISION_PATTERN.test(workflowRevision) && + candidateRevision !== workflowRevision && + CORRELATION_ID_PATTERN.test(environment.NEMOCLAW_E2E_CORRELATION_ID?.trim() ?? "") + ); +} + +function publicNvidiaInferenceKey( + secrets: HostedInferenceSecrets, + environment: NodeJS.ProcessEnv, +): string { + const current = secrets.optional?.(PUBLIC_NVIDIA_CREDENTIAL_ENV); + const historical = secrets.optional?.(HOSTED_INFERENCE_SECRET); + if (current && historical && current !== historical) { + throw new Error("NVIDIA_API_KEY and NVIDIA_INFERENCE_API_KEY contain different values"); + } + if (current) return requirePublicNvidiaInferenceKey(current); + const source = + historical && usesTrustedMainManualPrController(environment) + ? HOSTED_INFERENCE_SECRET + : PUBLIC_NVIDIA_CREDENTIAL_ENV; + return requirePublicNvidiaInferenceKey(secrets.required(source)); +} + function joinEndpoint(baseUrl: string, suffix: string): string { return `${baseUrl.replace(/\/+$/, "")}/${suffix.replace(/^\/+/, "")}`; } @@ -438,9 +473,7 @@ export async function createE2EInferenceAdapter( artifacts: options.artifacts, }); } - const apiKey = requirePublicNvidiaInferenceKey( - options.secrets.required(PUBLIC_NVIDIA_CREDENTIAL_ENV), - ); + const apiKey = publicNvidiaInferenceKey(options.secrets, env); const model = env.NEMOCLAW_MODEL || DEFAULT_PUBLIC_NVIDIA_MODEL; return new PublicNvidiaInferenceAdapter({ apiKey, diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts index ff89d9b7ef9..41a43aebd1f 100644 --- a/test/e2e/fixtures/managed-image-receipt.ts +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -9,10 +10,14 @@ import { isShippedManagedImageAgent, MANAGED_IMAGE_PLATFORMS, MANAGED_IMAGE_REPOSITORIES, + parseManagedImageContractV1, SHIPPED_MANAGED_IMAGE_AGENTS, + type ManagedImageContractV1, + type ManagedImagePlatform, type ShippedManagedImageAgent, } from "../../../src/lib/onboard/managed-image/contract.ts"; import { readManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts"; +import { liveE2eManagedImageCatalog } from "../../../src/lib/onboard/workload/preparation.ts"; import { readConfigFile } from "../../../src/lib/state/config-io.ts"; import { parseSandboxRegistryEntries } from "../../../src/lib/state/registry-normalization.ts"; import { cloneSandboxWorkloadReceipt } from "../../../src/lib/state/registry/workload.ts"; @@ -20,6 +25,81 @@ import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +function readCandidateCatalog( + environment: NodeJS.ProcessEnv, +): ReadonlyMap { + const selected = liveE2eManagedImageCatalog(environment); + if (!selected) { + throw new Error("stock onboarding requires a selected candidate managed-image catalog"); + } + + let descriptor: number | null = null; + try { + descriptor = fs.openSync(selected.path, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + const metadata = fs.fstatSync(descriptor); + const pathMetadata = fs.lstatSync(selected.path); + if ( + pathMetadata.isSymbolicLink() || + !metadata.isFile() || + metadata.dev !== pathMetadata.dev || + metadata.ino !== pathMetadata.ino || + metadata.size < 2 || + metadata.size > 64 * 1024 + ) { + throw new Error(); + } + const parsed = JSON.parse(fs.readFileSync(descriptor, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); + const catalog = parsed as Record; + if ( + JSON.stringify(Object.keys(catalog).sort()) !== + JSON.stringify([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()) + ) { + throw new Error(); + } + + const contracts = new Map(); + let cohort: string | null = null; + let platform: ManagedImagePlatform | null = null; + let release: string | null = null; + for (const agent of SHIPPED_MANAGED_IMAGE_AGENTS) { + const contract = parseManagedImageContractV1(catalog[agent], agent); + cohort ??= contract.source.cohort; + platform ??= contract.platform; + release ??= contract.source.release; + if ( + contract.source.revision !== selected.revision || + contract.source.cohort !== cohort || + contract.platform !== platform || + contract.source.release !== release + ) { + throw new Error(); + } + contracts.set(agent, contract); + } + return contracts; + } catch { + throw new Error("stock onboarding candidate managed-image catalog is invalid"); + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + } +} + +function selectedManagedImageRevision(environment: NodeJS.ProcessEnv): string { + const revision = environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; + if (revision) { + if (!REVISION_PATTERN.test(revision)) { + throw new Error("stock onboarding requires one exact managed-image cohort revision"); + } + return revision; + } + const catalog = liveE2eManagedImageCatalog(environment); + if (!catalog) { + throw new Error("stock onboarding requires one exact managed-image cohort revision"); + } + return catalog.revision; +} + export function assertManagedImageReceiptMatchesSelectedCohort(options: { readonly environment: NodeJS.ProcessEnv; readonly expectedAgent: ShippedManagedImageAgent; @@ -27,6 +107,33 @@ export function assertManagedImageReceiptMatchesSelectedCohort(options: { }): void { const revision = options.environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; const rawReceipt = options.environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT?.trim() ?? ""; + if (!revision) { + if (rawReceipt) { + throw new Error( + "stock onboarding requires the complete selected managed-image cohort receipt", + ); + } + const platform = options.workload?.platform; + if ( + typeof platform !== "string" || + !(MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) + ) { + throw new Error("stock onboarding candidate managed-image catalog is invalid"); + } + const contract = readCandidateCatalog(options.environment).get(options.expectedAgent); + if ( + !contract || + contract.platform !== platform || + options.workload?.kind !== "managed-image" || + options.workload.reference !== contract.reference || + options.workload.release !== contract.source.release || + options.workload.sourceRevision !== contract.source.revision || + options.workload.sourceCohort !== contract.source.cohort + ) { + throw new Error("stock onboarding must use the exact agent image from the selected cohort"); + } + return; + } if (!REVISION_PATTERN.test(revision) || !rawReceipt || Buffer.byteLength(rawReceipt) > 8 * 1024) { throw new Error("stock onboarding requires the complete selected managed-image cohort receipt"); } @@ -114,10 +221,7 @@ export function assertStockManagedImageReceipt(options: { readonly sandboxName: string; }): StockManagedImageReceiptEvidence { const environment = options.environment ?? process.env; - const revision = environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; - if (!REVISION_PATTERN.test(revision)) { - throw new Error("stock onboarding requires one exact managed-image cohort revision"); - } + const revision = selectedManagedImageRevision(environment); const home = environment.HOME?.trim() || os.homedir(); const registryPath = path.join( nemoclawStateRoot(home, gatewayPort(environment)), @@ -169,7 +273,9 @@ export function shouldAssertStockManagedImageReceipt( args: readonly string[], environment: NodeJS.ProcessEnv, ): boolean { - if (!environment.E2E_MANAGED_IMAGE_REVISION?.trim()) return false; + const selectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim(); + const selectedCatalog = selectedRevision ? null : liveE2eManagedImageCatalog(environment); + if (!selectedRevision && !selectedCatalog) return false; const selectedAgent = environment.NEMOCLAW_AGENT?.trim(); if (selectedAgent && !isShippedManagedImageAgent(selectedAgent)) return false; if (environment.NEMOCLAW_FROM_DOCKERFILE?.trim()) return false; diff --git a/test/e2e/live/full-e2e-workload-evidence.ts b/test/e2e/live/full-e2e-workload-evidence.ts index 8d7f6263250..56df7bb6250 100644 --- a/test/e2e/live/full-e2e-workload-evidence.ts +++ b/test/e2e/live/full-e2e-workload-evidence.ts @@ -1,37 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts"; -import { load as loadSandboxRegistry } from "../../../src/lib/state/registry/persistence.ts"; +import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; export function readFullE2eColdWorkloadEvidence( sandboxName: string, usedBuildKitPrebuild: boolean, environment: NodeJS.ProcessEnv = process.env, ) { - const entry = loadSandboxRegistry().sandboxes[sandboxName]; - if (!entry) { - throw new Error(`full E2E sandbox '${sandboxName}' is missing from the registry`); - } - - const managedAuthority = readManagedWorkloadAuthority(entry); - if (!managedAuthority) { - throw new Error("full E2E cold onboarding must register a managed-image workload receipt"); - } if (usedBuildKitPrebuild) { throw new Error("managed-image cold onboarding must not use a local BuildKit prebuild"); } - const expectedRevision = environment.E2E_MANAGED_IMAGE_REVISION?.trim() ?? ""; - if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { - throw new Error("full E2E cold onboarding requires one exact managed-image cohort revision"); - } - if (managedAuthority.receipt.sourceRevision !== expectedRevision) { - throw new Error("full E2E cold onboarding did not use the selected managed-image cohort"); - } + const receipt = assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName, + }); return { - kind: managedAuthority.receipt.kind, - reference: managedAuthority.receipt.reference, - sourceCohort: managedAuthority.receipt.sourceCohort, - sourceRevision: managedAuthority.receipt.sourceRevision, + kind: "managed-image", + reference: receipt.reference, + sourceCohort: receipt.sourceCohort, + sourceRevision: receipt.sourceRevision, } as const; } diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index 5bdfe61edd4..fcf911569f3 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -460,15 +460,14 @@ function receiveSlackSocketEvent() { return new Promise((resolve, reject) => { const socket = proxy ? net.createConnection({ host: proxy.host, port: proxy.port }) : net.createConnection({ host, port }); const timer = setTimeout(() => { socket.destroy(); reject(new Error("timed out waiting for fake Slack Socket Mode event")); }, 30000); - let proxyHandshake = Buffer.alloc(0); let handshake = Buffer.alloc(0); let framed = Buffer.alloc(0); - let tunnelEstablished = !proxy; let upgraded = false; - function sendUpgradeRequest() { + socket.on("connect", () => { const key = crypto.randomBytes(16).toString("base64"); + const requestTarget = proxy ? "http://" + host + ":" + port + "/socket-mode" : "/socket-mode"; socket.write([ - "GET /socket-mode HTTP/1.1", + "GET " + requestTarget + " HTTP/1.1", "Host: " + host + ":" + port, "Upgrade: websocket", "Connection: Upgrade", @@ -476,36 +475,8 @@ function receiveSlackSocketEvent() { "Sec-WebSocket-Version: 13", "\r\n", ].join("\r\n")); - } - socket.on("connect", () => { - if (!proxy) { - sendUpgradeRequest(); - return; - } - socket.write([ - "CONNECT " + host + ":" + port + " HTTP/1.1", - "Host: " + host + ":" + port, - "\r\n", - ].join("\r\n")); }); socket.on("data", (chunk) => { - if (!tunnelEstablished) { - proxyHandshake = Buffer.concat([proxyHandshake, chunk]); - const end = proxyHandshake.indexOf("\r\n\r\n"); - if (end === -1) return; - const statusLine = proxyHandshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; - if (!/^HTTP\/1\.[01] 200(?: |$)/.test(statusLine)) { - clearTimeout(timer); - socket.destroy(); - reject(new Error("OpenShell proxy CONNECT failed: " + statusLine)); - return; - } - tunnelEstablished = true; - chunk = proxyHandshake.slice(end + 4); - proxyHandshake = Buffer.alloc(0); - sendUpgradeRequest(); - if (chunk.length === 0) return; - } if (!upgraded) { handshake = Buffer.concat([handshake, chunk]); const end = handshake.indexOf("\r\n\r\n"); diff --git a/test/e2e/support/fixtures/slack-connect-proxy.ts b/test/e2e/support/fixtures/slack-forward-proxy.ts similarity index 86% rename from test/e2e/support/fixtures/slack-connect-proxy.ts rename to test/e2e/support/fixtures/slack-forward-proxy.ts index b1b049b1d4a..f8999eb2133 100644 --- a/test/e2e/support/fixtures/slack-connect-proxy.ts +++ b/test/e2e/support/fixtures/slack-forward-proxy.ts @@ -53,7 +53,7 @@ export async function closeServer(server: net.Server): Promise { await new Promise((resolve) => server.close(() => resolve())); } -export function createSuccessfulSlackConnectProxy(envelope: Record): { +export function createSuccessfulSlackForwardProxy(envelope: Record): { server: net.Server; requests: string[]; websocketBytes: () => number; @@ -62,9 +62,9 @@ export function createSuccessfulSlackConnectProxy(envelope: Record { let buffer = Buffer.alloc(0); - let phase: "connect" | "upgrade" | "websocket" = "connect"; + let upgraded = false; socket.on("data", (chunk) => { - if (phase === "websocket") { + if (upgraded) { receivedWebsocketBytes += chunk.length; return; } @@ -72,14 +72,7 @@ export function createSuccessfulSlackConnectProxy(envelope: Record socket.write("\r\n")); - return; - } - phase = "websocket"; + upgraded = true; socket.write( Buffer.concat([ Buffer.from( @@ -98,7 +91,7 @@ export function createSuccessfulSlackConnectProxy(envelope: Record { socket.once("data", () => { socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index 8dec4d09cdc..84e516349e7 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -35,6 +35,7 @@ function artifacts(): ArtifactSink { function secrets(values: Record) { return { + optional: (name: string) => values[name], required: (name: string) => { const value = values[name]; return ( @@ -384,6 +385,46 @@ describe("E2E inference adapter", () => { ).rejects.toThrow(/missing NVIDIA_API_KEY/); }); + it("accepts the public credential alias from a trusted-main manual PR controller", async () => { + const adapter = await createAdapter({ + env: { + GITHUB_ACTIONS: "true", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_SHA: "a".repeat(40), + NEMOCLAW_E2E_CORRELATION_ID: "6ce998a3-52f6-4e91-8b2d-089110070e74", + NEMOCLAW_E2E_EXPECTED_SHA: "b".repeat(40), + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", + }, + secrets: { NVIDIA_INFERENCE_API_KEY: "nvapi-trusted-controller-key" }, + }); + + expect(adapter.env()).toMatchObject({ + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", + NVIDIA_INFERENCE_API_KEY: "nvapi-trusted-controller-key", + }); + }); + + it("rejects conflicting public credential sources without exposing either value", async () => { + const create = createAdapter({ + env: { + GITHUB_ACTIONS: "true", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_SHA: "a".repeat(40), + NEMOCLAW_E2E_CORRELATION_ID: "6ce998a3-52f6-4e91-8b2d-089110070e74", + NEMOCLAW_E2E_EXPECTED_SHA: "b".repeat(40), + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", + }, + secrets: { + NVIDIA_API_KEY: "nvapi-current-controller-key", + NVIDIA_INFERENCE_API_KEY: "nvapi-historical-controller-key", + }, + }); + + await expect(create).rejects.toThrow( + /^NVIDIA_API_KEY and NVIDIA_INFERENCE_API_KEY contain different values$/u, + ); + }); + it("rejects unknown explicit modes instead of silently falling back", async () => { await expect( createAdapter({ env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvida" } }), diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts index c78e4d198d6..5cc0e85ccfd 100644 --- a/test/e2e/support/managed-image-receipt.test.ts +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -11,8 +11,11 @@ import { afterEach, describe, expect, it } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + SHIPPED_MANAGED_IMAGE_AGENTS, } from "../../../src/lib/onboard/managed-image/contract.ts"; import { encodeManagedStartupProfile } from "../../../src/lib/onboard/managed-startup/profile.ts"; import { nemoclawStateRoot } from "../../../src/lib/state/state-root.ts"; @@ -20,11 +23,17 @@ import { assertStockManagedImageReceipt, shouldAssertStockManagedImageReceipt, } from "../fixtures/managed-image-receipt.ts"; +import { readFullE2eColdWorkloadEvidence } from "../live/full-e2e-workload-evidence.ts"; const SANDBOX_NAME = "managed-only-stock"; const REVISION = "d".repeat(40); const COHORT = "ghrun-32707920950-1"; const REFERENCE = `${MANAGED_IMAGE_REPOSITORIES.openclaw}@sha256:${"a".repeat(64)}`; +const CATALOG_REFERENCES = { + openclaw: REFERENCE, + hermes: `${MANAGED_IMAGE_REPOSITORIES.hermes}@sha256:${"c".repeat(64)}`, + "langchain-deepagents-code": `${MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"]}@sha256:${"e".repeat(64)}`, +} as const; const temporaryHomes: string[] = []; afterEach(() => { @@ -80,6 +89,42 @@ function selectedEnvironment(home: string): NodeJS.ProcessEnv { }; } +function candidateCatalogEnvironment(home: string): NodeJS.ProcessEnv { + const catalog = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent) => { + const reference = CATALOG_REFERENCES[agent]; + return [ + agent, + { + agent, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + digest: reference.slice(reference.indexOf("@") + 1), + image: MANAGED_IMAGE_REPOSITORIES[agent], + platform: "linux/amd64", + reference, + source: { + cohort: COHORT, + release: "v0.0.100", + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: REVISION, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + }, + ]; + }), + ); + const catalogPath = path.join(home, "candidate-managed-image-catalog.json"); + fs.writeFileSync(catalogPath, `${JSON.stringify(catalog)}\n`, "utf8"); + return { + GITHUB_ACTIONS: "true", + HOME: home, + NEMOCLAW_E2E_EXPECTED_SHA: REVISION, + NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, + NEMOCLAW_RUN_LIVE_E2E: "1", + }; +} + function writeRegistry(workload: Record): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-only-receipt-")); temporaryHomes.push(home); @@ -116,6 +161,144 @@ describe("stock E2E managed-image receipt assertion", () => { ).toMatchObject({ agent: "openclaw", sourceRevision: REVISION }); }); + it("accepts the durable receipt from the trusted candidate catalog", () => { + const home = writeRegistry(managedReceipt()); + + expect( + assertStockManagedImageReceipt({ + environment: candidateCatalogEnvironment(home), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toMatchObject({ agent: "openclaw", sourceRevision: REVISION }); + }); + + it("uses the trusted candidate catalog for full E2E workload evidence", () => { + const home = writeRegistry(managedReceipt()); + + expect( + readFullE2eColdWorkloadEvidence(SANDBOX_NAME, false, candidateCatalogEnvironment(home)), + ).toMatchObject({ kind: "managed-image", sourceRevision: REVISION }); + }); + + it("rejects a candidate catalog whose source revision differs from the exact candidate revision", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + const catalogPath = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG!; + const catalog = JSON.parse(fs.readFileSync(catalogPath, "utf8")) as Record< + string, + { source: { revision: string } } + >; + catalog.openclaw!.source.revision = "b".repeat(40); + catalog.hermes!.source.revision = "b".repeat(40); + catalog["langchain-deepagents-code"]!.source.revision = "b".repeat(40); + fs.writeFileSync(catalogPath, `${JSON.stringify(catalog)}\n`, "utf8"); + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("candidate managed-image catalog is invalid"); + }); + + it("rejects a candidate catalog that mixes releases", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + const catalogPath = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG!; + const catalog = JSON.parse(fs.readFileSync(catalogPath, "utf8")) as Record< + string, + { source: { release: string } } + >; + catalog.hermes!.source.release = "v0.0.101"; + fs.writeFileSync(catalogPath, `${JSON.stringify(catalog)}\n`, "utf8"); + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("candidate managed-image catalog is invalid"); + }); + + it("rejects a candidate catalog with an extra agent", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + const catalogPath = environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG!; + const catalog = JSON.parse(fs.readFileSync(catalogPath, "utf8")) as Record; + catalog.extra = catalog.openclaw; + fs.writeFileSync(catalogPath, `${JSON.stringify(catalog)}\n`, "utf8"); + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("candidate managed-image catalog is invalid"); + }); + + it("rejects a candidate catalog that exceeds the size limit", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + fs.writeFileSync( + environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG!, + JSON.stringify({ padding: "x".repeat(64 * 1024) }), + "utf8", + ); + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("candidate managed-image catalog is invalid"); + }); + + it("rejects a durable workload release that differs from the candidate catalog", () => { + const home = writeRegistry({ ...managedReceipt(), release: "v0.0.101" }); + + expect(() => + assertStockManagedImageReceipt({ + environment: candidateCatalogEnvironment(home), + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("exact agent image from the selected cohort"); + }); + + it("does not replace a missing selected-cohort receipt with the candidate catalog", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + environment.E2E_MANAGED_IMAGE_REVISION = REVISION; + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("complete selected managed-image cohort receipt"); + }); + + it("does not replace a missing selected revision with the candidate catalog", () => { + const home = writeRegistry(managedReceipt()); + const environment = candidateCatalogEnvironment(home); + environment.E2E_MANAGED_IMAGE_COHORT_RECEIPT = + selectedEnvironment(home).E2E_MANAGED_IMAGE_COHORT_RECEIPT; + + expect(() => + assertStockManagedImageReceipt({ + environment, + expectedAgent: "openclaw", + sandboxName: SANDBOX_NAME, + }), + ).toThrow("complete selected managed-image cohort receipt"); + }); + it("rejects a stock legacy Dockerfile receipt", () => { const home = writeRegistry({ schemaVersion: 1, @@ -213,6 +396,23 @@ describe("stock E2E managed-image receipt assertion", () => { NEMOCLAW_FROM_DOCKERFILE: "/workspace/CustomDockerfile", }), ).toBe(false); + const home = writeRegistry(managedReceipt()); + expect( + shouldAssertStockManagedImageReceipt( + "/workspace/bin/nemoclaw.js", + ["onboard"], + candidateCatalogEnvironment(home), + ), + ).toBe(true); + const nonLiveEnvironment = candidateCatalogEnvironment(home); + nonLiveEnvironment.GITHUB_ACTIONS = "false"; + expect( + shouldAssertStockManagedImageReceipt( + "/workspace/bin/nemoclaw.js", + ["onboard"], + nonLiveEnvironment, + ), + ).toBe(false); expect( shouldAssertStockManagedImageReceipt( "/workspace/bin/nemoclaw.js", diff --git a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts index cf826a8d143..156fedcb8b5 100644 --- a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts +++ b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts @@ -11,11 +11,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { closeServer, - createRejectedSlackConnectProxy, + createRejectedSlackForwardProxy, createSlackSocketClient, - createSuccessfulSlackConnectProxy, + createSuccessfulSlackForwardProxy, listenOnLoopback, -} from "./fixtures/slack-connect-proxy.ts"; +} from "./fixtures/slack-forward-proxy.ts"; import { buildPairingApproveCommand, buildPairingPendingCommand, @@ -132,10 +132,10 @@ async function sendDiscordIdentify(port: number, token: string): Promise { } describe("OpenClaw pairing helper contracts", () => { - it("establishes an HTTP CONNECT tunnel before the fake Slack WebSocket upgrade", async () => { + it("sends an absolute-form fake Slack WebSocket upgrade through the proxy", async () => { const targetPort = 4443; const envelope = { payload: { event: { type: "message" } } }; - const proxy = createSuccessfulSlackConnectProxy(envelope); + const proxy = createSuccessfulSlackForwardProxy(envelope); const proxyPort = await listenOnLoopback(proxy.server); try { @@ -144,24 +144,25 @@ describe("OpenClaw pairing helper contracts", () => { interval: 10, timeout: 1_000, }); - expect(proxy.requests).toHaveLength(2); + expect(proxy.requests).toHaveLength(1); expect(proxy.requests[0]).toMatch( - new RegExp(`^CONNECT host\\.openshell\\.internal:${targetPort} HTTP/1\\.1`, "u"), + new RegExp( + `^GET http://host\\.openshell\\.internal:${targetPort}/socket-mode HTTP/1\\.1`, + "u", + ), ); - expect(proxy.requests[1]).toMatch(/^GET \/socket-mode HTTP\/1\.1/u); - expect(proxy.requests[1]).not.toContain("http://"); } finally { await closeServer(proxy.server); } }); - it("rejects a non-200 OpenShell proxy CONNECT response", async () => { - const proxy = createRejectedSlackConnectProxy(); + it("rejects a non-101 fake Slack WebSocket proxy response", async () => { + const proxy = createRejectedSlackForwardProxy(); const proxyPort = await listenOnLoopback(proxy); try { await expect(createSlackSocketClient(proxyPort, 4443)()).rejects.toThrow( - "OpenShell proxy CONNECT failed: HTTP/1.1 502 Bad Gateway", + "fake Slack websocket upgrade failed: HTTP/1.1 502 Bad Gateway", ); } finally { await closeServer(proxy); From d6630ed8735960c4ecd5459572cc10c304d64a94 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:53:34 -0700 Subject: [PATCH 30/37] test(onboard): stay within recreation file budget Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/onboarding/onboard-sandbox-recreation.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/onboarding/onboard-sandbox-recreation.test.ts b/test/onboarding/onboard-sandbox-recreation.test.ts index dfbce57b215..64413c07870 100644 --- a/test/onboarding/onboard-sandbox-recreation.test.ts +++ b/test/onboarding/onboard-sandbox-recreation.test.ts @@ -57,10 +57,7 @@ runner.runCapture = (command) => { if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.getSandbox = () => ({ - name: "my-assistant", - toolDisclosure: "progressive", -}); +registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive" }); childProcess.spawn = () => { throw new Error("unexpected sandbox create"); }; From 43e0b977d6ba892bf26625c58e10adfd14d3dffe Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:47:19 -0700 Subject: [PATCH 31/37] test(e2e): use scoped Slack credential references Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/fixtures/hosted-inference.ts | 1 - test/e2e/fixtures/inference-adapter.ts | 43 ++----------- test/e2e/live/openclaw-pairing-helpers.ts | 60 ++++++++++++------- .../support/fixtures/slack-forward-proxy.ts | 52 ++++++++++++---- test/e2e/support/inference-adapter.test.ts | 41 ------------- .../openclaw-discord-pairing-helpers.test.ts | 51 +++++++++++++++- 6 files changed, 136 insertions(+), 112 deletions(-) diff --git a/test/e2e/fixtures/hosted-inference.ts b/test/e2e/fixtures/hosted-inference.ts index a32d9b3a5e1..1c594ab74e0 100644 --- a/test/e2e/fixtures/hosted-inference.ts +++ b/test/e2e/fixtures/hosted-inference.ts @@ -21,7 +21,6 @@ export const DEFAULT_HOSTED_INFERENCE_MODEL = "nvidia/nvidia/nemotron-3-ultra"; const PORTABLE_DESCRIPTOR_VALIDITY_MS = 60 * 60_000; export interface HostedInferenceSecrets { - optional?(name: string): string | undefined; required(name: string): string; } diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index c7b1fcb6d8e..5e4ca42209c 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -29,10 +29,8 @@ import type { TestProgress, TestProgressCapability } from "./progress.ts"; * as compatible inference and rejects endpoint overrides outside its static * allowlist; `public-nvidia` reads the public `NVIDIA_API_KEY` credential and * stages it only under the runtime's historical `NVIDIA_INFERENCE_API_KEY` - * alias. A trusted-main manual PR controller can supply the public credential - * through the historical alias until it supplies `NVIDIA_API_KEY`. Every mode - * registers its credential for artifact redaction and removes credentials owned - * by the other modes. + * alias. Every mode registers its credential for artifact redaction and + * removes credentials owned by the other modes. * * Tests normally consume the `inference` fixture from `e2e-test.ts`, pass * `inference.env()` to install/onboard commands, use its model and provider @@ -86,9 +84,6 @@ const INTERNAL_NVIDIA_ALLOWED_HOSTS = ["inference-api.nvidia.com"] as const; const MODEL_PROBE_TIMEOUT_MS = 30_000; const PUBLIC_NVIDIA_ALLOWED_HOSTS = ["integrate.api.nvidia.com"] as const; const SANDBOX_HOST_ALIAS = "host.openshell.internal"; -const REVISION_PATTERN = /^[0-9a-f]{40}$/u; -const CORRELATION_ID_PATTERN = - /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; export function normalizeMode(env: NodeJS.ProcessEnv): E2EInferenceMode { const raw = env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase(); @@ -108,36 +103,6 @@ export function requirePublicNvidiaInferenceKey(value: string): string { return value; } -function usesTrustedMainManualPrController(environment: NodeJS.ProcessEnv): boolean { - const candidateRevision = environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; - const workflowRevision = environment.GITHUB_SHA?.trim() ?? ""; - return ( - environment.GITHUB_ACTIONS === "true" && - environment.GITHUB_EVENT_NAME === "workflow_dispatch" && - REVISION_PATTERN.test(candidateRevision) && - REVISION_PATTERN.test(workflowRevision) && - candidateRevision !== workflowRevision && - CORRELATION_ID_PATTERN.test(environment.NEMOCLAW_E2E_CORRELATION_ID?.trim() ?? "") - ); -} - -function publicNvidiaInferenceKey( - secrets: HostedInferenceSecrets, - environment: NodeJS.ProcessEnv, -): string { - const current = secrets.optional?.(PUBLIC_NVIDIA_CREDENTIAL_ENV); - const historical = secrets.optional?.(HOSTED_INFERENCE_SECRET); - if (current && historical && current !== historical) { - throw new Error("NVIDIA_API_KEY and NVIDIA_INFERENCE_API_KEY contain different values"); - } - if (current) return requirePublicNvidiaInferenceKey(current); - const source = - historical && usesTrustedMainManualPrController(environment) - ? HOSTED_INFERENCE_SECRET - : PUBLIC_NVIDIA_CREDENTIAL_ENV; - return requirePublicNvidiaInferenceKey(secrets.required(source)); -} - function joinEndpoint(baseUrl: string, suffix: string): string { return `${baseUrl.replace(/\/+$/, "")}/${suffix.replace(/^\/+/, "")}`; } @@ -473,7 +438,9 @@ export async function createE2EInferenceAdapter( artifacts: options.artifacts, }); } - const apiKey = publicNvidiaInferenceKey(options.secrets, env); + const apiKey = requirePublicNvidiaInferenceKey( + options.secrets.required(PUBLIC_NVIDIA_CREDENTIAL_ENV), + ); const model = env.NEMOCLAW_MODEL || DEFAULT_PUBLIC_NVIDIA_MODEL; return new PublicNvidiaInferenceAdapter({ apiKey, diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index fcf911569f3..626b5ff5029 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -369,18 +369,17 @@ console.log("DISCORD_PAIRING_E2E_RESULT " + JSON.stringify({ code: result.code, NODE `.replace("__LOAD_CONVERSATION_RUNTIME_SOURCE__", LOAD_CONVERSATION_RUNTIME_SOURCE); -// Source-of-truth boundary: the Slack live probe owns only validation for its -// localized fake API port and proxy environment because those values are injected -// by the Vitest harness before the probe opens direct Node socket/http clients. -// Invalid state: a malformed fake port or proxy env, or a proxy destination other -// than the NemoClaw/OpenShell gateway proxy emitted by scripts/nemoclaw-start.sh, -// would otherwise hide the real pairing failure behind a low-level network error -// or route the fake Slack websocket through an unexpected host. Source-fix -// constraint: do not change global sandbox proxy generation for this probe; fail -// closed here before network access. Support tests cover malformed values and an -// unexpected-but-valid HTTP proxy host. Remove this localized parser once the -// Slack probe delegates Socket Mode/REST traffic to a shared fake-provider client -// instead of hand-rolled sockets. +// Source-of-truth boundary: the Slack live probe validates its localized fake API +// ports, proxy environment, and the revision-scoped credential references issued +// to the sandbox before it opens direct Node.js socket and HTTP clients. Invalid +// state would otherwise hide a pairing failure behind a low-level network error, +// route the fake Slack WebSocket through an unexpected host, or send a credential +// reference without a revision that OpenShell must reject for an endpoint-bound provider. +// Source-fix constraint: do not change global sandbox proxy generation for this +// probe; fail closed here before network access. Support tests cover malformed +// values, an unexpected HTTP proxy host, and invalid credential references. Remove +// this localized parser once the probe delegates Socket Mode and REST traffic to a +// shared fake-provider client instead of custom socket clients. export const SLACK_PROBE_INPUT_VALIDATION_SOURCE = String.raw` function parseFakeSlackPort(name) { const raw = process.env[name] || ""; @@ -403,6 +402,14 @@ function parseProxyTarget() { if (parsed.hostname !== "10.200.0.1" || port !== 3128) throw new Error("unexpected HTTP proxy for Slack pairing probe"); return { host: parsed.hostname, port }; } +function parseManagedCredentialReference(name) { + if (name !== "SLACK_APP_TOKEN" && name !== "SLACK_BOT_TOKEN") throw new Error("unexpected Slack credential reference name"); + const value = process.env[name] || ""; + if (!new RegExp("^openshell:resolve:env:v[0-9]{1,20}_" + name + "$").test(value)) { + throw new Error(name + " must be the revision-scoped OpenShell credential reference issued to the sandbox"); + } + return value; +} `; export const SLACK_PAIRING_SCRIPT = String.raw` @@ -417,7 +424,7 @@ slack_pairing_user="$3" : "${"$"}{OPENCLAW_STATE_DIR:?OPENCLAW_STATE_DIR missing}" : "${"$"}{OPENCLAW_CONFIG_PATH:?OPENCLAW_CONFIG_PATH missing}" : "${"$"}{OPENCLAW_OAUTH_DIR:?OPENCLAW_OAUTH_DIR missing}" -exec env HOME=/sandbox PATH="/usr/local/bin:/usr/bin:/bin:${"$"}{PATH:-}" OPENCLAW_HOME="$OPENCLAW_HOME" OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" HTTP_PROXY="${"$"}{HTTP_PROXY:-}" HTTPS_PROXY="${"$"}{HTTPS_PROXY:-}" http_proxy="${"$"}{http_proxy:-}" https_proxy="${"$"}{https_proxy:-}" NO_PROXY="${"$"}{NO_PROXY:-}" no_proxy="${"$"}{no_proxy:-}" NODE_OPTIONS="${"$"}{NODE_OPTIONS:-}" FAKE_SLACK_API_HOST="host.openshell.internal" FAKE_SLACK_REST_PORT="$fake_slack_rest_port" FAKE_SLACK_WEBSOCKET_PORT="$fake_slack_websocket_port" SLACK_PAIRING_USER="$slack_pairing_user" node --input-type=module <<'NODE' +exec env HOME=/sandbox PATH="/usr/local/bin:/usr/bin:/bin:${"$"}{PATH:-}" OPENCLAW_HOME="$OPENCLAW_HOME" OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" HTTP_PROXY="${"$"}{HTTP_PROXY:-}" HTTPS_PROXY="${"$"}{HTTPS_PROXY:-}" http_proxy="${"$"}{http_proxy:-}" https_proxy="${"$"}{https_proxy:-}" NO_PROXY="${"$"}{NO_PROXY:-}" no_proxy="${"$"}{no_proxy:-}" NODE_OPTIONS="${"$"}{NODE_OPTIONS:-}" FAKE_SLACK_API_HOST="host.openshell.internal" FAKE_SLACK_REST_PORT="$fake_slack_rest_port" FAKE_SLACK_WEBSOCKET_PORT="$fake_slack_websocket_port" SLACK_PAIRING_USER="$slack_pairing_user" SLACK_APP_TOKEN="${"$"}{SLACK_APP_TOKEN:-}" SLACK_BOT_TOKEN="${"$"}{SLACK_BOT_TOKEN:-}" node --input-type=module <<'NODE' __LOAD_CONVERSATION_RUNTIME_SOURCE__ import crypto from "node:crypto"; import http from "node:http"; @@ -457,9 +464,18 @@ function receiveSlackSocketEvent() { const host = "host.openshell.internal"; const port = parseFakeSlackPort("FAKE_SLACK_WEBSOCKET_PORT"); const proxy = parseProxyTarget(); + const appToken = parseManagedCredentialReference("SLACK_APP_TOKEN"); return new Promise((resolve, reject) => { const socket = proxy ? net.createConnection({ host: proxy.host, port: proxy.port }) : net.createConnection({ host, port }); - const timer = setTimeout(() => { socket.destroy(); reject(new Error("timed out waiting for fake Slack Socket Mode event")); }, 30000); + let settled = false; + const fail = (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + reject(error); + }; + const timer = setTimeout(() => fail(new Error("timed out waiting for fake Slack Socket Mode event")), 30000); let handshake = Buffer.alloc(0); let framed = Buffer.alloc(0); let upgraded = false; @@ -483,14 +499,12 @@ function receiveSlackSocketEvent() { if (end === -1) return; const statusLine = handshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; if (!statusLine.includes("101")) { - clearTimeout(timer); - socket.destroy(); - reject(new Error("fake Slack websocket upgrade failed: " + statusLine)); + fail(new Error("fake Slack websocket upgrade failed: " + statusLine)); return; } upgraded = true; framed = Buffer.concat([framed, handshake.slice(end + 4)]); - socket.write(encodeClientText(JSON.stringify({ type: "socket_mode_client_hello", token: "xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN" }))); + socket.write(encodeClientText(JSON.stringify({ type: "socket_mode_client_hello", token: appToken }))); } else { framed = Buffer.concat([framed, chunk]); } @@ -498,9 +512,14 @@ function receiveSlackSocketEvent() { const frame = decodeServerFrame(framed); if (!frame) break; framed = framed.slice(frame.totalLength); + if (frame.opcode === 8) { + fail(new Error("fake Slack websocket closed before the Socket Mode event")); + return; + } if (frame.opcode !== 1) continue; const envelope = JSON.parse(frame.payload.toString("utf8")); socket.write(encodeClientText(JSON.stringify({ envelope_id: envelope.envelope_id }))); + settled = true; clearTimeout(timer); socket.end(); socket.destroy(); @@ -508,13 +527,14 @@ function receiveSlackSocketEvent() { return; } }); - socket.on("error", (error) => { clearTimeout(timer); reject(error); }); + socket.on("error", fail); + socket.on("close", () => fail(new Error("fake Slack websocket closed before the Socket Mode event"))); }); } function postPairingReply(text, channel) { const host = "host.openshell.internal"; const port = parseFakeSlackPort("FAKE_SLACK_REST_PORT"); - const token = "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"; + const token = parseManagedCredentialReference("SLACK_BOT_TOKEN"); const data = new URLSearchParams({ token, channel, text }).toString(); return new Promise((resolve, reject) => { const req = http.request({ diff --git a/test/e2e/support/fixtures/slack-forward-proxy.ts b/test/e2e/support/fixtures/slack-forward-proxy.ts index f8999eb2133..cac51d004cb 100644 --- a/test/e2e/support/fixtures/slack-forward-proxy.ts +++ b/test/e2e/support/fixtures/slack-forward-proxy.ts @@ -26,6 +26,7 @@ export function createSlackSocketClient(proxyPort: number, targetPort: number) { env: { FAKE_SLACK_WEBSOCKET_PORT: String(targetPort), HTTP_PROXY: "http://10.200.0.1:3128", + SLACK_APP_TOKEN: "openshell:resolve:env:v42_SLACK_APP_TOKEN", http_proxy: "", }, }, @@ -39,6 +40,30 @@ function encodeServerText(payload: Record): Buffer { return Buffer.concat([Buffer.from([0x81, body.length]), body]); } +function decodeClientText(buffer: Buffer): { payload: string; totalLength: number } | null { + if (buffer.length < 6 || (buffer[0] & 0x0f) !== 1 || (buffer[1] & 0x80) === 0) return null; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + if (payloadLength === 126) { + if (buffer.length < 8) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 14) return null; + const longLength = buffer.readBigUInt64BE(2); + if (longLength > BigInt(Number.MAX_SAFE_INTEGER)) + throw new Error("test WebSocket payload is too large"); + payloadLength = Number(longLength); + offset = 10; + } + const totalLength = offset + 4 + payloadLength; + if (buffer.length < totalLength) return null; + const mask = buffer.subarray(offset, offset + 4); + const body = Buffer.from(buffer.subarray(offset + 4, totalLength)); + for (let index = 0; index < body.length; index += 1) body[index] ^= mask[index % 4]; + return { payload: body.toString("utf8"), totalLength }; +} + export async function listenOnLoopback(server: net.Server): Promise { await new Promise((resolve, reject) => { server.once("error", reject); @@ -56,16 +81,23 @@ export async function closeServer(server: net.Server): Promise { export function createSuccessfulSlackForwardProxy(envelope: Record): { server: net.Server; requests: string[]; - websocketBytes: () => number; + websocketMessages: () => string[]; } { const requests: string[] = []; - let receivedWebsocketBytes = 0; + const websocketMessages: string[] = []; const server = net.createServer((socket) => { let buffer = Buffer.alloc(0); let upgraded = false; socket.on("data", (chunk) => { if (upgraded) { - receivedWebsocketBytes += chunk.length; + buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]); + while (buffer.length > 0) { + const frame = decodeClientText(buffer); + if (!frame) return; + buffer = buffer.subarray(frame.totalLength); + websocketMessages.push(frame.payload); + if (websocketMessages.length === 1) socket.write(encodeServerText(envelope)); + } return; } buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]); @@ -73,21 +105,19 @@ export function createSuccessfulSlackForwardProxy(envelope: Record receivedWebsocketBytes, + websocketMessages: () => websocketMessages, }; } diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index 84e516349e7..8dec4d09cdc 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -35,7 +35,6 @@ function artifacts(): ArtifactSink { function secrets(values: Record) { return { - optional: (name: string) => values[name], required: (name: string) => { const value = values[name]; return ( @@ -385,46 +384,6 @@ describe("E2E inference adapter", () => { ).rejects.toThrow(/missing NVIDIA_API_KEY/); }); - it("accepts the public credential alias from a trusted-main manual PR controller", async () => { - const adapter = await createAdapter({ - env: { - GITHUB_ACTIONS: "true", - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_SHA: "a".repeat(40), - NEMOCLAW_E2E_CORRELATION_ID: "6ce998a3-52f6-4e91-8b2d-089110070e74", - NEMOCLAW_E2E_EXPECTED_SHA: "b".repeat(40), - NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", - }, - secrets: { NVIDIA_INFERENCE_API_KEY: "nvapi-trusted-controller-key" }, - }); - - expect(adapter.env()).toMatchObject({ - NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", - NVIDIA_INFERENCE_API_KEY: "nvapi-trusted-controller-key", - }); - }); - - it("rejects conflicting public credential sources without exposing either value", async () => { - const create = createAdapter({ - env: { - GITHUB_ACTIONS: "true", - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_SHA: "a".repeat(40), - NEMOCLAW_E2E_CORRELATION_ID: "6ce998a3-52f6-4e91-8b2d-089110070e74", - NEMOCLAW_E2E_EXPECTED_SHA: "b".repeat(40), - NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", - }, - secrets: { - NVIDIA_API_KEY: "nvapi-current-controller-key", - NVIDIA_INFERENCE_API_KEY: "nvapi-historical-controller-key", - }, - }); - - await expect(create).rejects.toThrow( - /^NVIDIA_API_KEY and NVIDIA_INFERENCE_API_KEY contain different values$/u, - ); - }); - it("rejects unknown explicit modes instead of silently falling back", async () => { await expect( createAdapter({ env: { NEMOCLAW_E2E_INFERENCE_MODE: "public-nvida" } }), diff --git a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts index 156fedcb8b5..7823d5cdf33 100644 --- a/test/e2e/support/openclaw-discord-pairing-helpers.test.ts +++ b/test/e2e/support/openclaw-discord-pairing-helpers.test.ts @@ -140,7 +140,7 @@ describe("OpenClaw pairing helper contracts", () => { try { await expect(createSlackSocketClient(proxyPort, targetPort)()).resolves.toEqual(envelope); - await vi.waitFor(() => expect(proxy.websocketBytes()).toBeGreaterThan(0), { + await vi.waitFor(() => expect(proxy.websocketMessages()).not.toHaveLength(0), { interval: 10, timeout: 1_000, }); @@ -151,6 +151,10 @@ describe("OpenClaw pairing helper contracts", () => { "u", ), ); + expect(JSON.parse(proxy.websocketMessages()[0] ?? "{}")).toEqual({ + type: "socket_mode_client_hello", + token: "openshell:resolve:env:v42_SLACK_APP_TOKEN", + }); } finally { await closeServer(proxy.server); } @@ -304,6 +308,51 @@ describe("OpenClaw pairing helper contracts", () => { expect(result.stderr).toEqual(expect.stringContaining("NETWORK_ATTEMPTED=false")); }); + it.each(["SLACK_APP_TOKEN", "SLACK_BOT_TOKEN"])( + "accepts the revision-scoped OpenShell credential reference for %s", + (name) => { + const result = spawnSync(process.execPath, ["--input-type=module"], { + input: `${SLACK_PROBE_INPUT_VALIDATION_SOURCE}\nparseManagedCredentialReference(${JSON.stringify(name)}); console.log("VALID");\n`, + encoding: "utf8", + env: { ...process.env, [name]: `openshell:resolve:env:v42_${name}` }, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("VALID\n"); + expect(result.stdout).not.toContain("openshell:resolve:env:"); + }, + ); + + it.each([ + { name: "missing", value: "" }, + { name: "raw secret", value: "xapp-raw-secret" }, + { name: "identityless canonical reference", value: "openshell:resolve:env:SLACK_APP_TOKEN" }, + { + name: "identityless provider alias", + value: "xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", + }, + { + name: "wrong credential key", + value: "openshell:resolve:env:v42_SLACK_BOT_TOKEN", + }, + ])( + "rejects an invalid Slack app credential reference before network access: $name", + ({ value }) => { + const result = spawnSync(process.execPath, ["--input-type=module"], { + input: `${SLACK_PROBE_INPUT_VALIDATION_SOURCE}\nlet networkAttempted = false; try { parseManagedCredentialReference("SLACK_APP_TOKEN"); networkAttempted = true; } catch (error) { console.error(error.message); console.error("NETWORK_ATTEMPTED=" + networkAttempted); process.exit(1); }\n`, + encoding: "utf8", + env: { ...process.env, SLACK_APP_TOKEN: value }, + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "SLACK_APP_TOKEN must be the revision-scoped OpenShell credential reference issued to the sandbox", + ); + expect(result.stderr).toContain("NETWORK_ATTEMPTED=false"); + expect(result.stderr).not.toContain(value || "xapp-raw-secret"); + }, + ); + it("keeps Discord Gateway proof source valid for sandbox node heredoc", () => { const result = spawnSync(process.execPath, ["--input-type=module", "--check"], { input: DISCORD_GATEWAY_PROOF_SOURCE, From 53ff2b9e953d5ca0ff8ae2ea73136f2691de7d0e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:38:08 -0700 Subject: [PATCH 32/37] test(e2e): wait for sandbox readiness before recovery Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/live/gateway-guard-recovery.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index a84d3a414ae..3b3d1f9d708 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -355,6 +355,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) timeoutMs: 120_000, }); expect(restart.exitCode, resultText(restart)).toBe(0); + await waitForSandboxExecReady(host, instance.sandboxName, progress, "restart-openshell-ready"); progress.phase("recover managed supervisor and inference"); const credentialCanary = "nemoclaw-e2e-recovery-secret-restart"; From 40a3c234f9ff0dc365d0d7379f7219926deab3e7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 26 Aug 2026 11:24:56 -0700 Subject: [PATCH 33/37] fix(e2e): accept base image workflow rename --- test/e2e/support/base-image-publication.test.ts | 17 ++++++++++++++++- tools/e2e/base-image-publication.mts | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 69285ba7e7f..14cd15726bf 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -345,6 +345,7 @@ describe("base-image publication evidence", () => { it("binds API evidence to the active checked-in workflow identity (#7372)", () => { expect(validateWorkflow(workflowMetadata())).toBe(WORKFLOW_ID); + expect(validateWorkflow(workflowMetadata({ name: "Images / Base Images" }))).toBe(WORKFLOW_ID); expect(() => validateWorkflow(workflowMetadata({ state: "disabled_manually" }))).toThrow( /state must be active/u, ); @@ -503,6 +504,20 @@ describe("base-image publication evidence", () => { }); }); + it("accepts the renamed trusted workflow while selecting branch reuse", () => { + const selection = selectPublicationRun( + runsPayload([workflowRun({ name: "Images / Base Images" })]), + history(), + WORKFLOW_ID, + { completedSuccessOnly: true }, + ); + + expect(selection).toMatchObject({ + state: "selected", + run: { id: RUN_ID, headSha: RELEVANT_SHA, conclusion: "success" }, + }); + }); + it("does not select an incomplete or failed publication for branch reuse", () => { expect( selectPublicationRun( @@ -547,7 +562,7 @@ describe("base-image publication evidence", () => { history(), WORKFLOW_ID, ), - ).toThrow(/name must be Images \/ Publish Base and Managed Images/u); + ).toThrow(/name must be one of Images \/ Publish Base and Managed Images, Images \/ Base Images/u); }); it("selects an in-progress trusted publication run (#9549)", () => { diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index e26a7cdc9bf..1ab01f22527 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -11,7 +11,10 @@ const REPOSITORY = "NVIDIA/NemoClaw"; const MAIN_BRANCH = "main"; const WORKFLOW_PATH = ".github/workflows/base-image.yaml"; const WORKFLOW_FILE = "base-image.yaml"; -const WORKFLOW_NAME = "Images / Publish Base and Managed Images"; +const WORKFLOW_NAMES = new Set([ + "Images / Publish Base and Managed Images", + "Images / Base Images", +]); const API_ROOT = "https://api.github.com"; const RUN_URL_ROOT = `https://github.com/${REPOSITORY}/actions/runs`; const WORKFLOW_URL = `https://github.com/${REPOSITORY}/blob/${MAIN_BRANCH}/${WORKFLOW_PATH}`; @@ -151,6 +154,13 @@ function exactString(value: unknown, expected: string, label: string): string { return expected; } +function trustedWorkflowName(value: unknown, label: string): string { + if (typeof value !== "string" || !WORKFLOW_NAMES.has(value)) { + throw new Error(`${label} must be one of ${[...WORKFLOW_NAMES].join(", ")}`); + } + return value; +} + function sha(value: unknown, label: string): string { if (typeof value !== "string" || !SHA_PATTERN.test(value)) { throw new Error(`${label} must be a lowercase 40-character SHA`); @@ -344,7 +354,7 @@ export function resolveFirstParentHistory( export function validateWorkflow(payload: unknown): number { const workflow = asRecord(payload); const workflowId = positiveSafeInteger(workflow.id, "base-image workflow id"); - exactString(workflow.name, WORKFLOW_NAME, "base-image workflow name"); + trustedWorkflowName(workflow.name, "base-image workflow name"); exactString(workflow.path, WORKFLOW_PATH, "base-image workflow path"); exactString(workflow.state, "active", "base-image workflow state"); exactString(workflow.html_url, WORKFLOW_URL, "base-image workflow URL"); @@ -369,7 +379,7 @@ function validateRun(value: unknown, index: number, expectedWorkflowId: number): exactString(run.event, "push", `workflow run ${index} event`); exactString(run.head_branch, MAIN_BRANCH, `workflow run ${index} branch`); exactString(run.path, WORKFLOW_PATH, `workflow run ${index} path`); - exactString(run.name, WORKFLOW_NAME, `workflow run ${index} name`); + trustedWorkflowName(run.name, `workflow run ${index} name`); exactString(asRecord(run.repository).full_name, REPOSITORY, `workflow run ${index} repository`); exactString( asRecord(run.head_repository).full_name, From 550e8078268dba2136bdf1fc12c458b7ff2d0d7d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 26 Aug 2026 11:31:08 -0700 Subject: [PATCH 34/37] fix(e2e): accept renamed publisher jobs --- test/e2e/support/base-image-publication.test.ts | 10 ++++++++++ tools/e2e/base-image-publication.mts | 10 +++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 14cd15726bf..8b713fe8373 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -633,6 +633,16 @@ describe("base-image publication evidence", () => { ).toThrow(/provenance does not match/u); }); + it("accepts the renamed trusted publisher jobs", () => { + const jobs = [ + publisherJob("Manifests / OpenClaw", { id: 1 }), + publisherJob("Manifests / Hermes", { id: 2 }), + publisherJob("Manifests / Deep Agents Code", { id: 3 }), + ]; + + expect(validatePublisherJobs({ total_count: jobs.length, jobs }, selectedRun())).toBe("ready"); + }); + it("classifies an incomplete required publisher as pending only while the selected run is in progress (#9549)", () => { const jobs = successfulJobs().map((job) => job.name === "Build and push Hermes base image" diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index 1ab01f22527..8d0f6935969 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -78,6 +78,14 @@ export const REQUIRED_PUBLISHER_JOBS = [ "Build and push Hermes base image", "Build and push Deep Agents Code base image", ] as const; +const PUBLISHER_JOB_ALIASES = new Map([ + ["Build and push OpenClaw base image", "Build and push OpenClaw base image"], + ["Manifests / OpenClaw", "Build and push OpenClaw base image"], + ["Build and push Hermes base image", "Build and push Hermes base image"], + ["Manifests / Hermes", "Build and push Hermes base image"], + ["Build and push Deep Agents Code base image", "Build and push Deep Agents Code base image"], + ["Manifests / Deep Agents Code", "Build and push Deep Agents Code base image"], +]); type JsonRecord = Record; @@ -479,7 +487,7 @@ export function validatePublisherJobs(payload: unknown, run: PublicationRun): "p if (typeof job.name !== "string" || job.name.length === 0) { throw new Error(`publisher job ${index} name is invalid`); } - const requiredName = REQUIRED_PUBLISHER_JOBS.find((name) => name === job.name); + const requiredName = PUBLISHER_JOB_ALIASES.get(job.name); if (!requiredName) continue; if (typeof job.status !== "string") { throw new Error(`publisher job ${requiredName} status is invalid; ${run.url}`); From a09600f962b8ba47f197b4f3649f466bf1845534 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 26 Aug 2026 18:11:16 -0700 Subject: [PATCH 35/37] fix(e2e): close qualification trust gaps --- .github/workflows/e2e.yaml | 2 +- test/e2e/live/jetson-nvmap-gpu.test.ts | 14 +++++++------- ...ted-target-routing-workflow-boundary.test.ts | 17 ++++++++++++++++- tools/e2e/workflow-boundary.mts | 7 +++++-- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b5086ff5ccc..f8c306d17fc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2810,7 +2810,7 @@ jobs: - name: Run live E2E tests env: E2E_TARGET_ID: ${{ matrix.id }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} TARGET_ID: ${{ matrix.id }} run: | set -euo pipefail diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index 3790e04d28d..24175d06ff6 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -16,7 +16,6 @@ import type { HostCliClient } from "../fixtures/clients/host.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; -import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import type { FakeOpenAiCompatibleRequest } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; @@ -286,15 +285,12 @@ fi`, }); await artifacts.writeText("install-jetson-nvmap.log", resultText(install)); expect(install.exitCode, resultText(install)).toBe(0); - assertStockManagedImageReceipt({ - environment: env(inferenceEnv), - expectedAgent: "openclaw", - sandboxName: SANDBOX_NAME, - }); // #3508 failed after Jetson onboarding silently fell back to building // Dockerfile.base locally. Prove this buildless path instead registered - // the dispatched, immutable published linux/arm64 managed image. + // the dispatched, immutable published linux/arm64 managed image. The v2 + // Jetson dispatch contract carries the exact revision, so bind its remote + // durable receipt to the immutable image's cohort label here. expect(resultText(install)).not.toContain( "Building OpenClaw sandbox base image locally because no compatible published base image was found.", ); @@ -304,11 +300,14 @@ fi`, "u", ); expect(managedImage.imageTag).toMatch(expectedReference); + const sourceCohort = managedImage.workload.sourceCohort; + expect(sourceCohort).toMatch(/^ghrun-[1-9][0-9]*-[1-9][0-9]*$/u); expect(managedImage.workload).toMatchObject({ kind: "managed-image", platform: "linux/arm64", reference: managedImage.imageTag, shared: true, + sourceCohort, sourceRevision: MANAGED_IMAGE_SOURCE_REVISION, }); const managedImageLabels = await host.command( @@ -325,6 +324,7 @@ fi`, expect(labels).toMatchObject({ "io.nvidia.nemoclaw.agent": "openclaw", "io.nvidia.nemoclaw.managed-image.capabilities": "1", + "io.nvidia.nemoclaw.managed-image.cohort": sourceCohort, "io.nvidia.nemoclaw.managed-image.contract": "1", "io.nvidia.nemoclaw.managed-image.platform": "linux/arm64", "io.nvidia.nemoclaw.managed-image.startup-profile": "1", diff --git a/test/e2e/support/trusted-target-routing-workflow-boundary.test.ts b/test/e2e/support/trusted-target-routing-workflow-boundary.test.ts index 0d43054b59e..048280539b9 100644 --- a/test/e2e/support/trusted-target-routing-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-target-routing-workflow-boundary.test.ts @@ -7,7 +7,10 @@ import { readWorkflow } from "../../helpers/e2e-workflow-contract"; import { requireFixture } from "./require-fixture"; type ControllerWorkflow = { - jobs: Record }>; + jobs: Record< + string, + { steps: Array<{ env?: Record; id?: string; name?: string; run?: string }> } + >; }; const EXPECTED_ERROR = "trusted controller matrix must pin typed target runner to ubuntu-latest"; @@ -90,4 +93,16 @@ describe("trusted E2E target routing boundary (#7824)", () => { expect(validateE2eWorkflow(workflow)).toContain(EXPECTED_ERROR); }); + + it("rejects an inference credential exposed to an unauthorized PR candidate", () => { + const { workflow } = fixture(); + const run = workflow.jobs.live!.steps.find((step) => step.name === "Run live E2E tests")!; + const validationError = + "live E2E step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main run or an authorized NVIDIA-owned PR dispatch"; + + expect(validateE2eWorkflow(workflow)).not.toContain(validationError); + run.env!.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; + + expect(validateE2eWorkflow(workflow)).toContain(validationError); + }); }); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 5c496cfe043..8bca7f44ca5 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -222,6 +222,7 @@ const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} & const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const GUARDED_LIVE_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; const RUNNER_ROUTING_SCRIPT = [ @@ -3016,8 +3017,10 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (runVitestEnv.TARGET_ID !== "${{ matrix.id }}") { errors.push("live E2E step must pass matrix.id through TARGET_ID env"); } - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets"); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== GUARDED_LIVE_E2E_INFERENCE_KEY) { + errors.push( + "live E2E step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main run or an authorized NVIDIA-owned PR dispatch", + ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); requireRunContains(errors, runVitest, "test/e2e/live/registry-targets.test.ts"); From 6ab7a354323cfc9b23b3c7458e84765f41893ae1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 26 Aug 2026 18:22:31 -0700 Subject: [PATCH 36/37] refactor(e2e): reuse managed cohort receipt checks --- test/e2e/live/mcp-bridge-onboard-env.ts | 67 ++--------------- .../support/mcp-bridge-onboard-env.test.ts | 72 +++++++++++++++---- 2 files changed, 65 insertions(+), 74 deletions(-) diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index c41c9b2783c..527f32609e1 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -1,15 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; - -import { - MANAGED_IMAGE_PLATFORMS, - MANAGED_IMAGE_REPOSITORIES, - parseManagedImageContractV1, - type ManagedImagePlatform, - type ShippedManagedImageAgent, -} from "../../../src/lib/onboard/managed-image/contract.ts"; +import type { ShippedManagedImageAgent } from "../../../src/lib/onboard/managed-image/contract.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertManagedImageReceiptMatchesSelectedCohort } from "../fixtures/managed-image-receipt.ts"; @@ -63,58 +55,11 @@ export function assertMcpBridgeManagedImageReceipt(options: { if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { throw new Error("managed-image MCP qualification requires an exact cohort revision"); } - - const workloadPlatform = options.workload?.platform; - if ( - typeof workloadPlatform !== "string" || - !(MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(workloadPlatform) - ) { - throw new Error("managed-image MCP qualification requires an exact workload platform"); - } - const expectedPlatform = workloadPlatform as ManagedImagePlatform; - - let expectedReference: string; - let expectedCohort: string; - if (selectedRevision) { - assertManagedImageReceiptMatchesSelectedCohort({ - environment, - expectedAgent: options.expectedAgent, - workload: options.workload, - }); - return; - } else { - let catalog: Record; - try { - const parsed = JSON.parse(fs.readFileSync(exactCandidateCatalog!, "utf8")) as unknown; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(); - catalog = parsed as Record; - } catch { - throw new Error("managed-image MCP qualification catalog is invalid"); - } - const contract = parseManagedImageContractV1( - catalog[options.expectedAgent], - options.expectedAgent, - expectedPlatform, - ); - if (contract.source.revision !== expectedRevision) { - throw new Error("managed-image MCP qualification catalog revision is invalid"); - } - expectedReference = contract.reference; - expectedCohort = contract.source.cohort; - } - - if ( - typeof expectedReference !== "string" || - !expectedReference.startsWith(`${MANAGED_IMAGE_REPOSITORIES[options.expectedAgent]}@sha256:`) || - options.workload?.kind !== "managed-image" || - options.workload.sourceRevision !== expectedRevision || - options.workload.sourceCohort !== expectedCohort || - options.workload.reference !== expectedReference - ) { - throw new Error( - "MCP qualification must use the exact agent image from the selected cohort receipt", - ); - } + assertManagedImageReceiptMatchesSelectedCohort({ + environment, + expectedAgent: options.expectedAgent, + workload: options.workload, + }); } export function buildMcpBridgeExactMainEnv(options: { diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index d50a9f337b1..04d17737eea 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -165,33 +165,55 @@ describe("MCP bridge onboarding environment", () => { const catalogPath = path.join(fixtureRoot, "catalog.json"); fs.writeFileSync( catalogPath, - JSON.stringify({ - "langchain-deepagents-code": { - contractVersion: 1, - agent: "langchain-deepagents-code", - platform: PLATFORM, - image: MANAGED_IMAGE_REPOSITORIES["langchain-deepagents-code"], - digest: `sha256:${"d".repeat(64)}`, - reference, - source: { repository: "NVIDIA/NemoClaw", revision, release: "v0.0.114", cohort }, - startupProfileContractVersion: 1, - capabilityContractVersion: 1, - }, - }), + JSON.stringify( + Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, index) => { + const digest = + agent === "langchain-deepagents-code" + ? `sha256:${"d".repeat(64)}` + : `sha256:${String(index + 1).repeat(64)}`; + return [ + agent, + { + contractVersion: 1, + agent, + platform: PLATFORM, + image: MANAGED_IMAGE_REPOSITORIES[agent], + digest, + reference: + agent === "langchain-deepagents-code" + ? reference + : `${MANAGED_IMAGE_REPOSITORIES[agent]}@${digest}`, + source: { + repository: "NVIDIA/NemoClaw", + revision, + release: "v0.0.114", + cohort, + }, + startupProfileContractVersion: 1, + capabilityContractVersion: 1, + }, + ]; + }), + ), + ), { mode: 0o600 }, ); try { expect(() => assertMcpBridgeManagedImageReceipt({ environment: { + GITHUB_ACTIONS: "true", NEMOCLAW_E2E_EXPECTED_SHA: revision, NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, + NEMOCLAW_RUN_LIVE_E2E: "1", }, expectedAgent: "langchain-deepagents-code", workload: { kind: "managed-image", platform: PLATFORM, reference, + release: "v0.0.114", sourceCohort: cohort, sourceRevision: revision, }, @@ -202,6 +224,30 @@ describe("MCP bridge onboarding environment", () => { } }); + it("rejects a symbolic-link candidate catalog", () => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-catalog-link-")); + const targetPath = path.join(fixtureRoot, "catalog.json"); + const linkPath = path.join(fixtureRoot, "selected.json"); + fs.writeFileSync(targetPath, "{}", { mode: 0o600 }); + fs.symlinkSync(targetPath, linkPath); + try { + expect(() => + assertMcpBridgeManagedImageReceipt({ + environment: { + GITHUB_ACTIONS: "true", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: linkPath, + NEMOCLAW_RUN_LIVE_E2E: "1", + }, + expectedAgent: "langchain-deepagents-code", + workload: selectedWorkload("langchain-deepagents-code"), + }), + ).toThrow("candidate managed-image catalog is invalid"); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + it("accepts the selected cross-release managed-image cohort revision", () => { expect(() => assertMcpBridgeManagedImageReceipt({ From 0fa7dc96622c80367e932124ee5978b20e9276c4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 26 Aug 2026 18:37:49 -0700 Subject: [PATCH 37/37] test(onboard): allow loaded shard headroom --- src/lib/onboard/sandbox-create-plan.test.ts | 61 +++++++++++---------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 49e3e07d72a..09a90f5333b 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; +import { testTimeoutOptions } from "../../../test/helpers/timeouts"; import { MESSAGING_CREDENTIAL_PROVIDER_TYPE } from "../messaging/provider-profile"; import type { MessagingTokenDef } from "./messaging-prep"; import { materializeHermesPortableCreatePlan } from "./sandbox-create-plan-materialization"; @@ -315,34 +316,38 @@ describe("resolveSandboxCreateIntent", () => { expect(JSON.stringify(first)).not.toContain("/tmp/"); }); - it("omits Discord create-time effects when an unselected credential is available", () => { - const { intent, messagingTokenDefs } = resolveDiscordCreateIntent({ - selected: false, - reusable: true, - }); - const upsertMessagingProviders = vi.fn((tokenDefs: MessagingTokenDef[]) => - tokenDefs.map(({ name }) => name), - ); - - const plan = materializeDiscordCreatePlan( - { intent, messagingTokenDefs }, - { upsertMessagingProviders }, - ); - - expect(intent.reusableMessagingProviders).toEqual([]); - expect(plan.activeMessagingChannels).toEqual([]); - expect(plan.initialSandboxPolicy.appliedPresets).not.toContain("discord"); - expect(plan.initialSandboxPolicy.credentialBindingProviders ?? []).not.toContain( - discordProviderName, - ); - expect(plan.messagingProviders).not.toContain(discordProviderName); - expect(plan.createArgs).not.toContain(discordProviderName); - expect(upsertMessagingProviders).toHaveBeenCalledWith([], { - replaceExisting: true, - allowedSandboxes: ["sandbox"], - }); - plan.initialSandboxPolicy.cleanup?.(); - }); + it( + "omits Discord create-time effects when an unselected credential is available", + testTimeoutOptions(15_000), + () => { + const { intent, messagingTokenDefs } = resolveDiscordCreateIntent({ + selected: false, + reusable: true, + }); + const upsertMessagingProviders = vi.fn((tokenDefs: MessagingTokenDef[]) => + tokenDefs.map(({ name }) => name), + ); + + const plan = materializeDiscordCreatePlan( + { intent, messagingTokenDefs }, + { upsertMessagingProviders }, + ); + + expect(intent.reusableMessagingProviders).toEqual([]); + expect(plan.activeMessagingChannels).toEqual([]); + expect(plan.initialSandboxPolicy.appliedPresets).not.toContain("discord"); + expect(plan.initialSandboxPolicy.credentialBindingProviders ?? []).not.toContain( + discordProviderName, + ); + expect(plan.messagingProviders).not.toContain(discordProviderName); + expect(plan.createArgs).not.toContain(discordProviderName); + expect(upsertMessagingProviders).toHaveBeenCalledWith([], { + replaceExisting: true, + allowedSandboxes: ["sandbox"], + }); + plan.initialSandboxPolicy.cleanup?.(); + }, + ); it("attaches the selected Discord provider to its create-time policy", () => { const { intent, messagingTokenDefs } = resolveDiscordCreateIntent({