From e2afccde281fde182c15ab6953e855f542331669 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 20:15:24 -0500 Subject: [PATCH 01/71] test(e2e): execute native runtime qualification Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 136 ++- ...-native-runtime-installer-qualification.sh | 86 +- .../native-qualification-authority.ts | 2 +- ...tive-runtime-qualification-case-helpers.ts | 367 ++++++ .../native-runtime-qualification-case.test.ts | 1077 +++++++++++++++++ ...runtime-qualification-case-helpers.test.ts | 160 +++ ...ve-runtime-qualification-collector.test.ts | 4 +- ...e-qualification-producer-aggregate.test.ts | 354 ++++++ ...me-qualification-producer-evidence.test.ts | 101 +- ...me-qualification-producer-workflow.test.ts | 60 +- tools/e2e/check-semantic-phases.mts | 4 + ...ntime-qualification-producer-aggregate.mts | 365 ++++++ ...untime-qualification-producer-evidence.mts | 580 ++++++++- tools/e2e/operations-workflow-boundary.mts | 6 +- 14 files changed, 3188 insertions(+), 114 deletions(-) create mode 100644 test/e2e/live/native-runtime-qualification-case-helpers.ts create mode 100644 test/e2e/live/native-runtime-qualification-case.test.ts create mode 100644 test/e2e/support/native-runtime-qualification-case-helpers.test.ts create mode 100644 test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts create mode 100644 tools/e2e/native-runtime-qualification-producer-aggregate.mts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9aead5db647..d98491bae9e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1009,10 +1009,6 @@ jobs: with: node-version: 22.19.0 - - name: Install locked candidate test dependencies without scripts - working-directory: .candidate-runtime - run: npm ci --ignore-scripts - - name: Prepare the credential-free execution account and disable Docker id: boundary env: @@ -1020,7 +1016,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in git jq node pgrep podman sha256sum systemctl; do + for command in git jq node npm pgrep podman sha256sum systemctl; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1046,6 +1042,11 @@ jobs: runtime_dir="/run/user/${uid}" sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir" sudo chown -R "$uid:$uid" "$CANDIDATE_DIRECTORY" + node_directory="$(dirname "$(command -v node)")" + [[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || { + echo "::error::Pinned Node toolchain path is invalid" >&2 + exit 1 + } guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard" install -d -m 0700 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" @@ -1053,6 +1054,23 @@ jobs: printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" + printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" + + - name: Install locked candidate test dependencies without scripts + env: + ACCOUNT: ${{ steps.boundary.outputs.account }} + CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} + QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + shell: bash + run: | + set -euo pipefail + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + npm --prefix "$CANDIDATE_DIRECTORY" ci --ignore-scripts - name: Run the authenticated installer qualification env: @@ -1084,13 +1102,13 @@ jobs: echo "::error::Installer receipt directory is missing or invalid" >&2 exit 1 } - sudo chown -R -h root:root "$INSTALLER_RECEIPT_PARENT/receipts" - name: Execute the candidate qualification case without credentials env: ACCOUNT: ${{ steps.boundary.outputs.account }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} shell: bash @@ -1113,14 +1131,13 @@ jobs: NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \ NEMOCLAW_RUN_LIVE_E2E=1 \ - PATH="$GUARD_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \ --config "$CANDIDATE_DIRECTORY/vitest.config.ts" \ --project e2e-live \ "$live_test" sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true - sudo chown -R root:root "$receipt_directory" - name: Verify Docker stayed unavailable shell: bash @@ -1136,14 +1153,14 @@ jobs: EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} shell: bash run: | set -euo pipefail - sudo chown -R "$(id -u):$(id -g)" \ - "$INSTALLER_RECEIPT_DIRECTORY" \ - "$(dirname "$EXECUTION_RECEIPT_PATH")" - node --experimental-strip-types --no-warnings \ + sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ + "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ .trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts + sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources if: always() @@ -1163,11 +1180,101 @@ jobs: fi - name: Upload the qualification case evidence - if: always() + if: success() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: ${{ matrix.artifactName }} - path: ${{ runner.temp }}/native-runtime-evidence/evidence.json + path: ${{ runner.temp }}/native-runtime-evidence/ + + native-runtime-qualification-producer-aggregate: + name: Aggregate native runtime qualification evidence + needs: + [ + generate-matrix, + native-runtime-qualification-producer-plan, + native-runtime-qualification-producer, + ] + if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + actions: read + contents: read + pull-requests: read + steps: + - name: Check out the trusted qualification aggregator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .trusted-qualification-aggregate + persist-credentials: false + sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts + test/e2e/registry/native-runtime-qualification.ts + tools/e2e/native-runtime-qualification-producer-aggregate.mts + sparse-checkout-cone-mode: false + + - name: Download the exact case evidence cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-* + path: ${{ runner.temp }}/native-runtime-case-artifacts + merge-multiple: false + + - name: Resolve this aggregate job identity + id: aggregate-job + env: + GH_TOKEN: ${{ github.token }} + PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} + PRODUCER_RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + jobs="$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \ + -f per_page=100)" + job_id="$(jq -er \ + --arg name 'Aggregate native runtime qualification evidence' \ + --argjson runId "$PRODUCER_RUN_ID" \ + --argjson attempt "$PRODUCER_RUN_ATTEMPT" ' + select(.total_count <= 100) | + [.jobs[] | select( + .name == $name and + .run_id == $runId and + .run_attempt == $attempt and + .status == "in_progress" + )] | + select(length == 1) | + .[0].id + ' <<<"$jobs")" || { + echo "::error::Could not resolve one in-progress aggregate job identity" >&2 + exit 1 + } + [[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || { + echo "::error::Aggregate job identity is invalid" >&2 + exit 1 + } + printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT" + + - name: Validate and aggregate all 24 case receipts + working-directory: .trusted-qualification-aggregate + env: + AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }} + CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts + EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate + QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/native-runtime-qualification-producer-aggregate.mts + + - name: Upload the immutable aggregate evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-runtime-qualification-${{ inputs.checkout_sha }} + path: ${{ runner.temp }}/native-runtime-aggregate/ + if-no-files-found: error + retention-days: 30 + compression-level: 9 retired-selector-compatibility: needs: generate-matrix @@ -4147,6 +4254,7 @@ jobs: openclaw-plugin-runtime-exdev-release, openclaw-plugin-runtime-exdev, native-runtime-qualification-producer, + native-runtime-qualification-producer-aggregate, ] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} permissions: diff --git a/scripts/checks/run-native-runtime-installer-qualification.sh b/scripts/checks/run-native-runtime-installer-qualification.sh index 0dfd610e6e8..9d6d5aa513a 100755 --- a/scripts/checks/run-native-runtime-installer-qualification.sh +++ b/scripts/checks/run-native-runtime-installer-qualification.sh @@ -98,6 +98,7 @@ verify_checkout() { *) fail "$label has an unexpected origin repository." ;; esac assert_checkout_has_no_git_credentials "$checkout" "$label" + printf '%s\n' "$revision" } verify_committed_file() { @@ -191,14 +192,51 @@ assert_docker_unavailable() { [[ ! -S "$socket_path" ]] \ || fail "A Docker socket exists during the ${phase} check." done < <(docker_socket_paths) + + printf '%s\n' \ + '{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}' } run_native_runtime_installer_qualification() { - candidate_checkout="" - candidate_sha="" - expected_installer_sha256="" - expected_architecture="" - artifact_dir_input="" + local candidate_checkout="" + local candidate_sha="" + local expected_installer_sha256="" + local expected_architecture="" + local artifact_dir_input="" + local artifact_parent="" + local artifact_name="" + local artifact_dir="" + local runner_architecture="" + local candidate_installer="" + local candidate_setup_script="" + local qualification_root="" + local qualification_home="" + local qualification_tmp="" + local docker_guard_dir="" + local managed_payload_root="" + local verified_script_dir="" + local verified_installer="" + local verified_setup_script="" + local installed_checkout="" + local receipt_stage="" + local docker_guard="" + local docker_guard_sha256="" + local candidate_status=0 + local verified_candidate_revision="" + local installed_revision="" + local pre_execution_docker_posture="" + local post_execution_docker_posture="" + + cleanup() { + if [[ -n "$receipt_stage" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then + rm -rf -- "$receipt_stage" + fi + if [[ -n "$qualification_root" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then + rm -rf -- "$qualification_root" + fi + } + trap cleanup EXIT + while [[ "$#" -gt 0 ]]; do case "$1" in --candidate-checkout) @@ -272,7 +310,9 @@ run_native_runtime_installer_qualification() { candidate_installer="${candidate_checkout}/scripts/install.sh" candidate_setup_script="${candidate_checkout}/scripts/setup-jetson.sh" - verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout" + verified_candidate_revision="$( + verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout" + )" verify_installer \ "$candidate_checkout" \ "$candidate_sha" \ @@ -303,16 +343,6 @@ run_native_runtime_installer_qualification() { "$managed_payload_root" \ "$verified_script_dir" - cleanup() { - if [[ -n "${receipt_stage:-}" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then - rm -rf -- "$receipt_stage" - fi - if [[ -n "${qualification_root:-}" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then - rm -rf -- "$qualification_root" - fi - } - trap cleanup EXIT - cp -- "$candidate_installer" "$verified_installer" cp -- "$candidate_setup_script" "$verified_setup_script" chmod 500 "$verified_installer" "$verified_setup_script" @@ -336,9 +366,10 @@ run_native_runtime_installer_qualification() { PATH="${docker_guard_dir}:${PATH}" export PATH - assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256" + pre_execution_docker_posture="$( + assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256" + )" - candidate_status=0 # The child shell expands positional parameters inside this literal program. # shellcheck disable=SC2016 env -i \ @@ -366,11 +397,15 @@ run_native_runtime_installer_qualification() { install_nemoclaw_before_onboarding ' _ "$verified_installer" "$verified_script_dir" || candidate_status=$? - assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256" + post_execution_docker_posture="$( + assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256" + )" [[ "$candidate_status" -eq 0 ]] \ || fail "The candidate installer phase executor exited with status ${candidate_status}." - verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout" + installed_revision="$( + verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout" + )" verify_installer \ "$installed_checkout" \ "$candidate_sha" \ @@ -382,16 +417,16 @@ run_native_runtime_installer_qualification() { "$expected_installer_sha256" "$candidate_sha" "$runner_architecture" \ >"${receipt_stage}/invocation.json" printf '{"receiptVersion":1,"repository":"%s","revision":"%s","installerSha256":"%s"}\n' \ - "$CANONICAL_REPOSITORY" "$candidate_sha" "$expected_installer_sha256" \ + "$CANONICAL_REPOSITORY" "$verified_candidate_revision" "$expected_installer_sha256" \ >"${receipt_stage}/candidate-source.json" printf '{"receiptVersion":1,"repository":"%s","requestedRevision":"%s","installedRevision":"%s","installMode":"managed","installerSha256":"%s"}\n' \ - "$CANONICAL_REPOSITORY" "$candidate_sha" "$candidate_sha" "$expected_installer_sha256" \ + "$CANONICAL_REPOSITORY" "$candidate_sha" "$installed_revision" "$expected_installer_sha256" \ >"${receipt_stage}/installed-source.json" printf '{"receiptVersion":1,"requested":"%s","runner":"%s"}\n' \ "$expected_architecture" "$runner_architecture" \ >"${receipt_stage}/architecture.json" - printf '%s\n' \ - '{"receiptVersion":1,"preExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true},"postExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}}' \ + printf '{"receiptVersion":1,"preExecution":%s,"postExecution":%s}\n' \ + "$pre_execution_docker_posture" "$post_execution_docker_posture" \ >"${receipt_stage}/docker-absence.json" bounded_file "${receipt_stage}/installer.sh" "$MAX_INSTALLER_BYTES" @@ -409,6 +444,9 @@ run_native_runtime_installer_qualification() { receipt_stage="" printf 'Native runtime installer qualification receipts: %s\n' "$artifact_dir" + cleanup + trap - EXIT + unset -f cleanup } if [[ "${BASH_SOURCE[0]:-}" == "$0" ]]; then diff --git a/src/lib/onboard/runtime-provider/native-qualification-authority.ts b/src/lib/onboard/runtime-provider/native-qualification-authority.ts index 4c811882afc..b6fe3d5b5fd 100644 --- a/src/lib/onboard/runtime-provider/native-qualification-authority.ts +++ b/src/lib/onboard/runtime-provider/native-qualification-authority.ts @@ -10,7 +10,7 @@ export const NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY = "NVIDIA/NemoClaw"; /** The trusted collector is separate and rejects evidence emitted by its own workflow. */ export const NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW = - ".github/workflows/native-runtime-qualification.yaml"; + ".github/workflows/e2e.yaml"; export interface NativeRuntimeQualificationProtectedRun { readonly repository: string; diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts new file mode 100644 index 00000000000..e95fe4caf91 --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -0,0 +1,367 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { constants, closeSync, fstatSync, openSync, readFileSync } from "node:fs"; + +import { + NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, + NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS, + type NativeRuntimeQualificationProducerPlanRow, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, + type NativeRuntimeQualificationAcceleration, + type NativeRuntimeQualificationAgent, + type NativeRuntimeQualificationArchitecture, + type NativeRuntimeQualificationInference, +} from "../registry/native-runtime-qualification.ts"; + +export const NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT = + "/etc/nemoclaw/native-runtime-qualification-v1.json"; + +const SHA = /^[a-f0-9]{40}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const OCI_DIGEST = /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; +const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,511}$/u; +const ABSOLUTE_CACHE = + /^\/var\/lib\/nemoclaw\/native-runtime-qualification\/[A-Za-z0-9._/-]{1,384}$/u; +const SENSITIVE_ENVIRONMENT = + /^(?:GH_TOKEN|GITHUB_TOKEN|NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|NGC_API_KEY|NIM_NGC_API_KEY|HF_TOKEN|HUGGING_FACE_HUB_TOKEN|SSH_AUTH_SOCK|DOCKER_CERT_PATH|DOCKER_CONFIG|DOCKER_CONTEXT|DOCKER_HOST|DOCKER_TLS_VERIFY|CONTAINER_HOST|AWS_.+|AZURE_.+|GOOGLE_.+|.*(?:_API_KEY|_ACCESS_TOKEN|_AUTH_TOKEN|_PASSWORD|_PRIVATE_KEY|_SECRET|_SECRET_KEY))$/u; + +const AGENT_IMAGES = Object.freeze({ + amd64: Object.freeze({ + openclaw: + "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:2bca5955feb48f9b9170e51bbd5114c8ec481714b95a804d213957d4f5c3d069", + hermes: + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:28b9578ab9676ef046de37fa6feb9b7b61824b87d77fd08978758bd01c03cb54", + "langchain-deepagents-code": + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:f7ad7ddc95cea260cff02d26b873903805806ccfef5d27436cbec4eba3455eff", + }), + arm64: Object.freeze({ + openclaw: + "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:2f5bd4025b7cb61502d48f1fa02dd282d6d1156e7818c56e62ee675dc418c207", + hermes: + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:119076205d8ac366a1e0309a4c6a3822d616151d0c657b53df1e308ba690d46b", + "langchain-deepagents-code": + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:3ba92564fb17de4082745b9f32608188e28504cf421a8af7c5ef18e13680028e", + }), +}) satisfies Readonly< + Record< + NativeRuntimeQualificationArchitecture, + Readonly> + > +>; + +const OLLAMA_IMAGES = Object.freeze({ + amd64: + "docker.io/ollama/ollama@sha256:268c47cdc4718ded54babcd842579a7295ad79fd8d5c2ea64d7ba2e76872de6b", + arm64: + "docker.io/ollama/ollama@sha256:bcf5adbfacc0e13a975f981810959d05b6ee95632da0f27e5343bc868ad2c82d", +}) satisfies Readonly>; + +export interface NativeRuntimeQualificationRunnerContract { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-runner-v1"; + readonly architecture: NativeRuntimeQualificationArchitecture; + readonly gpuProbeImageRef: string; + readonly nim: { + readonly imageRef: string; + readonly model: string; + readonly cachePath: string; + }; + readonly vllm: { + readonly imageRef: string; + readonly model: string; + readonly modelPath: string; + }; +} + +type UnknownRecord = Record; + +function record(value: unknown, label: string): UnknownRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as UnknownRecord; +} + +function exactKeys(value: UnknownRecord, expected: readonly string[], label: string): void { + if (Object.keys(value).sort().join("\n") !== [...expected].sort().join("\n")) { + throw new Error(`${label} fields are invalid`); + } +} + +function exactJson(actual: unknown, expected: unknown, label: string): void { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label} does not match the trusted qualification definition`); + } +} + +export function parseNativeRuntimeQualificationRow( + serialized: string, +): NativeRuntimeQualificationProducerPlanRow { + let parsed: unknown; + try { + parsed = JSON.parse(serialized) as unknown; + } catch { + throw new Error("Native runtime qualification row is not valid JSON"); + } + const row = record(parsed, "Native runtime qualification row"); + exactKeys( + row, + [ + "id", + "jobName", + "artifactName", + "runner", + "installerSha256", + "source", + "case", + "rootModes", + "focusedOperations", + ], + "Native runtime qualification row", + ); + const qualificationCase = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.cases.find( + (entry) => entry.id === row.id, + ); + if (!qualificationCase) throw new Error("Native runtime qualification case is not canonical"); + exactJson(row.case, qualificationCase, "Native runtime qualification case"); + const source = record(row.source, "Native runtime qualification source"); + exactKeys( + source, + [ + "repository", + "producerWorkflow", + "pullRequestNumber", + "candidateRepository", + "candidateSha", + "baseRef", + "baseSha", + "workflowSha", + "producerRunId", + "producerRunAttempt", + "dispatchArtifact", + ], + "Native runtime qualification source", + ); + if ( + source.repository !== "NVIDIA/NemoClaw" || + source.producerWorkflow !== ".github/workflows/e2e.yaml" || + source.candidateRepository !== "NVIDIA/NemoClaw" || + source.baseRef !== "main" || + !Number.isSafeInteger(source.pullRequestNumber) || + Number(source.pullRequestNumber) < 1 || + typeof source.candidateSha !== "string" || + !SHA.test(source.candidateSha) || + typeof source.baseSha !== "string" || + !SHA.test(source.baseSha) || + source.candidateSha === source.baseSha || + source.workflowSha !== source.baseSha || + !/^[1-9][0-9]{0,19}$/u.test(String(source.producerRunId)) || + source.producerRunAttempt !== 1 + ) { + throw new Error("Native runtime qualification source identity is invalid"); + } + const artifact = record(source.dispatchArtifact, "Native runtime dispatch artifact"); + exactKeys(artifact, ["id", "name", "digest", "sizeInBytes"], "Native runtime dispatch artifact"); + if ( + !/^[1-9][0-9]{0,19}$/u.test(String(artifact.id)) || + artifact.name !== `e2e-dispatch-${String(source.producerRunId)}-1` || + typeof artifact.digest !== "string" || + !/^sha256:[a-f0-9]{64}$/u.test(artifact.digest) || + !Number.isSafeInteger(artifact.sizeInBytes) || + Number(artifact.sizeInBytes) < 1 || + Number(artifact.sizeInBytes) > 1_048_576 + ) { + throw new Error("Native runtime dispatch artifact identity is invalid"); + } + const focused = row.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE; + exactJson(row.rootModes, focused ? ["rootless", "rootful"] : ["rootless"], "Root modes"); + exactJson( + row.focusedOperations, + focused ? NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS : [], + "Focused operations", + ); + if ( + row.jobName !== `Native runtime qualification / ${String(row.id)}` || + row.artifactName !== + `native-runtime-qualification-evidence-${String(source.candidateSha)}-${String(row.id)}` || + typeof row.runner !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(row.runner) || + typeof row.installerSha256 !== "string" || + !SHA256.test(row.installerSha256) + ) { + throw new Error("Native runtime qualification plan metadata is invalid"); + } + return parsed as NativeRuntimeQualificationProducerPlanRow; +} + +export function assertCredentialFreeQualificationEnvironment(environment: NodeJS.ProcessEnv): void { + const present = Object.keys(environment).filter((name) => SENSITIVE_ENVIRONMENT.test(name)); + if (present.length > 0) { + throw new Error( + `Native runtime qualification environment contains forbidden credential names: ${present.sort().join(", ")}`, + ); + } +} + +function exactRunnerRuntime( + value: unknown, + label: "NIM" | "vLLM", + cacheField: "cachePath" | "modelPath", +): { + readonly imageRef: string; + readonly model: string; + readonly cachePath: string; +} { + const runtime = record(value, `${label} runner contract`); + exactKeys(runtime, ["imageRef", "model", cacheField], `${label} runner contract`); + const cachePath = runtime[cacheField]; + if ( + typeof runtime.imageRef !== "string" || + !OCI_DIGEST.test(runtime.imageRef) || + typeof runtime.model !== "string" || + !MODEL.test(runtime.model) || + typeof cachePath !== "string" || + !ABSOLUTE_CACHE.test(cachePath) || + cachePath.includes("//") || + cachePath.split("/").some((segment) => segment === "." || segment === "..") + ) { + throw new Error(`${label} runner contract is invalid`); + } + return Object.freeze({ + imageRef: runtime.imageRef, + model: runtime.model, + cachePath, + }); +} + +export function parseNativeRuntimeQualificationRunnerContract( + value: unknown, + architecture: NativeRuntimeQualificationArchitecture, +): NativeRuntimeQualificationRunnerContract { + const contract = record(value, "Native runtime qualification runner contract"); + exactKeys( + contract, + ["schemaVersion", "kind", "architecture", "gpuProbeImageRef", "nim", "vllm"], + "Native runtime qualification runner contract", + ); + if ( + contract.schemaVersion !== 1 || + contract.kind !== "nemoclaw-native-runtime-qualification-runner-v1" || + contract.architecture !== architecture || + typeof contract.gpuProbeImageRef !== "string" || + !OCI_DIGEST.test(contract.gpuProbeImageRef) + ) { + throw new Error("Native runtime qualification runner contract identity is invalid"); + } + const nim = exactRunnerRuntime(contract.nim, "NIM", "cachePath"); + const vllm = exactRunnerRuntime(contract.vllm, "vLLM", "modelPath"); + return Object.freeze({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture, + gpuProbeImageRef: contract.gpuProbeImageRef, + nim: Object.freeze({ + imageRef: nim.imageRef, + model: nim.model, + cachePath: nim.cachePath, + }), + vllm: Object.freeze({ + imageRef: vllm.imageRef, + model: vllm.model, + modelPath: vllm.cachePath, + }), + }); +} + +export function readNativeRuntimeQualificationRunnerContract( + architecture: NativeRuntimeQualificationArchitecture, + file = NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT, +): NativeRuntimeQualificationRunnerContract { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== 0n || + (before.mode & 0o022n) !== 0n || + before.size < 1n || + before.size > 65_536n + ) { + throw new Error("runner contract must be a bounded root-owned regular file"); + } + const bytes = readFileSync(descriptor); + const after = fstatSync(descriptor, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mode !== after.mode || + before.uid !== after.uid || + before.gid !== after.gid || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) { + throw new Error("runner contract changed during its stable read"); + } + return parseNativeRuntimeQualificationRunnerContract( + JSON.parse(bytes.toString("utf8")) as unknown, + architecture, + ); + } catch (error) { + throw new Error( + `Native runtime qualification runner contract is invalid: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +export function nativeRuntimeQualificationAgentImage( + architecture: NativeRuntimeQualificationArchitecture, + agent: NativeRuntimeQualificationAgent, +): string { + return AGENT_IMAGES[architecture][agent]; +} + +export function nativeRuntimeQualificationInferenceImage(input: { + readonly architecture: NativeRuntimeQualificationArchitecture; + readonly acceleration: NativeRuntimeQualificationAcceleration; + readonly inference: NativeRuntimeQualificationInference; + readonly runnerContract?: NativeRuntimeQualificationRunnerContract; +}): { + readonly imageRef: string; + readonly model: string; + readonly cachePath?: string; +} { + if (input.inference === "ollama") { + return Object.freeze({ + imageRef: OLLAMA_IMAGES[input.architecture], + model: "qwen3:0.6b", + }); + } + if (input.acceleration !== "nvidia-gpu" || !input.runnerContract) { + throw new Error(`${input.inference} qualification requires the reviewed GPU runner contract`); + } + if (input.inference === "nim") { + return Object.freeze({ + imageRef: input.runnerContract.nim.imageRef, + model: input.runnerContract.nim.model, + cachePath: input.runnerContract.nim.cachePath, + }); + } + return Object.freeze({ + imageRef: input.runnerContract.vllm.imageRef, + model: input.runnerContract.vllm.model, + cachePath: input.runnerContract.vllm.modelPath, + }); +} + +export function digestFromImageReference(reference: string): string { + if (!OCI_DIGEST.test(reference)) throw new Error("Managed image reference is not immutable"); + return reference.slice(reference.lastIndexOf("@") + 1); +} diff --git a/test/e2e/live/native-runtime-qualification-case.test.ts b/test/e2e/live/native-runtime-qualification-case.test.ts new file mode 100644 index 00000000000..b208bfaad0b --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case.test.ts @@ -0,0 +1,1077 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + capturePodmanSocketAuthority, + createPodmanContainerEngine, + type PodmanBoundContainerEngine, +} from "../../../src/lib/adapters/podman/index.ts"; +import type { RuntimeProviderLifecycleInput } from "../../../src/lib/onboard/runtime-provider/contract.ts"; +import { createPodmanRuntimeProviderBundle } from "../../../src/lib/onboard/runtime-provider/podman.ts"; +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE, + PODMAN_SANDBOX_NAMESPACE_LABEL, + PODMAN_SANDBOX_WORKSPACE, + PODMAN_SANDBOX_WORKSPACE_LABEL, +} from "../../../src/lib/onboard/runtime-provider/podman-lifecycle.ts"; +import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; +import type { NativeRuntimeQualificationObligation } from "../registry/native-runtime-qualification.ts"; +import { + assertCredentialFreeQualificationEnvironment, + digestFromImageReference, + nativeRuntimeQualificationAgentImage, + nativeRuntimeQualificationInferenceImage, + parseNativeRuntimeQualificationRow, + readNativeRuntimeQualificationRunnerContract, +} from "./native-runtime-qualification-case-helpers.ts"; + +const ENABLED = + process.env.NEMOCLAW_RUN_LIVE_E2E === "1" && + typeof process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW === "string"; +const FULL_ID = /^[a-f0-9]{64}$/u; +const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; +const COMMAND_TIMEOUT = 60_000; +const INFERENCE_TIMEOUT = 900_000; +const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const E2E_PHASES = [ + "validate credential-free Docker-unavailable isolation", + "bind the rootless Podman engine", + "launch exact local inference", + "onboard the managed agent image", + "exercise sandbox lifecycle and state recovery", + "restart and reconcile inference", + "prove exact cleanup", + "emit bounded case evidence", +] as const; + +interface PodmanNetworkAuthority { + readonly id: string; + readonly name: string; + readonly gateway: string; +} + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +interface GpuComputeProcess { + readonly gpuUuid: string; + readonly pid: number; + readonly processName: string; + readonly usedMemoryMiB: number; +} + +function bounded(value: string): string { + return value.replace(CONTROL, " ").replace(/\s+/gu, " ").trim().slice(-500); +} + +function command(command: string, args: readonly string[]): CommandResult { + const result = spawnSync(command, [...args], { + encoding: "utf8", + env: process.env, + timeout: 10_000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + }); + return { + status: result.status ?? (result.signal ? 128 : 127), + stdout: result.stdout ?? "", + stderr: result.stderr ?? result.error?.message ?? "", + }; +} + +function requireCommand(executable: string, args: readonly string[], label: string): string { + const result = command(executable, args); + if (result.status !== 0) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout)}`, + ); + } + return result.stdout.trim(); +} + +function capture( + engine: PodmanBoundContainerEngine, + args: readonly string[], + label: string, + timeout = COMMAND_TIMEOUT, +): string { + const result = engine.capture(args, timeout); + if (result.status !== 0 || result.error) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout || result.error?.message || "unknown failure")}`, + ); + } + return result.stdout.trim(); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function exactDirectory(directory: string): void { + const metadata = fs.lstatSync(directory); + const uid = process.getuid?.() ?? -1; + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + metadata.uid !== uid || + (metadata.mode & 0o077) !== 0 + ) { + throw new Error("Qualification receipt directory must be private and current-user owned"); + } +} + +function writeJson(directory: string, file: string, value: unknown): void { + const target = path.join(directory, file); + const temporary = `${target}.tmp`; + const serialized = `${JSON.stringify(value, null, 2)}\n`; + fs.writeFileSync(temporary, serialized, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, target); +} + +function operationFile(id: NativeRuntimeQualificationObligation): string { + return `operation-${id.replaceAll(".", "-")}.json`; +} + +function assertDockerUnavailable(): Record { + const guarded = command("docker", ["version"]); + if (guarded.status !== 97) { + throw new Error(`Docker PATH invocation guard returned ${String(guarded.status)}, expected 97`); + } + for (const executable of ["/usr/bin/docker", "/usr/local/bin/docker", "/snap/bin/docker"]) { + if (!fs.existsSync(executable)) continue; + const result = command(executable, ["version"]); + if (result.status === 0) + throw new Error(`Absolute Docker client remained usable: ${executable}`); + } + for (const socket of ["/var/run/docker.sock", "/run/docker.sock"]) { + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) throw new Error(`Docker socket remained available: ${socket}`); + } + for (const unit of ["docker.service", "docker.socket"]) { + if (command("systemctl", ["is-active", "--quiet", unit]).status === 0) { + throw new Error(`Docker unit remained active: ${unit}`); + } + } + const proc = fs.readdirSync("/proc").filter((entry) => /^[1-9][0-9]*$/u.test(entry)); + for (const pid of proc) { + try { + if (fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim() === "dockerd") { + throw new Error("Docker daemon process remained active"); + } + } catch (error) { + if (error instanceof Error && error.message === "Docker daemon process remained active") { + throw error; + } + } + } + return { + dockerCommandGuarded: true, + dockerServiceInactive: true, + dockerSocketUnitInactive: true, + dockerdProcessNameAbsent: true, + defaultSocketPathsAbsent: true, + }; +} + +async function waitForSocket(socket: string, child: ChildProcess): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("Rootless Podman API service exited before its socket became ready"); + } + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("Rootless Podman API service did not create its socket"); +} + +function startPodmanQualificationService(socket: string, progress: TestProgress): ChildProcess { + return spawnObservedChild("podman", ["system", "service", "--time=0", `unix://${socket}`], { + activityLabel: "command: rootless Podman qualification service", + progress, + spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, + }); +} + +async function stopService(child: ChildProcess | null, socket: string): Promise { + if (child && child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } + fs.rmSync(socket, { force: true }); +} + +function createProviderNetwork( + engine: PodmanBoundContainerEngine, + name: string, + caseId: string, +): PodmanNetworkAuthority { + const id = capture( + engine, + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], + "provider network creation", + ); + if (!FULL_ID.test(id)) throw new Error("Provider network did not return a full immutable ID"); + const inspected = JSON.parse( + capture(engine, ["network", "inspect", id], "provider network inspection"), + ) as Array<{ + id?: unknown; + name?: unknown; + subnets?: Array<{ gateway?: unknown }>; + }>; + const entry = inspected[0]; + const gateway = entry?.subnets?.[0]?.gateway; + if ( + inspected.length !== 1 || + entry?.id !== id || + entry.name !== name || + typeof gateway !== "string" + ) { + throw new Error("Provider network inspection lacks exact identity"); + } + return Object.freeze({ id, name, gateway }); +} + +function pullPublicImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["pull", imageRef], `pull ${imageRef}`, INFERENCE_TIMEOUT); + capture(engine, ["image", "exists", imageRef], `inspect pulled image ${imageRef}`); +} + +function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); +} + +function rootOwnedReadOnlyDirectory(directory: string): void { + const canonical = fs.realpathSync(directory); + if (canonical !== directory) { + throw new Error(`Runner model resource is not canonical: ${directory}`); + } + const boundary = "/var/lib/nemoclaw/native-runtime-qualification"; + let current = directory; + while (current.startsWith(`${boundary}/`) || current === boundary) { + const metadata = fs.lstatSync(current); + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + metadata.uid !== 0 || + (metadata.mode & 0o022) !== 0 + ) { + throw new Error( + `Runner model resource is not an exact root-owned read-only directory: ${current}`, + ); + } + if (current === boundary) return; + current = path.dirname(current); + } + throw new Error(`Runner model resource escapes its reviewed root: ${directory}`); +} + +function proveGpuDevices( + engine: PodmanBoundContainerEngine, + probeImageRef: string, +): readonly string[] { + const devices = capture( + engine, + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + probeImageRef, + "nvidia-smi", + "--query-gpu=uuid", + "--format=csv,noheader", + ], + "NVIDIA CDI runtime proof", + ) + .split(/\r?\n/u) + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); + if ( + devices.length === 0 || + new Set(devices).size !== devices.length || + devices.some((device) => !/^GPU-[0-9A-Fa-f-]{36}$/u.test(device)) + ) { + throw new Error("NVIDIA CDI runtime proof did not return exact physical GPU UUIDs"); + } + return Object.freeze(devices); +} + +function proveGpuBackedInference( + engine: PodmanBoundContainerEngine, + containerId: string, + selectedDevices: readonly string[], +): readonly GpuComputeProcess[] { + const output = capture( + engine, + [ + "exec", + containerId, + "nvidia-smi", + "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", + "--format=csv,noheader,nounits", + ], + "GPU-backed inference process proof", + ); + const processes = output + .split(/\r?\n/u) + .map((line) => line.split(",").map((field) => field.trim())) + .filter((fields) => fields.length === 4) + .map(([gpuUuid, pid, processName, usedMemoryMiB]) => ({ + gpuUuid: gpuUuid ?? "", + pid: Number(pid), + processName: processName ?? "", + usedMemoryMiB: Number(usedMemoryMiB), + })) + .filter( + (entry) => + selectedDevices.includes(entry.gpuUuid) && + Number.isSafeInteger(entry.pid) && + entry.pid > 0 && + entry.processName.length > 0 && + !/[\u0000-\u001f\u007f-\u009f]/u.test(entry.processName) && + Number.isSafeInteger(entry.usedMemoryMiB) && + entry.usedMemoryMiB > 0, + ); + if (processes.length === 0) { + throw new Error("Inference turn did not leave an exact GPU compute process proof"); + } + return Object.freeze(processes.map((entry) => Object.freeze(entry))); +} + +function createAgentContainer(input: { + readonly engine: PodmanBoundContainerEngine; + readonly imageRef: string; + readonly name: string; + readonly network: string; + readonly qualificationId: string; + readonly sandboxId: string; + readonly sandboxName: string; + readonly volume: string; +}): string { + const id = capture( + input.engine, + [ + "run", + "--detach", + "--pull=never", + "--name", + input.name, + "--network", + input.network, + "--label", + `${PODMAN_MANAGED_LABEL}=true`, + "--label", + `${PODMAN_SANDBOX_NAME_LABEL}=${input.sandboxName}`, + "--label", + `${PODMAN_SANDBOX_ID_LABEL}=${input.sandboxId}`, + "--label", + `${PODMAN_SANDBOX_NAMESPACE_LABEL}=${PODMAN_SANDBOX_NAMESPACE}`, + "--label", + `${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, + "--label", + `${QUALIFICATION_LABEL}=${input.qualificationId}`, + "--volume", + `${input.volume}:/qualification`, + "--entrypoint", + "/bin/sh", + input.imageRef, + "-c", + "while :; do sleep 3600; done", + ], + "agent container creation", + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(id)) throw new Error("Agent container did not return a full immutable ID"); + return id; +} + +async function agentTurn( + engine: PodmanBoundContainerEngine, + containerId: string, + endpoint: string, + model: string, +): Promise { + const body = JSON.stringify({ + model, + messages: [{ role: "user", content: "Reply with the single word qualified." }], + max_tokens: 32, + stream: false, + }); + const args = [ + "exec", + containerId, + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--connect-timeout", + "5", + "--max-time", + "60", + "--header", + "Content-Type: application/json", + "--data-binary", + body, + `${endpoint}/v1/chat/completions`, + ]; + const deadline = Date.now() + 600_000; + let output = ""; + let lastFailure = "inference request was not attempted"; + while (Date.now() < deadline) { + const result = engine.capture(args, 90_000); + if (result.status === 0 && !result.error) { + output = result.stdout.trim(); + break; + } + lastFailure = bounded(result.stderr || result.stdout || result.error?.message || "failed"); + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + if (!output) throw new Error(`Agent inference turn did not become ready: ${lastFailure}`); + const response = JSON.parse(output) as { + model?: unknown; + choices?: Array<{ + finish_reason?: unknown; + message?: { content?: unknown; tool_calls?: unknown }; + }>; + }; + const first = response.choices?.[0]; + if ( + response.model !== model || + typeof first?.finish_reason !== "string" || + first.finish_reason === "length" || + (typeof first.message?.content !== "string" && !Array.isArray(first.message?.tool_calls)) + ) { + throw new Error("Agent turn did not return a complete exact-model inference response"); + } + return sha256(output); +} + +function lifecycleInput(agent: string, sandboxName: string): RuntimeProviderLifecycleInput { + const sandbox: SandboxEntry = { + agent, + name: sandboxName, + openshellDriver: "podman", + }; + return { + environment: process.env, + log: () => undefined, + sandbox, + sandboxName, + }; +} + +function assertNoQualificationResidue(engine: PodmanBoundContainerEngine, caseId: string): void { + for (const [resource, args] of [ + ["container", ["ps", "--all", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["volume", ["volume", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["network", ["network", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ] as const) { + if (capture(engine, args, `qualification ${resource} residue inspection`) !== "") { + throw new Error(`Qualification cleanup left an owned ${resource}`); + } + } +} + +test.skipIf(!ENABLED)( + "executes one exact credential-free native runtime qualification case", + { meta: { e2ePhases: E2E_PHASES }, timeout: 1_800_000 }, + async ({ progress }) => { + progress.phase("validate credential-free Docker-unavailable isolation"); + assertCredentialFreeQualificationEnvironment(process.env); + expect(process.platform).toBe("linux"); + const uid = process.getuid?.() ?? 0; + expect(uid, "Native runtime qualification must execute as an unprivileged UID").toBeGreaterThan( + 0, + ); + const row = parseNativeRuntimeQualificationRow( + process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW ?? "", + ); + const expectedArchitecture = process.arch === "x64" ? "amd64" : process.arch; + expect(expectedArchitecture).toBe(row.case.architecture); + const receiptPath = process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT ?? ""; + expect(path.basename(receiptPath)).toBe("execution.json"); + const receiptDirectory = path.dirname(receiptPath); + exactDirectory(receiptDirectory); + + const dockerBefore = assertDockerUnavailable(); + const runtimeDirectory = process.env.XDG_RUNTIME_DIR ?? ""; + expect(runtimeDirectory).toBe(`/run/user/${String(uid)}`); + const socket = path.join(runtimeDirectory, "podman", "podman.sock"); + fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); + + let service: ChildProcess | null = startPodmanQualificationService(socket, progress); + let hostEngine: PodmanBoundContainerEngine | null = null; + let inferenceEngine: PodmanBoundContainerEngine | null = null; + let lifecycleEngine: PodmanBoundContainerEngine | null = null; + const ownedContainers = new Set(); + const ownedVolumes = new Set(); + const ownedNetworks = new Set(); + let inferenceContainerId: string | null = null; + let gpuDevices: readonly string[] = []; + let gpuComputeProcesses: readonly GpuComputeProcess[] = []; + let completed = false; + const operationDetails = new Map< + NativeRuntimeQualificationObligation, + Record + >(); + + try { + progress.phase("bind the rootless Podman engine"); + await waitForSocket(socket, service); + const socketAuthority = capturePodmanSocketAuthority(socket); + hostEngine = createPodmanContainerEngine({ + operation: "host-doctor", + socketAuthority, + }); + inferenceEngine = createPodmanContainerEngine({ + operation: "host-local-inference", + socketAuthority, + }); + lifecycleEngine = createPodmanContainerEngine({ + operation: "sandbox-lifecycle", + socketAuthority, + }); + const bundle = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: hostEngine, + sandboxLifecycle: lifecycleEngine, + }, + }); + expect(bundle.identity.id).toBe("podman"); + expect(bundle.workload.profile.support).toBeNull(); + expect(bundle.preflightDoctor.inspectHost()).toMatchObject({ + status: "ok", + }); + const caseSuffix = sha256(row.id).slice(0, 12); + const networkName = `nemoclaw-q-${caseSuffix}`; + const network = createProviderNetwork(inferenceEngine, networkName, row.id); + ownedNetworks.add(network.id); + const hostPort = 20_000 + (Number.parseInt(caseSuffix.slice(0, 4), 16) % 20_000); + const runnerContract = + row.case.acceleration === "nvidia-gpu" + ? readNativeRuntimeQualificationRunnerContract(row.case.architecture) + : undefined; + const agentImage = nativeRuntimeQualificationAgentImage( + row.case.architecture, + row.case.agent, + ); + const inference = nativeRuntimeQualificationInferenceImage({ + architecture: row.case.architecture, + acceleration: row.case.acceleration, + inference: row.case.inference, + ...(runnerContract ? { runnerContract } : {}), + }); + pullPublicImage(inferenceEngine, agentImage); + if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); + else { + requirePreloadedImage(inferenceEngine, inference.imageRef); + rootOwnedReadOnlyDirectory(inference.cachePath ?? ""); + } + if (runnerContract) { + requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const inferenceName = `nemoclaw-inference-${caseSuffix}`; + const endpoint = `http://${network.gateway}:${String(hostPort)}`; + progress.phase("launch exact local inference"); + const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; + const inferenceArguments = [ + "run", + "--detach", + "--pull=never", + "--name", + inferenceName, + "--network", + network.name, + "--publish", + `127.0.0.1:${String(hostPort)}:${String(inferencePort)}`, + "--publish", + `${network.gateway}:${String(hostPort)}:${String(inferencePort)}`, + "--label", + `${QUALIFICATION_LABEL}=${row.id}`, + ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), + ...(row.case.inference === "nim" + ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/opt/nim/.cache:ro`] + : []), + ...(row.case.inference === "vllm" + ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/models:ro`] + : []), + inference.imageRef, + ...(row.case.inference === "vllm" + ? [ + "--model", + "/models", + "--served-model-name", + inference.model, + "--host", + "0.0.0.0", + "--port", + String(inferencePort), + "--max-model-len", + "2048", + ] + : []), + ]; + inferenceContainerId = capture( + inferenceEngine, + inferenceArguments, + `${row.case.inference} container start`, + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(inferenceContainerId)) { + throw new Error("Inference container did not return a full immutable ID"); + } + ownedContainers.add(inferenceContainerId); + if (row.case.inference === "ollama") { + capture( + inferenceEngine, + ["exec", inferenceContainerId, "ollama", "pull", inference.model], + "Ollama model acquisition", + INFERENCE_TIMEOUT, + ); + } + if (row.case.acceleration === "nvidia-gpu") { + if (!runnerContract) throw new Error("GPU runner contract is unavailable"); + gpuDevices = proveGpuDevices(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const sandboxId = caseSuffix; + progress.phase("onboard the managed agent image"); + const sandboxName = `qualification-${row.case.agent}`; + const agentName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; + const volumeName = `nemoclaw-q-state-${caseSuffix}`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "agent volume creation", + ); + ownedVolumes.add(volumeName); + let agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedContainers.add(agentId); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' qualified >/qualification/state"], + "agent state initialization", + ); + operationDetails.set("agent.onboard", { + containerId: agentId, + agent: row.case.agent, + imageDigest: digestFromImageReference(agentImage), + }); + + const turnSha256 = await agentTurn(lifecycleEngine, agentId, endpoint, inference.model); + if (row.case.acceleration === "nvidia-gpu") { + gpuComputeProcesses = proveGpuBackedInference( + inferenceEngine, + inferenceContainerId, + gpuDevices, + ); + } + operationDetails.set("agent.turn", { + protocol: "openai-chat-completions", + model: inference.model, + responseSha256: turnSha256, + route: "provider-network-gateway", + }); + + if (!bundle.lifecycle.supported) throw new Error("Podman lifecycle surface is unavailable"); + progress.phase("exercise sandbox lifecycle and state recovery"); + const lifecycle = bundle.lifecycle; + const input = lifecycleInput(row.case.agent, sandboxName); + let beforeStopCalled = false; + expect( + lifecycle.stop(input, { + beforeStop: () => { + beforeStopCalled = true; + }, + }), + ).toMatchObject({ exitCode: 0, state: "stopped" }); + expect(beforeStopCalled).toBe(true); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + operationDetails.set("sandbox.stop-start", { + containerId: agentId, + executionPath: "runtime-provider-bundle", + stoppedAndStarted: true, + }); + + const snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); + expect(lifecycle.stop(input, { beforeStop: () => undefined })).toMatchObject({ exitCode: 0 }); + capture( + lifecycleEngine, + ["volume", "export", "--output", snapshot, volumeName], + "sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + const snapshotBytes = fs.readFileSync(snapshot); + const snapshotSha256 = sha256(snapshotBytes); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' drifted >/qualification/state"], + "sandbox state mutation", + ); + capture(lifecycleEngine, ["rm", "--force", agentId], "remove sandbox before rebuild"); + ownedContainers.delete(agentId); + capture(lifecycleEngine, ["volume", "rm", volumeName], "remove sandbox volume"); + ownedVolumes.delete(volumeName); + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "recreate sandbox volume", + ); + ownedVolumes.add(volumeName); + capture( + lifecycleEngine, + ["volume", "import", volumeName, snapshot], + "restore sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedContainers.add(agentId); + expect( + capture( + lifecycleEngine, + ["exec", agentId, "cat", "/qualification/state"], + "restored state", + ), + ).toBe("qualified"); + operationDetails.set("sandbox.snapshot-restore", { + snapshotSha256, + restoredStateSha256: sha256("qualified\n"), + }); + operationDetails.set("sandbox.rebuild", { + priorContainerReplaced: true, + rebuiltContainerId: agentId, + preservedState: true, + }); + + const focusedResults: Record = Object.create(null); + if (row.focusedOperations.length > 0) { + const cloneVolume = `${volumeName}-clone`; + const cloneName = `${agentName}-clone`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, cloneVolume], + "clone volume creation", + ); + ownedVolumes.add(cloneVolume); + capture( + lifecycleEngine, + ["volume", "import", cloneVolume, snapshot], + "clone volume restore", + INFERENCE_TIMEOUT, + ); + const cloneId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: cloneName, + network: network.name, + qualificationId: row.id, + sandboxId: `${sandboxId}c`, + sandboxName: `${sandboxName}-clone`, + volume: cloneVolume, + }); + ownedContainers.add(cloneId); + expect( + capture(lifecycleEngine, ["exec", cloneId, "cat", "/qualification/state"], "clone state"), + ).toBe("qualified"); + const duplicate = lifecycleEngine.capture([ + "run", + "--detach", + "--pull=never", + "--name", + agentName, + "--entrypoint", + "/bin/sh", + agentImage, + "-c", + "exit 0", + ]); + if (duplicate.status === 0) throw new Error("Podman allowed unsafe managed-name reuse"); + capture(lifecycleEngine, ["kill", "--signal", "KILL", agentId], "sandbox crash injection"); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + expect( + capture( + lifecycleEngine, + ["exec", agentId, "cat", "/qualification/state"], + "recovered state", + ), + ).toBe("qualified"); + focusedResults.clone = { cloneContainerId: cloneId, restored: true }; + focusedResults.backup = { + sha256: snapshotSha256, + bytes: snapshotBytes.length, + }; + focusedResults["crash-recovery"] = { + signal: "SIGKILL", + recovered: true, + }; + focusedResults.rollback = { restoredSnapshotSha256: snapshotSha256 }; + focusedResults["name-reuse"] = { rejected: true }; + } + + if (!inferenceContainerId) throw new Error("Inference runtime identity is missing"); + progress.phase("restart and reconcile inference"); + capture( + inferenceEngine, + ["restart", inferenceContainerId], + `${row.case.inference} runtime restart`, + INFERENCE_TIMEOUT, + ); + const reconciledTurnSha256 = await agentTurn( + lifecycleEngine, + agentId, + endpoint, + inference.model, + ); + const reconciledGpuComputeProcesses = + row.case.acceleration === "nvidia-gpu" + ? proveGpuBackedInference(inferenceEngine, inferenceContainerId, gpuDevices) + : []; + operationDetails.set("runtime.restart-reconcile", { + service: row.case.inference, + runtimeIdentity: inferenceContainerId, + responseSha256: reconciledTurnSha256, + gpuComputeProcesses: reconciledGpuComputeProcesses, + revalidated: true, + }); + + operationDetails.set("installer.install", { + authority: "trusted-installer-step", + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + }); + const rootfulSelectionDenied = row.rootModes.includes("rootful") + ? command("podman", ["--root", "/var/lib/containers/storage", "info"]).status !== 0 + : true; + if (!rootfulSelectionDenied) { + throw new Error( + "Unprivileged qualification unexpectedly obtained a rootful Podman storage authority", + ); + } + operationDetails.set("runtime.docker-unavailable", { + beforeCandidate: dockerBefore, + rootfulSelectionDenied, + executedRootMode: "rootless", + }); + + progress.phase("prove exact cleanup"); + capture(lifecycleEngine, ["rm", "--force", agentId], "agent cleanup"); + ownedContainers.delete(agentId); + for (const containerId of [...ownedContainers]) { + if (containerId === inferenceContainerId) continue; + capture(lifecycleEngine, ["rm", "--force", containerId], "focused container cleanup"); + ownedContainers.delete(containerId); + } + for (const volume of [...ownedVolumes]) { + capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); + ownedVolumes.delete(volume); + } + if (inferenceContainerId) { + capture( + inferenceEngine, + ["rm", "--force", inferenceContainerId], + "inference runtime cleanup", + ); + ownedContainers.delete(inferenceContainerId); + } + capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); + ownedNetworks.delete(network.id); + fs.rmSync(snapshot, { force: true }); + assertNoQualificationResidue(lifecycleEngine, row.id); + operationDetails.set("cleanup.exact", { + containersRemaining: 0, + volumesRemaining: 0, + networksRemaining: 0, + }); + + if (row.focusedOperations.length > 0) { + Object.assign(focusedResults, { + restart: operationDetails.get("runtime.restart-reconcile"), + rebuild: operationDetails.get("sandbox.rebuild"), + "snapshot-restore": operationDetails.get("sandbox.snapshot-restore"), + installer: operationDetails.get("installer.install"), + cleanup: operationDetails.get("cleanup.exact"), + }); + const missing = row.focusedOperations.filter( + (operation) => !Object.hasOwn(focusedResults, operation), + ); + if (missing.length > 0) { + throw new Error(`Focused qualification operations are incomplete: ${missing.join(", ")}`); + } + } + + const dockerAfter = assertDockerUnavailable(); + progress.phase("emit bounded case evidence"); + const podmanVersion = requireCommand("podman", ["--version"], "Podman version"); + const managedImages = [ + { role: "agent", digest: digestFromImageReference(agentImage) }, + { + role: "inference", + digest: digestFromImageReference(inference.imageRef), + }, + ...(runnerContract + ? [ + { + role: "gpu-probe", + digest: digestFromImageReference(runnerContract.gpuProbeImageRef), + }, + ] + : []), + ]; + writeJson(receiptDirectory, "runtime-result.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { + providerId: "podman", + executionPath: "runtime-provider-bundle", + rootMode: "rootless", + podmanVersion, + inferenceService: row.case.inference, + focusedOperations: focusedResults, + dockerBefore, + dockerAfter, + }, + }); + for (const obligation of row.case.obligations) { + const details = operationDetails.get(obligation); + if (!details) throw new Error(`Qualification operation '${obligation}' was not executed`); + writeJson(receiptDirectory, operationFile(obligation), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: obligation, + result: "passed", + details, + }); + } + if (row.case.acceleration === "nvidia-gpu") { + writeJson(receiptDirectory, "nvidia-cdi.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { + requested: "nvidia.com/gpu=all", + selectedDevices: gpuDevices, + inferenceRuntimeId: inferenceContainerId, + inferenceComputeProcesses: gpuComputeProcesses, + }, + }); + } + writeJson(receiptDirectory, "case-evidence.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: podmanVersion.replace(/^podman version\s+/u, ""), + managedImages, + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: operationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { + nvidiaCdi: { + device: "nvidia.com/gpu=all", + file: "nvidia-cdi.json", + }, + } + : {}), + }); + writeJson(receiptDirectory, "execution.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-execution-v1", + caseId: row.id, + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + agent: row.case.agent, + inference: row.case.inference, + rootModes: row.rootModes, + obligations: row.case.obligations, + focusedOperations: row.focusedOperations, + evidenceKinds: row.case.evidenceKinds, + dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, + credentialBoundary: { + githubCredentialsAbsent: true, + modelCredentialsAbsent: true, + isolatedUid: true, + }, + result: "passed", + }); + completed = true; + } finally { + if (!completed) { + if (lifecycleEngine) { + for (const containerId of ownedContainers) { + lifecycleEngine.capture(["rm", "--force", containerId], COMMAND_TIMEOUT); + } + for (const volume of ownedVolumes) { + lifecycleEngine.capture(["volume", "rm", "--force", volume], COMMAND_TIMEOUT); + } + } + if (inferenceEngine) { + for (const networkId of ownedNetworks) { + inferenceEngine.capture(["network", "rm", "--force", networkId], COMMAND_TIMEOUT); + } + } + } + await stopService(service, socket); + service = null; + } + }, +); diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts new file mode 100644 index 00000000000..d4f7e9b9c20 --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildNativeRuntimeQualificationProducerPlan, + type NativeRuntimeQualificationProducerPlanInput, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + assertCredentialFreeQualificationEnvironment, + digestFromImageReference, + nativeRuntimeQualificationAgentImage, + nativeRuntimeQualificationInferenceImage, + parseNativeRuntimeQualificationRow, + parseNativeRuntimeQualificationRunnerContract, +} from "../live/native-runtime-qualification-case-helpers.ts"; + +const SOURCE = { + repository: "NVIDIA/NemoClaw", + producerWorkflow: ".github/workflows/e2e.yaml", + pullRequestNumber: 9144, + candidateRepository: "NVIDIA/NemoClaw", + candidateSha: "a".repeat(40), + baseRef: "main", + baseSha: "b".repeat(40), + workflowSha: "b".repeat(40), + producerRunId: "123456", + producerRunAttempt: 1, + dispatchArtifact: { + id: "987654", + name: "e2e-dispatch-123456-1", + digest: `sha256:${"c".repeat(64)}`, + sizeInBytes: 4096, + }, +} as const; + +function row() { + return buildNativeRuntimeQualificationProducerPlan({ + source: SOURCE, + installerSha256: "d".repeat(64), + arm64GpuRunner: "native-arm64-gpu", + } satisfies NativeRuntimeQualificationProducerPlanInput).include[0]!; +} + +function runnerContract() { + return { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture: "amd64", + gpuProbeImageRef: `nvcr.io/nvidia/cuda@sha256:${"3".repeat(64)}`, + nim: { + imageRef: `nvcr.io/nim/nvidia/model@sha256:${"1".repeat(64)}`, + model: "nvidia/model", + cachePath: "/var/lib/nemoclaw/native-runtime-qualification/nim/cache", + }, + vllm: { + imageRef: `docker.io/vllm/vllm-openai@sha256:${"2".repeat(64)}`, + model: "qualification", + modelPath: "/var/lib/nemoclaw/native-runtime-qualification/vllm/model", + }, + } as const; +} + +describe("native runtime qualification case boundaries", () => { + it("accepts only an exact canonical trusted-plan row", () => { + const expected = row(); + expect(parseNativeRuntimeQualificationRow(JSON.stringify(expected))).toEqual(expected); + + const forged = JSON.parse(JSON.stringify(expected)) as Record; + forged.rootModes = ["rootless", "rootful"]; + expect(() => parseNativeRuntimeQualificationRow(JSON.stringify(forged))).toThrow( + "Root modes does not match", + ); + }); + + it("rejects credential and alternate runtime authority environment names", () => { + expect(() => + assertCredentialFreeQualificationEnvironment({ + HOME: "/tmp/home", + PATH: "/usr/bin", + }), + ).not.toThrow(); + for (const name of [ + "GITHUB_TOKEN", + "NGC_API_KEY", + "HF_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "SSH_AUTH_SOCK", + "DOCKER_CONFIG", + "DOCKER_HOST", + "CUSTOM_API_KEY", + ]) { + expect(() => assertCredentialFreeQualificationEnvironment({ [name]: "forbidden" })).toThrow( + name, + ); + } + }); + + it("accepts only typed immutable GPU runner resources", () => { + const parsed = parseNativeRuntimeQualificationRunnerContract(runnerContract(), "amd64"); + expect(parsed.nim.imageRef).toContain("@sha256:"); + expect(parsed.vllm.modelPath).toMatch(/^\/var\/lib\/nemoclaw\/native-runtime-qualification\//u); + + expect(() => + parseNativeRuntimeQualificationRunnerContract( + { + ...runnerContract(), + nim: { ...runnerContract().nim, command: ["bash", "-c", "id"] }, + }, + "amd64", + ), + ).toThrow("NIM runner contract fields are invalid"); + expect(() => + parseNativeRuntimeQualificationRunnerContract( + { + ...runnerContract(), + vllm: { ...runnerContract().vllm, modelPath: "/tmp/model" }, + }, + "amd64", + ), + ).toThrow("vLLM runner contract is invalid"); + }); + + it("pins every public case image to architecture-specific immutable digests", () => { + for (const architecture of ["amd64", "arm64"] as const) { + for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + expect(nativeRuntimeQualificationAgentImage(architecture, agent)).toMatch( + /@sha256:[a-f0-9]{64}$/u, + ); + } + const ollama = nativeRuntimeQualificationInferenceImage({ + architecture, + acceleration: "cpu", + inference: "ollama", + }); + expect(ollama).toMatchObject({ model: "qwen3:0.6b" }); + expect(digestFromImageReference(ollama.imageRef)).toMatch(/^sha256:[a-f0-9]{64}$/u); + } + }); + + it("requires the root-owned typed contract for NIM and vLLM", () => { + expect(() => + nativeRuntimeQualificationInferenceImage({ + architecture: "amd64", + acceleration: "nvidia-gpu", + inference: "nim", + }), + ).toThrow("reviewed GPU runner contract"); + const contract = parseNativeRuntimeQualificationRunnerContract(runnerContract(), "amd64"); + expect( + nativeRuntimeQualificationInferenceImage({ + architecture: "amd64", + acceleration: "nvidia-gpu", + inference: "vllm", + runnerContract: contract, + }), + ).toMatchObject({ model: "qualification" }); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-collector.test.ts b/test/e2e/support/native-runtime-qualification-collector.test.ts index 7ac90810263..cb8bf4526db 100644 --- a/test/e2e/support/native-runtime-qualification-collector.test.ts +++ b/test/e2e/support/native-runtime-qualification-collector.test.ts @@ -22,7 +22,7 @@ import type { NativeRuntimeQualificationEvidenceEnvelope } from "../registry/nat const REPOSITORY = "NVIDIA/NemoClaw"; const ACTOR = "maintainer"; -const WORKFLOW = ".github/workflows/native-runtime-qualification.yaml"; +const WORKFLOW = ".github/workflows/e2e.yaml"; const JOB_NAME = "Aggregate native runtime qualification evidence"; const ARTIFACT_NAME = "native-runtime-qualification-9143"; @@ -137,7 +137,7 @@ function githubFixture( [`repos/${REPOSITORY}/pulls/9143`, pull], [`repos/${REPOSITORY}/commits/main`, { sha: NATIVE_QUALIFICATION_BASE_SHA }], [ - `repos/${REPOSITORY}/actions/workflows/native-runtime-qualification.yaml`, + `repos/${REPOSITORY}/actions/workflows/e2e.yaml`, { id: 101, path: WORKFLOW, state: "active" }, ], [`repos/${REPOSITORY}/actions/runs/7001`, run], diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts new file mode 100644 index 00000000000..3e5e969e6f2 --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -0,0 +1,354 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + aggregateNativeRuntimeQualificationProducerEvidence, + NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE, +} from "../../../tools/e2e/native-runtime-qualification-producer-aggregate.mts"; +import { writeNativeRuntimeQualificationProducerEvidence } from "../../../tools/e2e/native-runtime-qualification-producer-evidence.mts"; +import { + buildNativeRuntimeQualificationProducerPlan, + type NativeRuntimeQualificationProducerPlanRow, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; + +const roots: string[] = []; +const INSTALLER = "#!/usr/bin/env bash\nexit 0\n"; +const INSTALLER_SHA256 = createHash("sha256").update(INSTALLER).digest("hex"); +const AGGREGATE_TEST_OPTIONS = { timeout: 15_000 } as const; + +function writeJson(file: string, value: unknown): void { + fs.writeFileSync(file, `${JSON.stringify(value)}\n`); +} + +function operationFile(id: string): string { + return `operation-${id.replaceAll(".", "-")}.json`; +} + +function installerReceipts( + directory: string, + row: NativeRuntimeQualificationProducerPlanRow, +): void { + fs.mkdirSync(directory); + fs.writeFileSync(path.join(directory, "installer.sh"), INSTALLER); + writeJson(path.join(directory, "invocation.json"), { + receiptVersion: 1, + script: "scripts/install.sh", + scriptSha256: INSTALLER_SHA256, + candidateSha: row.source.candidateSha, + architecture: row.case.architecture, + }); + writeJson(path.join(directory, "candidate-source.json"), { + receiptVersion: 1, + repository: "https://github.com/NVIDIA/NemoClaw.git", + revision: row.source.candidateSha, + installerSha256: INSTALLER_SHA256, + }); + writeJson(path.join(directory, "installed-source.json"), { + receiptVersion: 1, + repository: "https://github.com/NVIDIA/NemoClaw.git", + requestedRevision: row.source.candidateSha, + installedRevision: row.source.candidateSha, + installMode: "managed", + installerSha256: INSTALLER_SHA256, + }); + writeJson(path.join(directory, "architecture.json"), { + receiptVersion: 1, + requested: row.case.architecture, + runner: row.case.architecture, + }); + const posture = { + dockerCommandGuarded: true, + dockerEnvironmentVariablesUnset: true, + dockerServiceInactive: true, + dockerSocketUnitInactive: true, + dockerdProcessNameAbsent: true, + defaultSocketPathsAbsent: true, + }; + writeJson(path.join(directory, "docker-absence.json"), { + receiptVersion: 1, + preExecution: posture, + postExecution: posture, + }); +} + +function candidateReceipts( + directory: string, + row: NativeRuntimeQualificationProducerPlanRow, +): string { + fs.mkdirSync(directory); + const executionPath = path.join(directory, "execution.json"); + writeJson(executionPath, { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-execution-v1", + caseId: row.id, + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + agent: row.case.agent, + inference: row.case.inference, + rootModes: row.rootModes, + obligations: row.case.obligations, + focusedOperations: row.focusedOperations, + evidenceKinds: row.case.evidenceKinds, + dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, + credentialBoundary: { + githubCredentialsAbsent: true, + modelCredentialsAbsent: true, + isolatedUid: true, + }, + result: "passed", + }); + writeJson(path.join(directory, "runtime-result.json"), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { engineAuthority: `podman-sha256:${"9".repeat(64)}` }, + }); + for (const id of row.case.obligations) { + writeJson(path.join(directory, operationFile(id)), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: id, + result: "passed", + details: { proof: id }, + }); + } + if (row.case.acceleration === "nvidia-gpu") { + writeJson(path.join(directory, "nvidia-cdi.json"), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }); + } + writeJson(path.join(directory, "case-evidence.json"), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: "5.6.2", + managedImages: [ + { role: "agent", digest: `sha256:${"1".repeat(64)}` }, + { role: "inference", digest: `sha256:${"2".repeat(64)}` }, + ], + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: operationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { nvidiaCdi: { device: "nvidia.com/gpu=all", file: "nvidia-cdi.json" } } + : {}), + }); + return executionPath; +} + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-aggregate-")); + roots.push(root); + const plan = buildNativeRuntimeQualificationProducerPlan({ + source: { + repository: "NVIDIA/NemoClaw", + producerWorkflow: ".github/workflows/e2e.yaml", + pullRequestNumber: 9144, + candidateRepository: "NVIDIA/NemoClaw", + candidateSha: "a".repeat(40), + baseRef: "main", + baseSha: "b".repeat(40), + workflowSha: "b".repeat(40), + producerRunId: "7001", + producerRunAttempt: 1, + dispatchArtifact: { + id: "42", + name: "e2e-dispatch-7001-1", + digest: `sha256:${"c".repeat(64)}`, + sizeInBytes: 4096, + }, + }, + installerSha256: INSTALLER_SHA256, + arm64GpuRunner: "reviewed-arm64-gpu", + }); + const artifactRoot = path.join(root, "case-artifacts"); + fs.mkdirSync(artifactRoot); + for (const row of plan.include) { + const rowRoot = path.join(root, "rows", row.id); + fs.mkdirSync(rowRoot, { recursive: true }); + const installer = path.join(rowRoot, "installer"); + const candidate = path.join(rowRoot, "candidate"); + installerReceipts(installer, row); + const execution = candidateReceipts(candidate, row); + writeNativeRuntimeQualificationProducerEvidence( + row, + installer, + execution, + path.join(artifactRoot, row.artifactName), + ); + } + const evidenceDirectory = path.join(root, "aggregate"); + return { artifactRoot, evidenceDirectory, plan, root }; +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +describe("native runtime qualification producer aggregate", () => { + it( + "binds the exact 24-case cohort to one protected aggregate job", + AGGREGATE_TEST_OPTIONS, + () => { + const value = fixture(); + + const envelope = aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }); + + expect(envelope.cases).toHaveLength(24); + expect(new Set(envelope.cases.map((entry) => entry.caseId)).size).toBe(24); + expect( + envelope.cases.every( + (entry) => + entry.protectedRun.runId === 7001 && + entry.protectedRun.attempt === 1 && + entry.protectedRun.jobId === 811, + ), + ).toBe(true); + expect( + fs.existsSync( + path.join(value.evidenceDirectory, NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE), + ), + ).toBe(true); + }, + ); + + it("rejects an omitted case artifact", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + fs.renameSync( + path.join(value.artifactRoot, value.plan.include[0]!.artifactName), + path.join(value.root, "omitted"), + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("cohort is incomplete or mixed"); + }); + + it("rejects a symlink substituted for a candidate receipt", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const row = value.plan.include[0]!; + const artifact = path.join(value.artifactRoot, row.artifactName); + const receipt = path.join(artifact, "receipts", row.id, "runtime", "runtime-result.json"); + const target = path.join(value.root, "substituted.json"); + fs.renameSync(receipt, target); + fs.symlinkSync(target, receipt); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("cannot contain symlinks"); + }); + + it("rejects a trusted plan with a mixed source cohort", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const first = value.plan.include[0]!; + const mixedPlan = { + include: [ + { ...first, source: { ...first.source, candidateSha: "d".repeat(40) } }, + ...value.plan.include.slice(1), + ], + }; + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: mixedPlan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("source cohort"); + }); + + it("rejects an unexpected file in a case artifact", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + fs.writeFileSync( + path.join(value.artifactRoot, value.plan.include[0]!.artifactName, "candidate.log"), + "unexpected", + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("invalid files"); + }); + + it( + "rejects a receipt whose bytes no longer match its trusted fragment", + AGGREGATE_TEST_OPTIONS, + () => { + const value = fixture(); + const row = value.plan.include[0]!; + fs.appendFileSync( + path.join( + value.artifactRoot, + row.artifactName, + "receipts", + row.id, + "runtime", + "runtime-result.json", + ), + " ", + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("does not match its SHA-256 digest"); + }, + ); + + it("rejects an invalid aggregate job identity", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 0, + }), + ).toThrow("aggregate job id is invalid"); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts index 62babcb43aa..09a1e08a3bb 100644 --- a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts @@ -44,9 +44,11 @@ function fixture() { arm64GpuRunner: "reviewed-native-arm64-gpu-runner", }).include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; const installerDirectory = path.join(root, "installer"); - const executionPath = path.join(root, "execution.json"); + const executionDirectory = path.join(root, "candidate"); + const executionPath = path.join(executionDirectory, "execution.json"); const evidenceDirectory = path.join(root, "evidence"); fs.mkdirSync(installerDirectory); + fs.mkdirSync(executionDirectory); fs.writeFileSync(path.join(installerDirectory, "installer.sh"), INSTALLER); fs.writeFileSync( path.join(installerDirectory, "invocation.json"), @@ -121,6 +123,48 @@ function fixture() { result: "passed", }; fs.writeFileSync(executionPath, JSON.stringify(execution)); + const operationFile = (id: string) => `operation-${id.replaceAll(".", "-")}.json`; + fs.writeFileSync( + path.join(executionDirectory, "runtime-result.json"), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { endpointAuthority: `podman-sha256:${"f".repeat(64)}` }, + }), + ); + for (const id of row.case.obligations) { + fs.writeFileSync( + path.join(executionDirectory, operationFile(id)), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: id, + result: "passed", + details: { proof: id }, + }), + ); + } + fs.writeFileSync( + path.join(executionDirectory, "case-evidence.json"), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: "5.6.2", + managedImages: [ + { role: "agent", digest: `sha256:${"1".repeat(64)}` }, + { role: "inference", digest: `sha256:${"2".repeat(64)}` }, + ], + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ id, file: operationFile(id) })), + }), + ); return { evidenceDirectory, execution, executionPath, installerDirectory, root, row }; } @@ -139,21 +183,44 @@ describe("native runtime qualification producer evidence", () => { value.evidenceDirectory, ); - expect(fs.readdirSync(value.evidenceDirectory)).toEqual(["evidence.json"]); + expect(fs.readdirSync(value.evidenceDirectory).sort()).toEqual([ + "case-fragment.json", + "receipts", + ]); expect( - JSON.parse(fs.readFileSync(path.join(value.evidenceDirectory, "evidence.json"), "utf8")), - ).toEqual({ + JSON.parse(fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8")), + ).toMatchObject({ schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-case-evidence-v1", + kind: "nemoclaw-native-runtime-qualification-case-fragment-v1", qualificationId: "podman-protected-host-local-inference", providerId: "podman", source: value.row.source, case: value.row.case, - result: "passed", + installer: { + architecture: "amd64", + dockerAvailability: "unavailable", + exitCode: 0, + providerId: "podman", + }, + runtime: { + agent: "openclaw", + engineName: "Podman", + engineVersion: "5.6.2", + providerId: "podman", + }, }); - expect(fs.statSync(path.join(value.evidenceDirectory, "evidence.json")).mode & 0o777).toBe( - 0o600, - ); + expect(fs.statSync(path.join(value.evidenceDirectory, "case-fragment.json")).mode & 0o777).toBe(0o600); + expect( + fs.existsSync( + path.join( + value.evidenceDirectory, + "receipts", + value.row.id, + "installer", + "installer.sh", + ), + ), + ).toBe(true); }); it.each([ @@ -247,6 +314,20 @@ describe("native runtime qualification producer evidence", () => { value.executionPath, value.evidenceDirectory, ), - ).toThrow("receipt is missing or invalid"); + ).toThrow("receipt file is invalid"); + }); + + it("rejects an unexpected candidate-controlled receipt file", () => { + const value = fixture(); + fs.writeFileSync(path.join(path.dirname(value.executionPath), "candidate.log"), "candidate output"); + + expect(() => + writeNativeRuntimeQualificationProducerEvidence( + value.row, + value.installerDirectory, + value.executionPath, + value.evidenceDirectory, + ), + ).toThrow("receipt files are invalid"); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index d6cbd25fb79..5d330473112 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -81,6 +81,10 @@ describe("native runtime qualification producer workflow", () => { producer, "Prepare the credential-free execution account and disable Docker", ); + const dependencies = step( + producer, + "Install locked candidate test dependencies without scripts", + ); const installer = step(producer, "Run the authenticated installer qualification"); const execute = step(producer, "Execute the candidate qualification case without credentials"); const validate = step(producer, "Validate receipts and emit bounded evidence"); @@ -88,7 +92,6 @@ describe("native runtime qualification producer workflow", () => { const cleanup = step(producer, "Remove qualification resources"); const source = JSON.stringify(producer); const boundaryRun = boundary.run ?? ""; - const installerRun = installer.run ?? ""; expect(producer.name).toBe("${{ matrix.jobName }}"); expect(producer["runs-on"]).toBe("${{ matrix.runner }}"); @@ -100,20 +103,29 @@ describe("native runtime qualification producer workflow", () => { expect(boundaryRun.indexOf("printf 'account=%s")).toBeLessThan( boundaryRun.indexOf("useradd --create-home"), ); + expect(dependencies.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(dependencies.run).toContain("npm --prefix"); + expect(dependencies.run).toContain("ci --ignore-scripts"); expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); expect(installer.run).toContain("run-native-runtime-installer-qualification.sh"); - expect(installerRun.indexOf("pkill -KILL -u")).toBeLessThan( - installerRun.indexOf("chown -R -h root:root"), - ); + expect(installer.run).not.toContain("chown -R"); expect(installer.run).toContain('[[ -d "$INSTALLER_RECEIPT_PARENT/receipts" && ! -L'); expect(execute.run).toContain('sudo -u "$ACCOUNT" env -i'); expect(execute.run).toContain("native-runtime-qualification-case.test.ts"); expect(execute.run).not.toContain("GITHUB_TOKEN"); expect(execute.run).not.toContain("GH_TOKEN"); + expect(execute.run).not.toContain("chown -R"); + expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(execute.run).toContain( + 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', + ); + expect(validate.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(validate.run).toContain("sudo --preserve-env="); + expect(validate.run).toContain('"$NODE_DIRECTORY/node"'); expect(validate.run).toContain("native-runtime-qualification-producer-evidence.mts"); expect(upload.with).toMatchObject({ name: "${{ matrix.artifactName }}", - path: "${{ runner.temp }}/native-runtime-evidence/evidence.json", + path: "${{ runner.temp }}/native-runtime-evidence/", }); expect(cleanup.if).toBe("always()"); expect(cleanup.run).toContain('account="${ACCOUNT:-nemoclawq}"'); @@ -121,4 +133,42 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); }); + + it("aggregates the exact successful 24-case cohort in a separate trusted job", () => { + const aggregate = job("native-runtime-qualification-producer-aggregate"); + const download = step(aggregate, "Download the exact case evidence cohort"); + const identity = step(aggregate, "Resolve this aggregate job identity"); + const collect = step(aggregate, "Validate and aggregate all 24 case receipts"); + const upload = step(aggregate, "Upload the immutable aggregate evidence"); + + expect(aggregate.name).toBe("Aggregate native runtime qualification evidence"); + expect(aggregate.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-producer-plan", + "native-runtime-qualification-producer", + ]); + expect(aggregate.if).toContain( + "needs.native-runtime-qualification-producer.result == 'success'", + ); + expect(aggregate.permissions).toEqual({ + actions: "read", + contents: "read", + "pull-requests": "read", + }); + expect(download.with).toMatchObject({ + pattern: "native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*", + "merge-multiple": false, + }); + expect(identity.run).toContain('.status == "in_progress"'); + expect(identity.run).toContain("select(length == 1)"); + expect(collect.run).toContain("native-runtime-qualification-producer-aggregate.mts"); + expect(collect.env?.QUALIFICATION_PLAN).toBe( + "${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}", + ); + expect(upload.with).toMatchObject({ + name: "native-runtime-qualification-${{ inputs.checkout_sha }}", + path: "${{ runner.temp }}/native-runtime-aggregate/", + "if-no-files-found": "error", + }); + }); }); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 51863863d93..ab0360995d2 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -391,6 +391,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map; + +function record(value: unknown, label: string): UnknownRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as UnknownRecord; +} + +function exactKeys(value: UnknownRecord, keys: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label} fields are invalid`); + } +} + +function readBoundedBytes(file: string, maximum: number): Buffer { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const status = fstatSync(descriptor); + if (!status.isFile() || status.size < 1 || status.size > maximum) { + throw new Error(`Native runtime qualification aggregate input is invalid: ${file}`); + } + return readFileSync(descriptor); + } catch (error) { + if (error instanceof Error && error.message.startsWith("Native runtime qualification")) { + throw error; + } + throw new Error(`Native runtime qualification aggregate input is invalid: ${file}`); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function readJson(file: string, maximum = MAX_FRAGMENT_BYTES): unknown { + try { + return JSON.parse(readBoundedBytes(file, maximum).toString("utf8")) as unknown; + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Native runtime qualification aggregate input is not JSON: ${file}`); + } + throw error; + } +} + +function positiveInteger(value: string, label: string): number { + if (!POSITIVE_INTEGER.test(value)) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } + return parsed; +} + +function exactJson(actual: unknown, expected: unknown, label: string): void { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`Native runtime qualification ${label} does not match the trusted plan`); + } +} + +function assertDirectory(directory: string, label: string): void { + const status = lstatSync(directory, { throwIfNoEntry: false }); + if (!status?.isDirectory() || status.isSymbolicLink()) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } +} + +function walkRegularFiles(root: string): readonly string[] { + const files: string[] = []; + const visit = (directory: string, relative: string): void => { + assertDirectory(directory, "aggregate artifact directory"); + for (const name of readdirSync(directory).sort()) { + const child = path.join(directory, name); + const childRelative = relative ? `${relative}/${name}` : name; + const status = lstatSync(child); + if (status.isSymbolicLink()) { + throw new Error( + `Native runtime qualification aggregate input cannot contain symlinks: ${childRelative}`, + ); + } + if (status.isDirectory()) visit(child, childRelative); + else if (status.isFile() && status.size >= 1 && status.size <= MAX_RECEIPT_BYTES) + files.push(childRelative); + else + throw new Error( + `Native runtime qualification aggregate input is invalid: ${childRelative}`, + ); + } + }; + visit(root, ""); + return Object.freeze(files); +} + +function expectedReceiptFiles(caseId: string, gpu: boolean): readonly string[] { + const operations = [ + "installer-install", + "runtime-docker-unavailable", + "agent-onboard", + "agent-turn", + "sandbox-stop-start", + "sandbox-snapshot-restore", + "sandbox-rebuild", + "runtime-restart-reconcile", + "cleanup-exact", + ].map((id) => `receipts/${caseId}/operations/operation-${id}.json`); + return Object.freeze([ + "case-fragment.json", + `receipts/${caseId}/installer/architecture.json`, + `receipts/${caseId}/installer/candidate-source.json`, + `receipts/${caseId}/installer/docker-absence.json`, + `receipts/${caseId}/installer/installed-source.json`, + `receipts/${caseId}/installer/installer.sh`, + `receipts/${caseId}/installer/invocation.json`, + `receipts/${caseId}/runtime/runtime-result.json`, + ...operations, + ...(gpu ? [`receipts/${caseId}/runtime/nvidia-cdi.json`] : []), + ]); +} + +function validateFragment( + value: unknown, + row: NativeRuntimeQualificationProducerPlan["include"][number], +): NativeRuntimeQualificationCaseFragment { + const fragment = record(value, `Native runtime qualification fragment '${row.id}'`); + exactKeys( + fragment, + [ + "schemaVersion", + "kind", + "qualificationId", + "providerId", + "source", + "case", + "installer", + "runtime", + "operations", + ...(row.case.acceleration === "nvidia-gpu" ? ["nvidiaCdi"] : []), + ], + `Native runtime qualification fragment '${row.id}'`, + ); + if ( + fragment.schemaVersion !== 1 || + fragment.kind !== "nemoclaw-native-runtime-qualification-case-fragment-v1" || + fragment.qualificationId !== "podman-protected-host-local-inference" || + fragment.providerId !== "podman" + ) { + throw new Error(`Native runtime qualification fragment '${row.id}' identity is invalid`); + } + exactJson(fragment.source, row.source, `fragment '${row.id}' source`); + exactJson(fragment.case, row.case, `fragment '${row.id}' case`); + return fragment as unknown as NativeRuntimeQualificationCaseFragment; +} + +function expectedSource( + plan: NativeRuntimeQualificationProducerPlan, + aggregateJobId: number, +): NativeRuntimeQualificationExpectedSource { + const first = plan.include[0]; + if (!first) throw new Error("Native runtime qualification plan is empty"); + return Object.freeze({ + repository: first.source.repository, + workflow: first.source.producerWorkflow, + pullRequestNumber: first.source.pullRequestNumber, + candidateRepository: first.source.candidateRepository, + headSha: first.source.candidateSha, + baseRef: "main" as const, + baseSha: first.source.baseSha, + runId: positiveInteger(first.source.producerRunId, "producer run id"), + attempt: first.source.producerRunAttempt, + jobId: aggregateJobId, + // The aggregate job cannot know the GitHub artifact identity before upload. + // The collector replaces these placeholders with the independently resolved identity. + artifact: Object.freeze({ + id: 1, + name: "native-runtime-qualification-pre-upload", + digest: `sha256:${"0".repeat(64)}`, + }), + }); +} + +function caseEvidence( + fragment: NativeRuntimeQualificationCaseFragment, + source: NativeRuntimeQualificationExpectedSource, +): NativeRuntimeQualificationEvidenceEnvelope["cases"][number] { + return Object.freeze({ + schemaVersion: 1, + caseId: fragment.case.id, + protectedRun: Object.freeze({ + repository: source.repository, + workflow: source.workflow, + pullRequestNumber: source.pullRequestNumber, + candidateRepository: source.candidateRepository, + headSha: source.headSha, + baseRef: source.baseRef, + baseSha: source.baseSha, + runId: source.runId, + attempt: source.attempt, + jobId: source.jobId, + }), + installer: + fragment.installer as NativeRuntimeQualificationEvidenceEnvelope["cases"][number]["installer"], + runtime: + fragment.runtime as NativeRuntimeQualificationEvidenceEnvelope["cases"][number]["runtime"], + operations: fragment.operations, + ...(fragment.nvidiaCdi ? { nvidiaCdi: fragment.nvidiaCdi } : {}), + }); +} + +export function aggregateNativeRuntimeQualificationProducerEvidence(input: { + readonly plan: NativeRuntimeQualificationProducerPlan; + readonly caseArtifactRoot: string; + readonly evidenceDirectory: string; + readonly aggregateJobId: number; +}): NativeRuntimeQualificationEvidenceEnvelope { + const { plan } = input; + if (plan.include.length !== 24 || new Set(plan.include.map((row) => row.id)).size !== 24) { + throw new Error("Native runtime qualification aggregate requires the exact 24-case plan"); + } + const first = plan.include[0]!; + for (const row of plan.include) { + exactJson(row.source, first.source, `plan row '${row.id}' source cohort`); + } + if (!Number.isSafeInteger(input.aggregateJobId) || input.aggregateJobId < 1) { + throw new Error("Native runtime qualification aggregate job id is invalid"); + } + assertDirectory(input.caseArtifactRoot, "aggregate artifact root"); + const expectedDirectories = plan.include.map((row) => row.artifactName).sort(); + const actualDirectories = readdirSync(input.caseArtifactRoot).sort(); + if (JSON.stringify(actualDirectories) !== JSON.stringify(expectedDirectories)) { + throw new Error( + "Native runtime qualification aggregate artifact cohort is incomplete or mixed", + ); + } + if (lstatSync(input.evidenceDirectory, { throwIfNoEntry: false })) { + throw new Error("Native runtime qualification aggregate output must not already exist"); + } + const outputParent = path.dirname(input.evidenceDirectory); + assertDirectory(outputParent, "aggregate output parent"); + mkdirSync(input.evidenceDirectory, { mode: 0o700 }); + + const source = expectedSource(plan, input.aggregateJobId); + const cases: NativeRuntimeQualificationEvidenceEnvelope["cases"][number][] = []; + let totalBytes = 0; + const copiedPaths = new Set(); + for (const row of plan.include) { + const artifactDirectory = path.join(input.caseArtifactRoot, row.artifactName); + assertDirectory(artifactDirectory, `case artifact '${row.artifactName}'`); + const actualFiles = walkRegularFiles(artifactDirectory); + const expectedFiles = [ + ...expectedReceiptFiles(row.id, row.case.acceleration === "nvidia-gpu"), + ].sort(); + if (JSON.stringify([...actualFiles].sort()) !== JSON.stringify(expectedFiles)) { + throw new Error(`Native runtime qualification case artifact '${row.id}' has invalid files`); + } + const fragment = validateFragment( + readJson(path.join(artifactDirectory, "case-fragment.json")), + row, + ); + for (const relativePath of actualFiles.filter((file) => file !== "case-fragment.json")) { + if (copiedPaths.has(relativePath)) { + throw new Error(`Native runtime qualification aggregate repeats receipt '${relativePath}'`); + } + copiedPaths.add(relativePath); + const bytes = readBoundedBytes(path.join(artifactDirectory, relativePath), MAX_RECEIPT_BYTES); + totalBytes += bytes.length; + if (totalBytes > MAX_TOTAL_BYTES) { + throw new Error("Native runtime qualification aggregate receipts exceed their byte limit"); + } + const target = path.join(input.evidenceDirectory, relativePath); + mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + writeFileSync(target, bytes, { flag: "wx", mode: 0o600 }); + } + cases.push(caseEvidence(fragment, source)); + } + const envelope: NativeRuntimeQualificationEvidenceEnvelope = Object.freeze({ + schemaVersion: 1, + qualificationId: "podman-protected-host-local-inference", + providerId: "podman", + cases: Object.freeze(cases), + }); + const definition = compileNativeRuntimeQualification( + nativeRuntimeQualificationDefinition("podman"), + ); + consumeNativeRuntimeQualificationEvidence(definition, envelope, source, (receiptPath) => { + try { + return readBoundedBytes(path.join(input.evidenceDirectory, receiptPath), MAX_RECEIPT_BYTES); + } catch { + return null; + } + }); + writeFileSync( + path.join(input.evidenceDirectory, NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE), + `${JSON.stringify(envelope)}\n`, + { flag: "wx", mode: 0o600 }, + ); + return envelope; +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Native runtime qualification environment '${name}' is missing`); + return value; +} + +if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? "")) { + try { + if (process.argv.length !== 2) { + throw new Error("Usage: native-runtime-qualification-producer-aggregate.mts"); + } + const plan = JSON.parse( + requiredEnvironment("QUALIFICATION_PLAN"), + ) as NativeRuntimeQualificationProducerPlan; + aggregateNativeRuntimeQualificationProducerEvidence({ + plan, + caseArtifactRoot: requiredEnvironment("CASE_ARTIFACT_ROOT"), + evidenceDirectory: requiredEnvironment("EVIDENCE_DIRECTORY"), + aggregateJobId: positiveInteger(requiredEnvironment("AGGREGATE_JOB_ID"), "aggregate job id"), + }); + } catch (error) { + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/tools/e2e/native-runtime-qualification-producer-evidence.mts b/tools/e2e/native-runtime-qualification-producer-evidence.mts index a17f36095f2..7874cb95294 100644 --- a/tools/e2e/native-runtime-qualification-producer-evidence.mts +++ b/tools/e2e/native-runtime-qualification-producer-evidence.mts @@ -15,9 +15,14 @@ import { } from "node:fs"; import path from "node:path"; +import type { + NativeRuntimeQualificationObligation, + NativeRuntimeQualificationArtifactReceipt, +} from "../../test/e2e/registry/native-runtime-qualification.ts"; import type { NativeRuntimeQualificationProducerPlanRow } from "./native-runtime-qualification-producer-plan.mts"; const MAX_RECEIPT_BYTES = 65_536; +const MAX_INSTALLER_BYTES = 524_288; const MAX_RECEIPT_DIRECTORY_BYTES = 1_048_576; const EXPECTED_INSTALLER_FILES = [ "architecture.json", @@ -27,6 +32,15 @@ const EXPECTED_INSTALLER_FILES = [ "installer.sh", "invocation.json", ] as const; +const DETAIL_FILE = "case-evidence.json"; +const EXECUTION_FILE = "execution.json"; +const RUNTIME_FILE = "runtime-result.json"; +const CDI_FILE = "nvidia-cdi.json"; +const SHA256 = /^[a-f0-9]{64}$/u; +const IMAGE_DIGEST = /^sha256:[a-f0-9]{64}$/u; +const SAFE_ENGINE = /^[A-Za-z0-9][A-Za-z0-9 ._/-]{0,127}$/u; +const FORBIDDEN_RECEIPT_TEXT = + /(?:github_pat_|gh[pousr]_[A-Za-z0-9]{20}|nvapi-[A-Za-z0-9_-]{12}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----)/u; interface CaseExecutionReceipt { readonly schemaVersion: 1; @@ -54,14 +68,83 @@ interface CaseExecutionReceipt { readonly result: "passed"; } -function record(value: unknown, label: string): Record { +interface CandidateCaseDetails { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-case-details-v1"; + readonly caseId: string; + readonly runtime: { + readonly engineName: string; + readonly engineVersion: string; + readonly managedImages: readonly { + readonly role: string; + readonly digest: string; + }[]; + readonly resultFile: typeof RUNTIME_FILE; + }; + readonly operations: readonly { + readonly id: NativeRuntimeQualificationObligation; + readonly file: string; + }[]; + readonly nvidiaCdi?: { + readonly device: "nvidia.com/gpu=all"; + readonly file: typeof CDI_FILE; + }; +} + +export interface NativeRuntimeQualificationCaseFragment { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-case-fragment-v1"; + readonly qualificationId: string; + readonly providerId: string; + readonly source: NativeRuntimeQualificationProducerPlanRow["source"]; + readonly case: NativeRuntimeQualificationProducerPlanRow["case"]; + readonly installer: { + readonly providerId: string; + readonly architecture: string; + readonly dockerAvailability: "unavailable"; + readonly exitCode: 0; + readonly invocation: NativeRuntimeQualificationArtifactReceipt; + readonly script: NativeRuntimeQualificationArtifactReceipt; + }; + readonly runtime: { + readonly providerId: string; + readonly agent: string; + readonly inference: string; + readonly architecture: string; + readonly acceleration: string; + readonly rootMode: "rootless"; + readonly engineName: string; + readonly engineVersion: string; + readonly managedImages: readonly { + readonly role: string; + readonly digest: string; + }[]; + readonly result: NativeRuntimeQualificationArtifactReceipt; + }; + readonly operations: readonly { + readonly id: NativeRuntimeQualificationObligation; + readonly artifact: NativeRuntimeQualificationArtifactReceipt; + }[]; + readonly nvidiaCdi?: { + readonly device: "nvidia.com/gpu=all"; + readonly artifact: NativeRuntimeQualificationArtifactReceipt; + }; +} + +type UnknownRecord = Record; + +function record(value: unknown, label: string): UnknownRecord { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`${label} must be an object`); } - return value as Record; + return value as UnknownRecord; } -function exactKeys(value: Record, keys: readonly string[], label: string): void { +function exactKeys( + value: UnknownRecord, + keys: readonly string[], + label: string, +): void { const actual = Object.keys(value).sort(); const expected = [...keys].sort(); if (JSON.stringify(actual) !== JSON.stringify(expected)) { @@ -69,7 +152,11 @@ function exactKeys(value: Record, keys: readonly string[], labe } } -function exactStrings(actual: unknown, expected: readonly string[], label: string): void { +function exactStrings( + actual: unknown, + expected: readonly string[], + label: string, +): void { if ( !Array.isArray(actual) || actual.some((entry) => typeof entry !== "string") || @@ -79,71 +166,116 @@ function exactStrings(actual: unknown, expected: readonly string[], label: strin } } -function readBoundedFile(file: string, maximum = MAX_RECEIPT_BYTES): string { +function readBoundedBytes(file: string, maximum = MAX_RECEIPT_BYTES): Buffer { let descriptor: number | undefined; try { descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); const status = fstatSync(descriptor); if (!status.isFile() || status.size < 1 || status.size > maximum) { - throw new Error(`Native runtime qualification receipt is missing or invalid: ${file}`); + throw new Error( + `Native runtime qualification receipt is missing or invalid: ${file}`, + ); + } + const bytes = readFileSync(descriptor); + if (FORBIDDEN_RECEIPT_TEXT.test(bytes.toString("utf8"))) { + throw new Error( + `Native runtime qualification receipt contains credential material: ${file}`, + ); } - return readFileSync(descriptor, "utf8"); + return bytes; } catch (error) { - if (error instanceof Error && error.message.startsWith("Native runtime qualification")) { + if ( + error instanceof Error && + error.message.startsWith("Native runtime qualification") + ) { throw error; } - throw new Error(`Native runtime qualification receipt is missing or invalid: ${file}`); + throw new Error( + `Native runtime qualification receipt is missing or invalid: ${file}`, + ); } finally { if (descriptor !== undefined) closeSync(descriptor); } } -function readJson(file: string): unknown { +function parseJsonBytes(bytes: Buffer, file: string): unknown { try { - return JSON.parse(readBoundedFile(file)) as unknown; + return JSON.parse(bytes.toString("utf8")) as unknown; } catch (error) { if (error instanceof SyntaxError) { - throw new Error(`Native runtime qualification receipt is not valid JSON: ${file}`); + throw new Error( + `Native runtime qualification receipt is not valid JSON: ${file}`, + ); } throw error; } } -function validateDirectory(directory: string, expectedFiles: readonly string[]): void { +function validateDirectory( + directory: string, + expectedFiles: readonly string[], +): void { const status = lstatSync(directory, { throwIfNoEntry: false }); if (!status?.isDirectory() || status.isSymbolicLink()) { - throw new Error(`Native runtime qualification receipt directory is invalid: ${directory}`); + throw new Error( + `Native runtime qualification receipt directory is invalid: ${directory}`, + ); } const files = readdirSync(directory).sort(); if (JSON.stringify(files) !== JSON.stringify([...expectedFiles].sort())) { - throw new Error(`Native runtime qualification receipt files are invalid: ${directory}`); + throw new Error( + `Native runtime qualification receipt files are invalid: ${directory}`, + ); } let total = 0; for (const file of files) { const child = path.join(directory, file); const childStatus = lstatSync(child); - if (!childStatus.isFile() || childStatus.isSymbolicLink() || childStatus.size < 1) { - throw new Error(`Native runtime qualification receipt file is invalid: ${child}`); + if ( + !childStatus.isFile() || + childStatus.isSymbolicLink() || + childStatus.size < 1 + ) { + throw new Error( + `Native runtime qualification receipt file is invalid: ${child}`, + ); } total += childStatus.size; } if (total > MAX_RECEIPT_DIRECTORY_BYTES) { - throw new Error(`Native runtime qualification receipts exceed their size limit: ${directory}`); + throw new Error( + `Native runtime qualification receipts exceed their size limit: ${directory}`, + ); } } function validateInstallerReceipts( row: NativeRuntimeQualificationProducerPlanRow, directory: string, -) { +): Readonly> { validateDirectory(directory, EXPECTED_INSTALLER_FILES); + const receipts = Object.fromEntries( + EXPECTED_INSTALLER_FILES.map((file) => [ + file, + readBoundedBytes( + path.join(directory, file), + file === "installer.sh" ? MAX_INSTALLER_BYTES : MAX_RECEIPT_BYTES, + ), + ]), + ) as Record<(typeof EXPECTED_INSTALLER_FILES)[number], Buffer>; const invocation = record( - readJson(path.join(directory, "invocation.json")), + parseJsonBytes(receipts["invocation.json"], "invocation.json"), "Installer invocation", ); exactKeys( invocation, - ["receiptVersion", "script", "scriptSha256", "candidateSha", "architecture"], + [ + "receiptVersion", + "script", + "scriptSha256", + "candidateSha", + "architecture", + ], "Installer invocation", ); if ( @@ -153,26 +285,34 @@ function validateInstallerReceipts( invocation.candidateSha !== row.source.candidateSha || invocation.architecture !== row.case.architecture ) { - throw new Error("Native runtime qualification installer invocation is invalid"); + throw new Error( + "Native runtime qualification installer invocation is invalid", + ); } const architecture = record( - readJson(path.join(directory, "architecture.json")), + parseJsonBytes(receipts["architecture.json"], "architecture.json"), + "Installer architecture", + ); + exactKeys( + architecture, + ["receiptVersion", "requested", "runner"], "Installer architecture", ); - exactKeys(architecture, ["receiptVersion", "requested", "runner"], "Installer architecture"); if ( architecture.receiptVersion !== 1 || architecture.requested !== row.case.architecture || architecture.runner !== row.case.architecture ) { - throw new Error("Native runtime qualification installer architecture is invalid"); + throw new Error( + "Native runtime qualification installer architecture is invalid", + ); } const candidate = record( - readJson(path.join(directory, "candidate-source.json")), + parseJsonBytes(receipts["candidate-source.json"], "candidate-source.json"), "Installer candidate source", ); const installed = record( - readJson(path.join(directory, "installed-source.json")), + parseJsonBytes(receipts["installed-source.json"], "installed-source.json"), "Installed source", ); exactKeys( @@ -205,10 +345,12 @@ function validateInstallerReceipts( installed.installMode !== "managed" || installed.installerSha256 !== row.installerSha256 ) { - throw new Error("Native runtime qualification installer source identity is invalid"); + throw new Error( + "Native runtime qualification installer source identity is invalid", + ); } const docker = record( - readJson(path.join(directory, "docker-absence.json")), + parseJsonBytes(receipts["docker-absence.json"], "docker-absence.json"), "Installer Docker absence", ); exactKeys( @@ -228,26 +370,36 @@ function validateInstallerReceipts( const value = record(docker[phase], `Installer Docker absence ${phase}`); exactKeys(value, requiredDockerKeys, `Installer Docker absence ${phase}`); if (requiredDockerKeys.some((key) => value[key] !== true)) { - throw new Error("Native runtime qualification installer Docker absence is invalid"); + throw new Error( + "Native runtime qualification installer Docker absence is invalid", + ); } } if (docker.receiptVersion !== 1) { - throw new Error("Native runtime qualification installer Docker absence is invalid"); + throw new Error( + "Native runtime qualification installer Docker absence is invalid", + ); } - const installer = readBoundedFile(path.join(directory, "installer.sh"), 524_288); + const installer = receipts["installer.sh"]; if ( - !installer.startsWith("#!/") || + !installer.toString("utf8").startsWith("#!/") || createHash("sha256").update(installer).digest("hex") !== row.installerSha256 ) { - throw new Error("Native runtime qualification installer receipt is invalid"); + throw new Error( + "Native runtime qualification installer receipt is invalid", + ); } + return Object.freeze(receipts); } function validateCaseExecution( row: NativeRuntimeQualificationProducerPlanRow, value: unknown, ): CaseExecutionReceipt { - const receipt = record(value, "Native runtime qualification execution receipt"); + const receipt = record( + value, + "Native runtime qualification execution receipt", + ); exactKeys( receipt, [ @@ -270,15 +422,29 @@ function validateCaseExecution( ], "Native runtime qualification execution receipt", ); - const docker = record(receipt.dockerUnavailable, "Docker-unavailable execution receipt"); - const credentials = record(receipt.credentialBoundary, "Credential-boundary execution receipt"); - exactKeys(docker, ["beforeCandidate", "afterCandidate"], "Docker-unavailable execution receipt"); + const docker = record( + receipt.dockerUnavailable, + "Docker-unavailable execution receipt", + ); + const credentials = record( + receipt.credentialBoundary, + "Credential-boundary execution receipt", + ); + exactKeys( + docker, + ["beforeCandidate", "afterCandidate"], + "Docker-unavailable execution receipt", + ); exactKeys( credentials, ["githubCredentialsAbsent", "modelCredentialsAbsent", "isolatedUid"], "Credential-boundary execution receipt", ); - exactStrings(receipt.rootModes, row.rootModes, "Native runtime qualification root modes"); + exactStrings( + receipt.rootModes, + row.rootModes, + "Native runtime qualification root modes", + ); exactStrings( receipt.obligations, row.case.obligations, @@ -311,21 +477,238 @@ function validateCaseExecution( credentials.isolatedUid !== true || receipt.result !== "passed" ) { - throw new Error("Native runtime qualification execution receipt identity is invalid"); + throw new Error( + "Native runtime qualification execution receipt identity is invalid", + ); } return receipt as unknown as CaseExecutionReceipt; } +function operationFile(id: string): string { + return `operation-${id.replaceAll(".", "-")}.json`; +} + +function expectedCaseFiles( + row: NativeRuntimeQualificationProducerPlanRow, +): string[] { + return [ + DETAIL_FILE, + EXECUTION_FILE, + RUNTIME_FILE, + ...row.case.obligations.map(operationFile), + ...(row.case.acceleration === "nvidia-gpu" ? [CDI_FILE] : []), + ]; +} + +function validateEvidencePayload( + bytes: Buffer, + file: string, + expected: { + readonly caseId: string; + readonly kind: string; + readonly operationId?: string; + }, +): void { + const value = record( + parseJsonBytes(bytes, file), + `Candidate evidence '${path.basename(file)}'`, + ); + exactKeys( + value, + [ + "schemaVersion", + "kind", + "caseId", + ...(expected.operationId ? ["operationId"] : []), + "result", + "details", + ], + `Candidate evidence '${path.basename(file)}'`, + ); + record(value.details, `Candidate evidence '${path.basename(file)}' details`); + if ( + value.schemaVersion !== 1 || + value.kind !== expected.kind || + value.caseId !== expected.caseId || + value.result !== "passed" || + (expected.operationId !== undefined && + value.operationId !== expected.operationId) + ) { + throw new Error( + `Native runtime qualification candidate evidence is invalid: ${file}`, + ); + } +} + +function validateCandidateDetails( + row: NativeRuntimeQualificationProducerPlanRow, + directory: string, +): { + readonly details: CandidateCaseDetails; + readonly receipts: Readonly>; +} { + const expectedFiles = expectedCaseFiles(row); + validateDirectory(directory, expectedFiles); + const receipts = Object.fromEntries( + expectedFiles.map((file) => [ + file, + readBoundedBytes(path.join(directory, file)), + ]), + ) as Record; + const details = record( + parseJsonBytes(receipts[DETAIL_FILE]!, DETAIL_FILE), + "Candidate case details", + ); + exactKeys( + details, + [ + "schemaVersion", + "kind", + "caseId", + "runtime", + "operations", + ...(row.case.acceleration === "nvidia-gpu" ? ["nvidiaCdi"] : []), + ], + "Candidate case details", + ); + const runtime = record(details.runtime, "Candidate runtime details"); + exactKeys( + runtime, + ["engineName", "engineVersion", "managedImages", "resultFile"], + "Candidate runtime details", + ); + if ( + details.schemaVersion !== 1 || + details.kind !== "nemoclaw-native-runtime-qualification-case-details-v1" || + details.caseId !== row.id || + typeof runtime.engineName !== "string" || + !SAFE_ENGINE.test(runtime.engineName) || + typeof runtime.engineVersion !== "string" || + !SAFE_ENGINE.test(runtime.engineVersion) || + runtime.resultFile !== RUNTIME_FILE || + !Array.isArray(runtime.managedImages) || + runtime.managedImages.length < 2 || + runtime.managedImages.length > 8 + ) { + throw new Error( + "Native runtime qualification candidate runtime details are invalid", + ); + } + const roles = new Set(); + for (const entry of runtime.managedImages) { + const image = record(entry, "Candidate managed image"); + exactKeys(image, ["role", "digest"], "Candidate managed image"); + if ( + typeof image.role !== "string" || + !/^[a-z][a-z0-9-]{0,62}$/u.test(image.role) || + roles.has(image.role) || + typeof image.digest !== "string" || + !IMAGE_DIGEST.test(image.digest) + ) { + throw new Error( + "Native runtime qualification candidate managed image is invalid", + ); + } + roles.add(image.role); + } + if ( + !Array.isArray(details.operations) || + details.operations.length !== row.case.obligations.length + ) { + throw new Error( + "Native runtime qualification candidate operations are incomplete", + ); + } + const expectedOperations = row.case.obligations.map((id) => ({ + id, + file: operationFile(id), + })); + for (const [index, entry] of details.operations.entries()) { + const operation = record(entry, "Candidate operation detail"); + exactKeys(operation, ["id", "file"], "Candidate operation detail"); + if ( + operation.id !== expectedOperations[index]?.id || + operation.file !== expectedOperations[index]?.file + ) { + throw new Error( + "Native runtime qualification candidate operations do not match the trusted plan", + ); + } + const file = String(operation.file); + validateEvidencePayload(receipts[file]!, file, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + operationId: String(operation.id), + }); + } + validateEvidencePayload(receipts[RUNTIME_FILE]!, RUNTIME_FILE, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + }); + if (row.case.acceleration === "nvidia-gpu") { + const cdi = record(details.nvidiaCdi, "Candidate NVIDIA CDI details"); + exactKeys(cdi, ["device", "file"], "Candidate NVIDIA CDI details"); + if (cdi.device !== "nvidia.com/gpu=all" || cdi.file !== CDI_FILE) { + throw new Error( + "Native runtime qualification candidate NVIDIA CDI details are invalid", + ); + } + validateEvidencePayload(receipts[CDI_FILE]!, CDI_FILE, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + }); + } + return Object.freeze({ + details: details as unknown as CandidateCaseDetails, + receipts: Object.freeze(receipts), + }); +} + +function receiptPath(caseId: string, category: string, file: string): string { + return `receipts/${caseId}/${category}/${file}`; +} + +function copyReceipt( + bytes: Buffer, + outputRoot: string, + relativePath: string, +): NativeRuntimeQualificationArtifactReceipt { + const target = path.join(outputRoot, relativePath); + const parent = path.dirname(target); + mkdirSync(parent, { mode: 0o700, recursive: true }); + writeFileSync(target, bytes, { mode: 0o600, flag: "wx" }); + return Object.freeze({ + path: relativePath, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); +} + export function writeNativeRuntimeQualificationProducerEvidence( row: NativeRuntimeQualificationProducerPlanRow, installerReceiptDirectory: string, executionReceiptPath: string, evidenceDirectory: string, ): void { - validateInstallerReceipts(row, installerReceiptDirectory); - validateCaseExecution(row, readJson(executionReceiptPath)); + const installerBytes = validateInstallerReceipts( + row, + installerReceiptDirectory, + ); + const executionDirectory = path.dirname(executionReceiptPath); + if (path.basename(executionReceiptPath) !== EXECUTION_FILE) { + throw new Error( + "Native runtime qualification execution receipt path is invalid", + ); + } + const candidate = validateCandidateDetails(row, executionDirectory); + validateCaseExecution( + row, + parseJsonBytes(candidate.receipts[EXECUTION_FILE]!, EXECUTION_FILE), + ); + const { details } = candidate; if (lstatSync(evidenceDirectory, { throwIfNoEntry: false })) { - throw new Error("Native runtime qualification evidence directory must not already exist"); + throw new Error( + "Native runtime qualification evidence directory must not already exist", + ); } const parent = path.dirname(evidenceDirectory); const parentStatus = lstatSync(parent, { throwIfNoEntry: false }); @@ -337,25 +720,106 @@ export function writeNativeRuntimeQualificationProducerEvidence( throw new Error("Native runtime qualification evidence parent is invalid"); } mkdirSync(evidenceDirectory, { mode: 0o700 }); + + const installerReceipts = Object.fromEntries( + EXPECTED_INSTALLER_FILES.map((file) => [ + file, + copyReceipt( + installerBytes[file], + evidenceDirectory, + receiptPath(row.id, "installer", file), + ), + ]), + ) as Record< + (typeof EXPECTED_INSTALLER_FILES)[number], + NativeRuntimeQualificationArtifactReceipt + >; + const runtimeReceipt = copyReceipt( + candidate.receipts[RUNTIME_FILE]!, + evidenceDirectory, + receiptPath(row.id, "runtime", RUNTIME_FILE), + ); + const operations = row.case.obligations.map((id) => { + const file = operationFile(id); + return Object.freeze({ + id, + artifact: copyReceipt( + candidate.receipts[file]!, + evidenceDirectory, + receiptPath(row.id, "operations", file), + ), + }); + }); + const cdiReceipt = + row.case.acceleration === "nvidia-gpu" + ? copyReceipt( + candidate.receipts[CDI_FILE]!, + evidenceDirectory, + receiptPath(row.id, "runtime", CDI_FILE), + ) + : undefined; + const providerId = row.case.id.slice(0, row.case.id.indexOf("-")); + const fragment: NativeRuntimeQualificationCaseFragment = Object.freeze({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-fragment-v1", + qualificationId: `${providerId}-protected-host-local-inference`, + providerId, + source: row.source, + case: row.case, + installer: Object.freeze({ + providerId, + architecture: row.case.architecture, + dockerAvailability: "unavailable", + exitCode: 0, + invocation: installerReceipts["invocation.json"], + script: installerReceipts["installer.sh"], + }), + runtime: Object.freeze({ + providerId, + agent: row.case.agent, + inference: row.case.inference, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + rootMode: "rootless", + engineName: details.runtime.engineName, + engineVersion: details.runtime.engineVersion, + managedImages: Object.freeze( + details.runtime.managedImages.map((entry) => + Object.freeze({ ...entry }), + ), + ), + result: runtimeReceipt, + }), + operations: Object.freeze(operations), + ...(cdiReceipt + ? { + nvidiaCdi: Object.freeze({ + device: "nvidia.com/gpu=all" as const, + artifact: cdiReceipt, + }), + } + : {}), + }); writeFileSync( - path.join(evidenceDirectory, "evidence.json"), - `${JSON.stringify({ - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-case-evidence-v1", - qualificationId: `${row.case.id.slice(0, row.case.id.indexOf("-"))}-protected-host-local-inference`, - providerId: row.case.id.slice(0, row.case.id.indexOf("-")), - source: row.source, - case: row.case, - result: "passed", - })}\n`, - { mode: 0o600 }, + path.join(evidenceDirectory, "case-fragment.json"), + `${JSON.stringify(fragment)}\n`, + { + mode: 0o600, + flag: "wx", + }, ); } -if (process.argv[1]?.endsWith("native-runtime-qualification-producer-evidence.mts")) { +if ( + process.argv[1]?.endsWith( + "native-runtime-qualification-producer-evidence.mts", + ) +) { try { if (process.argv.length !== 2) { - throw new Error("Usage: native-runtime-qualification-producer-evidence.mts"); + throw new Error( + "Usage: native-runtime-qualification-producer-evidence.mts", + ); } const row = JSON.parse( process.env.QUALIFICATION_ROW ?? "null", @@ -367,7 +831,9 @@ if (process.argv[1]?.endsWith("native-runtime-qualification-producer-evidence.mt process.env.EVIDENCE_DIRECTORY ?? "", ); } catch (error) { - console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + console.error( + `::error::${error instanceof Error ? error.message : String(error)}`, + ); process.exitCode = 1; } } diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index ff57174011e..dbc52ff621f 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -57,6 +57,7 @@ const GH_API_WRITE_METHOD = /\bgh\s+api\b[\s\S]{0,160}?(?:(?:--method|-X)\s+(?:POST|PUT|PATCH|DELETE)\b|graphql\b[\s\S]{0,160}?\bmutation\b)/iu; const NATIVE_RUNTIME_QUALIFICATION_READ_JOBS = new Set([ "native-runtime-qualification-producer-plan", + "native-runtime-qualification-producer-aggregate", ]); const GENERIC_ISSUE_REST_MUTATION = /github\.request\s*\(\s*["'`](?:POST|PATCH|PUT|DELETE)\s+\/repos\/[^/\s]+\/[^/\s]+\/issues(?:\/|\b)/u; @@ -489,7 +490,10 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow (jobName === "native-runtime-qualification-producer" && step.name === "Check out the candidate commit" && step.with?.repository === "${{ matrix.source.candidateRepository }}" && - step.with?.ref === "${{ matrix.source.candidateSha }}")); + step.with?.ref === "${{ matrix.source.candidateSha }}") || + (jobName === "native-runtime-qualification-producer-aggregate" && + step.name === "Check out the trusted qualification aggregator" && + step.with?.ref === "${{ github.workflow_sha }}")); const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || From c3327ac0de492b0534c3b7e344ad9f97edcbde98 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 20:38:05 -0500 Subject: [PATCH 02/71] fix(e2e): authorize admin candidate qualification --- .github/workflows/e2e.yaml | 83 +++++++++++++++---- ...tive-runtime-qualification-case-helpers.ts | 2 +- .../e2e-operations-workflow-boundary.test.ts | 61 ++++++++++++++ ...runtime-qualification-case-helpers.test.ts | 11 +++ ...untime-qualification-producer-plan.test.ts | 14 +++- ...me-qualification-producer-workflow.test.ts | 25 +++++- ...ad-e2e-artifacts-workflow-boundary.test.ts | 13 +++ ...ve-runtime-qualification-producer-plan.mts | 2 +- tools/e2e/operations-workflow-boundary.mts | 18 +++- ...upload-e2e-artifacts-workflow-boundary.mts | 24 +++++- 10 files changed, 232 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d98491bae9e..2fae2aee54d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -84,7 +84,7 @@ on: default: "" type: string workflow_sha: - description: Optional trusted main workflow SHA for manual exact-revision E2E. + description: Optional exact workflow SHA for manual exact-revision E2E. Candidate-ref native runtime qualification requires an administrator and must equal checkout_sha. required: false default: "" type: string @@ -125,11 +125,23 @@ jobs: env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} EVENT_NAME: ${{ github.event_name }} + JOBS: ${{ inputs.jobs }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} + WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail + if [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && + "$REF" != "refs/heads/main" && + "$EVENT_NAME" == "workflow_dispatch" && + "$JOBS" == "native-runtime-qualification-producer" && + -n "$CHECKOUT_SHA" && + "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]]; then + required=0 + printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}" + exit 0 + fi case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) required=1 @@ -306,8 +318,26 @@ jobs: esac } - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { - echo "::error::Manual PR E2E must be dispatched from main" >&2 + require_admin() { + local administrator="$1" + [[ "$administrator" =~ ^[A-Za-z0-9-]{1,39}$ && "$administrator" != -* && "$administrator" != *- ]] || { + echo "::error::Candidate-workflow native runtime qualification actor is invalid" >&2 + exit 1 + } + local permission_json + permission_json="$(curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${administrator}/permission")" + [[ "$(jq -r '.role_name // ""' <<< "$permission_json")" == "admin" ]] || { + echo "::error::Candidate-workflow native runtime qualification requires a repository administrator" >&2 + exit 1 + } + } + + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" ]] || { + echo "::error::Manual PR E2E must use workflow_dispatch" >&2 exit 1 } [[ "$RUN_ATTEMPT" == "1" ]] || { echo "::error::Manual PR E2E cannot be rerun" >&2; exit 1; } @@ -325,13 +355,7 @@ jobs: [[ "$REVIEW_REASON" =~ ^[[:print:]]+$ ]] && (( ${#REVIEW_REASON} >= 10 && ${#REVIEW_REASON} <= 500 )) || { echo "::error::review_reason must contain 10 to 500 printable characters" >&2; exit 1; } - [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; } - - - require_maintainer "$ACTOR" - if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then - require_maintainer "$TRIGGERING_ACTOR" - fi + [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the executing workflow SHA" >&2; exit 1; } pull_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ @@ -342,6 +366,35 @@ jobs: [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR head repository" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the PR head SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } + candidate_workflow=false + if [[ "$WORKFLOW_REF" != "refs/heads/main" ]]; then + candidate_workflow=true + [[ "$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS" ]] || { + echo "::error::Candidate-workflow dispatch accepts only native-runtime-qualification-producer" >&2 + exit 1 + } + [[ "$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA" ]] || { + echo "::error::Candidate-workflow native runtime qualification must execute the exact same-repository PR head" >&2 + exit 1 + } + head_ref="$(jq -r '.head.ref // ""' <<< "$pull_json")" + [[ -n "$head_ref" && "$WORKFLOW_REF" == "refs/heads/${head_ref}" ]] || { + echo "::error::Candidate-workflow ref must match the exact PR head branch" >&2 + exit 1 + } + fi + + if [[ "$candidate_workflow" == "true" ]]; then + require_admin "$ACTOR" + if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then + require_admin "$TRIGGERING_ACTOR" + fi + else + require_maintainer "$ACTOR" + if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then + require_maintainer "$TRIGGERING_ACTOR" + fi + fi - name: Authorize release qualification waiver if: ${{ github.event_name == 'workflow_dispatch' && (inputs.release_qualification_waived_jobs != '' || inputs.release_qualification_waiver_reason != '') }} @@ -828,7 +881,7 @@ jobs: native-runtime-qualification-producer-plan: needs: generate-matrix - if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }} + if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && (github.ref == 'refs/heads/main' || github.workflow_sha == inputs.checkout_sha) && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -844,6 +897,7 @@ jobs: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts test/e2e/registry/native-runtime-qualification.ts tools/e2e/native-runtime-qualification-producer-plan.mts sparse-checkout-cone-mode: false @@ -868,6 +922,7 @@ jobs: BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha }} + DISPATCH_WORKFLOW_SHA: ${{ github.workflow_sha }} GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ inputs.pr_number }} PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} @@ -884,8 +939,8 @@ jobs: echo "::error::Native runtime qualification producer runs cannot be rerun" >&2 exit 1 } - [[ "$BASE_SHA" == "$WORKFLOW_SHA" && "$CANDIDATE_SHA" != "$WORKFLOW_SHA" ]] || { - echo "::error::Native runtime qualification requires base_sha to equal workflow_sha and checkout_sha to name a different commit" >&2 + [[ "$CANDIDATE_SHA" != "$BASE_SHA" && ( "$WORKFLOW_SHA" == "$BASE_SHA" || "$WORKFLOW_SHA" == "$CANDIDATE_SHA" ) ]] || { + echo "::error::Native runtime qualification requires workflow_sha to equal the exact base or candidate commit" >&2 exit 1 } [[ "$(git -C .candidate-source rev-parse --verify 'HEAD^{commit}')" == "$CANDIDATE_SHA" ]] || { @@ -915,7 +970,7 @@ jobs: artifact="$(jq -ce \ --arg name "$artifact_name" \ --arg runId "$PRODUCER_RUN_ID" \ - --arg workflowSha "$WORKFLOW_SHA" ' + --arg workflowSha "$DISPATCH_WORKFLOW_SHA" ' select(.total_count == 1 and (.artifacts | length) == 1) | .artifacts[0] | select(.name == $name and .expired == false) | diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts index e95fe4caf91..a3283649b51 100644 --- a/test/e2e/live/native-runtime-qualification-case-helpers.ts +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -157,7 +157,7 @@ export function parseNativeRuntimeQualificationRow( typeof source.baseSha !== "string" || !SHA.test(source.baseSha) || source.candidateSha === source.baseSha || - source.workflowSha !== source.baseSha || + (source.workflowSha !== source.baseSha && source.workflowSha !== source.candidateSha) || !/^[1-9][0-9]{0,19}$/u.test(String(source.producerRunId)) || source.producerRunAttempt !== 1 ) { diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 2d43b6e76c2..6ceaba3cbfb 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -498,6 +498,67 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }, ); + it.each([ + ["admin", "refs/heads/feat/native", 0, ""], + [ + "maintain", + "refs/heads/feat/native", + 1, + "requires a repository administrator", + ], + ["admin", "refs/heads/feat/other", 1, "must match the exact PR head branch"], + ])( + "requires admin-bound exact-head candidate workflow execution for %s on %s", + (role, workflowRef, expectedStatus, expectedStderr) => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const headSha = "a".repeat(40); + const baseSha = "b".repeat(40); + const prefix = [ + "curl() {", + ' case "${@: -1}" in', + ` *collaborators*) printf '%s' '{"role_name":"${role}"}' ;;`, + ` *pulls/42) printf '%s' '{"state":"open","head":{"ref":"feat/native","repo":{"full_name":"NVIDIA/NemoClaw"},"sha":"${headSha}"},"base":{"sha":"${baseSha}"}}' ;;`, + " *) return 1 ;;", + " esac", + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${authentication.run}`], + { + encoding: "utf8", + env: { + ...process.env, + ACTOR: "administrator", + ALLOW_JETSON_DISPATCH: "false", + BASE_SHA: baseSha, + CHECKOUT_REPOSITORY: "NVIDIA/NemoClaw", + CHECKOUT_SHA: headSha, + EXPECTED_WORKFLOW_SHA: headSha, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + INCLUDE_LAUNCHABLE: "false", + JOBS: "native-runtime-qualification-producer", + PR_NUMBER: "42", + REVIEW_REASON: "Administrator candidate execution", + RUN_ATTEMPT: "1", + TARGETS: "", + TRIGGERING_ACTOR: "administrator", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: workflowRef, + WORKFLOW_SHA: headSha, + }, + }, + ); + + expect(result.status, result.stderr).toBe(expectedStatus); + expect(result.stderr).toContain(expectedStderr); + }, + ); + it("uses central maintainer authorization for protected managed-image qualification", () => { const workflow = readE2eOperationsWorkflow(); const guards = [ diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index d4f7e9b9c20..73629b85431 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -74,6 +74,17 @@ describe("native runtime qualification case boundaries", () => { ); }); + it("accepts an exact administrator-authorized candidate workflow row", () => { + const candidateSource = { ...SOURCE, workflowSha: SOURCE.candidateSha }; + const candidateRow = buildNativeRuntimeQualificationProducerPlan({ + source: candidateSource, + installerSha256: "d".repeat(64), + arm64GpuRunner: "native-arm64-gpu", + } satisfies NativeRuntimeQualificationProducerPlanInput).include[0]!; + + expect(parseNativeRuntimeQualificationRow(JSON.stringify(candidateRow))).toEqual(candidateRow); + }); + it("rejects credential and alternate runtime authority environment names", () => { expect(() => assertCredentialFreeQualificationEnvironment({ diff --git a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts index 7075a8c7424..3602ed19bc3 100644 --- a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts @@ -94,10 +94,22 @@ describe("native runtime qualification producer plan", () => { ).toBe(true); }); + it("accepts an exact candidate workflow SHA for administrator-authorized execution", () => { + const baseInput = input(); + const candidateWorkflow = { + ...baseInput, + source: { ...baseInput.source, workflowSha: CANDIDATE_SHA }, + } satisfies NativeRuntimeQualificationProducerPlanInput; + const plan = buildNativeRuntimeQualificationProducerPlan(candidateWorkflow); + + expect(plan.include).toHaveLength(24); + expect(plan.include.every((entry) => entry.source.workflowSha === CANDIDATE_SHA)).toBe(true); + }); + it.each([ ["fork candidate", { source: { ...input().source, candidateRepository: "fork/NemoClaw" } }], ["candidate commit", { source: { ...input().source, candidateSha: "A".repeat(40) } }], - ["base authority", { source: { ...input().source, workflowSha: "e".repeat(40) } }], + ["unbound workflow", { source: { ...input().source, workflowSha: "e".repeat(40) } }], ["run attempt", { source: { ...input().source, producerRunAttempt: 2 } }], ["installer digest", { installerSha256: "short" }], ["ARM64 GPU runner", { arm64GpuRunner: "" }], diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 5d330473112..ab1b6f64563 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -62,7 +62,9 @@ describe("native runtime qualification producer workflow", () => { "pull-requests": "read", }); expect(authenticate.run).toContain('"$CANDIDATE_REPOSITORY" == "NVIDIA/NemoClaw"'); - expect(authenticate.run).toContain('"$BASE_SHA" == "$WORKFLOW_SHA"'); + expect(authenticate.run).toContain('"$WORKFLOW_SHA" == "$BASE_SHA"'); + expect(authenticate.run).toContain('"$WORKFLOW_SHA" == "$CANDIDATE_SHA"'); + expect(authenticate.env?.DISPATCH_WORKFLOW_SHA).toBe("${{ github.workflow_sha }}"); expect(authenticate.run).toContain(".head.sha == $candidateSha"); expect(authenticate.run).toContain(".base.sha == $baseSha"); expect(authenticate.run).toContain(".total_count == 1"); @@ -73,6 +75,27 @@ describe("native runtime qualification producer workflow", () => { ); expect(compile.run).toContain("native-runtime-qualification-producer-plan.mts --ci-output"); expect(JSON.stringify(plan)).not.toContain("linux-arm64-gpu-dgx-spark-gb10-protected-1"); + const trustedCheckout = step(plan, "Check out the trusted qualification producer"); + expect(trustedCheckout.with?.["sparse-checkout"]).toContain( + "src/lib/onboard/runtime-provider/native-qualification-authority.ts", + ); + }); + + it("limits candidate-workflow protected execution to the exact PR head and administrators", () => { + const generate = job("generate-matrix"); + const authenticate = step(generate, "Authenticate manual PR dispatch"); + const source = authenticate.run ?? ""; + + expect(source).toContain( + "Candidate-workflow native runtime qualification requires a repository administrator", + ); + expect(source).toContain( + '"$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA"', + ); + expect(source).toContain('"$WORKFLOW_REF" == "refs/heads/${head_ref}"'); + expect(source).toContain( + '"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"', + ); }); it("runs each candidate case in an isolated account and emits one trusted artifact", () => { diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 0d7b8db2710..81e1e4a24c0 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -151,6 +151,19 @@ describe("E2E artifact uploads", () => { ); }); + it("allows only the exact 30-day native runtime aggregate upload", () => { + const workflow = mutableWorkflow(); + const upload = workflow.jobs["native-runtime-qualification-producer-aggregate"].steps?.find( + (step) => step.name === "Upload the immutable aggregate evidence", + ); + expect(upload).toBeDefined(); + upload!.with!["retention-days"] = 14; + + expect(validateUploadE2eArtifactsInvocations(workflow)).toContain( + "native-runtime-qualification-producer-aggregate must not invoke actions/upload-artifact directly", + ); + }); + it.each([ ["name", "another-cache-artifact"], ["path", "another-cache-path/"], diff --git a/tools/e2e/native-runtime-qualification-producer-plan.mts b/tools/e2e/native-runtime-qualification-producer-plan.mts index 26b282e4414..8c16bda0277 100644 --- a/tools/e2e/native-runtime-qualification-producer-plan.mts +++ b/tools/e2e/native-runtime-qualification-producer-plan.mts @@ -105,7 +105,7 @@ function validateSource( !COMMIT_SHA.test(value.baseSha) || !COMMIT_SHA.test(value.workflowSha) || value.candidateSha === value.baseSha || - value.baseSha !== value.workflowSha || + (value.workflowSha !== value.baseSha && value.workflowSha !== value.candidateSha) || !RUN_ID.test(value.producerRunId) || value.producerRunAttempt !== 1 ) { diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index dbc52ff621f..403a1c4a9ff 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -34,6 +34,16 @@ const PUBLICATION_REQUIRED_CONDITION = "${{ steps.publication_mode.outputs.requi const PUBLICATION_CLASSIFIER_SCRIPT = [ "set -euo pipefail", + 'if [[ "$REPOSITORY" == "NVIDIA/NemoClaw" &&', + ' "$REF" != "refs/heads/main" &&', + ' "$EVENT_NAME" == "workflow_dispatch" &&', + ' "$JOBS" == "native-runtime-qualification-producer" &&', + ' -n "$CHECKOUT_SHA" &&', + ' "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]]; then', + " required=0", + ' printf \'required=%s\\n\' "${required}" >> "${GITHUB_OUTPUT}"', + " exit 0", + "fi", 'case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in', " NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)", " required=1", @@ -333,7 +343,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const acceptedJobNames = `${acceptedNames.slice(0, -1).join(", ")}, or ${acceptedNames.at(-1)}`; for (const fragment of [ '"$WORKFLOW_EVENT" == "workflow_dispatch"', - '"$WORKFLOW_REF" == "refs/heads/main"', + '"$WORKFLOW_REF" != "refs/heads/main"', '"$RUN_ATTEMPT" == "1"', '"$PR_NUMBER" =~ ^[1-9][0-9]*$', '"$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$', @@ -344,6 +354,10 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "${#REVIEW_REASON} <= 500", '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', "Manual PR E2E requires a repository maintainer or administrator", + "Candidate-workflow native runtime qualification requires a repository administrator", + '"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"', + '"$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA"', + '"$WORKFLOW_REF" == "refs/heads/${head_ref}"', `${acceptedJobCases}) ;;`, `Manual PR E2E accepts only empty selectors, ${acceptedJobNames}`, "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", @@ -546,8 +560,10 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): env: { CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EVENT_NAME: "${{ github.event_name }}", + JOBS: "${{ inputs.jobs }}", REF: "${{ github.ref }}", REPOSITORY: "${{ github.repository }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", }, shell: "bash", run: PUBLICATION_CLASSIFIER_SCRIPT, diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 9d237248327..6b83c8c6be1 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -52,6 +52,17 @@ const RELEASE_QUALIFICATION_WAIVER_UPLOAD_CONTRACT: WorkflowStep = { "retention-days": 30, }, }; +const NATIVE_RUNTIME_AGGREGATE_UPLOAD_CONTRACT: WorkflowStep = { + name: "Upload the immutable aggregate evidence", + uses: UPLOAD_ARTIFACT_ACTION, + with: { + name: "native-runtime-qualification-${{ inputs.checkout_sha }}", + path: "${{ runner.temp }}/native-runtime-aggregate/", + "if-no-files-found": "error", + "retention-days": 30, + "compression-level": 9, + }, +}; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const RETIRED_SELECTOR_COMPATIBILITY_JOB = "retired-selector-compatibility"; @@ -111,6 +122,13 @@ function isExactReleaseQualificationWaiverUpload( ); } +function isExactNativeRuntimeAggregateUpload(jobName: string, step: WorkflowStep): boolean { + return ( + jobName === "native-runtime-qualification-producer-aggregate" && + isDeepStrictEqual(step, NATIVE_RUNTIME_AGGREGATE_UPLOAD_CONTRACT) + ); +} + const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "generate-matrix", @@ -186,7 +204,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ "native-runtime-qualification-producer", { name: "${{ matrix.artifactName }}", - path: "${{ runner.temp }}/native-runtime-evidence/evidence.json", + path: "${{ runner.temp }}/native-runtime-evidence/", }, ], [ @@ -242,6 +260,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["generate-matrix", "${{ github.event_name == 'workflow_dispatch' }}"], + ["native-runtime-qualification-producer", "success()"], ["staging-brev-launchable", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], @@ -441,7 +460,8 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX) && !isExactCommitCliArtifactUpload && !isExactManagedImageBuildCacheUpload(jobName, step) && - !isExactReleaseQualificationWaiverUpload(jobName, step) + !isExactReleaseQualificationWaiverUpload(jobName, step) && + !isExactNativeRuntimeAggregateUpload(jobName, step) ) { errors.push(`${jobName} must not invoke actions/upload-artifact directly`); } From 5db381876a0c448d1156474b9df9bde2a1a72d3c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 20:51:10 -0500 Subject: [PATCH 03/71] fix(e2e): harden qualification evidence --- .github/workflows/e2e.yaml | 30 ++- ...-native-runtime-installer-qualification.sh | 4 +- .../native-runtime-qualification-case.test.ts | 30 ++- .../registry/native-runtime-qualification.ts | 10 +- .../e2e-operations-workflow-boundary.test.ts | 21 +- ...runtime-qualification-case-helpers.test.ts | 5 +- ...ve-runtime-qualification-collector.test.ts | 2 +- ...e-qualification-producer-aggregate.test.ts | 12 +- ...me-qualification-producer-evidence.test.ts | 92 +++++-- ...untime-qualification-producer-plan.test.ts | 4 + ...me-qualification-producer-workflow.test.ts | 21 +- ...ad-e2e-artifacts-workflow-boundary.test.ts | 1 - ...ntime-qualification-producer-aggregate.mts | 46 ++-- ...untime-qualification-producer-evidence.mts | 235 +++++------------- ...ve-runtime-qualification-producer-plan.mts | 15 ++ tools/e2e/operations-workflow-boundary.mts | 33 ++- ...upload-e2e-artifacts-workflow-boundary.mts | 5 +- 17 files changed, 265 insertions(+), 301 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fae2aee54d..2072695de84 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1046,7 +1046,10 @@ jobs: path: .trusted-qualification persist-credentials: false sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts scripts/checks/run-native-runtime-installer-qualification.sh + test/e2e/registry/native-runtime-qualification.ts + tools/e2e/native-runtime-qualification-producer-plan.mts tools/e2e/native-runtime-qualification-producer-evidence.mts sparse-checkout-cone-mode: false @@ -1103,9 +1106,9 @@ jobs: exit 1 } guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard" - install -d -m 0700 "$guard_dir" + install -d -m 0755 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" - chmod 0500 "$guard_dir/docker" + chmod 0555 "$guard_dir/docker" printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" @@ -1212,10 +1215,11 @@ jobs: shell: bash run: | set -euo pipefail - sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ - "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ + sudo chown -R -h "$(id -u):$(id -g)" \ + "$INSTALLER_RECEIPT_DIRECTORY" \ + "$(dirname "$EXECUTION_RECEIPT_PATH")" + "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ .trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts - sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources if: always() @@ -1260,12 +1264,14 @@ jobs: - name: Check out the trusted qualification aggregator uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} path: .trusted-qualification-aggregate persist-credentials: false sparse-checkout: | src/lib/onboard/runtime-provider/native-qualification-authority.ts test/e2e/registry/native-runtime-qualification.ts + tools/e2e/native-runtime-qualification-producer-plan.mts tools/e2e/native-runtime-qualification-producer-aggregate.mts sparse-checkout-cone-mode: false @@ -1288,11 +1294,18 @@ jobs: jobs="$(gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \ -f per_page=100)" + total_count="$(jq -er '.total_count | select(type == "number" and . >= 1)' <<<"$jobs")" || { + echo "::error::Aggregate job lookup returned an invalid job count" >&2 + exit 1 + } + (( total_count <= 100 )) || { + echo "::error::Aggregate job lookup exceeds the bounded 100-job page" >&2 + exit 1 + } job_id="$(jq -er \ --arg name 'Aggregate native runtime qualification evidence' \ --argjson runId "$PRODUCER_RUN_ID" \ --argjson attempt "$PRODUCER_RUN_ATTEMPT" ' - select(.total_count <= 100) | [.jobs[] | select( .name == $name and .run_id == $runId and @@ -1311,6 +1324,11 @@ jobs: } printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT" + - name: Set up Node for qualification aggregation + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + - name: Validate and aggregate all 24 case receipts working-directory: .trusted-qualification-aggregate env: diff --git a/scripts/checks/run-native-runtime-installer-qualification.sh b/scripts/checks/run-native-runtime-installer-qualification.sh index 9d6d5aa513a..61db3065be6 100755 --- a/scripts/checks/run-native-runtime-installer-qualification.sh +++ b/scripts/checks/run-native-runtime-installer-qualification.sh @@ -197,7 +197,7 @@ assert_docker_unavailable() { '{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}' } -run_native_runtime_installer_qualification() { +run_native_runtime_installer_qualification() ( local candidate_checkout="" local candidate_sha="" local expected_installer_sha256="" @@ -447,7 +447,7 @@ run_native_runtime_installer_qualification() { cleanup trap - EXIT unset -f cleanup -} +) if [[ "${BASH_SOURCE[0]:-}" == "$0" ]]; then run_native_runtime_installer_qualification "$@" diff --git a/test/e2e/live/native-runtime-qualification-case.test.ts b/test/e2e/live/native-runtime-qualification-case.test.ts index b208bfaad0b..2a5e1d01d0e 100644 --- a/test/e2e/live/native-runtime-qualification-case.test.ts +++ b/test/e2e/live/native-runtime-qualification-case.test.ts @@ -29,6 +29,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import type { NativeRuntimeQualificationObligation } from "../registry/native-runtime-qualification.ts"; +import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; import { assertCredentialFreeQualificationEnvironment, digestFromImageReference, @@ -149,10 +150,6 @@ function writeJson(directory: string, file: string, value: unknown): void { fs.renameSync(temporary, target); } -function operationFile(id: NativeRuntimeQualificationObligation): string { - return `operation-${id.replaceAll(".", "-")}.json`; -} - function assertDockerUnavailable(): Record { const guarded = command("docker", ["version"]); if (guarded.status !== 97) { @@ -535,7 +532,7 @@ test.skipIf(!ENABLED)( const ownedContainers = new Set(); const ownedVolumes = new Set(); const ownedNetworks = new Set(); - let inferenceContainerId: string | null = null; + let inferenceContainerId: string; let gpuDevices: readonly string[] = []; let gpuComputeProcesses: readonly GpuComputeProcess[] = []; let completed = false; @@ -593,8 +590,11 @@ test.skipIf(!ENABLED)( pullPublicImage(inferenceEngine, agentImage); if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); else { + if (!inference.cachePath || !path.isAbsolute(inference.cachePath)) { + throw new Error("Native runtime qualification GPU cache path must be absolute"); + } requirePreloadedImage(inferenceEngine, inference.imageRef); - rootOwnedReadOnlyDirectory(inference.cachePath ?? ""); + rootOwnedReadOnlyDirectory(inference.cachePath); } if (runnerContract) { requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); @@ -914,14 +914,12 @@ test.skipIf(!ENABLED)( capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); ownedVolumes.delete(volume); } - if (inferenceContainerId) { - capture( - inferenceEngine, - ["rm", "--force", inferenceContainerId], - "inference runtime cleanup", - ); - ownedContainers.delete(inferenceContainerId); - } + capture( + inferenceEngine, + ["rm", "--force", inferenceContainerId], + "inference runtime cleanup", + ); + ownedContainers.delete(inferenceContainerId); capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); ownedNetworks.delete(network.id); fs.rmSync(snapshot, { force: true }); @@ -985,7 +983,7 @@ test.skipIf(!ENABLED)( for (const obligation of row.case.obligations) { const details = operationDetails.get(obligation); if (!details) throw new Error(`Qualification operation '${obligation}' was not executed`); - writeJson(receiptDirectory, operationFile(obligation), { + writeJson(receiptDirectory, nativeRuntimeQualificationOperationFile(obligation), { schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-operation-v1", caseId: row.id, @@ -1020,7 +1018,7 @@ test.skipIf(!ENABLED)( }, operations: row.case.obligations.map((id) => ({ id, - file: operationFile(id), + file: nativeRuntimeQualificationOperationFile(id), })), ...(row.case.acceleration === "nvidia-gpu" ? { diff --git a/test/e2e/registry/native-runtime-qualification.ts b/test/e2e/registry/native-runtime-qualification.ts index a5d2c3b6e8c..22db5a3ff43 100644 --- a/test/e2e/registry/native-runtime-qualification.ts +++ b/test/e2e/registry/native-runtime-qualification.ts @@ -6,18 +6,18 @@ import { createHash } from "node:crypto"; export { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; import type { NativeRuntimeQualificationAuthority, NativeRuntimeQualificationExpectedSource, NativeRuntimeQualificationProtectedRun, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; export type { NativeRuntimeQualificationAuthority, NativeRuntimeQualificationExpectedSource, NativeRuntimeQualificationProtectedRun, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; export const NATIVE_RUNTIME_QUALIFICATION_AGENTS = [ "openclaw", @@ -436,7 +436,9 @@ function validatedArtifactReceipt( } const contents = readReceipt(artifactPath); if (contents === null) { - throw new Error(`${label} receipt '${artifactPath}' is missing from the authenticated artifact`); + throw new Error( + `${label} receipt '${artifactPath}' is missing from the authenticated artifact`, + ); } const actualSha256 = createHash("sha256").update(contents).digest("hex"); if (actualSha256 !== artifact.sha256) { diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 6ceaba3cbfb..53fc1ec08fb 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -350,15 +350,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; writeFileSync(output, ""); const result = spawnSync( "bash", - [ - "--noprofile", - "--norc", - "-e", - "-o", - "pipefail", - "-c", - credentialAuthorization.run!, - ], + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", credentialAuthorization.run!], { encoding: "utf8", env: { @@ -376,7 +368,9 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(`allowed=${expectedAllowed ? "true" : "false"}\n`); + expect(readFileSync(output, "utf8")).toBe( + `allowed=${expectedAllowed ? "true" : "false"}\n`, + ); } finally { rmSync(directory, { force: true, recursive: true }); } @@ -500,12 +494,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; it.each([ ["admin", "refs/heads/feat/native", 0, ""], - [ - "maintain", - "refs/heads/feat/native", - 1, - "requires a repository administrator", - ], + ["maintain", "refs/heads/feat/native", 1, "requires a repository administrator"], ["admin", "refs/heads/feat/other", 1, "must match the exact PR head branch"], ])( "requires admin-bound exact-head candidate workflow execution for %s on %s", diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index 73629b85431..d27346f7dd0 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -36,11 +36,12 @@ const SOURCE = { } as const; function row() { - return buildNativeRuntimeQualificationProducerPlan({ + const plan = buildNativeRuntimeQualificationProducerPlan({ source: SOURCE, installerSha256: "d".repeat(64), arm64GpuRunner: "native-arm64-gpu", - } satisfies NativeRuntimeQualificationProducerPlanInput).include[0]!; + } satisfies NativeRuntimeQualificationProducerPlanInput); + return plan.include.find((entry) => entry.id === "podman-hermes-linux-amd64-cpu-ollama")!; } function runnerContract() { diff --git a/test/e2e/support/native-runtime-qualification-collector.test.ts b/test/e2e/support/native-runtime-qualification-collector.test.ts index cb8bf4526db..61b73abd412 100644 --- a/test/e2e/support/native-runtime-qualification-collector.test.ts +++ b/test/e2e/support/native-runtime-qualification-collector.test.ts @@ -137,7 +137,7 @@ function githubFixture( [`repos/${REPOSITORY}/pulls/9143`, pull], [`repos/${REPOSITORY}/commits/main`, { sha: NATIVE_QUALIFICATION_BASE_SHA }], [ - `repos/${REPOSITORY}/actions/workflows/e2e.yaml`, + `repos/${REPOSITORY}/actions/workflows/${WORKFLOW.split("/").at(-1)!}`, { id: 101, path: WORKFLOW, state: "active" }, ], [`repos/${REPOSITORY}/actions/runs/7001`, run], diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts index 3e5e969e6f2..3f0412cfc66 100644 --- a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW } from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; import { aggregateNativeRuntimeQualificationProducerEvidence, NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE, @@ -15,6 +16,7 @@ import { import { writeNativeRuntimeQualificationProducerEvidence } from "../../../tools/e2e/native-runtime-qualification-producer-evidence.mts"; import { buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, type NativeRuntimeQualificationProducerPlanRow, } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; @@ -27,10 +29,6 @@ function writeJson(file: string, value: unknown): void { fs.writeFileSync(file, `${JSON.stringify(value)}\n`); } -function operationFile(id: string): string { - return `operation-${id.replaceAll(".", "-")}.json`; -} - function installerReceipts( directory: string, row: NativeRuntimeQualificationProducerPlanRow, @@ -114,7 +112,7 @@ function candidateReceipts( details: { engineAuthority: `podman-sha256:${"9".repeat(64)}` }, }); for (const id of row.case.obligations) { - writeJson(path.join(directory, operationFile(id)), { + writeJson(path.join(directory, nativeRuntimeQualificationOperationFile(id)), { schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-operation-v1", caseId: row.id, @@ -147,7 +145,7 @@ function candidateReceipts( }, operations: row.case.obligations.map((id) => ({ id, - file: operationFile(id), + file: nativeRuntimeQualificationOperationFile(id), })), ...(row.case.acceleration === "nvidia-gpu" ? { nvidiaCdi: { device: "nvidia.com/gpu=all", file: "nvidia-cdi.json" } } @@ -162,7 +160,7 @@ function fixture() { const plan = buildNativeRuntimeQualificationProducerPlan({ source: { repository: "NVIDIA/NemoClaw", - producerWorkflow: ".github/workflows/e2e.yaml", + producerWorkflow: NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, pullRequestNumber: 9144, candidateRepository: "NVIDIA/NemoClaw", candidateSha: "a".repeat(40), diff --git a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts index 09a1e08a3bb..72f49b69ed7 100644 --- a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts @@ -8,9 +8,11 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW } from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; import { writeNativeRuntimeQualificationProducerEvidence } from "../../../tools/e2e/native-runtime-qualification-producer-evidence.mts"; import { buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; @@ -18,13 +20,13 @@ const roots: string[] = []; const INSTALLER = "#!/usr/bin/env bash\nexit 0\n"; const INSTALLER_SHA256 = createHash("sha256").update(INSTALLER).digest("hex"); -function fixture() { +function fixture(options: { readonly gpu?: boolean } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-producer-evidence-")); roots.push(root); - const row = buildNativeRuntimeQualificationProducerPlan({ + const plan = buildNativeRuntimeQualificationProducerPlan({ source: { repository: "NVIDIA/NemoClaw", - producerWorkflow: ".github/workflows/e2e.yaml", + producerWorkflow: NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, pullRequestNumber: 8064, candidateRepository: "NVIDIA/NemoClaw", candidateSha: "a".repeat(40), @@ -42,7 +44,12 @@ function fixture() { }, installerSha256: INSTALLER_SHA256, arm64GpuRunner: "reviewed-native-arm64-gpu-runner", - }).include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; + }); + const row = options.gpu + ? plan.include.find( + (entry) => entry.case.architecture === "amd64" && entry.case.acceleration === "nvidia-gpu", + )! + : plan.include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; const installerDirectory = path.join(root, "installer"); const executionDirectory = path.join(root, "candidate"); const executionPath = path.join(executionDirectory, "execution.json"); @@ -82,7 +89,11 @@ function fixture() { ); fs.writeFileSync( path.join(installerDirectory, "architecture.json"), - JSON.stringify({ receiptVersion: 1, requested: "amd64", runner: "amd64" }), + JSON.stringify({ + receiptVersion: 1, + requested: row.case.architecture, + runner: row.case.architecture, + }), ); const dockerPosture = { dockerCommandGuarded: true, @@ -123,7 +134,6 @@ function fixture() { result: "passed", }; fs.writeFileSync(executionPath, JSON.stringify(execution)); - const operationFile = (id: string) => `operation-${id.replaceAll(".", "-")}.json`; fs.writeFileSync( path.join(executionDirectory, "runtime-result.json"), JSON.stringify({ @@ -136,7 +146,7 @@ function fixture() { ); for (const id of row.case.obligations) { fs.writeFileSync( - path.join(executionDirectory, operationFile(id)), + path.join(executionDirectory, nativeRuntimeQualificationOperationFile(id)), JSON.stringify({ schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-operation-v1", @@ -147,6 +157,18 @@ function fixture() { }), ); } + if (row.case.acceleration === "nvidia-gpu") { + fs.writeFileSync( + path.join(executionDirectory, "nvidia-cdi.json"), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }), + ); + } fs.writeFileSync( path.join(executionDirectory, "case-evidence.json"), JSON.stringify({ @@ -162,7 +184,13 @@ function fixture() { ], resultFile: "runtime-result.json", }, - operations: row.case.obligations.map((id) => ({ id, file: operationFile(id) })), + operations: row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { nvidiaCdi: { device: "nvidia.com/gpu=all", file: "nvidia-cdi.json" } } + : {}), }), ); return { evidenceDirectory, execution, executionPath, installerDirectory, root, row }; @@ -209,18 +237,45 @@ describe("native runtime qualification producer evidence", () => { providerId: "podman", }, }); - expect(fs.statSync(path.join(value.evidenceDirectory, "case-fragment.json")).mode & 0o777).toBe(0o600); + expect(fs.statSync(path.join(value.evidenceDirectory, "case-fragment.json")).mode & 0o777).toBe( + 0o600, + ); expect( fs.existsSync( - path.join( - value.evidenceDirectory, - "receipts", - value.row.id, - "installer", - "installer.sh", - ), + path.join(value.evidenceDirectory, "receipts", value.row.id, "installer", "installer.sh"), ), ).toBe(true); + const fragment = JSON.parse( + fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8"), + ) as { installer: { script: { path: string; sha256: string } } }; + const copied = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.installer.script.path), + ); + expect(createHash("sha256").update(copied).digest("hex")).toBe( + fragment.installer.script.sha256, + ); + }); + + it("emits the NVIDIA CDI receipt for a GPU case", () => { + const value = fixture({ gpu: true }); + + writeNativeRuntimeQualificationProducerEvidence( + value.row, + value.installerDirectory, + value.executionPath, + value.evidenceDirectory, + ); + + const fragment = JSON.parse( + fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8"), + ) as { nvidiaCdi: { artifact: { path: string; sha256: string } } }; + const copied = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.nvidiaCdi.artifact.path), + ); + expect(fragment.nvidiaCdi.artifact.path).toContain("/runtime/nvidia-cdi.json"); + expect(createHash("sha256").update(copied).digest("hex")).toBe( + fragment.nvidiaCdi.artifact.sha256, + ); }); it.each([ @@ -319,7 +374,10 @@ describe("native runtime qualification producer evidence", () => { it("rejects an unexpected candidate-controlled receipt file", () => { const value = fixture(); - fs.writeFileSync(path.join(path.dirname(value.executionPath), "candidate.log"), "candidate output"); + fs.writeFileSync( + path.join(path.dirname(value.executionPath), "candidate.log"), + "candidate output", + ); expect(() => writeNativeRuntimeQualificationProducerEvidence( diff --git a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts index 3602ed19bc3..33c0855a3c9 100644 --- a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS, type NativeRuntimeQualificationProducerPlanInput, @@ -51,6 +52,9 @@ describe("native runtime qualification producer plan", () => { expect(entry.source.candidateSha).toBe(CANDIDATE_SHA); expect(entry.source.baseSha).toBe(entry.source.workflowSha); expect(entry.case.id).toBe(entry.id); + expect( + new Set(entry.case.obligations.map(nativeRuntimeQualificationOperationFile)).size, + ).toBe(entry.case.obligations.length); expect(Object.isFrozen(entry)).toBe(true); } expect( diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index ab1b6f64563..c5a8735e58c 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -93,13 +93,12 @@ describe("native runtime qualification producer workflow", () => { '"$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA"', ); expect(source).toContain('"$WORKFLOW_REF" == "refs/heads/${head_ref}"'); - expect(source).toContain( - '"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"', - ); + expect(source).toContain('"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"'); }); it("runs each candidate case in an isolated account and emits one trusted artifact", () => { const producer = job("native-runtime-qualification-producer"); + const harness = step(producer, "Check out the trusted qualification harness"); const boundary = step( producer, "Prepare the credential-free execution account and disable Docker", @@ -120,9 +119,17 @@ describe("native runtime qualification producer workflow", () => { expect(producer["runs-on"]).toBe("${{ matrix.runner }}"); expect(producer.permissions).toEqual({ contents: "read" }); expect(producer.strategy).toMatchObject({ "fail-fast": false }); + expect(harness.with?.["sparse-checkout"]).toContain( + "tools/e2e/native-runtime-qualification-producer-plan.mts", + ); + expect(harness.with?.["sparse-checkout"]).toContain( + "test/e2e/registry/native-runtime-qualification.ts", + ); expect(source).not.toMatch(/NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); + expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); + expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); expect(boundaryRun.indexOf("printf 'account=%s")).toBeLessThan( boundaryRun.indexOf("useradd --create-home"), ); @@ -143,7 +150,8 @@ describe("native runtime qualification producer workflow", () => { 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', ); expect(validate.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); - expect(validate.run).toContain("sudo --preserve-env="); + expect(validate.run).toContain('sudo chown -R -h "$(id -u):$(id -g)"'); + expect(validate.run).not.toContain("sudo --preserve-env="); expect(validate.run).toContain('"$NODE_DIRECTORY/node"'); expect(validate.run).toContain("native-runtime-qualification-producer-evidence.mts"); expect(upload.with).toMatchObject({ @@ -161,8 +169,10 @@ describe("native runtime qualification producer workflow", () => { const aggregate = job("native-runtime-qualification-producer-aggregate"); const download = step(aggregate, "Download the exact case evidence cohort"); const identity = step(aggregate, "Resolve this aggregate job identity"); + const setupNode = step(aggregate, "Set up Node for qualification aggregation"); const collect = step(aggregate, "Validate and aggregate all 24 case receipts"); const upload = step(aggregate, "Upload the immutable aggregate evidence"); + const aggregateCheckout = step(aggregate, "Check out the trusted qualification aggregator"); expect(aggregate.name).toBe("Aggregate native runtime qualification evidence"); expect(aggregate.needs).toEqual([ @@ -178,12 +188,15 @@ describe("native runtime qualification producer workflow", () => { contents: "read", "pull-requests": "read", }); + expect(aggregateCheckout.with?.repository).toBe("${{ github.repository }}"); expect(download.with).toMatchObject({ pattern: "native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*", "merge-multiple": false, }); expect(identity.run).toContain('.status == "in_progress"'); expect(identity.run).toContain("select(length == 1)"); + expect(identity.run).toContain("Aggregate job lookup exceeds the bounded 100-job page"); + expect(setupNode.with?.["node-version"]).toBe("22.19.0"); expect(collect.run).toContain("native-runtime-qualification-producer-aggregate.mts"); expect(collect.env?.QUALIFICATION_PLAN).toBe( "${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}", diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 81e1e4a24c0..8c6efb1cf60 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -95,7 +95,6 @@ describe("E2E artifact uploads", () => { expect(policyErrors).toContain( "upload-e2e-artifacts must preserve artifact defaults, hidden-file policy, missing-file behavior, and retention", ); - }); it("uploads artifacts even when an earlier step fails", () => { diff --git a/tools/e2e/native-runtime-qualification-producer-aggregate.mts b/tools/e2e/native-runtime-qualification-producer-aggregate.mts index eb72f43f97b..b9540725f9b 100644 --- a/tools/e2e/native-runtime-qualification-producer-aggregate.mts +++ b/tools/e2e/native-runtime-qualification-producer-aggregate.mts @@ -16,14 +16,16 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { - compileNativeRuntimeQualification, consumeNativeRuntimeQualificationEvidence, - nativeRuntimeQualificationDefinition, + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, type NativeRuntimeQualificationEvidenceEnvelope, type NativeRuntimeQualificationExpectedSource, } from "../../test/e2e/registry/native-runtime-qualification.ts"; import type { NativeRuntimeQualificationCaseFragment } from "./native-runtime-qualification-producer-evidence.mts"; -import type { NativeRuntimeQualificationProducerPlan } from "./native-runtime-qualification-producer-plan.mts"; +import { + nativeRuntimeQualificationOperationFile, + type NativeRuntimeQualificationProducerPlan, +} from "./native-runtime-qualification-producer-plan.mts"; export const NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_JOB_NAME = "Aggregate native runtime qualification evidence"; @@ -34,7 +36,6 @@ const MAX_FRAGMENT_BYTES = 256 * 1024; const MAX_RECEIPT_BYTES = 524_288; const MAX_TOTAL_BYTES = 32 * 1024 * 1024; const POSITIVE_INTEGER = /^[1-9][0-9]{0,19}$/u; -const SHA256 = /^[a-f0-9]{64}$/u; type UnknownRecord = Record; @@ -133,18 +134,13 @@ function walkRegularFiles(root: string): readonly string[] { return Object.freeze(files); } -function expectedReceiptFiles(caseId: string, gpu: boolean): readonly string[] { - const operations = [ - "installer-install", - "runtime-docker-unavailable", - "agent-onboard", - "agent-turn", - "sandbox-stop-start", - "sandbox-snapshot-restore", - "sandbox-rebuild", - "runtime-restart-reconcile", - "cleanup-exact", - ].map((id) => `receipts/${caseId}/operations/operation-${id}.json`); +function expectedReceiptFiles( + row: NativeRuntimeQualificationProducerPlan["include"][number], +): readonly string[] { + const caseId = row.id; + const operations = row.case.obligations.map( + (id) => `receipts/${caseId}/operations/${nativeRuntimeQualificationOperationFile(id)}`, + ); return Object.freeze([ "case-fragment.json", `receipts/${caseId}/installer/architecture.json`, @@ -155,7 +151,9 @@ function expectedReceiptFiles(caseId: string, gpu: boolean): readonly string[] { `receipts/${caseId}/installer/invocation.json`, `receipts/${caseId}/runtime/runtime-result.json`, ...operations, - ...(gpu ? [`receipts/${caseId}/runtime/nvidia-cdi.json`] : []), + ...(row.case.acceleration === "nvidia-gpu" + ? [`receipts/${caseId}/runtime/nvidia-cdi.json`] + : []), ]); } @@ -255,7 +253,11 @@ export function aggregateNativeRuntimeQualificationProducerEvidence(input: { readonly aggregateJobId: number; }): NativeRuntimeQualificationEvidenceEnvelope { const { plan } = input; - if (plan.include.length !== 24 || new Set(plan.include.map((row) => row.id)).size !== 24) { + const expectedCaseCount = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.cases.length; + if ( + plan.include.length !== expectedCaseCount || + new Set(plan.include.map((row) => row.id)).size !== expectedCaseCount + ) { throw new Error("Native runtime qualification aggregate requires the exact 24-case plan"); } const first = plan.include[0]!; @@ -288,9 +290,7 @@ export function aggregateNativeRuntimeQualificationProducerEvidence(input: { const artifactDirectory = path.join(input.caseArtifactRoot, row.artifactName); assertDirectory(artifactDirectory, `case artifact '${row.artifactName}'`); const actualFiles = walkRegularFiles(artifactDirectory); - const expectedFiles = [ - ...expectedReceiptFiles(row.id, row.case.acceleration === "nvidia-gpu"), - ].sort(); + const expectedFiles = [...expectedReceiptFiles(row)].sort(); if (JSON.stringify([...actualFiles].sort()) !== JSON.stringify(expectedFiles)) { throw new Error(`Native runtime qualification case artifact '${row.id}' has invalid files`); } @@ -320,9 +320,7 @@ export function aggregateNativeRuntimeQualificationProducerEvidence(input: { providerId: "podman", cases: Object.freeze(cases), }); - const definition = compileNativeRuntimeQualification( - nativeRuntimeQualificationDefinition("podman"), - ); + const definition = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION; consumeNativeRuntimeQualificationEvidence(definition, envelope, source, (receiptPath) => { try { return readBoundedBytes(path.join(input.evidenceDirectory, receiptPath), MAX_RECEIPT_BYTES); diff --git a/tools/e2e/native-runtime-qualification-producer-evidence.mts b/tools/e2e/native-runtime-qualification-producer-evidence.mts index 7874cb95294..85b2f12adf1 100644 --- a/tools/e2e/native-runtime-qualification-producer-evidence.mts +++ b/tools/e2e/native-runtime-qualification-producer-evidence.mts @@ -19,7 +19,12 @@ import type { NativeRuntimeQualificationObligation, NativeRuntimeQualificationArtifactReceipt, } from "../../test/e2e/registry/native-runtime-qualification.ts"; -import type { NativeRuntimeQualificationProducerPlanRow } from "./native-runtime-qualification-producer-plan.mts"; +import { + nativeRuntimeQualificationOperationFile, + NATIVE_RUNTIME_QUALIFICATION_ID, + NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID, + type NativeRuntimeQualificationProducerPlanRow, +} from "./native-runtime-qualification-producer-plan.mts"; const MAX_RECEIPT_BYTES = 65_536; const MAX_INSTALLER_BYTES = 524_288; @@ -36,7 +41,6 @@ const DETAIL_FILE = "case-evidence.json"; const EXECUTION_FILE = "execution.json"; const RUNTIME_FILE = "runtime-result.json"; const CDI_FILE = "nvidia-cdi.json"; -const SHA256 = /^[a-f0-9]{64}$/u; const IMAGE_DIGEST = /^sha256:[a-f0-9]{64}$/u; const SAFE_ENGINE = /^[A-Za-z0-9][A-Za-z0-9 ._/-]{0,127}$/u; const FORBIDDEN_RECEIPT_TEXT = @@ -140,11 +144,7 @@ function record(value: unknown, label: string): UnknownRecord { return value as UnknownRecord; } -function exactKeys( - value: UnknownRecord, - keys: readonly string[], - label: string, -): void { +function exactKeys(value: UnknownRecord, keys: readonly string[], label: string): void { const actual = Object.keys(value).sort(); const expected = [...keys].sort(); if (JSON.stringify(actual) !== JSON.stringify(expected)) { @@ -152,11 +152,7 @@ function exactKeys( } } -function exactStrings( - actual: unknown, - expected: readonly string[], - label: string, -): void { +function exactStrings(actual: unknown, expected: readonly string[], label: string): void { if ( !Array.isArray(actual) || actual.some((entry) => typeof entry !== "string") || @@ -172,27 +168,18 @@ function readBoundedBytes(file: string, maximum = MAX_RECEIPT_BYTES): Buffer { descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); const status = fstatSync(descriptor); if (!status.isFile() || status.size < 1 || status.size > maximum) { - throw new Error( - `Native runtime qualification receipt is missing or invalid: ${file}`, - ); + throw new Error(`Native runtime qualification receipt is missing or invalid: ${file}`); } const bytes = readFileSync(descriptor); if (FORBIDDEN_RECEIPT_TEXT.test(bytes.toString("utf8"))) { - throw new Error( - `Native runtime qualification receipt contains credential material: ${file}`, - ); + throw new Error(`Native runtime qualification receipt contains credential material: ${file}`); } return bytes; } catch (error) { - if ( - error instanceof Error && - error.message.startsWith("Native runtime qualification") - ) { + if (error instanceof Error && error.message.startsWith("Native runtime qualification")) { throw error; } - throw new Error( - `Native runtime qualification receipt is missing or invalid: ${file}`, - ); + throw new Error(`Native runtime qualification receipt is missing or invalid: ${file}`); } finally { if (descriptor !== undefined) closeSync(descriptor); } @@ -203,49 +190,32 @@ function parseJsonBytes(bytes: Buffer, file: string): unknown { return JSON.parse(bytes.toString("utf8")) as unknown; } catch (error) { if (error instanceof SyntaxError) { - throw new Error( - `Native runtime qualification receipt is not valid JSON: ${file}`, - ); + throw new Error(`Native runtime qualification receipt is not valid JSON: ${file}`); } throw error; } } -function validateDirectory( - directory: string, - expectedFiles: readonly string[], -): void { +function validateDirectory(directory: string, expectedFiles: readonly string[]): void { const status = lstatSync(directory, { throwIfNoEntry: false }); if (!status?.isDirectory() || status.isSymbolicLink()) { - throw new Error( - `Native runtime qualification receipt directory is invalid: ${directory}`, - ); + throw new Error(`Native runtime qualification receipt directory is invalid: ${directory}`); } const files = readdirSync(directory).sort(); if (JSON.stringify(files) !== JSON.stringify([...expectedFiles].sort())) { - throw new Error( - `Native runtime qualification receipt files are invalid: ${directory}`, - ); + throw new Error(`Native runtime qualification receipt files are invalid: ${directory}`); } let total = 0; for (const file of files) { const child = path.join(directory, file); const childStatus = lstatSync(child); - if ( - !childStatus.isFile() || - childStatus.isSymbolicLink() || - childStatus.size < 1 - ) { - throw new Error( - `Native runtime qualification receipt file is invalid: ${child}`, - ); + if (!childStatus.isFile() || childStatus.isSymbolicLink() || childStatus.size < 1) { + throw new Error(`Native runtime qualification receipt file is invalid: ${child}`); } total += childStatus.size; } if (total > MAX_RECEIPT_DIRECTORY_BYTES) { - throw new Error( - `Native runtime qualification receipts exceed their size limit: ${directory}`, - ); + throw new Error(`Native runtime qualification receipts exceed their size limit: ${directory}`); } } @@ -269,13 +239,7 @@ function validateInstallerReceipts( ); exactKeys( invocation, - [ - "receiptVersion", - "script", - "scriptSha256", - "candidateSha", - "architecture", - ], + ["receiptVersion", "script", "scriptSha256", "candidateSha", "architecture"], "Installer invocation", ); if ( @@ -285,27 +249,19 @@ function validateInstallerReceipts( invocation.candidateSha !== row.source.candidateSha || invocation.architecture !== row.case.architecture ) { - throw new Error( - "Native runtime qualification installer invocation is invalid", - ); + throw new Error("Native runtime qualification installer invocation is invalid"); } const architecture = record( parseJsonBytes(receipts["architecture.json"], "architecture.json"), "Installer architecture", ); - exactKeys( - architecture, - ["receiptVersion", "requested", "runner"], - "Installer architecture", - ); + exactKeys(architecture, ["receiptVersion", "requested", "runner"], "Installer architecture"); if ( architecture.receiptVersion !== 1 || architecture.requested !== row.case.architecture || architecture.runner !== row.case.architecture ) { - throw new Error( - "Native runtime qualification installer architecture is invalid", - ); + throw new Error("Native runtime qualification installer architecture is invalid"); } const candidate = record( parseJsonBytes(receipts["candidate-source.json"], "candidate-source.json"), @@ -345,9 +301,7 @@ function validateInstallerReceipts( installed.installMode !== "managed" || installed.installerSha256 !== row.installerSha256 ) { - throw new Error( - "Native runtime qualification installer source identity is invalid", - ); + throw new Error("Native runtime qualification installer source identity is invalid"); } const docker = record( parseJsonBytes(receipts["docker-absence.json"], "docker-absence.json"), @@ -370,24 +324,18 @@ function validateInstallerReceipts( const value = record(docker[phase], `Installer Docker absence ${phase}`); exactKeys(value, requiredDockerKeys, `Installer Docker absence ${phase}`); if (requiredDockerKeys.some((key) => value[key] !== true)) { - throw new Error( - "Native runtime qualification installer Docker absence is invalid", - ); + throw new Error("Native runtime qualification installer Docker absence is invalid"); } } if (docker.receiptVersion !== 1) { - throw new Error( - "Native runtime qualification installer Docker absence is invalid", - ); + throw new Error("Native runtime qualification installer Docker absence is invalid"); } const installer = receipts["installer.sh"]; if ( !installer.toString("utf8").startsWith("#!/") || createHash("sha256").update(installer).digest("hex") !== row.installerSha256 ) { - throw new Error( - "Native runtime qualification installer receipt is invalid", - ); + throw new Error("Native runtime qualification installer receipt is invalid"); } return Object.freeze(receipts); } @@ -396,10 +344,7 @@ function validateCaseExecution( row: NativeRuntimeQualificationProducerPlanRow, value: unknown, ): CaseExecutionReceipt { - const receipt = record( - value, - "Native runtime qualification execution receipt", - ); + const receipt = record(value, "Native runtime qualification execution receipt"); exactKeys( receipt, [ @@ -422,29 +367,15 @@ function validateCaseExecution( ], "Native runtime qualification execution receipt", ); - const docker = record( - receipt.dockerUnavailable, - "Docker-unavailable execution receipt", - ); - const credentials = record( - receipt.credentialBoundary, - "Credential-boundary execution receipt", - ); - exactKeys( - docker, - ["beforeCandidate", "afterCandidate"], - "Docker-unavailable execution receipt", - ); + const docker = record(receipt.dockerUnavailable, "Docker-unavailable execution receipt"); + const credentials = record(receipt.credentialBoundary, "Credential-boundary execution receipt"); + exactKeys(docker, ["beforeCandidate", "afterCandidate"], "Docker-unavailable execution receipt"); exactKeys( credentials, ["githubCredentialsAbsent", "modelCredentialsAbsent", "isolatedUid"], "Credential-boundary execution receipt", ); - exactStrings( - receipt.rootModes, - row.rootModes, - "Native runtime qualification root modes", - ); + exactStrings(receipt.rootModes, row.rootModes, "Native runtime qualification root modes"); exactStrings( receipt.obligations, row.case.obligations, @@ -477,25 +408,17 @@ function validateCaseExecution( credentials.isolatedUid !== true || receipt.result !== "passed" ) { - throw new Error( - "Native runtime qualification execution receipt identity is invalid", - ); + throw new Error("Native runtime qualification execution receipt identity is invalid"); } return receipt as unknown as CaseExecutionReceipt; } -function operationFile(id: string): string { - return `operation-${id.replaceAll(".", "-")}.json`; -} - -function expectedCaseFiles( - row: NativeRuntimeQualificationProducerPlanRow, -): string[] { +function expectedCaseFiles(row: NativeRuntimeQualificationProducerPlanRow): string[] { return [ DETAIL_FILE, EXECUTION_FILE, RUNTIME_FILE, - ...row.case.obligations.map(operationFile), + ...row.case.obligations.map(nativeRuntimeQualificationOperationFile), ...(row.case.acceleration === "nvidia-gpu" ? [CDI_FILE] : []), ]; } @@ -509,10 +432,7 @@ function validateEvidencePayload( readonly operationId?: string; }, ): void { - const value = record( - parseJsonBytes(bytes, file), - `Candidate evidence '${path.basename(file)}'`, - ); + const value = record(parseJsonBytes(bytes, file), `Candidate evidence '${path.basename(file)}'`); exactKeys( value, [ @@ -531,12 +451,9 @@ function validateEvidencePayload( value.kind !== expected.kind || value.caseId !== expected.caseId || value.result !== "passed" || - (expected.operationId !== undefined && - value.operationId !== expected.operationId) + (expected.operationId !== undefined && value.operationId !== expected.operationId) ) { - throw new Error( - `Native runtime qualification candidate evidence is invalid: ${file}`, - ); + throw new Error(`Native runtime qualification candidate evidence is invalid: ${file}`); } } @@ -550,10 +467,7 @@ function validateCandidateDetails( const expectedFiles = expectedCaseFiles(row); validateDirectory(directory, expectedFiles); const receipts = Object.fromEntries( - expectedFiles.map((file) => [ - file, - readBoundedBytes(path.join(directory, file)), - ]), + expectedFiles.map((file) => [file, readBoundedBytes(path.join(directory, file))]), ) as Record; const details = record( parseJsonBytes(receipts[DETAIL_FILE]!, DETAIL_FILE), @@ -590,9 +504,7 @@ function validateCandidateDetails( runtime.managedImages.length < 2 || runtime.managedImages.length > 8 ) { - throw new Error( - "Native runtime qualification candidate runtime details are invalid", - ); + throw new Error("Native runtime qualification candidate runtime details are invalid"); } const roles = new Set(); for (const entry of runtime.managedImages) { @@ -605,9 +517,7 @@ function validateCandidateDetails( typeof image.digest !== "string" || !IMAGE_DIGEST.test(image.digest) ) { - throw new Error( - "Native runtime qualification candidate managed image is invalid", - ); + throw new Error("Native runtime qualification candidate managed image is invalid"); } roles.add(image.role); } @@ -615,13 +525,11 @@ function validateCandidateDetails( !Array.isArray(details.operations) || details.operations.length !== row.case.obligations.length ) { - throw new Error( - "Native runtime qualification candidate operations are incomplete", - ); + throw new Error("Native runtime qualification candidate operations are incomplete"); } const expectedOperations = row.case.obligations.map((id) => ({ id, - file: operationFile(id), + file: nativeRuntimeQualificationOperationFile(id), })); for (const [index, entry] of details.operations.entries()) { const operation = record(entry, "Candidate operation detail"); @@ -649,9 +557,7 @@ function validateCandidateDetails( const cdi = record(details.nvidiaCdi, "Candidate NVIDIA CDI details"); exactKeys(cdi, ["device", "file"], "Candidate NVIDIA CDI details"); if (cdi.device !== "nvidia.com/gpu=all" || cdi.file !== CDI_FILE) { - throw new Error( - "Native runtime qualification candidate NVIDIA CDI details are invalid", - ); + throw new Error("Native runtime qualification candidate NVIDIA CDI details are invalid"); } validateEvidencePayload(receipts[CDI_FILE]!, CDI_FILE, { caseId: row.id, @@ -689,26 +595,16 @@ export function writeNativeRuntimeQualificationProducerEvidence( executionReceiptPath: string, evidenceDirectory: string, ): void { - const installerBytes = validateInstallerReceipts( - row, - installerReceiptDirectory, - ); + const installerBytes = validateInstallerReceipts(row, installerReceiptDirectory); const executionDirectory = path.dirname(executionReceiptPath); if (path.basename(executionReceiptPath) !== EXECUTION_FILE) { - throw new Error( - "Native runtime qualification execution receipt path is invalid", - ); + throw new Error("Native runtime qualification execution receipt path is invalid"); } const candidate = validateCandidateDetails(row, executionDirectory); - validateCaseExecution( - row, - parseJsonBytes(candidate.receipts[EXECUTION_FILE]!, EXECUTION_FILE), - ); + validateCaseExecution(row, parseJsonBytes(candidate.receipts[EXECUTION_FILE]!, EXECUTION_FILE)); const { details } = candidate; if (lstatSync(evidenceDirectory, { throwIfNoEntry: false })) { - throw new Error( - "Native runtime qualification evidence directory must not already exist", - ); + throw new Error("Native runtime qualification evidence directory must not already exist"); } const parent = path.dirname(evidenceDirectory); const parentStatus = lstatSync(parent, { throwIfNoEntry: false }); @@ -724,23 +620,16 @@ export function writeNativeRuntimeQualificationProducerEvidence( const installerReceipts = Object.fromEntries( EXPECTED_INSTALLER_FILES.map((file) => [ file, - copyReceipt( - installerBytes[file], - evidenceDirectory, - receiptPath(row.id, "installer", file), - ), + copyReceipt(installerBytes[file], evidenceDirectory, receiptPath(row.id, "installer", file)), ]), - ) as Record< - (typeof EXPECTED_INSTALLER_FILES)[number], - NativeRuntimeQualificationArtifactReceipt - >; + ) as Record<(typeof EXPECTED_INSTALLER_FILES)[number], NativeRuntimeQualificationArtifactReceipt>; const runtimeReceipt = copyReceipt( candidate.receipts[RUNTIME_FILE]!, evidenceDirectory, receiptPath(row.id, "runtime", RUNTIME_FILE), ); const operations = row.case.obligations.map((id) => { - const file = operationFile(id); + const file = nativeRuntimeQualificationOperationFile(id); return Object.freeze({ id, artifact: copyReceipt( @@ -758,11 +647,11 @@ export function writeNativeRuntimeQualificationProducerEvidence( receiptPath(row.id, "runtime", CDI_FILE), ) : undefined; - const providerId = row.case.id.slice(0, row.case.id.indexOf("-")); + const providerId = NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID; const fragment: NativeRuntimeQualificationCaseFragment = Object.freeze({ schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-case-fragment-v1", - qualificationId: `${providerId}-protected-host-local-inference`, + qualificationId: NATIVE_RUNTIME_QUALIFICATION_ID, providerId, source: row.source, case: row.case, @@ -784,9 +673,7 @@ export function writeNativeRuntimeQualificationProducerEvidence( engineName: details.runtime.engineName, engineVersion: details.runtime.engineVersion, managedImages: Object.freeze( - details.runtime.managedImages.map((entry) => - Object.freeze({ ...entry }), - ), + details.runtime.managedImages.map((entry) => Object.freeze({ ...entry })), ), result: runtimeReceipt, }), @@ -810,16 +697,10 @@ export function writeNativeRuntimeQualificationProducerEvidence( ); } -if ( - process.argv[1]?.endsWith( - "native-runtime-qualification-producer-evidence.mts", - ) -) { +if (process.argv[1]?.endsWith("native-runtime-qualification-producer-evidence.mts")) { try { if (process.argv.length !== 2) { - throw new Error( - "Usage: native-runtime-qualification-producer-evidence.mts", - ); + throw new Error("Usage: native-runtime-qualification-producer-evidence.mts"); } const row = JSON.parse( process.env.QUALIFICATION_ROW ?? "null", @@ -831,9 +712,7 @@ if ( process.env.EVIDENCE_DIRECTORY ?? "", ); } catch (error) { - console.error( - `::error::${error instanceof Error ? error.message : String(error)}`, - ); + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; } } diff --git a/tools/e2e/native-runtime-qualification-producer-plan.mts b/tools/e2e/native-runtime-qualification-producer-plan.mts index 8c16bda0277..68e7ed3ff4f 100644 --- a/tools/e2e/native-runtime-qualification-producer-plan.mts +++ b/tools/e2e/native-runtime-qualification-producer-plan.mts @@ -31,6 +31,15 @@ export const NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS = [ "cleanup", ] as const; +export const NATIVE_RUNTIME_QUALIFICATION_ID = + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.id; +export const NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID = + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.providerId; + +export function nativeRuntimeQualificationOperationFile(id: string): string { + return `operation-${id.replaceAll(".", "-")}.json`; +} + export interface NativeRuntimeQualificationDispatchArtifact { readonly id: string; readonly name: string; @@ -132,6 +141,12 @@ function runnerForCase(entry: NativeRuntimeQualificationCase, arm64GpuRunner: st } function immutableCase(value: NativeRuntimeQualificationCase): NativeRuntimeQualificationCase { + const operationFiles = value.obligations.map(nativeRuntimeQualificationOperationFile); + if (new Set(operationFiles).size !== operationFiles.length) { + throw new Error( + `Native runtime qualification case '${value.id}' has colliding operation files`, + ); + } return Object.freeze({ ...value, capabilities: Object.freeze([...value.capabilities]), diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 403a1c4a9ff..e507c56efe7 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -393,8 +393,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const credentialAuthorization = credentialAuthorizationIndex >= 0 ? steps[credentialAuthorizationIndex] : {}; if ( - matrixJob.outputs?.e2e_credentials_allowed !== - "${{ steps.e2e_credentials.outputs.allowed }}" || + matrixJob.outputs?.e2e_credentials_allowed !== "${{ steps.e2e_credentials.outputs.allowed }}" || credentialAuthorization.id !== "e2e_credentials" || credentialAuthorization.if !== "${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}" || @@ -411,12 +410,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow WORKFLOW_REPOSITORY: "${{ github.repository }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", }; - if ( - !isDeepStrictEqual( - credentialAuthorization.env, - expectedCredentialAuthorizationEnvironment, - ) - ) { + if (!isDeepStrictEqual(credentialAuthorization.env, expectedCredentialAuthorizationEnvironment)) { errors.push( "Manual PR credential authorization must bind the workflow and checkout identities", ); @@ -495,19 +489,20 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.ref === "${{ inputs.workflow_sha || github.workflow_sha }}" && step.with?.path === ".trusted-openshell-dev-artifact"; const trustedNativeRuntimeCheckout = - ((jobName === "native-runtime-qualification-producer-plan" && + (jobName === "native-runtime-qualification-producer-plan" && step.name === "Check out the trusted qualification producer" && step.with?.ref === "${{ github.workflow_sha }}") || - (jobName === "native-runtime-qualification-producer" && - step.name === "Check out the trusted qualification harness" && - step.with?.ref === "${{ matrix.source.workflowSha }}") || - (jobName === "native-runtime-qualification-producer" && - step.name === "Check out the candidate commit" && - step.with?.repository === "${{ matrix.source.candidateRepository }}" && - step.with?.ref === "${{ matrix.source.candidateSha }}") || - (jobName === "native-runtime-qualification-producer-aggregate" && - step.name === "Check out the trusted qualification aggregator" && - step.with?.ref === "${{ github.workflow_sha }}")); + (jobName === "native-runtime-qualification-producer" && + step.name === "Check out the trusted qualification harness" && + step.with?.ref === "${{ matrix.source.workflowSha }}") || + (jobName === "native-runtime-qualification-producer" && + step.name === "Check out the candidate commit" && + step.with?.repository === "${{ matrix.source.candidateRepository }}" && + step.with?.ref === "${{ matrix.source.candidateSha }}") || + (jobName === "native-runtime-qualification-producer-aggregate" && + step.name === "Check out the trusted qualification aggregator" && + step.with?.repository === "${{ github.repository }}" && + step.with?.ref === "${{ github.workflow_sha }}"); const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 6b83c8c6be1..186db84a190 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -112,10 +112,7 @@ function isExactManagedImageBuildCacheUpload(jobName: string, step: WorkflowStep ); } -function isExactReleaseQualificationWaiverUpload( - jobName: string, - step: WorkflowStep, -): boolean { +function isExactReleaseQualificationWaiverUpload(jobName: string, step: WorkflowStep): boolean { return ( jobName === "release-qualification" && isDeepStrictEqual(step, RELEASE_QUALIFICATION_WAIVER_UPLOAD_CONTRACT) From 26f41109bed9c764ba693560c66907dce1b884ba Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 20:54:38 -0500 Subject: [PATCH 04/71] fix(e2e): preserve receipt read boundary --- .github/workflows/e2e.yaml | 7 +++---- .../native-runtime-qualification-producer-workflow.test.ts | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2072695de84..810ffdcb23e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1215,11 +1215,10 @@ jobs: shell: bash run: | set -euo pipefail - sudo chown -R -h "$(id -u):$(id -g)" \ - "$INSTALLER_RECEIPT_DIRECTORY" \ - "$(dirname "$EXECUTION_RECEIPT_PATH")" - "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ + sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ + "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ .trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts + sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources if: always() diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index c5a8735e58c..05c4ce70937 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -150,8 +150,8 @@ describe("native runtime qualification producer workflow", () => { 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', ); expect(validate.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); - expect(validate.run).toContain('sudo chown -R -h "$(id -u):$(id -g)"'); - expect(validate.run).not.toContain("sudo --preserve-env="); + expect(validate.run).not.toContain('chown -R -h "$(id -u):$(id -g)"'); + expect(validate.run).toContain("sudo --preserve-env="); expect(validate.run).toContain('"$NODE_DIRECTORY/node"'); expect(validate.run).toContain("native-runtime-qualification-producer-evidence.mts"); expect(upload.with).toMatchObject({ From c3d65f9e702aadc6b93ab5a7e7dd65b79888b140 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 21:07:24 -0500 Subject: [PATCH 05/71] refactor(e2e): linearize native qualification test --- ...ive-runtime-qualification-case-executor.ts | 1054 ++++++++++++++++ .../native-runtime-qualification-case.test.ts | 1069 +---------------- ...e-qualification-producer-aggregate.test.ts | 22 +- ...me-qualification-producer-evidence.test.ts | 25 +- tools/e2e/check-semantic-phases.mts | 2 +- 5 files changed, 1089 insertions(+), 1083 deletions(-) create mode 100644 test/e2e/live/native-runtime-qualification-case-executor.ts diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts new file mode 100644 index 00000000000..b7677f23b93 --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -0,0 +1,1054 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + capturePodmanSocketAuthority, + createPodmanContainerEngine, + type PodmanBoundContainerEngine, +} from "../../../src/lib/adapters/podman/index.ts"; +import type { RuntimeProviderLifecycleInput } from "../../../src/lib/onboard/runtime-provider/contract.ts"; +import { createPodmanRuntimeProviderBundle } from "../../../src/lib/onboard/runtime-provider/podman.ts"; +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE, + PODMAN_SANDBOX_NAMESPACE_LABEL, + PODMAN_SANDBOX_WORKSPACE, + PODMAN_SANDBOX_WORKSPACE_LABEL, +} from "../../../src/lib/onboard/runtime-provider/podman-lifecycle.ts"; +import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; +import type { NativeRuntimeQualificationObligation } from "../registry/native-runtime-qualification.ts"; +import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + assertCredentialFreeQualificationEnvironment, + digestFromImageReference, + nativeRuntimeQualificationAgentImage, + nativeRuntimeQualificationInferenceImage, + parseNativeRuntimeQualificationRow, + readNativeRuntimeQualificationRunnerContract, +} from "./native-runtime-qualification-case-helpers.ts"; + +const FULL_ID = /^[a-f0-9]{64}$/u; +const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; +const COMMAND_TIMEOUT = 60_000; +const INFERENCE_TIMEOUT = 900_000; +const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +export const NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES = [ + "validate credential-free Docker-unavailable isolation", + "bind the rootless Podman engine", + "launch exact local inference", + "onboard the managed agent image", + "exercise sandbox lifecycle and state recovery", + "restart and reconcile inference", + "prove exact cleanup", + "emit bounded case evidence", +] as const; + +interface PodmanNetworkAuthority { + readonly id: string; + readonly name: string; + readonly gateway: string; +} + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +interface GpuComputeProcess { + readonly gpuUuid: string; + readonly pid: number; + readonly processName: string; + readonly usedMemoryMiB: number; +} + +function bounded(value: string): string { + return value.replace(CONTROL, " ").replace(/\s+/gu, " ").trim().slice(-500); +} + +function command(command: string, args: readonly string[]): CommandResult { + const result = spawnSync(command, [...args], { + encoding: "utf8", + env: process.env, + timeout: 10_000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + }); + return { + status: result.status ?? (result.signal ? 128 : 127), + stdout: result.stdout ?? "", + stderr: result.stderr ?? result.error?.message ?? "", + }; +} + +function requireCommand(executable: string, args: readonly string[], label: string): string { + const result = command(executable, args); + if (result.status !== 0) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout)}`, + ); + } + return result.stdout.trim(); +} + +function capture( + engine: PodmanBoundContainerEngine, + args: readonly string[], + label: string, + timeout = COMMAND_TIMEOUT, +): string { + const result = engine.capture(args, timeout); + if (result.status !== 0 || result.error) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout || result.error?.message || "unknown failure")}`, + ); + } + return result.stdout.trim(); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function exactDirectory(directory: string): void { + const metadata = fs.lstatSync(directory); + const uid = process.getuid?.() ?? -1; + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + metadata.uid !== uid || + (metadata.mode & 0o077) !== 0 + ) { + throw new Error("Qualification receipt directory must be private and current-user owned"); + } +} + +function writeJson(directory: string, file: string, value: unknown): void { + const target = path.join(directory, file); + const temporary = `${target}.tmp`; + const serialized = `${JSON.stringify(value, null, 2)}\n`; + fs.writeFileSync(temporary, serialized, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, target); +} + +function assertDockerUnavailable(): Record { + const guarded = command("docker", ["version"]); + if (guarded.status !== 97) { + throw new Error(`Docker PATH invocation guard returned ${String(guarded.status)}, expected 97`); + } + for (const executable of ["/usr/bin/docker", "/usr/local/bin/docker", "/snap/bin/docker"]) { + if (!fs.existsSync(executable)) continue; + const result = command(executable, ["version"]); + if (result.status === 0) + throw new Error(`Absolute Docker client remained usable: ${executable}`); + } + for (const socket of ["/var/run/docker.sock", "/run/docker.sock"]) { + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) throw new Error(`Docker socket remained available: ${socket}`); + } + for (const unit of ["docker.service", "docker.socket"]) { + if (command("systemctl", ["is-active", "--quiet", unit]).status === 0) { + throw new Error(`Docker unit remained active: ${unit}`); + } + } + const proc = fs.readdirSync("/proc").filter((entry) => /^[1-9][0-9]*$/u.test(entry)); + for (const pid of proc) { + try { + if (fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim() === "dockerd") { + throw new Error("Docker daemon process remained active"); + } + } catch (error) { + if (error instanceof Error && error.message === "Docker daemon process remained active") { + throw error; + } + } + } + return { + dockerCommandGuarded: true, + dockerServiceInactive: true, + dockerSocketUnitInactive: true, + dockerdProcessNameAbsent: true, + defaultSocketPathsAbsent: true, + }; +} + +async function waitForSocket(socket: string, child: ChildProcess): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("Rootless Podman API service exited before its socket became ready"); + } + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("Rootless Podman API service did not create its socket"); +} + +function startPodmanQualificationService(socket: string, progress: TestProgress): ChildProcess { + return spawnObservedChild("podman", ["system", "service", "--time=0", `unix://${socket}`], { + activityLabel: "command: rootless Podman qualification service", + progress, + spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, + }); +} + +async function stopService(child: ChildProcess | null, socket: string): Promise { + if (child && child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } + fs.rmSync(socket, { force: true }); +} + +function createProviderNetwork( + engine: PodmanBoundContainerEngine, + name: string, + caseId: string, +): PodmanNetworkAuthority { + const id = capture( + engine, + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], + "provider network creation", + ); + if (!FULL_ID.test(id)) throw new Error("Provider network did not return a full immutable ID"); + const inspected = JSON.parse( + capture(engine, ["network", "inspect", id], "provider network inspection"), + ) as Array<{ + id?: unknown; + name?: unknown; + subnets?: Array<{ gateway?: unknown }>; + }>; + const entry = inspected[0]; + const gateway = entry?.subnets?.[0]?.gateway; + if ( + inspected.length !== 1 || + entry?.id !== id || + entry.name !== name || + typeof gateway !== "string" + ) { + throw new Error("Provider network inspection lacks exact identity"); + } + return Object.freeze({ id, name, gateway }); +} + +function pullPublicImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["pull", imageRef], `pull ${imageRef}`, INFERENCE_TIMEOUT); + capture(engine, ["image", "exists", imageRef], `inspect pulled image ${imageRef}`); +} + +function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); +} + +function rootOwnedReadOnlyDirectory(directory: string): void { + const canonical = fs.realpathSync(directory); + if (canonical !== directory) { + throw new Error(`Runner model resource is not canonical: ${directory}`); + } + const boundary = "/var/lib/nemoclaw/native-runtime-qualification"; + let current = directory; + while (current.startsWith(`${boundary}/`) || current === boundary) { + const metadata = fs.lstatSync(current); + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + metadata.uid !== 0 || + (metadata.mode & 0o022) !== 0 + ) { + throw new Error( + `Runner model resource is not an exact root-owned read-only directory: ${current}`, + ); + } + if (current === boundary) return; + current = path.dirname(current); + } + throw new Error(`Runner model resource escapes its reviewed root: ${directory}`); +} + +function proveGpuDevices( + engine: PodmanBoundContainerEngine, + probeImageRef: string, +): readonly string[] { + const devices = capture( + engine, + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + probeImageRef, + "nvidia-smi", + "--query-gpu=uuid", + "--format=csv,noheader", + ], + "NVIDIA CDI runtime proof", + ) + .split(/\r?\n/u) + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); + if ( + devices.length === 0 || + new Set(devices).size !== devices.length || + devices.some((device) => !/^GPU-[0-9A-Fa-f-]{36}$/u.test(device)) + ) { + throw new Error("NVIDIA CDI runtime proof did not return exact physical GPU UUIDs"); + } + return Object.freeze(devices); +} + +function proveGpuBackedInference( + engine: PodmanBoundContainerEngine, + containerId: string, + selectedDevices: readonly string[], +): readonly GpuComputeProcess[] { + const output = capture( + engine, + [ + "exec", + containerId, + "nvidia-smi", + "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", + "--format=csv,noheader,nounits", + ], + "GPU-backed inference process proof", + ); + const processes = output + .split(/\r?\n/u) + .map((line) => line.split(",").map((field) => field.trim())) + .filter((fields) => fields.length === 4) + .map(([gpuUuid, pid, processName, usedMemoryMiB]) => ({ + gpuUuid: gpuUuid ?? "", + pid: Number(pid), + processName: processName ?? "", + usedMemoryMiB: Number(usedMemoryMiB), + })) + .filter( + (entry) => + selectedDevices.includes(entry.gpuUuid) && + Number.isSafeInteger(entry.pid) && + entry.pid > 0 && + entry.processName.length > 0 && + !/[\u0000-\u001f\u007f-\u009f]/u.test(entry.processName) && + Number.isSafeInteger(entry.usedMemoryMiB) && + entry.usedMemoryMiB > 0, + ); + if (processes.length === 0) { + throw new Error("Inference turn did not leave an exact GPU compute process proof"); + } + return Object.freeze(processes.map((entry) => Object.freeze(entry))); +} + +function createAgentContainer(input: { + readonly engine: PodmanBoundContainerEngine; + readonly imageRef: string; + readonly name: string; + readonly network: string; + readonly qualificationId: string; + readonly sandboxId: string; + readonly sandboxName: string; + readonly volume: string; +}): string { + const id = capture( + input.engine, + [ + "run", + "--detach", + "--pull=never", + "--name", + input.name, + "--network", + input.network, + "--label", + `${PODMAN_MANAGED_LABEL}=true`, + "--label", + `${PODMAN_SANDBOX_NAME_LABEL}=${input.sandboxName}`, + "--label", + `${PODMAN_SANDBOX_ID_LABEL}=${input.sandboxId}`, + "--label", + `${PODMAN_SANDBOX_NAMESPACE_LABEL}=${PODMAN_SANDBOX_NAMESPACE}`, + "--label", + `${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, + "--label", + `${QUALIFICATION_LABEL}=${input.qualificationId}`, + "--volume", + `${input.volume}:/qualification`, + "--entrypoint", + "/bin/sh", + input.imageRef, + "-c", + "while :; do sleep 3600; done", + ], + "agent container creation", + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(id)) throw new Error("Agent container did not return a full immutable ID"); + return id; +} + +async function agentTurn( + engine: PodmanBoundContainerEngine, + containerId: string, + endpoint: string, + model: string, +): Promise { + const body = JSON.stringify({ + model, + messages: [{ role: "user", content: "Reply with the single word qualified." }], + max_tokens: 32, + stream: false, + }); + const args = [ + "exec", + containerId, + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--connect-timeout", + "5", + "--max-time", + "60", + "--header", + "Content-Type: application/json", + "--data-binary", + body, + `${endpoint}/v1/chat/completions`, + ]; + const deadline = Date.now() + 600_000; + let output = ""; + let lastFailure = "inference request was not attempted"; + while (Date.now() < deadline) { + const result = engine.capture(args, 90_000); + if (result.status === 0 && !result.error) { + output = result.stdout.trim(); + break; + } + lastFailure = bounded(result.stderr || result.stdout || result.error?.message || "failed"); + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + if (!output) throw new Error(`Agent inference turn did not become ready: ${lastFailure}`); + const response = JSON.parse(output) as { + model?: unknown; + choices?: Array<{ + finish_reason?: unknown; + message?: { content?: unknown; tool_calls?: unknown }; + }>; + }; + const first = response.choices?.[0]; + if ( + response.model !== model || + typeof first?.finish_reason !== "string" || + first.finish_reason === "length" || + (typeof first.message?.content !== "string" && !Array.isArray(first.message?.tool_calls)) + ) { + throw new Error("Agent turn did not return a complete exact-model inference response"); + } + return sha256(output); +} + +function lifecycleInput(agent: string, sandboxName: string): RuntimeProviderLifecycleInput { + const sandbox: SandboxEntry = { + agent, + name: sandboxName, + openshellDriver: "podman", + }; + return { + environment: process.env, + log: () => undefined, + sandbox, + sandboxName, + }; +} + +function assertNoQualificationResidue(engine: PodmanBoundContainerEngine, caseId: string): void { + for (const [resource, args] of [ + ["container", ["ps", "--all", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["volume", ["volume", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["network", ["network", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ] as const) { + if (capture(engine, args, `qualification ${resource} residue inspection`) !== "") { + throw new Error(`Qualification cleanup left an owned ${resource}`); + } + } +} + +export async function executeNativeRuntimeQualificationCase(progress: TestProgress): Promise { + progress.phase("validate credential-free Docker-unavailable isolation"); + assertCredentialFreeQualificationEnvironment(process.env); + expect(process.platform).toBe("linux"); + const uid = process.getuid?.() ?? 0; + expect(uid, "Native runtime qualification must execute as an unprivileged UID").toBeGreaterThan( + 0, + ); + const row = parseNativeRuntimeQualificationRow( + process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW ?? "", + ); + const expectedArchitecture = process.arch === "x64" ? "amd64" : process.arch; + expect(expectedArchitecture).toBe(row.case.architecture); + const receiptPath = process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT ?? ""; + expect(path.basename(receiptPath)).toBe("execution.json"); + const receiptDirectory = path.dirname(receiptPath); + exactDirectory(receiptDirectory); + + const dockerBefore = assertDockerUnavailable(); + const runtimeDirectory = process.env.XDG_RUNTIME_DIR ?? ""; + expect(runtimeDirectory).toBe(`/run/user/${String(uid)}`); + const socket = path.join(runtimeDirectory, "podman", "podman.sock"); + fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); + + let service: ChildProcess | null = startPodmanQualificationService(socket, progress); + let hostEngine: PodmanBoundContainerEngine | null = null; + let inferenceEngine: PodmanBoundContainerEngine | null = null; + let lifecycleEngine: PodmanBoundContainerEngine | null = null; + const ownedContainers = new Set(); + const ownedVolumes = new Set(); + const ownedNetworks = new Set(); + let inferenceContainerId: string; + let gpuDevices: readonly string[] = []; + let gpuComputeProcesses: readonly GpuComputeProcess[] = []; + let completed = false; + const operationDetails = new Map>(); + + try { + progress.phase("bind the rootless Podman engine"); + await waitForSocket(socket, service); + const socketAuthority = capturePodmanSocketAuthority(socket); + hostEngine = createPodmanContainerEngine({ + operation: "host-doctor", + socketAuthority, + }); + inferenceEngine = createPodmanContainerEngine({ + operation: "host-local-inference", + socketAuthority, + }); + lifecycleEngine = createPodmanContainerEngine({ + operation: "sandbox-lifecycle", + socketAuthority, + }); + const bundle = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: hostEngine, + sandboxLifecycle: lifecycleEngine, + }, + }); + expect(bundle.identity.id).toBe("podman"); + expect(bundle.workload.profile.support).toBeNull(); + expect(bundle.preflightDoctor.inspectHost()).toMatchObject({ + status: "ok", + }); + const caseSuffix = sha256(row.id).slice(0, 12); + const networkName = `nemoclaw-q-${caseSuffix}`; + const network = createProviderNetwork(inferenceEngine, networkName, row.id); + ownedNetworks.add(network.id); + const hostPort = 20_000 + (Number.parseInt(caseSuffix.slice(0, 4), 16) % 20_000); + const runnerContract = + row.case.acceleration === "nvidia-gpu" + ? readNativeRuntimeQualificationRunnerContract(row.case.architecture) + : undefined; + const agentImage = nativeRuntimeQualificationAgentImage(row.case.architecture, row.case.agent); + const inference = nativeRuntimeQualificationInferenceImage({ + architecture: row.case.architecture, + acceleration: row.case.acceleration, + inference: row.case.inference, + ...(runnerContract ? { runnerContract } : {}), + }); + pullPublicImage(inferenceEngine, agentImage); + if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); + else { + if (!inference.cachePath || !path.isAbsolute(inference.cachePath)) { + throw new Error("Native runtime qualification GPU cache path must be absolute"); + } + requirePreloadedImage(inferenceEngine, inference.imageRef); + rootOwnedReadOnlyDirectory(inference.cachePath); + } + if (runnerContract) { + requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const inferenceName = `nemoclaw-inference-${caseSuffix}`; + const endpoint = `http://${network.gateway}:${String(hostPort)}`; + progress.phase("launch exact local inference"); + const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; + const inferenceArguments = [ + "run", + "--detach", + "--pull=never", + "--name", + inferenceName, + "--network", + network.name, + "--publish", + `127.0.0.1:${String(hostPort)}:${String(inferencePort)}`, + "--publish", + `${network.gateway}:${String(hostPort)}:${String(inferencePort)}`, + "--label", + `${QUALIFICATION_LABEL}=${row.id}`, + ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), + ...(row.case.inference === "nim" + ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/opt/nim/.cache:ro`] + : []), + ...(row.case.inference === "vllm" + ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/models:ro`] + : []), + inference.imageRef, + ...(row.case.inference === "vllm" + ? [ + "--model", + "/models", + "--served-model-name", + inference.model, + "--host", + "0.0.0.0", + "--port", + String(inferencePort), + "--max-model-len", + "2048", + ] + : []), + ]; + inferenceContainerId = capture( + inferenceEngine, + inferenceArguments, + `${row.case.inference} container start`, + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(inferenceContainerId)) { + throw new Error("Inference container did not return a full immutable ID"); + } + ownedContainers.add(inferenceContainerId); + if (row.case.inference === "ollama") { + capture( + inferenceEngine, + ["exec", inferenceContainerId, "ollama", "pull", inference.model], + "Ollama model acquisition", + INFERENCE_TIMEOUT, + ); + } + if (row.case.acceleration === "nvidia-gpu") { + if (!runnerContract) throw new Error("GPU runner contract is unavailable"); + gpuDevices = proveGpuDevices(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const sandboxId = caseSuffix; + progress.phase("onboard the managed agent image"); + const sandboxName = `qualification-${row.case.agent}`; + const agentName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; + const volumeName = `nemoclaw-q-state-${caseSuffix}`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "agent volume creation", + ); + ownedVolumes.add(volumeName); + let agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedContainers.add(agentId); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' qualified >/qualification/state"], + "agent state initialization", + ); + operationDetails.set("agent.onboard", { + containerId: agentId, + agent: row.case.agent, + imageDigest: digestFromImageReference(agentImage), + }); + + const turnSha256 = await agentTurn(lifecycleEngine, agentId, endpoint, inference.model); + if (row.case.acceleration === "nvidia-gpu") { + gpuComputeProcesses = proveGpuBackedInference( + inferenceEngine, + inferenceContainerId, + gpuDevices, + ); + } + operationDetails.set("agent.turn", { + protocol: "openai-chat-completions", + model: inference.model, + responseSha256: turnSha256, + route: "provider-network-gateway", + }); + + if (!bundle.lifecycle.supported) throw new Error("Podman lifecycle surface is unavailable"); + progress.phase("exercise sandbox lifecycle and state recovery"); + const lifecycle = bundle.lifecycle; + const input = lifecycleInput(row.case.agent, sandboxName); + let beforeStopCalled = false; + expect( + lifecycle.stop(input, { + beforeStop: () => { + beforeStopCalled = true; + }, + }), + ).toMatchObject({ exitCode: 0, state: "stopped" }); + expect(beforeStopCalled).toBe(true); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + operationDetails.set("sandbox.stop-start", { + containerId: agentId, + executionPath: "runtime-provider-bundle", + stoppedAndStarted: true, + }); + + const snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); + expect(lifecycle.stop(input, { beforeStop: () => undefined })).toMatchObject({ exitCode: 0 }); + capture( + lifecycleEngine, + ["volume", "export", "--output", snapshot, volumeName], + "sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + const snapshotBytes = fs.readFileSync(snapshot); + const snapshotSha256 = sha256(snapshotBytes); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' drifted >/qualification/state"], + "sandbox state mutation", + ); + capture(lifecycleEngine, ["rm", "--force", agentId], "remove sandbox before rebuild"); + ownedContainers.delete(agentId); + capture(lifecycleEngine, ["volume", "rm", volumeName], "remove sandbox volume"); + ownedVolumes.delete(volumeName); + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "recreate sandbox volume", + ); + ownedVolumes.add(volumeName); + capture( + lifecycleEngine, + ["volume", "import", volumeName, snapshot], + "restore sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedContainers.add(agentId); + expect( + capture(lifecycleEngine, ["exec", agentId, "cat", "/qualification/state"], "restored state"), + ).toBe("qualified"); + operationDetails.set("sandbox.snapshot-restore", { + snapshotSha256, + restoredStateSha256: sha256("qualified\n"), + }); + operationDetails.set("sandbox.rebuild", { + priorContainerReplaced: true, + rebuiltContainerId: agentId, + preservedState: true, + }); + + const focusedResults: Record = Object.create(null); + if (row.focusedOperations.length > 0) { + const cloneVolume = `${volumeName}-clone`; + const cloneName = `${agentName}-clone`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, cloneVolume], + "clone volume creation", + ); + ownedVolumes.add(cloneVolume); + capture( + lifecycleEngine, + ["volume", "import", cloneVolume, snapshot], + "clone volume restore", + INFERENCE_TIMEOUT, + ); + const cloneId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: cloneName, + network: network.name, + qualificationId: row.id, + sandboxId: `${sandboxId}c`, + sandboxName: `${sandboxName}-clone`, + volume: cloneVolume, + }); + ownedContainers.add(cloneId); + expect( + capture(lifecycleEngine, ["exec", cloneId, "cat", "/qualification/state"], "clone state"), + ).toBe("qualified"); + const duplicate = lifecycleEngine.capture([ + "run", + "--detach", + "--pull=never", + "--name", + agentName, + "--entrypoint", + "/bin/sh", + agentImage, + "-c", + "exit 0", + ]); + if (duplicate.status === 0) throw new Error("Podman allowed unsafe managed-name reuse"); + capture(lifecycleEngine, ["kill", "--signal", "KILL", agentId], "sandbox crash injection"); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + expect( + capture( + lifecycleEngine, + ["exec", agentId, "cat", "/qualification/state"], + "recovered state", + ), + ).toBe("qualified"); + focusedResults.clone = { cloneContainerId: cloneId, restored: true }; + focusedResults.backup = { + sha256: snapshotSha256, + bytes: snapshotBytes.length, + }; + focusedResults["crash-recovery"] = { + signal: "SIGKILL", + recovered: true, + }; + focusedResults.rollback = { restoredSnapshotSha256: snapshotSha256 }; + focusedResults["name-reuse"] = { rejected: true }; + } + + if (!inferenceContainerId) throw new Error("Inference runtime identity is missing"); + progress.phase("restart and reconcile inference"); + capture( + inferenceEngine, + ["restart", inferenceContainerId], + `${row.case.inference} runtime restart`, + INFERENCE_TIMEOUT, + ); + const reconciledTurnSha256 = await agentTurn( + lifecycleEngine, + agentId, + endpoint, + inference.model, + ); + const reconciledGpuComputeProcesses = + row.case.acceleration === "nvidia-gpu" + ? proveGpuBackedInference(inferenceEngine, inferenceContainerId, gpuDevices) + : []; + operationDetails.set("runtime.restart-reconcile", { + service: row.case.inference, + runtimeIdentity: inferenceContainerId, + responseSha256: reconciledTurnSha256, + gpuComputeProcesses: reconciledGpuComputeProcesses, + revalidated: true, + }); + + operationDetails.set("installer.install", { + authority: "trusted-installer-step", + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + }); + const rootfulSelectionDenied = row.rootModes.includes("rootful") + ? command("podman", ["--root", "/var/lib/containers/storage", "info"]).status !== 0 + : true; + if (!rootfulSelectionDenied) { + throw new Error( + "Unprivileged qualification unexpectedly obtained a rootful Podman storage authority", + ); + } + operationDetails.set("runtime.docker-unavailable", { + beforeCandidate: dockerBefore, + rootfulSelectionDenied, + executedRootMode: "rootless", + }); + + progress.phase("prove exact cleanup"); + capture(lifecycleEngine, ["rm", "--force", agentId], "agent cleanup"); + ownedContainers.delete(agentId); + for (const containerId of [...ownedContainers]) { + if (containerId === inferenceContainerId) continue; + capture(lifecycleEngine, ["rm", "--force", containerId], "focused container cleanup"); + ownedContainers.delete(containerId); + } + for (const volume of [...ownedVolumes]) { + capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); + ownedVolumes.delete(volume); + } + capture(inferenceEngine, ["rm", "--force", inferenceContainerId], "inference runtime cleanup"); + ownedContainers.delete(inferenceContainerId); + capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); + ownedNetworks.delete(network.id); + fs.rmSync(snapshot, { force: true }); + assertNoQualificationResidue(lifecycleEngine, row.id); + operationDetails.set("cleanup.exact", { + containersRemaining: 0, + volumesRemaining: 0, + networksRemaining: 0, + }); + + if (row.focusedOperations.length > 0) { + Object.assign(focusedResults, { + restart: operationDetails.get("runtime.restart-reconcile"), + rebuild: operationDetails.get("sandbox.rebuild"), + "snapshot-restore": operationDetails.get("sandbox.snapshot-restore"), + installer: operationDetails.get("installer.install"), + cleanup: operationDetails.get("cleanup.exact"), + }); + const missing = row.focusedOperations.filter( + (operation) => !Object.hasOwn(focusedResults, operation), + ); + if (missing.length > 0) { + throw new Error(`Focused qualification operations are incomplete: ${missing.join(", ")}`); + } + } + + const dockerAfter = assertDockerUnavailable(); + progress.phase("emit bounded case evidence"); + const podmanVersion = requireCommand("podman", ["--version"], "Podman version"); + const managedImages = [ + { role: "agent", digest: digestFromImageReference(agentImage) }, + { + role: "inference", + digest: digestFromImageReference(inference.imageRef), + }, + ...(runnerContract + ? [ + { + role: "gpu-probe", + digest: digestFromImageReference(runnerContract.gpuProbeImageRef), + }, + ] + : []), + ]; + writeJson(receiptDirectory, "runtime-result.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { + providerId: "podman", + executionPath: "runtime-provider-bundle", + rootMode: "rootless", + podmanVersion, + inferenceService: row.case.inference, + focusedOperations: focusedResults, + dockerBefore, + dockerAfter, + }, + }); + for (const obligation of row.case.obligations) { + const details = operationDetails.get(obligation); + if (!details) throw new Error(`Qualification operation '${obligation}' was not executed`); + writeJson(receiptDirectory, nativeRuntimeQualificationOperationFile(obligation), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: obligation, + result: "passed", + details, + }); + } + if (row.case.acceleration === "nvidia-gpu") { + writeJson(receiptDirectory, "nvidia-cdi.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { + requested: "nvidia.com/gpu=all", + selectedDevices: gpuDevices, + inferenceRuntimeId: inferenceContainerId, + inferenceComputeProcesses: gpuComputeProcesses, + }, + }); + } + writeJson(receiptDirectory, "case-evidence.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: podmanVersion.replace(/^podman version\s+/u, ""), + managedImages, + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { + nvidiaCdi: { + device: "nvidia.com/gpu=all", + file: "nvidia-cdi.json", + }, + } + : {}), + }); + writeJson(receiptDirectory, "execution.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-execution-v1", + caseId: row.id, + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + agent: row.case.agent, + inference: row.case.inference, + rootModes: row.rootModes, + obligations: row.case.obligations, + focusedOperations: row.focusedOperations, + evidenceKinds: row.case.evidenceKinds, + dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, + credentialBoundary: { + githubCredentialsAbsent: true, + modelCredentialsAbsent: true, + isolatedUid: true, + }, + result: "passed", + }); + completed = true; + } finally { + if (!completed) { + if (lifecycleEngine) { + for (const containerId of ownedContainers) { + lifecycleEngine.capture(["rm", "--force", containerId], COMMAND_TIMEOUT); + } + for (const volume of ownedVolumes) { + lifecycleEngine.capture(["volume", "rm", "--force", volume], COMMAND_TIMEOUT); + } + } + if (inferenceEngine) { + for (const networkId of ownedNetworks) { + inferenceEngine.capture(["network", "rm", "--force", networkId], COMMAND_TIMEOUT); + } + } + } + await stopService(service, socket); + service = null; + } +} diff --git a/test/e2e/live/native-runtime-qualification-case.test.ts b/test/e2e/live/native-runtime-qualification-case.test.ts index 2a5e1d01d0e..677832c6508 100644 --- a/test/e2e/live/native-runtime-qualification-case.test.ts +++ b/test/e2e/live/native-runtime-qualification-case.test.ts @@ -1,1075 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync, type ChildProcess } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { - capturePodmanSocketAuthority, - createPodmanContainerEngine, - type PodmanBoundContainerEngine, -} from "../../../src/lib/adapters/podman/index.ts"; -import type { RuntimeProviderLifecycleInput } from "../../../src/lib/onboard/runtime-provider/contract.ts"; -import { createPodmanRuntimeProviderBundle } from "../../../src/lib/onboard/runtime-provider/podman.ts"; -import { - PODMAN_MANAGED_LABEL, - PODMAN_SANDBOX_CONTAINER_PREFIX, - PODMAN_SANDBOX_ID_LABEL, - PODMAN_SANDBOX_NAME_LABEL, - PODMAN_SANDBOX_NAMESPACE, - PODMAN_SANDBOX_NAMESPACE_LABEL, - PODMAN_SANDBOX_WORKSPACE, - PODMAN_SANDBOX_WORKSPACE_LABEL, -} from "../../../src/lib/onboard/runtime-provider/podman-lifecycle.ts"; -import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; -import type { TestProgress } from "../fixtures/progress.ts"; -import type { NativeRuntimeQualificationObligation } from "../registry/native-runtime-qualification.ts"; -import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { test } from "../fixtures/e2e-test.ts"; import { - assertCredentialFreeQualificationEnvironment, - digestFromImageReference, - nativeRuntimeQualificationAgentImage, - nativeRuntimeQualificationInferenceImage, - parseNativeRuntimeQualificationRow, - readNativeRuntimeQualificationRunnerContract, -} from "./native-runtime-qualification-case-helpers.ts"; + executeNativeRuntimeQualificationCase, + NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES, +} from "./native-runtime-qualification-case-executor.ts"; const ENABLED = process.env.NEMOCLAW_RUN_LIVE_E2E === "1" && typeof process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW === "string"; -const FULL_ID = /^[a-f0-9]{64}$/u; -const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; -const COMMAND_TIMEOUT = 60_000; -const INFERENCE_TIMEOUT = 900_000; -const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; -const E2E_PHASES = [ - "validate credential-free Docker-unavailable isolation", - "bind the rootless Podman engine", - "launch exact local inference", - "onboard the managed agent image", - "exercise sandbox lifecycle and state recovery", - "restart and reconcile inference", - "prove exact cleanup", - "emit bounded case evidence", -] as const; - -interface PodmanNetworkAuthority { - readonly id: string; - readonly name: string; - readonly gateway: string; -} - -interface CommandResult { - readonly status: number; - readonly stdout: string; - readonly stderr: string; -} - -interface GpuComputeProcess { - readonly gpuUuid: string; - readonly pid: number; - readonly processName: string; - readonly usedMemoryMiB: number; -} - -function bounded(value: string): string { - return value.replace(CONTROL, " ").replace(/\s+/gu, " ").trim().slice(-500); -} - -function command(command: string, args: readonly string[]): CommandResult { - const result = spawnSync(command, [...args], { - encoding: "utf8", - env: process.env, - timeout: 10_000, - killSignal: "SIGKILL", - maxBuffer: 1024 * 1024, - }); - return { - status: result.status ?? (result.signal ? 128 : 127), - stdout: result.stdout ?? "", - stderr: result.stderr ?? result.error?.message ?? "", - }; -} - -function requireCommand(executable: string, args: readonly string[], label: string): string { - const result = command(executable, args); - if (result.status !== 0) { - throw new Error( - `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout)}`, - ); - } - return result.stdout.trim(); -} - -function capture( - engine: PodmanBoundContainerEngine, - args: readonly string[], - label: string, - timeout = COMMAND_TIMEOUT, -): string { - const result = engine.capture(args, timeout); - if (result.status !== 0 || result.error) { - throw new Error( - `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout || result.error?.message || "unknown failure")}`, - ); - } - return result.stdout.trim(); -} - -function sha256(value: string | Buffer): string { - return createHash("sha256").update(value).digest("hex"); -} - -function exactDirectory(directory: string): void { - const metadata = fs.lstatSync(directory); - const uid = process.getuid?.() ?? -1; - if ( - !metadata.isDirectory() || - metadata.isSymbolicLink() || - metadata.uid !== uid || - (metadata.mode & 0o077) !== 0 - ) { - throw new Error("Qualification receipt directory must be private and current-user owned"); - } -} - -function writeJson(directory: string, file: string, value: unknown): void { - const target = path.join(directory, file); - const temporary = `${target}.tmp`; - const serialized = `${JSON.stringify(value, null, 2)}\n`; - fs.writeFileSync(temporary, serialized, { - encoding: "utf8", - flag: "wx", - mode: 0o600, - }); - fs.renameSync(temporary, target); -} - -function assertDockerUnavailable(): Record { - const guarded = command("docker", ["version"]); - if (guarded.status !== 97) { - throw new Error(`Docker PATH invocation guard returned ${String(guarded.status)}, expected 97`); - } - for (const executable of ["/usr/bin/docker", "/usr/local/bin/docker", "/snap/bin/docker"]) { - if (!fs.existsSync(executable)) continue; - const result = command(executable, ["version"]); - if (result.status === 0) - throw new Error(`Absolute Docker client remained usable: ${executable}`); - } - for (const socket of ["/var/run/docker.sock", "/run/docker.sock"]) { - const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); - if (metadata?.isSocket()) throw new Error(`Docker socket remained available: ${socket}`); - } - for (const unit of ["docker.service", "docker.socket"]) { - if (command("systemctl", ["is-active", "--quiet", unit]).status === 0) { - throw new Error(`Docker unit remained active: ${unit}`); - } - } - const proc = fs.readdirSync("/proc").filter((entry) => /^[1-9][0-9]*$/u.test(entry)); - for (const pid of proc) { - try { - if (fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim() === "dockerd") { - throw new Error("Docker daemon process remained active"); - } - } catch (error) { - if (error instanceof Error && error.message === "Docker daemon process remained active") { - throw error; - } - } - } - return { - dockerCommandGuarded: true, - dockerServiceInactive: true, - dockerSocketUnitInactive: true, - dockerdProcessNameAbsent: true, - defaultSocketPathsAbsent: true, - }; -} - -async function waitForSocket(socket: string, child: ChildProcess): Promise { - const deadline = Date.now() + 30_000; - while (Date.now() < deadline) { - if (child.exitCode !== null || child.signalCode !== null) { - throw new Error("Rootless Podman API service exited before its socket became ready"); - } - const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); - if (metadata?.isSocket()) return; - await new Promise((resolve) => setTimeout(resolve, 100)); - } - throw new Error("Rootless Podman API service did not create its socket"); -} - -function startPodmanQualificationService(socket: string, progress: TestProgress): ChildProcess { - return spawnObservedChild("podman", ["system", "service", "--time=0", `unix://${socket}`], { - activityLabel: "command: rootless Podman qualification service", - progress, - spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, - }); -} - -async function stopService(child: ChildProcess | null, socket: string): Promise { - if (child && child.exitCode === null && child.signalCode === null) { - child.kill("SIGTERM"); - const deadline = Date.now() + 10_000; - while (Date.now() < deadline && child.exitCode === null && child.signalCode === null) { - await new Promise((resolve) => setTimeout(resolve, 50)); - } - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); - } - fs.rmSync(socket, { force: true }); -} - -function createProviderNetwork( - engine: PodmanBoundContainerEngine, - name: string, - caseId: string, -): PodmanNetworkAuthority { - const id = capture( - engine, - ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], - "provider network creation", - ); - if (!FULL_ID.test(id)) throw new Error("Provider network did not return a full immutable ID"); - const inspected = JSON.parse( - capture(engine, ["network", "inspect", id], "provider network inspection"), - ) as Array<{ - id?: unknown; - name?: unknown; - subnets?: Array<{ gateway?: unknown }>; - }>; - const entry = inspected[0]; - const gateway = entry?.subnets?.[0]?.gateway; - if ( - inspected.length !== 1 || - entry?.id !== id || - entry.name !== name || - typeof gateway !== "string" - ) { - throw new Error("Provider network inspection lacks exact identity"); - } - return Object.freeze({ id, name, gateway }); -} - -function pullPublicImage(engine: PodmanBoundContainerEngine, imageRef: string): void { - capture(engine, ["pull", imageRef], `pull ${imageRef}`, INFERENCE_TIMEOUT); - capture(engine, ["image", "exists", imageRef], `inspect pulled image ${imageRef}`); -} - -function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: string): void { - capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); -} - -function rootOwnedReadOnlyDirectory(directory: string): void { - const canonical = fs.realpathSync(directory); - if (canonical !== directory) { - throw new Error(`Runner model resource is not canonical: ${directory}`); - } - const boundary = "/var/lib/nemoclaw/native-runtime-qualification"; - let current = directory; - while (current.startsWith(`${boundary}/`) || current === boundary) { - const metadata = fs.lstatSync(current); - if ( - !metadata.isDirectory() || - metadata.isSymbolicLink() || - metadata.uid !== 0 || - (metadata.mode & 0o022) !== 0 - ) { - throw new Error( - `Runner model resource is not an exact root-owned read-only directory: ${current}`, - ); - } - if (current === boundary) return; - current = path.dirname(current); - } - throw new Error(`Runner model resource escapes its reviewed root: ${directory}`); -} - -function proveGpuDevices( - engine: PodmanBoundContainerEngine, - probeImageRef: string, -): readonly string[] { - const devices = capture( - engine, - [ - "run", - "--rm", - "--pull=never", - "--device", - "nvidia.com/gpu=all", - probeImageRef, - "nvidia-smi", - "--query-gpu=uuid", - "--format=csv,noheader", - ], - "NVIDIA CDI runtime proof", - ) - .split(/\r?\n/u) - .map((entry) => entry.trim()) - .filter(Boolean) - .sort(); - if ( - devices.length === 0 || - new Set(devices).size !== devices.length || - devices.some((device) => !/^GPU-[0-9A-Fa-f-]{36}$/u.test(device)) - ) { - throw new Error("NVIDIA CDI runtime proof did not return exact physical GPU UUIDs"); - } - return Object.freeze(devices); -} - -function proveGpuBackedInference( - engine: PodmanBoundContainerEngine, - containerId: string, - selectedDevices: readonly string[], -): readonly GpuComputeProcess[] { - const output = capture( - engine, - [ - "exec", - containerId, - "nvidia-smi", - "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", - "--format=csv,noheader,nounits", - ], - "GPU-backed inference process proof", - ); - const processes = output - .split(/\r?\n/u) - .map((line) => line.split(",").map((field) => field.trim())) - .filter((fields) => fields.length === 4) - .map(([gpuUuid, pid, processName, usedMemoryMiB]) => ({ - gpuUuid: gpuUuid ?? "", - pid: Number(pid), - processName: processName ?? "", - usedMemoryMiB: Number(usedMemoryMiB), - })) - .filter( - (entry) => - selectedDevices.includes(entry.gpuUuid) && - Number.isSafeInteger(entry.pid) && - entry.pid > 0 && - entry.processName.length > 0 && - !/[\u0000-\u001f\u007f-\u009f]/u.test(entry.processName) && - Number.isSafeInteger(entry.usedMemoryMiB) && - entry.usedMemoryMiB > 0, - ); - if (processes.length === 0) { - throw new Error("Inference turn did not leave an exact GPU compute process proof"); - } - return Object.freeze(processes.map((entry) => Object.freeze(entry))); -} - -function createAgentContainer(input: { - readonly engine: PodmanBoundContainerEngine; - readonly imageRef: string; - readonly name: string; - readonly network: string; - readonly qualificationId: string; - readonly sandboxId: string; - readonly sandboxName: string; - readonly volume: string; -}): string { - const id = capture( - input.engine, - [ - "run", - "--detach", - "--pull=never", - "--name", - input.name, - "--network", - input.network, - "--label", - `${PODMAN_MANAGED_LABEL}=true`, - "--label", - `${PODMAN_SANDBOX_NAME_LABEL}=${input.sandboxName}`, - "--label", - `${PODMAN_SANDBOX_ID_LABEL}=${input.sandboxId}`, - "--label", - `${PODMAN_SANDBOX_NAMESPACE_LABEL}=${PODMAN_SANDBOX_NAMESPACE}`, - "--label", - `${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, - "--label", - `${QUALIFICATION_LABEL}=${input.qualificationId}`, - "--volume", - `${input.volume}:/qualification`, - "--entrypoint", - "/bin/sh", - input.imageRef, - "-c", - "while :; do sleep 3600; done", - ], - "agent container creation", - INFERENCE_TIMEOUT, - ); - if (!FULL_ID.test(id)) throw new Error("Agent container did not return a full immutable ID"); - return id; -} - -async function agentTurn( - engine: PodmanBoundContainerEngine, - containerId: string, - endpoint: string, - model: string, -): Promise { - const body = JSON.stringify({ - model, - messages: [{ role: "user", content: "Reply with the single word qualified." }], - max_tokens: 32, - stream: false, - }); - const args = [ - "exec", - containerId, - "curl", - "--fail-with-body", - "--silent", - "--show-error", - "--connect-timeout", - "5", - "--max-time", - "60", - "--header", - "Content-Type: application/json", - "--data-binary", - body, - `${endpoint}/v1/chat/completions`, - ]; - const deadline = Date.now() + 600_000; - let output = ""; - let lastFailure = "inference request was not attempted"; - while (Date.now() < deadline) { - const result = engine.capture(args, 90_000); - if (result.status === 0 && !result.error) { - output = result.stdout.trim(); - break; - } - lastFailure = bounded(result.stderr || result.stdout || result.error?.message || "failed"); - await new Promise((resolve) => setTimeout(resolve, 2_000)); - } - if (!output) throw new Error(`Agent inference turn did not become ready: ${lastFailure}`); - const response = JSON.parse(output) as { - model?: unknown; - choices?: Array<{ - finish_reason?: unknown; - message?: { content?: unknown; tool_calls?: unknown }; - }>; - }; - const first = response.choices?.[0]; - if ( - response.model !== model || - typeof first?.finish_reason !== "string" || - first.finish_reason === "length" || - (typeof first.message?.content !== "string" && !Array.isArray(first.message?.tool_calls)) - ) { - throw new Error("Agent turn did not return a complete exact-model inference response"); - } - return sha256(output); -} - -function lifecycleInput(agent: string, sandboxName: string): RuntimeProviderLifecycleInput { - const sandbox: SandboxEntry = { - agent, - name: sandboxName, - openshellDriver: "podman", - }; - return { - environment: process.env, - log: () => undefined, - sandbox, - sandboxName, - }; -} - -function assertNoQualificationResidue(engine: PodmanBoundContainerEngine, caseId: string): void { - for (const [resource, args] of [ - ["container", ["ps", "--all", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], - ["volume", ["volume", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], - ["network", ["network", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], - ] as const) { - if (capture(engine, args, `qualification ${resource} residue inspection`) !== "") { - throw new Error(`Qualification cleanup left an owned ${resource}`); - } - } -} test.skipIf(!ENABLED)( "executes one exact credential-free native runtime qualification case", - { meta: { e2ePhases: E2E_PHASES }, timeout: 1_800_000 }, - async ({ progress }) => { - progress.phase("validate credential-free Docker-unavailable isolation"); - assertCredentialFreeQualificationEnvironment(process.env); - expect(process.platform).toBe("linux"); - const uid = process.getuid?.() ?? 0; - expect(uid, "Native runtime qualification must execute as an unprivileged UID").toBeGreaterThan( - 0, - ); - const row = parseNativeRuntimeQualificationRow( - process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW ?? "", - ); - const expectedArchitecture = process.arch === "x64" ? "amd64" : process.arch; - expect(expectedArchitecture).toBe(row.case.architecture); - const receiptPath = process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT ?? ""; - expect(path.basename(receiptPath)).toBe("execution.json"); - const receiptDirectory = path.dirname(receiptPath); - exactDirectory(receiptDirectory); - - const dockerBefore = assertDockerUnavailable(); - const runtimeDirectory = process.env.XDG_RUNTIME_DIR ?? ""; - expect(runtimeDirectory).toBe(`/run/user/${String(uid)}`); - const socket = path.join(runtimeDirectory, "podman", "podman.sock"); - fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); - - let service: ChildProcess | null = startPodmanQualificationService(socket, progress); - let hostEngine: PodmanBoundContainerEngine | null = null; - let inferenceEngine: PodmanBoundContainerEngine | null = null; - let lifecycleEngine: PodmanBoundContainerEngine | null = null; - const ownedContainers = new Set(); - const ownedVolumes = new Set(); - const ownedNetworks = new Set(); - let inferenceContainerId: string; - let gpuDevices: readonly string[] = []; - let gpuComputeProcesses: readonly GpuComputeProcess[] = []; - let completed = false; - const operationDetails = new Map< - NativeRuntimeQualificationObligation, - Record - >(); - - try { - progress.phase("bind the rootless Podman engine"); - await waitForSocket(socket, service); - const socketAuthority = capturePodmanSocketAuthority(socket); - hostEngine = createPodmanContainerEngine({ - operation: "host-doctor", - socketAuthority, - }); - inferenceEngine = createPodmanContainerEngine({ - operation: "host-local-inference", - socketAuthority, - }); - lifecycleEngine = createPodmanContainerEngine({ - operation: "sandbox-lifecycle", - socketAuthority, - }); - const bundle = createPodmanRuntimeProviderBundle({ - engines: { - hostDoctor: hostEngine, - sandboxLifecycle: lifecycleEngine, - }, - }); - expect(bundle.identity.id).toBe("podman"); - expect(bundle.workload.profile.support).toBeNull(); - expect(bundle.preflightDoctor.inspectHost()).toMatchObject({ - status: "ok", - }); - const caseSuffix = sha256(row.id).slice(0, 12); - const networkName = `nemoclaw-q-${caseSuffix}`; - const network = createProviderNetwork(inferenceEngine, networkName, row.id); - ownedNetworks.add(network.id); - const hostPort = 20_000 + (Number.parseInt(caseSuffix.slice(0, 4), 16) % 20_000); - const runnerContract = - row.case.acceleration === "nvidia-gpu" - ? readNativeRuntimeQualificationRunnerContract(row.case.architecture) - : undefined; - const agentImage = nativeRuntimeQualificationAgentImage( - row.case.architecture, - row.case.agent, - ); - const inference = nativeRuntimeQualificationInferenceImage({ - architecture: row.case.architecture, - acceleration: row.case.acceleration, - inference: row.case.inference, - ...(runnerContract ? { runnerContract } : {}), - }); - pullPublicImage(inferenceEngine, agentImage); - if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); - else { - if (!inference.cachePath || !path.isAbsolute(inference.cachePath)) { - throw new Error("Native runtime qualification GPU cache path must be absolute"); - } - requirePreloadedImage(inferenceEngine, inference.imageRef); - rootOwnedReadOnlyDirectory(inference.cachePath); - } - if (runnerContract) { - requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); - } - - const inferenceName = `nemoclaw-inference-${caseSuffix}`; - const endpoint = `http://${network.gateway}:${String(hostPort)}`; - progress.phase("launch exact local inference"); - const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; - const inferenceArguments = [ - "run", - "--detach", - "--pull=never", - "--name", - inferenceName, - "--network", - network.name, - "--publish", - `127.0.0.1:${String(hostPort)}:${String(inferencePort)}`, - "--publish", - `${network.gateway}:${String(hostPort)}:${String(inferencePort)}`, - "--label", - `${QUALIFICATION_LABEL}=${row.id}`, - ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), - ...(row.case.inference === "nim" - ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/opt/nim/.cache:ro`] - : []), - ...(row.case.inference === "vllm" - ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/models:ro`] - : []), - inference.imageRef, - ...(row.case.inference === "vllm" - ? [ - "--model", - "/models", - "--served-model-name", - inference.model, - "--host", - "0.0.0.0", - "--port", - String(inferencePort), - "--max-model-len", - "2048", - ] - : []), - ]; - inferenceContainerId = capture( - inferenceEngine, - inferenceArguments, - `${row.case.inference} container start`, - INFERENCE_TIMEOUT, - ); - if (!FULL_ID.test(inferenceContainerId)) { - throw new Error("Inference container did not return a full immutable ID"); - } - ownedContainers.add(inferenceContainerId); - if (row.case.inference === "ollama") { - capture( - inferenceEngine, - ["exec", inferenceContainerId, "ollama", "pull", inference.model], - "Ollama model acquisition", - INFERENCE_TIMEOUT, - ); - } - if (row.case.acceleration === "nvidia-gpu") { - if (!runnerContract) throw new Error("GPU runner contract is unavailable"); - gpuDevices = proveGpuDevices(inferenceEngine, runnerContract.gpuProbeImageRef); - } - - const sandboxId = caseSuffix; - progress.phase("onboard the managed agent image"); - const sandboxName = `qualification-${row.case.agent}`; - const agentName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; - const volumeName = `nemoclaw-q-state-${caseSuffix}`; - capture( - lifecycleEngine, - ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], - "agent volume creation", - ); - ownedVolumes.add(volumeName); - let agentId = createAgentContainer({ - engine: lifecycleEngine, - imageRef: agentImage, - name: agentName, - network: network.name, - qualificationId: row.id, - sandboxId, - sandboxName, - volume: volumeName, - }); - ownedContainers.add(agentId); - capture( - lifecycleEngine, - ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' qualified >/qualification/state"], - "agent state initialization", - ); - operationDetails.set("agent.onboard", { - containerId: agentId, - agent: row.case.agent, - imageDigest: digestFromImageReference(agentImage), - }); - - const turnSha256 = await agentTurn(lifecycleEngine, agentId, endpoint, inference.model); - if (row.case.acceleration === "nvidia-gpu") { - gpuComputeProcesses = proveGpuBackedInference( - inferenceEngine, - inferenceContainerId, - gpuDevices, - ); - } - operationDetails.set("agent.turn", { - protocol: "openai-chat-completions", - model: inference.model, - responseSha256: turnSha256, - route: "provider-network-gateway", - }); - - if (!bundle.lifecycle.supported) throw new Error("Podman lifecycle surface is unavailable"); - progress.phase("exercise sandbox lifecycle and state recovery"); - const lifecycle = bundle.lifecycle; - const input = lifecycleInput(row.case.agent, sandboxName); - let beforeStopCalled = false; - expect( - lifecycle.stop(input, { - beforeStop: () => { - beforeStopCalled = true; - }, - }), - ).toMatchObject({ exitCode: 0, state: "stopped" }); - expect(beforeStopCalled).toBe(true); - expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); - operationDetails.set("sandbox.stop-start", { - containerId: agentId, - executionPath: "runtime-provider-bundle", - stoppedAndStarted: true, - }); - - const snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); - expect(lifecycle.stop(input, { beforeStop: () => undefined })).toMatchObject({ exitCode: 0 }); - capture( - lifecycleEngine, - ["volume", "export", "--output", snapshot, volumeName], - "sandbox volume snapshot", - INFERENCE_TIMEOUT, - ); - const snapshotBytes = fs.readFileSync(snapshot); - const snapshotSha256 = sha256(snapshotBytes); - expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); - capture( - lifecycleEngine, - ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' drifted >/qualification/state"], - "sandbox state mutation", - ); - capture(lifecycleEngine, ["rm", "--force", agentId], "remove sandbox before rebuild"); - ownedContainers.delete(agentId); - capture(lifecycleEngine, ["volume", "rm", volumeName], "remove sandbox volume"); - ownedVolumes.delete(volumeName); - capture( - lifecycleEngine, - ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], - "recreate sandbox volume", - ); - ownedVolumes.add(volumeName); - capture( - lifecycleEngine, - ["volume", "import", volumeName, snapshot], - "restore sandbox volume snapshot", - INFERENCE_TIMEOUT, - ); - agentId = createAgentContainer({ - engine: lifecycleEngine, - imageRef: agentImage, - name: agentName, - network: network.name, - qualificationId: row.id, - sandboxId, - sandboxName, - volume: volumeName, - }); - ownedContainers.add(agentId); - expect( - capture( - lifecycleEngine, - ["exec", agentId, "cat", "/qualification/state"], - "restored state", - ), - ).toBe("qualified"); - operationDetails.set("sandbox.snapshot-restore", { - snapshotSha256, - restoredStateSha256: sha256("qualified\n"), - }); - operationDetails.set("sandbox.rebuild", { - priorContainerReplaced: true, - rebuiltContainerId: agentId, - preservedState: true, - }); - - const focusedResults: Record = Object.create(null); - if (row.focusedOperations.length > 0) { - const cloneVolume = `${volumeName}-clone`; - const cloneName = `${agentName}-clone`; - capture( - lifecycleEngine, - ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, cloneVolume], - "clone volume creation", - ); - ownedVolumes.add(cloneVolume); - capture( - lifecycleEngine, - ["volume", "import", cloneVolume, snapshot], - "clone volume restore", - INFERENCE_TIMEOUT, - ); - const cloneId = createAgentContainer({ - engine: lifecycleEngine, - imageRef: agentImage, - name: cloneName, - network: network.name, - qualificationId: row.id, - sandboxId: `${sandboxId}c`, - sandboxName: `${sandboxName}-clone`, - volume: cloneVolume, - }); - ownedContainers.add(cloneId); - expect( - capture(lifecycleEngine, ["exec", cloneId, "cat", "/qualification/state"], "clone state"), - ).toBe("qualified"); - const duplicate = lifecycleEngine.capture([ - "run", - "--detach", - "--pull=never", - "--name", - agentName, - "--entrypoint", - "/bin/sh", - agentImage, - "-c", - "exit 0", - ]); - if (duplicate.status === 0) throw new Error("Podman allowed unsafe managed-name reuse"); - capture(lifecycleEngine, ["kill", "--signal", "KILL", agentId], "sandbox crash injection"); - expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); - expect( - capture( - lifecycleEngine, - ["exec", agentId, "cat", "/qualification/state"], - "recovered state", - ), - ).toBe("qualified"); - focusedResults.clone = { cloneContainerId: cloneId, restored: true }; - focusedResults.backup = { - sha256: snapshotSha256, - bytes: snapshotBytes.length, - }; - focusedResults["crash-recovery"] = { - signal: "SIGKILL", - recovered: true, - }; - focusedResults.rollback = { restoredSnapshotSha256: snapshotSha256 }; - focusedResults["name-reuse"] = { rejected: true }; - } - - if (!inferenceContainerId) throw new Error("Inference runtime identity is missing"); - progress.phase("restart and reconcile inference"); - capture( - inferenceEngine, - ["restart", inferenceContainerId], - `${row.case.inference} runtime restart`, - INFERENCE_TIMEOUT, - ); - const reconciledTurnSha256 = await agentTurn( - lifecycleEngine, - agentId, - endpoint, - inference.model, - ); - const reconciledGpuComputeProcesses = - row.case.acceleration === "nvidia-gpu" - ? proveGpuBackedInference(inferenceEngine, inferenceContainerId, gpuDevices) - : []; - operationDetails.set("runtime.restart-reconcile", { - service: row.case.inference, - runtimeIdentity: inferenceContainerId, - responseSha256: reconciledTurnSha256, - gpuComputeProcesses: reconciledGpuComputeProcesses, - revalidated: true, - }); - - operationDetails.set("installer.install", { - authority: "trusted-installer-step", - candidateSha: row.source.candidateSha, - installerSha256: row.installerSha256, - }); - const rootfulSelectionDenied = row.rootModes.includes("rootful") - ? command("podman", ["--root", "/var/lib/containers/storage", "info"]).status !== 0 - : true; - if (!rootfulSelectionDenied) { - throw new Error( - "Unprivileged qualification unexpectedly obtained a rootful Podman storage authority", - ); - } - operationDetails.set("runtime.docker-unavailable", { - beforeCandidate: dockerBefore, - rootfulSelectionDenied, - executedRootMode: "rootless", - }); - - progress.phase("prove exact cleanup"); - capture(lifecycleEngine, ["rm", "--force", agentId], "agent cleanup"); - ownedContainers.delete(agentId); - for (const containerId of [...ownedContainers]) { - if (containerId === inferenceContainerId) continue; - capture(lifecycleEngine, ["rm", "--force", containerId], "focused container cleanup"); - ownedContainers.delete(containerId); - } - for (const volume of [...ownedVolumes]) { - capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); - ownedVolumes.delete(volume); - } - capture( - inferenceEngine, - ["rm", "--force", inferenceContainerId], - "inference runtime cleanup", - ); - ownedContainers.delete(inferenceContainerId); - capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); - ownedNetworks.delete(network.id); - fs.rmSync(snapshot, { force: true }); - assertNoQualificationResidue(lifecycleEngine, row.id); - operationDetails.set("cleanup.exact", { - containersRemaining: 0, - volumesRemaining: 0, - networksRemaining: 0, - }); - - if (row.focusedOperations.length > 0) { - Object.assign(focusedResults, { - restart: operationDetails.get("runtime.restart-reconcile"), - rebuild: operationDetails.get("sandbox.rebuild"), - "snapshot-restore": operationDetails.get("sandbox.snapshot-restore"), - installer: operationDetails.get("installer.install"), - cleanup: operationDetails.get("cleanup.exact"), - }); - const missing = row.focusedOperations.filter( - (operation) => !Object.hasOwn(focusedResults, operation), - ); - if (missing.length > 0) { - throw new Error(`Focused qualification operations are incomplete: ${missing.join(", ")}`); - } - } - - const dockerAfter = assertDockerUnavailable(); - progress.phase("emit bounded case evidence"); - const podmanVersion = requireCommand("podman", ["--version"], "Podman version"); - const managedImages = [ - { role: "agent", digest: digestFromImageReference(agentImage) }, - { - role: "inference", - digest: digestFromImageReference(inference.imageRef), - }, - ...(runnerContract - ? [ - { - role: "gpu-probe", - digest: digestFromImageReference(runnerContract.gpuProbeImageRef), - }, - ] - : []), - ]; - writeJson(receiptDirectory, "runtime-result.json", { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-runtime-v1", - caseId: row.id, - result: "passed", - details: { - providerId: "podman", - executionPath: "runtime-provider-bundle", - rootMode: "rootless", - podmanVersion, - inferenceService: row.case.inference, - focusedOperations: focusedResults, - dockerBefore, - dockerAfter, - }, - }); - for (const obligation of row.case.obligations) { - const details = operationDetails.get(obligation); - if (!details) throw new Error(`Qualification operation '${obligation}' was not executed`); - writeJson(receiptDirectory, nativeRuntimeQualificationOperationFile(obligation), { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-operation-v1", - caseId: row.id, - operationId: obligation, - result: "passed", - details, - }); - } - if (row.case.acceleration === "nvidia-gpu") { - writeJson(receiptDirectory, "nvidia-cdi.json", { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", - caseId: row.id, - result: "passed", - details: { - requested: "nvidia.com/gpu=all", - selectedDevices: gpuDevices, - inferenceRuntimeId: inferenceContainerId, - inferenceComputeProcesses: gpuComputeProcesses, - }, - }); - } - writeJson(receiptDirectory, "case-evidence.json", { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-case-details-v1", - caseId: row.id, - runtime: { - engineName: "Podman", - engineVersion: podmanVersion.replace(/^podman version\s+/u, ""), - managedImages, - resultFile: "runtime-result.json", - }, - operations: row.case.obligations.map((id) => ({ - id, - file: nativeRuntimeQualificationOperationFile(id), - })), - ...(row.case.acceleration === "nvidia-gpu" - ? { - nvidiaCdi: { - device: "nvidia.com/gpu=all", - file: "nvidia-cdi.json", - }, - } - : {}), - }); - writeJson(receiptDirectory, "execution.json", { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-execution-v1", - caseId: row.id, - candidateSha: row.source.candidateSha, - installerSha256: row.installerSha256, - architecture: row.case.architecture, - acceleration: row.case.acceleration, - agent: row.case.agent, - inference: row.case.inference, - rootModes: row.rootModes, - obligations: row.case.obligations, - focusedOperations: row.focusedOperations, - evidenceKinds: row.case.evidenceKinds, - dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, - credentialBoundary: { - githubCredentialsAbsent: true, - modelCredentialsAbsent: true, - isolatedUid: true, - }, - result: "passed", - }); - completed = true; - } finally { - if (!completed) { - if (lifecycleEngine) { - for (const containerId of ownedContainers) { - lifecycleEngine.capture(["rm", "--force", containerId], COMMAND_TIMEOUT); - } - for (const volume of ownedVolumes) { - lifecycleEngine.capture(["volume", "rm", "--force", volume], COMMAND_TIMEOUT); - } - } - if (inferenceEngine) { - for (const networkId of ownedNetworks) { - inferenceEngine.capture(["network", "rm", "--force", networkId], COMMAND_TIMEOUT); - } - } - } - await stopService(service, socket); - service = null; - } - }, + { meta: { e2ePhases: NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES }, timeout: 1_800_000 }, + async ({ progress }) => executeNativeRuntimeQualificationCase(progress), ); diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts index 3f0412cfc66..5da9ddc123f 100644 --- a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -121,14 +121,20 @@ function candidateReceipts( details: { proof: id }, }); } - if (row.case.acceleration === "nvidia-gpu") { - writeJson(path.join(directory, "nvidia-cdi.json"), { - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", - caseId: row.id, - result: "passed", - details: { device: "nvidia.com/gpu=all" }, - }); + const cdiReceipts = + row.case.acceleration === "nvidia-gpu" + ? [ + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }, + ] + : []; + for (const receipt of cdiReceipts) { + writeJson(path.join(directory, "nvidia-cdi.json"), receipt); } writeJson(path.join(directory, "case-evidence.json"), { schemaVersion: 1, diff --git a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts index 72f49b69ed7..097763187ec 100644 --- a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts @@ -157,17 +157,20 @@ function fixture(options: { readonly gpu?: boolean } = {}) { }), ); } - if (row.case.acceleration === "nvidia-gpu") { - fs.writeFileSync( - path.join(executionDirectory, "nvidia-cdi.json"), - JSON.stringify({ - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", - caseId: row.id, - result: "passed", - details: { device: "nvidia.com/gpu=all" }, - }), - ); + const cdiReceipts = + row.case.acceleration === "nvidia-gpu" + ? [ + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }, + ] + : []; + for (const receipt of cdiReceipts) { + fs.writeFileSync(path.join(executionDirectory, "nvidia-cdi.json"), JSON.stringify(receipt)); } fs.writeFileSync( path.join(executionDirectory, "case-evidence.json"), diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index ab0360995d2..a7795771ccf 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -392,7 +392,7 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map Date: Sat, 15 Aug 2026 21:10:27 -0500 Subject: [PATCH 06/71] test(e2e): cover arm64 qualification receipts --- ...me-qualification-producer-evidence.test.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts index 097763187ec..b69391d3100 100644 --- a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts @@ -47,7 +47,7 @@ function fixture(options: { readonly gpu?: boolean } = {}) { }); const row = options.gpu ? plan.include.find( - (entry) => entry.case.architecture === "amd64" && entry.case.acceleration === "nvidia-gpu", + (entry) => entry.case.architecture === "arm64" && entry.case.acceleration === "nvidia-gpu", )! : plan.include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; const installerDirectory = path.join(root, "installer"); @@ -228,7 +228,7 @@ describe("native runtime qualification producer evidence", () => { source: value.row.source, case: value.row.case, installer: { - architecture: "amd64", + architecture: value.row.case.architecture, dockerAvailability: "unavailable", exitCode: 0, providerId: "podman", @@ -271,12 +271,22 @@ describe("native runtime qualification producer evidence", () => { const fragment = JSON.parse( fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8"), - ) as { nvidiaCdi: { artifact: { path: string; sha256: string } } }; - const copied = fs.readFileSync( + ) as { + installer: { architecture: string; script: { path: string; sha256: string } }; + nvidiaCdi: { artifact: { path: string; sha256: string } }; + }; + expect(fragment.installer.architecture).toBe("arm64"); + const copiedInstaller = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.installer.script.path), + ); + expect(createHash("sha256").update(copiedInstaller).digest("hex")).toBe( + fragment.installer.script.sha256, + ); + const copiedCdi = fs.readFileSync( path.join(value.evidenceDirectory, fragment.nvidiaCdi.artifact.path), ); expect(fragment.nvidiaCdi.artifact.path).toContain("/runtime/nvidia-cdi.json"); - expect(createHash("sha256").update(copied).digest("hex")).toBe( + expect(createHash("sha256").update(copiedCdi).digest("hex")).toBe( fragment.nvidiaCdi.artifact.sha256, ); }); From d0bfd86966d5ffd35f4c258be738fd6ffacbafb3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 21:16:30 -0500 Subject: [PATCH 07/71] fix(e2e): confirm Podman service cleanup --- .../native-runtime-qualification-case-executor.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index b7677f23b93..9e8462c5265 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -216,7 +216,18 @@ async function stopService(child: ChildProcess | null, socket: string): Promise< while (Date.now() < deadline && child.exitCode === null && child.signalCode === null) { await new Promise((resolve) => setTimeout(resolve, 50)); } - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + if (child.exitCode === null && child.signalCode === null) { + if (!child.kill("SIGKILL")) { + throw new Error("Rootless Podman API service rejected SIGKILL"); + } + const killDeadline = Date.now() + 10_000; + while (Date.now() < killDeadline && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (child.exitCode === null && child.signalCode === null) { + throw new Error("Rootless Podman API service remained alive after SIGKILL"); + } + } } fs.rmSync(socket, { force: true }); } From 2efe4e81e533eccc455ac9c97ae9c016d4a308aa Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 21:34:56 -0500 Subject: [PATCH 08/71] fix(e2e): prepare native qualification runners Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 40 ++++++++++++++++++- ...untime-qualification-producer-plan.test.ts | 2 +- ...me-qualification-producer-workflow.test.ts | 23 +++++++++++ ...ve-runtime-qualification-producer-plan.mts | 2 +- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 810ffdcb23e..ff35ba61fa7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1067,6 +1067,23 @@ jobs: with: node-version: 22.19.0 + - name: Install Podman 5 and rootless prerequisites from the signed runner OS repository + shell: bash + run: | + set -euo pipefail + [[ -x /usr/bin/apt-get ]] || { + echo "::error::Protected runner cannot install Podman from its signed OS repository" >&2 + exit 1 + } + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ + --yes --no-install-recommends \ + acl apparmor fuse-overlayfs passt podman slirp4netns uidmap + [[ "$(podman --version)" =~ ^podman\ version\ 5[.] ]] || { + echo "::error::Protected runner must provide Podman 5" >&2 + exit 1 + } + - name: Prepare the credential-free execution account and disable Docker id: boundary env: @@ -1074,7 +1091,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in git jq node npm pgrep podman sha256sum systemctl; do + for command in git jq node npm pgrep podman setfacl sha256sum systemctl; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1099,6 +1116,15 @@ jobs: home="$(getent passwd "$account" | cut -d: -f6)" runtime_dir="/run/user/${uid}" sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir" + ancestor="$(dirname "$CANDIDATE_DIRECTORY")" + while [[ "$ancestor" != "/home" ]]; do + [[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || { + echo "::error::Candidate checkout ancestor is outside the reviewed runner workspace" >&2 + exit 1 + } + sudo setfacl --modify "u:${account}:--x" "$ancestor" + ancestor="$(dirname "$ancestor")" + done sudo chown -R "$uid:$uid" "$CANDIDATE_DIRECTORY" node_directory="$(dirname "$(command -v node)")" [[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || { @@ -1128,7 +1154,17 @@ jobs: HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ - npm --prefix "$CANDIDATE_DIRECTORY" ci --ignore-scripts + /bin/bash --noprofile --norc -c ' + set -euo pipefail + cd "$1" + for file in package.json package-lock.json; do + [[ -f "$file" && ! -L "$file" && -O "$file" ]] || { + echo "Candidate dependency manifest is missing or invalid: $file" >&2 + exit 1 + } + done + exec npm --prefix "$1" ci --ignore-scripts + ' bash "$CANDIDATE_DIRECTORY" - name: Run the authenticated installer qualification env: diff --git a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts index 33c0855a3c9..b95e7779096 100644 --- a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts @@ -66,7 +66,7 @@ describe("native runtime qualification producer plan", () => { plan.include.find( (entry) => entry.case.architecture === "arm64" && entry.case.acceleration === "cpu", )?.runner, - ).toBe("ubuntu-24.04-arm"); + ).toBe("ubuntu-26.04-arm"); expect( plan.include.find( (entry) => entry.case.architecture === "amd64" && entry.case.acceleration === "nvidia-gpu", diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 05c4ce70937..544239f13ca 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -99,6 +99,10 @@ describe("native runtime qualification producer workflow", () => { it("runs each candidate case in an isolated account and emits one trusted artifact", () => { const producer = job("native-runtime-qualification-producer"); const harness = step(producer, "Check out the trusted qualification harness"); + const podman = step( + producer, + "Install Podman 5 and rootless prerequisites from the signed runner OS repository", + ); const boundary = step( producer, "Prepare the credential-free execution account and disable Docker", @@ -126,14 +130,33 @@ describe("native runtime qualification producer workflow", () => { "test/e2e/registry/native-runtime-qualification.ts", ); expect(source).not.toMatch(/NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u); + expect(podman.run).toContain("/usr/bin/apt-get install"); + for (const requiredPackage of [ + "acl", + "apparmor", + "fuse-overlayfs", + "passt", + "podman", + "slirp4netns", + "uidmap", + ]) { + expect(podman.run).toContain(requiredPackage); + } + expect(podman.run).toContain("^podman\\ version\\ 5[.]"); + expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); + expect(boundary.run).toContain('setfacl --modify "u:${account}:--x"'); + expect(boundary.run).not.toContain("chmod o+x"); expect(boundaryRun.indexOf("printf 'account=%s")).toBeLessThan( boundaryRun.indexOf("useradd --create-home"), ); expect(dependencies.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(dependencies.run).toContain('cd "$1"'); + expect(dependencies.run).toContain("package.json package-lock.json"); + expect(dependencies.run).toContain('! -L "$file" && -O "$file"'); expect(dependencies.run).toContain("npm --prefix"); expect(dependencies.run).toContain("ci --ignore-scripts"); expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); diff --git a/tools/e2e/native-runtime-qualification-producer-plan.mts b/tools/e2e/native-runtime-qualification-producer-plan.mts index 68e7ed3ff4f..b3006a71803 100644 --- a/tools/e2e/native-runtime-qualification-producer-plan.mts +++ b/tools/e2e/native-runtime-qualification-producer-plan.mts @@ -129,7 +129,7 @@ function validateSource( function runnerForCase(entry: NativeRuntimeQualificationCase, arm64GpuRunner: string): string { if (entry.architecture === "amd64" && entry.acceleration === "cpu") return "ubuntu-26.04"; if (entry.architecture === "arm64" && entry.acceleration === "cpu") { - return "ubuntu-24.04-arm"; + return "ubuntu-26.04-arm"; } if (entry.architecture === "amd64") return "linux-amd64-gpu-rtxpro6000-latest-1"; if (!RUNNER_LABEL.test(arm64GpuRunner)) { From b788822c5b454bd8d5f913aea7f13f2de5afdc49 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 21:48:47 -0500 Subject: [PATCH 09/71] fix(e2e): pin native Podman qualification Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 16 +++++++++------- test/e2e/mock-parity.json | 9 +++++++++ ...ntime-qualification-producer-workflow.test.ts | 5 ++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ff35ba61fa7..4243b503e3b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1068,6 +1068,8 @@ jobs: node-version: 22.19.0 - name: Install Podman 5 and rootless prerequisites from the signed runner OS repository + env: + PODMAN_APT_VERSION: "5.7.0+ds2-3build1" shell: bash run: | set -euo pipefail @@ -1078,11 +1080,11 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ - acl apparmor fuse-overlayfs passt podman slirp4netns uidmap - [[ "$(podman --version)" =~ ^podman\ version\ 5[.] ]] || { - echo "::error::Protected runner must provide Podman 5" >&2 - exit 1 - } + acl apparmor fuse-overlayfs passt "podman=$PODMAN_APT_VERSION" slirp4netns uidmap + package_version="$(dpkg-query --show --showformat='${Version}' podman)" + version="$(podman --version)" + [[ "$package_version" == "$PODMAN_APT_VERSION" ]] + [[ "$version" == "podman version 5.7.0" ]] - name: Prepare the credential-free execution account and disable Docker id: boundary @@ -1097,8 +1099,8 @@ jobs: exit 1 } done - [[ "$(podman --version)" =~ ^podman\ version\ 5[.] ]] || { - echo "::error::Protected runner must provide Podman 5" >&2 + [[ "$(podman --version)" == "podman version 5.7.0" ]] || { + echo "::error::Protected runner must provide Podman 5.7.0" >&2 exit 1 } sudo systemctl stop docker.service docker.socket 2>/dev/null || true diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 41b93ad979c..2c4788efc75 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -74,6 +74,15 @@ "test/runtime-provider-source-shape.test.ts" ] }, + { + "live": "test/e2e/live/native-runtime-qualification-case.test.ts", + "fast": [ + "test/e2e/support/native-runtime-qualification-case-helpers.test.ts", + "test/e2e/support/native-runtime-qualification-producer-workflow.test.ts", + "test/e2e/support/native-runtime-qualification.test.ts", + "test/install-native-runtime-qualification.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 544239f13ca..9601c91485f 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -142,7 +142,10 @@ describe("native runtime qualification producer workflow", () => { ]) { expect(podman.run).toContain(requiredPackage); } - expect(podman.run).toContain("^podman\\ version\\ 5[.]"); + expect(podman.env?.PODMAN_APT_VERSION).toBe("5.7.0+ds2-3build1"); + expect(podman.run).toContain('"podman=$PODMAN_APT_VERSION"'); + expect(podman.run).toContain('[[ "$package_version" == "$PODMAN_APT_VERSION" ]]'); + expect(podman.run).toContain('[[ "$version" == "podman version 5.7.0" ]]'); expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); From 20cdd8067b0ad3e64d9f278c0162da100a93b07c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 22:35:44 -0500 Subject: [PATCH 10/71] fix(e2e): build pinned Podman 6 qualification toolchain --- .github/workflows/e2e.yaml | 273 +++++++++++++++++- ...me-qualification-producer-workflow.test.ts | 105 ++++++- tools/e2e/operations-workflow-boundary.mts | 22 ++ ...upload-e2e-artifacts-workflow-boundary.mts | 9 + 4 files changed, 394 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4243b503e3b..3285e7f2cf6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1012,9 +1012,159 @@ jobs: WORKFLOW_SHA: ${{ github.workflow_sha }} run: node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-plan.mts --ci-output + native-runtime-qualification-podman-toolchain: + name: Build pinned native Podman toolchain / ${{ matrix.architecture }} + needs: [generate-matrix, native-runtime-qualification-producer-plan] + if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runner: ubuntu-24.04 + - architecture: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Check out the pinned Podman source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: podman-container-tools/podman + ref: cade97a52ebdf9dbf9e81de8009015776837a074 # v6.1.0 + path: .podman-source + fetch-depth: 1 + persist-credentials: false + + - name: Check out the pinned Netavark source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: containers/netavark + ref: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab # v2.1.0 + path: .netavark-source + fetch-depth: 1 + persist-credentials: false + + - name: Check out the pinned Aardvark DNS source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: containers/aardvark-dns + ref: cd7417681229219059939bdd9f0b3bd9ac9abb08 # v2.1.0 + path: .aardvark-source + fetch-depth: 1 + persist-credentials: false + + - name: Set up pinned Go for the Podman build + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: 1.25.9 + cache: false + + - name: Set up pinned Rust for the network helper builds + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.88.0 + + - name: Install build dependencies from the signed runner OS repository + shell: bash + run: | + set -euo pipefail + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ + --yes --no-install-recommends \ + gcc git libapparmor-dev libassuan-dev libbtrfs-dev libc6-dev \ + libdevmapper-dev libglib2.0-dev libgpg-error-dev libgpgme-dev \ + libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ + libsqlite3-dev libsubid-dev libsystemd-dev make pkg-config protobuf-compiler + + - name: Build and package the pinned native toolchain + env: + AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08 + EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} + NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab + PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074 + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain + shell: bash + run: | + set -euo pipefail + [[ "$(dpkg --print-architecture)" == "$EXPECTED_ARCHITECTURE" ]] + [[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]] + [[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]] + [[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]] + for source in .podman-source .netavark-source .aardvark-source; do + [[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]] + done + [[ "$(go version)" == go\ version\ go1.25.9\ * ]] + [[ "$(rustc --version)" == rustc\ 1.88.0\ * ]] + + SOURCE_DATE_EPOCH=1786554266 \ + BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \ + make --directory=.podman-source --jobs=2 \ + podman rootlessport PREFIX=/usr/local + SOURCE_DATE_EPOCH=1785940686 CI=1 \ + make --directory=.netavark-source --jobs=2 build + SOURCE_DATE_EPOCH=1785940850 CI=1 \ + make --directory=.aardvark-source --jobs=2 build + + install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" + install -D -m 0755 .podman-source/bin/rootlessport \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" + install -D -m 0755 .netavark-source/bin/netavark \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" + install -D -m 0755 .aardvark-source/bin/aardvark-dns \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" + install -D -m 0644 \ + .podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \ + "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" + + [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]] + [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]] + [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]] + jq -n \ + --arg architecture "$EXPECTED_ARCHITECTURE" \ + --arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \ + --arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \ + --arg podmanSourceSha "$PODMAN_SOURCE_SHA" ' + { + schemaVersion: 1, + kind: "nemoclaw-native-podman-toolchain-v1", + architecture: $architecture, + podmanVersion: "6.1.0", + podmanSourceSha: $podmanSourceSha, + netavarkVersion: "2.1.0", + netavarkSourceSha: $netavarkSourceSha, + aardvarkDnsVersion: "2.1.0", + aardvarkDnsSourceSha: $aardvarkDnsSourceSha, + goVersion: "1.25.9", + rustVersion: "1.88.0" + } + ' >"$TOOLCHAIN_DIRECTORY/manifest.json" + ( + cd "$TOOLCHAIN_DIRECTORY" + sha256sum \ + bin/podman \ + libexec/podman/aardvark-dns \ + libexec/podman/netavark \ + libexec/podman/rootlessport \ + manifest.json \ + share/containers/containers.conf >SHA256SUMS + ) + + - name: Upload the pinned native Podman toolchain + if: success() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: native-runtime-podman-toolchain-${{ matrix.architecture }} + path: ${{ runner.temp }}/native-runtime-podman-toolchain/ + native-runtime-qualification-producer: name: ${{ matrix.jobName }} - needs: [generate-matrix, native-runtime-qualification-producer-plan] + needs: + - generate-matrix + - native-runtime-qualification-podman-toolchain + - native-runtime-qualification-producer-plan if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} strategy: fail-fast: false @@ -1067,9 +1217,35 @@ jobs: with: node-version: 22.19.0 - - name: Install Podman 5 and rootless prerequisites from the signed runner OS repository + - name: Require a reviewed Ubuntu runtime host + shell: bash + run: | + set -euo pipefail + [[ -r /etc/os-release ]] || { + echo "::error::Protected runner does not expose an OS release identity" >&2 + exit 1 + } + # shellcheck disable=SC1091 + source /etc/os-release + [[ "${ID:-}" == "ubuntu" ]] || { + echo "::error::Protected runner must use a reviewed Ubuntu image" >&2 + exit 1 + } + [[ "${VERSION_ID:-}" == "24.04" || "${VERSION_ID:-}" == "26.04" ]] || { + echo "::error::Protected runner Ubuntu release is not reviewed for Podman qualification" >&2 + exit 1 + } + + - name: Download the pinned native Podman toolchain + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-runtime-podman-toolchain-${{ matrix.case.architecture }} + path: ${{ runner.temp }}/native-runtime-podman-toolchain + + - name: Install the pinned native Podman toolchain and rootless prerequisites env: - PODMAN_APT_VERSION: "5.7.0+ds2-3build1" + EXPECTED_ARCHITECTURE: ${{ matrix.case.architecture }} + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail @@ -1080,11 +1256,90 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ - acl apparmor fuse-overlayfs passt "podman=$PODMAN_APT_VERSION" slirp4netns uidmap - package_version="$(dpkg-query --show --showformat='${Version}' podman)" + acl apparmor btrfs-progs conmon fuse-overlayfs \ + golang-github-containers-common iptables nftables passt runc slirp4netns uidmap + + [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] + [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] + mapfile -t actual_files < <( + cd "$TOOLCHAIN_DIRECTORY" + find . -type f -print | LC_ALL=C sort + ) + expected_files=( + ./SHA256SUMS + ./bin/podman + ./libexec/podman/aardvark-dns + ./libexec/podman/netavark + ./libexec/podman/rootlessport + ./manifest.json + ./share/containers/containers.conf + ) + [[ "${actual_files[*]}" == "${expected_files[*]}" ]] || { + echo "::error::Downloaded native Podman toolchain contains unexpected files" >&2 + exit 1 + } + ( + cd "$TOOLCHAIN_DIRECTORY" + sha256sum --check --strict SHA256SUMS + ) + jq -e \ + --arg architecture "$EXPECTED_ARCHITECTURE" ' + type == "object" and + keys == [ + "aardvarkDnsSourceSha", + "aardvarkDnsVersion", + "architecture", + "goVersion", + "kind", + "netavarkSourceSha", + "netavarkVersion", + "podmanSourceSha", + "podmanVersion", + "rustVersion", + "schemaVersion" + ] and + .schemaVersion == 1 and + .kind == "nemoclaw-native-podman-toolchain-v1" and + .architecture == $architecture and + .podmanVersion == "6.1.0" and + .podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and + .netavarkVersion == "2.1.0" and + .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and + .aardvarkDnsVersion == "2.1.0" and + .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and + .goVersion == "1.25.9" and + .rustVersion == "1.88.0" + ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null + for target in \ + /usr/local/bin/podman \ + /usr/local/libexec/podman/aardvark-dns \ + /usr/local/libexec/podman/netavark \ + /usr/local/libexec/podman/rootlessport \ + /usr/share/containers/containers.conf; do + [[ ! -L "$target" ]] || { + echo "::error::Native Podman toolchain target must not be a symlink: $target" >&2 + exit 1 + } + done + sudo install --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/bin/podman" /usr/local/bin/podman + for helper in aardvark-dns netavark rootlessport; do + sudo install -D --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \ + "/usr/local/libexec/podman/$helper" + done + sudo install --owner=root --group=root --mode=0644 \ + "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \ + /usr/share/containers/containers.conf + [[ "$(command -v podman)" == "/usr/local/bin/podman" ]] + conmon_version="$(conmon --version | awk 'NR == 1 { print $NF }')" + runc_version="$(runc --version | awk 'NR == 1 { print $NF }')" + dpkg --compare-versions "$conmon_version" ge 2.1.7 + dpkg --compare-versions "$runc_version" ge 1.1.11 + [[ "$(/usr/local/libexec/podman/netavark --version)" == "netavark 2.1.0" ]] + [[ "$(/usr/local/libexec/podman/aardvark-dns --version)" == "aardvark-dns 2.1.0" ]] version="$(podman --version)" - [[ "$package_version" == "$PODMAN_APT_VERSION" ]] - [[ "$version" == "podman version 5.7.0" ]] + [[ "$version" == "podman version 6.1.0" ]] - name: Prepare the credential-free execution account and disable Docker id: boundary @@ -1099,8 +1354,8 @@ jobs: exit 1 } done - [[ "$(podman --version)" == "podman version 5.7.0" ]] || { - echo "::error::Protected runner must provide Podman 5.7.0" >&2 + [[ "$(podman --version)" == "podman version 6.1.0" ]] || { + echo "::error::Protected runner must provide Podman 6.1.0" >&2 exit 1 } sudo systemctl stop docker.service docker.socket 2>/dev/null || true diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 9601c91485f..0c072e9a3a6 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -96,12 +96,84 @@ describe("native runtime qualification producer workflow", () => { expect(source).toContain('"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"'); }); + it("builds one pinned Podman 6 toolchain for each qualified architecture", () => { + const toolchain = job("native-runtime-qualification-podman-toolchain"); + const podmanSource = step(toolchain, "Check out the pinned Podman source"); + const netavarkSource = step(toolchain, "Check out the pinned Netavark source"); + const aardvarkSource = step(toolchain, "Check out the pinned Aardvark DNS source"); + const setupGo = step(toolchain, "Set up pinned Go for the Podman build"); + const setupRust = step(toolchain, "Set up pinned Rust for the network helper builds"); + const build = step(toolchain, "Build and package the pinned native toolchain"); + const upload = step(toolchain, "Upload the pinned native Podman toolchain"); + + expect(toolchain.name).toBe( + "Build pinned native Podman toolchain / ${{ matrix.architecture }}", + ); + expect(toolchain.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-producer-plan", + ]); + expect(toolchain["runs-on"]).toBe("${{ matrix.runner }}"); + expect(toolchain.permissions).toEqual({ contents: "read" }); + expect(toolchain.strategy).toMatchObject({ + "fail-fast": false, + matrix: { + include: [ + { architecture: "amd64", runner: "ubuntu-24.04" }, + { architecture: "arm64", runner: "ubuntu-24.04-arm" }, + ], + }, + }); + expect(podmanSource.with).toMatchObject({ + repository: "podman-container-tools/podman", + ref: "cade97a52ebdf9dbf9e81de8009015776837a074", + path: ".podman-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(netavarkSource.with).toMatchObject({ + repository: "containers/netavark", + ref: "8e91ad1d947ed325327b638f0cb906bea1f7d0ab", + path: ".netavark-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(aardvarkSource.with).toMatchObject({ + repository: "containers/aardvark-dns", + ref: "cd7417681229219059939bdd9f0b3bd9ac9abb08", + path: ".aardvark-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(setupGo.uses).toBe("actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00"); + expect(setupGo.with).toEqual({ "go-version": "1.25.9", cache: false }); + expect(setupRust.uses).toBe("dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c"); + expect(setupRust.with).toEqual({ toolchain: "1.88.0" }); + expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); + expect(build.run).not.toContain("quadlet"); + expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); + expect(build.run).toContain("make --directory=.aardvark-source --jobs=2 build"); + expect(build.run).toContain("sha256sum"); + expect(build.run).toMatch(/sha256sum[\s\S]+manifest\.json/u); + expect(build.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); + expect(upload.with).toMatchObject({ + name: "native-runtime-podman-toolchain-${{ matrix.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain/", + }); + expect(upload.if).toBe("success()"); + expect(upload.uses).toBe( + "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57", + ); + }); + it("runs each candidate case in an isolated account and emits one trusted artifact", () => { const producer = job("native-runtime-qualification-producer"); const harness = step(producer, "Check out the trusted qualification harness"); + const podmanHost = step(producer, "Require a reviewed Ubuntu runtime host"); + const podmanDownload = step(producer, "Download the pinned native Podman toolchain"); const podman = step( producer, - "Install Podman 5 and rootless prerequisites from the signed runner OS repository", + "Install the pinned native Podman toolchain and rootless prerequisites", ); const boundary = step( producer, @@ -120,6 +192,11 @@ describe("native runtime qualification producer workflow", () => { const boundaryRun = boundary.run ?? ""; expect(producer.name).toBe("${{ matrix.jobName }}"); + expect(producer.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-podman-toolchain", + "native-runtime-qualification-producer-plan", + ]); expect(producer["runs-on"]).toBe("${{ matrix.runner }}"); expect(producer.permissions).toEqual({ contents: "read" }); expect(producer.strategy).toMatchObject({ "fail-fast": false }); @@ -130,22 +207,38 @@ describe("native runtime qualification producer workflow", () => { "test/e2e/registry/native-runtime-qualification.ts", ); expect(source).not.toMatch(/NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u); + expect(podmanHost.run).toContain('[[ "${ID:-}" == "ubuntu" ]]'); + expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "24.04"'); + expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "26.04"'); + expect(podmanHost.run).toContain("Ubuntu release is not reviewed"); + expect(podmanDownload.with).toMatchObject({ + name: "native-runtime-podman-toolchain-${{ matrix.case.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain", + }); expect(podman.run).toContain("/usr/bin/apt-get install"); for (const requiredPackage of [ "acl", "apparmor", + "conmon", "fuse-overlayfs", + "golang-github-containers-common", "passt", - "podman", + "runc", "slirp4netns", "uidmap", ]) { expect(podman.run).toContain(requiredPackage); } - expect(podman.env?.PODMAN_APT_VERSION).toBe("5.7.0+ds2-3build1"); - expect(podman.run).toContain('"podman=$PODMAN_APT_VERSION"'); - expect(podman.run).toContain('[[ "$package_version" == "$PODMAN_APT_VERSION" ]]'); - expect(podman.run).toContain('[[ "$version" == "podman version 5.7.0" ]]'); + expect(podman.run).toContain("find -P"); + expect(podman.run).toContain("sha256sum --check --strict SHA256SUMS"); + expect(podman.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); + expect(podman.run).toContain("Downloaded native Podman toolchain contains unexpected files"); + expect(podman.run).toContain("Native Podman toolchain target must not be a symlink"); + expect(podman.run).toContain('dpkg --compare-versions "$conmon_version" ge 2.1.7'); + expect(podman.run).toContain('dpkg --compare-versions "$runc_version" ge 1.1.11'); + expect(podman.run).toContain('"netavark 2.1.0"'); + expect(podman.run).toContain('"aardvark-dns 2.1.0"'); + expect(podman.run).toContain('[[ "$version" == "podman version 6.1.0" ]]'); expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index e507c56efe7..42c9e4bfb6c 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -16,6 +16,7 @@ const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); const DEFAULT_ADVISOR_PATH = join(REPO_ROOT, ".github", "workflows", "pr-review-advisor.yaml"); const META_JOBS = new Set([ + "native-runtime-qualification-podman-toolchain", "native-runtime-qualification-producer-plan", "release-qualification", "relevant-e2e", @@ -489,6 +490,27 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.ref === "${{ inputs.workflow_sha || github.workflow_sha }}" && step.with?.path === ".trusted-openshell-dev-artifact"; const trustedNativeRuntimeCheckout = + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Podman source" && + step.with?.repository === "podman-container-tools/podman" && + step.with?.ref === "cade97a52ebdf9dbf9e81de8009015776837a074" && + step.with?.path === ".podman-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Netavark source" && + step.with?.repository === "containers/netavark" && + step.with?.ref === "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" && + step.with?.path === ".netavark-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Aardvark DNS source" && + step.with?.repository === "containers/aardvark-dns" && + step.with?.ref === "cd7417681229219059939bdd9f0b3bd9ac9abb08" && + step.with?.path === ".aardvark-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || (jobName === "native-runtime-qualification-producer-plan" && step.name === "Check out the trusted qualification producer" && step.with?.ref === "${{ github.workflow_sha }}") || diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 186db84a190..4879d3f4f67 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -197,6 +197,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/managed-image-protected-runtime/", }, ], + [ + "native-runtime-qualification-podman-toolchain", + { + name: "native-runtime-podman-toolchain-${{ matrix.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain/", + }, + ], [ "native-runtime-qualification-producer", { @@ -257,6 +264,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["generate-matrix", "${{ github.event_name == 'workflow_dispatch' }}"], + ["native-runtime-qualification-podman-toolchain", "success()"], ["native-runtime-qualification-producer", "success()"], ["staging-brev-launchable", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], @@ -392,6 +400,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): jobName === "generate-matrix" || jobName === "jetson-nvmap-gpu" || jobName === "live" || + jobName === "native-runtime-qualification-podman-toolchain" || jobName === "openshell-dev-artifact" || jobName === RETIRED_SELECTOR_COMPATIBILITY_JOB || env.E2E_JOB === "1" || From 695c6561f3aca9fad93ee11e2e5dfe6233e05d51 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 22:41:20 -0500 Subject: [PATCH 11/71] fix(e2e): use allowed Rust toolchain action --- .github/workflows/e2e.yaml | 4 +++- .../native-runtime-qualification-producer-workflow.test.ts | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3285e7f2cf6..175da301060 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1063,9 +1063,11 @@ jobs: cache: false - name: Set up pinned Rust for the network helper builds - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: toolchain: 1.88.0 + cache: false + rustflags: "" - name: Install build dependencies from the signed runner OS repository shell: bash diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 0c072e9a3a6..1903e2c98e8 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -147,8 +147,10 @@ describe("native runtime qualification producer workflow", () => { }); expect(setupGo.uses).toBe("actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00"); expect(setupGo.with).toEqual({ "go-version": "1.25.9", cache: false }); - expect(setupRust.uses).toBe("dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c"); - expect(setupRust.with).toEqual({ toolchain: "1.88.0" }); + expect(setupRust.uses).toBe( + "actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659", + ); + expect(setupRust.with).toEqual({ toolchain: "1.88.0", cache: false, rustflags: "" }); expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); expect(build.run).not.toContain("quadlet"); expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); From 54a65fc48defa3487442b3833078383788e40d7d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 22:50:48 -0500 Subject: [PATCH 12/71] fix(e2e): keep Podman toolchain ABI portable --- .github/workflows/e2e.yaml | 12 +++++++++++- ...e-runtime-qualification-producer-workflow.test.ts | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 175da301060..69af558b790 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1079,7 +1079,7 @@ jobs: gcc git libapparmor-dev libassuan-dev libbtrfs-dev libc6-dev \ libdevmapper-dev libglib2.0-dev libgpg-error-dev libgpgme-dev \ libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ - libsqlite3-dev libsubid-dev libsystemd-dev make pkg-config protobuf-compiler + libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler - name: Build and package the pinned native toolchain env: @@ -1110,6 +1110,16 @@ jobs: SOURCE_DATE_EPOCH=1785940850 CI=1 \ make --directory=.aardvark-source --jobs=2 build + podman_dependencies="$(ldd .podman-source/bin/podman)" + if grep -F "not found" <<<"$podman_dependencies"; then + echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2 + exit 1 + fi + if grep -F "libsubid" <<<"$podman_dependencies"; then + echo "::error::Pinned Podman build must not require the optional libsubid ABI" >&2 + exit 1 + fi + install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" install -D -m 0755 .podman-source/bin/rootlessport \ "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 1903e2c98e8..aed32043587 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -103,6 +103,10 @@ describe("native runtime qualification producer workflow", () => { const aardvarkSource = step(toolchain, "Check out the pinned Aardvark DNS source"); const setupGo = step(toolchain, "Set up pinned Go for the Podman build"); const setupRust = step(toolchain, "Set up pinned Rust for the network helper builds"); + const buildDependencies = step( + toolchain, + "Install build dependencies from the signed runner OS repository", + ); const build = step(toolchain, "Build and package the pinned native toolchain"); const upload = step(toolchain, "Upload the pinned native Podman toolchain"); @@ -151,11 +155,14 @@ describe("native runtime qualification producer workflow", () => { "actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659", ); expect(setupRust.with).toEqual({ toolchain: "1.88.0", cache: false, rustflags: "" }); + expect(buildDependencies.run).not.toContain("libsubid-dev"); expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); expect(build.run).not.toContain("quadlet"); expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); expect(build.run).toContain("make --directory=.aardvark-source --jobs=2 build"); expect(build.run).toContain("sha256sum"); + expect(build.run).toContain("Pinned Podman build has an unresolved runtime dependency"); + expect(build.run).toContain("Pinned Podman build must not require the optional libsubid ABI"); expect(build.run).toMatch(/sha256sum[\s\S]+manifest\.json/u); expect(build.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); expect(upload.with).toMatchObject({ From 566ed84c3f8fb69fbf7bd6c20aaf75ae916088ec Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 23:00:49 -0500 Subject: [PATCH 13/71] fix(e2e): use portable Podman OpenPGP backend --- .github/workflows/e2e.yaml | 10 ++++++---- ...ive-runtime-qualification-producer-workflow.test.ts | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 69af558b790..848cb5a4d9e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1076,8 +1076,8 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ - gcc git libapparmor-dev libassuan-dev libbtrfs-dev libc6-dev \ - libdevmapper-dev libglib2.0-dev libgpg-error-dev libgpgme-dev \ + gcc git libapparmor-dev libbtrfs-dev libc6-dev \ + libdevmapper-dev libglib2.0-dev \ libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler @@ -1103,6 +1103,7 @@ jobs: SOURCE_DATE_EPOCH=1786554266 \ BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \ + EXTRA_BUILDTAGS=containers_image_openpgp \ make --directory=.podman-source --jobs=2 \ podman rootlessport PREFIX=/usr/local SOURCE_DATE_EPOCH=1785940686 CI=1 \ @@ -1111,12 +1112,13 @@ jobs: make --directory=.aardvark-source --jobs=2 build podman_dependencies="$(ldd .podman-source/bin/podman)" + printf '%s\n' "$podman_dependencies" if grep -F "not found" <<<"$podman_dependencies"; then echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2 exit 1 fi - if grep -F "libsubid" <<<"$podman_dependencies"; then - echo "::error::Pinned Podman build must not require the optional libsubid ABI" >&2 + if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then + echo "::error::Pinned Podman build must not require an optional host ABI" >&2 exit 1 fi diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index aed32043587..e0f924120eb 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -156,13 +156,18 @@ describe("native runtime qualification producer workflow", () => { ); expect(setupRust.with).toEqual({ toolchain: "1.88.0", cache: false, rustflags: "" }); expect(buildDependencies.run).not.toContain("libsubid-dev"); + expect(buildDependencies.run).not.toContain("libgpgme-dev"); + expect(buildDependencies.run).not.toContain("libassuan-dev"); + expect(buildDependencies.run).not.toContain("libgpg-error-dev"); expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); + expect(build.run).toContain("EXTRA_BUILDTAGS=containers_image_openpgp"); expect(build.run).not.toContain("quadlet"); expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); expect(build.run).toContain("make --directory=.aardvark-source --jobs=2 build"); expect(build.run).toContain("sha256sum"); expect(build.run).toContain("Pinned Podman build has an unresolved runtime dependency"); - expect(build.run).toContain("Pinned Podman build must not require the optional libsubid ABI"); + expect(build.run).toContain("Pinned Podman build must not require an optional host ABI"); + expect(build.run).toContain('grep -E "libgpgme|libsubid"'); expect(build.run).toMatch(/sha256sum[\s\S]+manifest\.json/u); expect(build.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); expect(upload.with).toMatchObject({ From 9e33b53d0159ed2e113ed13e2d295a3ed5e2aa83 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 23:13:49 -0500 Subject: [PATCH 14/71] fix(e2e): validate protected installer receipt metadata --- .github/workflows/e2e.yaml | 6 +++++- .../native-runtime-qualification-producer-workflow.test.ts | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 848cb5a4d9e..d5cca5a05da 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1463,10 +1463,14 @@ jobs: --architecture "$ARCHITECTURE" \ --artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts" sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true - [[ -d "$INSTALLER_RECEIPT_PARENT/receipts" && ! -L "$INSTALLER_RECEIPT_PARENT/receipts" ]] || { + sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { echo "::error::Installer receipt directory is missing or invalid" >&2 exit 1 } + sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts" || { + echo "::error::Installer receipt directory must not be a symlink" >&2 + exit 1 + } - name: Execute the candidate qualification case without credentials env: diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index e0f924120eb..217a49557d9 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -272,7 +272,8 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); expect(installer.run).toContain("run-native-runtime-installer-qualification.sh"); expect(installer.run).not.toContain("chown -R"); - expect(installer.run).toContain('[[ -d "$INSTALLER_RECEIPT_PARENT/receipts" && ! -L'); + expect(installer.run).toContain('sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts"'); + expect(installer.run).toContain('sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(execute.run).toContain('sudo -u "$ACCOUNT" env -i'); expect(execute.run).toContain("native-runtime-qualification-case.test.ts"); expect(execute.run).not.toContain("GITHUB_TOKEN"); From 7e89c811e4122e8d84bda51d8e1378402a3b3120 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 23:26:43 -0500 Subject: [PATCH 15/71] fix(e2e): execute qualification from candidate root --- .github/workflows/e2e.yaml | 5 +++-- ...native-runtime-qualification-producer-workflow.test.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d5cca5a05da..25f8a44a760 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1483,14 +1483,15 @@ jobs: shell: bash run: | set -euo pipefail - live_test="$CANDIDATE_DIRECTORY/test/e2e/live/native-runtime-qualification-case.test.ts" - [[ -f "$live_test" && ! -L "$live_test" ]] || { + live_test="test/e2e/live/native-runtime-qualification-case.test.ts" + [[ -f "$CANDIDATE_DIRECTORY/$live_test" && ! -L "$CANDIDATE_DIRECTORY/$live_test" ]] || { echo "::error::Candidate commit does not provide the native runtime qualification case executor" >&2 exit 1 } receipt_directory="${RUNNER_TEMP}/native-runtime-case" install -d -m 0700 "$receipt_directory" sudo chown "$ACCOUNT:$ACCOUNT" "$receipt_directory" + cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ E2E_DEFAULT_ENABLED=0 \ diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 217a49557d9..e5fd8686def 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -204,6 +204,7 @@ describe("native runtime qualification producer workflow", () => { const cleanup = step(producer, "Remove qualification resources"); const source = JSON.stringify(producer); const boundaryRun = boundary.run ?? ""; + const executeRun = execute.run ?? ""; expect(producer.name).toBe("${{ matrix.jobName }}"); expect(producer.needs).toEqual([ @@ -275,6 +276,13 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(installer.run).toContain('sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(execute.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(execute.run).toContain( + 'live_test="test/e2e/live/native-runtime-qualification-case.test.ts"', + ); + expect(execute.run).toContain('cd "$CANDIDATE_DIRECTORY"'); + expect(executeRun.indexOf('cd "$CANDIDATE_DIRECTORY"')).toBeLessThan( + executeRun.indexOf('sudo -u "$ACCOUNT" env -i'), + ); expect(execute.run).toContain("native-runtime-qualification-case.test.ts"); expect(execute.run).not.toContain("GITHUB_TOKEN"); expect(execute.run).not.toContain("GH_TOKEN"); From dcda59e2398a4b565a7bbc85f09e9f1e94c79419 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 23:36:31 -0500 Subject: [PATCH 16/71] fix(e2e): distinguish PR source branch identity --- .github/workflows/e2e.yaml | 16 ++++++++-------- .../e2e-operations-workflow-boundary.test.ts | 6 +++--- ...ntime-qualification-producer-workflow.test.ts | 4 ++-- tools/e2e/operations-workflow-boundary.mts | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 25f8a44a760..9c448e33413 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -64,12 +64,12 @@ on: default: false type: boolean checkout_sha: - description: Optional lowercase PR head SHA for manual exact-revision E2E. + description: Optional lowercase latest PR commit SHA for manual exact-revision E2E. required: false default: "" type: string checkout_repository: - description: Optional PR head repository for manual exact-revision E2E. + description: Optional PR source repository for manual exact-revision E2E. required: false default: "" type: string @@ -363,8 +363,8 @@ jobs: --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } - [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR head repository" >&2; exit 1; } - [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the PR head SHA" >&2; exit 1; } + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } candidate_workflow=false if [[ "$WORKFLOW_REF" != "refs/heads/main" ]]; then @@ -374,12 +374,12 @@ jobs: exit 1 } [[ "$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA" ]] || { - echo "::error::Candidate-workflow native runtime qualification must execute the exact same-repository PR head" >&2 + echo "::error::Candidate-workflow native runtime qualification must execute the latest commit on the same-repository PR source branch" >&2 exit 1 } - head_ref="$(jq -r '.head.ref // ""' <<< "$pull_json")" - [[ -n "$head_ref" && "$WORKFLOW_REF" == "refs/heads/${head_ref}" ]] || { - echo "::error::Candidate-workflow ref must match the exact PR head branch" >&2 + pr_source_ref="$(jq -r '.head.ref // ""' <<< "$pull_json")" + [[ -n "$pr_source_ref" && "$WORKFLOW_REF" == "refs/heads/${pr_source_ref}" ]] || { + echo "::error::Candidate-workflow ref must match the PR source branch" >&2 exit 1 } fi diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 53fc1ec08fb..54f46eb72a7 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -476,7 +476,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; INCLUDE_LAUNCHABLE: "false", JOBS: jobs, PR_NUMBER: "42", - REVIEW_REASON: "Reviewed PR head revision", + REVIEW_REASON: "Reviewed latest PR commit", RUN_ATTEMPT: "1", TARGETS: targets, TRIGGERING_ACTOR: "maintainer", @@ -495,9 +495,9 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; it.each([ ["admin", "refs/heads/feat/native", 0, ""], ["maintain", "refs/heads/feat/native", 1, "requires a repository administrator"], - ["admin", "refs/heads/feat/other", 1, "must match the exact PR head branch"], + ["admin", "refs/heads/feat/other", 1, "must match the PR source branch"], ])( - "requires admin-bound exact-head candidate workflow execution for %s on %s", + "requires the latest commit on an admin-controlled PR source branch for %s on %s", (role, workflowRef, expectedStatus, expectedStderr) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index e5fd8686def..288c976adc4 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -81,7 +81,7 @@ describe("native runtime qualification producer workflow", () => { ); }); - it("limits candidate-workflow protected execution to the exact PR head and administrators", () => { + it("limits candidate-workflow protected execution to the latest commit on an administrator-controlled PR source branch", () => { const generate = job("generate-matrix"); const authenticate = step(generate, "Authenticate manual PR dispatch"); const source = authenticate.run ?? ""; @@ -92,7 +92,7 @@ describe("native runtime qualification producer workflow", () => { expect(source).toContain( '"$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA"', ); - expect(source).toContain('"$WORKFLOW_REF" == "refs/heads/${head_ref}"'); + expect(source).toContain('"$WORKFLOW_REF" == "refs/heads/${pr_source_ref}"'); expect(source).toContain('"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"'); }); diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 42c9e4bfb6c..63de9ccc362 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -358,7 +358,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "Candidate-workflow native runtime qualification requires a repository administrator", '"$JOBS" == "native-runtime-qualification-producer" && -z "$TARGETS"', '"$CHECKOUT_REPOSITORY" == "$GITHUB_REPOSITORY" && "$WORKFLOW_SHA" == "$CHECKOUT_SHA" && "$BASE_SHA" != "$CHECKOUT_SHA"', - '"$WORKFLOW_REF" == "refs/heads/${head_ref}"', + '"$WORKFLOW_REF" == "refs/heads/${pr_source_ref}"', `${acceptedJobCases}) ;;`, `Manual PR E2E accepts only empty selectors, ${acceptedJobNames}`, "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", From 00e5a16f150bbe0090fd575cf7a81edce6cd744c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 15 Aug 2026 23:59:37 -0500 Subject: [PATCH 17/71] fix(e2e): provision rootless subordinate IDs --- .github/workflows/e2e.yaml | 55 ++++++++++++++++++- ...ive-runtime-qualification-case-executor.ts | 41 +++++++++++--- ...me-qualification-producer-workflow.test.ts | 6 ++ 3 files changed, 92 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9c448e33413..3a9cfc03441 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1362,7 +1362,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in git jq node npm pgrep podman setfacl sha256sum systemctl; do + for command in awk git jq node npm pgrep podman setfacl sha256sum systemctl usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1383,6 +1383,50 @@ jobs: account="nemoclawq" printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" sudo useradd --create-home --shell /usr/sbin/nologin "$account" + ensure_subordinate_range() { + local file="$1" + local option="$2" + local range_start=100000 + local range_end + local conflict_end + [[ -f "$file" && ! -L "$file" ]] || { + echo "::error::Rootless Podman subordinate-ID file is missing or invalid: $file" >&2 + exit 1 + } + if awk -F: -v account="$account" ' + $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } + END { exit found ? 0 : 1 } + ' "$file"; then + return + fi + while :; do + ((range_start <= 4294901760)) || { + echo "::error::Protected runner has no free subordinate-ID range for rootless Podman" >&2 + exit 1 + } + range_end=$((range_start + 65535)) + conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" ' + $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ { + current_end = $2 + $3 - 1 + if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end + } + END { if (maximum != "") print maximum } + ' "$file")" + [[ -n "$conflict_end" ]] || break + range_start=$((conflict_end + 1)) + done + range_end=$((range_start + 65535)) + sudo usermod "$option" "${range_start}-${range_end}" "$account" + awk -F: -v account="$account" ' + $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } + END { exit found ? 0 : 1 } + ' "$file" || { + echo "::error::Protected runner did not provision rootless Podman subordinate IDs" >&2 + exit 1 + } + } + ensure_subordinate_range /etc/subuid --add-subuids + ensure_subordinate_range /etc/subgid --add-subgids uid="$(id -u "$account")" home="$(getent passwd "$account" | cut -d: -f6)" runtime_dir="/run/user/${uid}" @@ -1406,6 +1450,15 @@ jobs: install -d -m 0755 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" chmod 0555 "$guard_dir/docker" + sudo -u "$account" env -i \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH="$guard_dir:/usr/local/bin:/usr/bin:/bin" \ + XDG_RUNTIME_DIR="$runtime_dir" \ + podman info --format json >/dev/null || { + echo "::error::Credential-free rootless Podman readiness failed" >&2 + exit 1 + } printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 9e8462c5265..5b3b5ecca82 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -61,6 +61,11 @@ interface PodmanNetworkAuthority { readonly gateway: string; } +interface PodmanQualificationService { + readonly child: ChildProcess; + readonly diagnostic: () => string; +} + interface CommandResult { readonly status: number; readonly stdout: string; @@ -188,11 +193,14 @@ function assertDockerUnavailable(): Record { }; } -async function waitForSocket(socket: string, child: ChildProcess): Promise { +async function waitForSocket(socket: string, service: PodmanQualificationService): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { + const { child } = service; if (child.exitCode !== null || child.signalCode !== null) { - throw new Error("Rootless Podman API service exited before its socket became ready"); + throw new Error( + `Rootless Podman API service exited before its socket became ready (exit=${String(child.exitCode)}, signal=${String(child.signalCode)}): ${service.diagnostic() || "no bounded diagnostic"}`, + ); } const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); if (metadata?.isSocket()) return; @@ -201,12 +209,24 @@ async function waitForSocket(socket: string, child: ChildProcess): Promise throw new Error("Rootless Podman API service did not create its socket"); } -function startPodmanQualificationService(socket: string, progress: TestProgress): ChildProcess { - return spawnObservedChild("podman", ["system", "service", "--time=0", `unix://${socket}`], { - activityLabel: "command: rootless Podman qualification service", - progress, - spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, +function startPodmanQualificationService( + socket: string, + progress: TestProgress, +): PodmanQualificationService { + let diagnostic = ""; + const child = spawnObservedChild( + "podman", + ["system", "service", "--time=0", `unix://${socket}`], + { + activityLabel: "command: rootless Podman qualification service", + progress, + spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, + }, + ); + child.stderr?.on("data", (value: Buffer | string) => { + diagnostic = bounded(`${diagnostic} ${String(value)}`); }); + return Object.freeze({ child, diagnostic: () => diagnostic }); } async function stopService(child: ChildProcess | null, socket: string): Promise { @@ -530,7 +550,10 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const socket = path.join(runtimeDirectory, "podman", "podman.sock"); fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); - let service: ChildProcess | null = startPodmanQualificationService(socket, progress); + let service: PodmanQualificationService | null = startPodmanQualificationService( + socket, + progress, + ); let hostEngine: PodmanBoundContainerEngine | null = null; let inferenceEngine: PodmanBoundContainerEngine | null = null; let lifecycleEngine: PodmanBoundContainerEngine | null = null; @@ -1059,7 +1082,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre } } } - await stopService(service, socket); + await stopService(service?.child ?? null, socket); service = null; } } diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 288c976adc4..5a0aff2e0ee 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -257,6 +257,12 @@ describe("native runtime qualification producer workflow", () => { expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); + expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); + expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); + expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); + expect(boundary.run).toContain('sudo -u "$account" env -i'); + expect(boundary.run).toContain("podman info --format json"); + expect(boundary.run).toContain("Credential-free rootless Podman readiness failed"); expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); expect(boundary.run).toContain('setfacl --modify "u:${account}:--x"'); From 724d7b87109309e3b47d3959cc65db530fd0643f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 00:17:51 -0500 Subject: [PATCH 18/71] fix(e2e): bind rootless qualification storage --- .github/workflows/e2e.yaml | 65 ++++++++++++++++++- ...me-qualification-producer-workflow.test.ts | 11 ++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3a9cfc03441..bf4eb406750 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1362,7 +1362,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in awk git jq node npm pgrep podman setfacl sha256sum systemctl usermod; do + for command in awk git jq node npm pgrep podman setfacl sha256sum stat systemctl tee usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1429,8 +1429,34 @@ jobs: ensure_subordinate_range /etc/subgid --add-subgids uid="$(id -u "$account")" home="$(getent passwd "$account" | cut -d: -f6)" + [[ "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { + echo "::error::Qualification account home is missing or invalid" >&2 + exit 1 + } runtime_dir="/run/user/${uid}" sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir" + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + storage_config="${storage_config_directory}/storage.conf" + [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration directory already exists" >&2 + exit 1 + } + sudo install -d -o root -g root -m 0755 "$storage_config_directory" + printf '%s\n' \ + '[storage]' \ + 'driver = "overlay"' \ + "runroot = \"${home}/.local/share/containers/runroot\"" \ + "graphroot = \"${home}/.local/share/containers/storage\"" \ + "rootless_storage_path = \"${home}/.local/share/containers/storage\"" \ + '' \ + '[storage.options.overlay]' \ + 'mount_program = "/usr/bin/fuse-overlayfs"' | sudo tee "$storage_config" >/dev/null + sudo chown root:root "$storage_config" + sudo chmod 0444 "$storage_config" + [[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || { + echo "::error::Qualification storage configuration is not root-owned and read-only" >&2 + exit 1 + } ancestor="$(dirname "$CANDIDATE_DIRECTORY")" while [[ "$ancestor" != "/home" ]]; do [[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || { @@ -1451,6 +1477,7 @@ jobs: printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" chmod 0555 "$guard_dir/docker" sudo -u "$account" env -i \ + CONTAINERS_STORAGE_CONF="$storage_config" \ HOME="$home" \ LANG=C.UTF-8 \ PATH="$guard_dir:/usr/local/bin:/usr/bin:/bin" \ @@ -1463,6 +1490,7 @@ jobs: printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" + printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" - name: Install locked candidate test dependencies without scripts env: @@ -1533,6 +1561,7 @@ jobs: NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} shell: bash run: | set -euo pipefail @@ -1547,6 +1576,7 @@ jobs: cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ E2E_DEFAULT_ENABLED=0 \ E2E_JOB=1 \ HOME="$QUALIFICATION_HOME" \ @@ -1593,14 +1623,45 @@ jobs: run: | set -euo pipefail account="${ACCOUNT:-nemoclawq}" + uid="" if id "$account" >/dev/null 2>&1; then - sudo pkill -KILL -u "$(id -u "$account")" 2>/dev/null || true + uid="$(id -u "$account")" + runtime_dir="/run/user/${uid}" + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + sudo pkill -KILL -u "$uid" 2>/dev/null || true + if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then + [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration cleanup target is invalid" >&2 + exit 1 + } + sudo rm -f -- "$storage_config_directory/storage.conf" + sudo rmdir "$storage_config_directory" + fi + if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then + [[ -d "$runtime_dir" && ! -L "$runtime_dir" ]] || { + echo "::error::Qualification runtime cleanup target is invalid" >&2 + exit 1 + } + sudo rmdir "$runtime_dir/podman" 2>/dev/null || true + sudo rmdir "$runtime_dir" + fi sudo userdel --remove "$account" fi if id "$account" >/dev/null 2>&1; then echo "::error::Qualification account still exists after cleanup" >&2 exit 1 fi + if [[ -n "$uid" ]]; then + [[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || { + echo "::error::Qualification runtime directory remains after cleanup" >&2 + exit 1 + } + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration remains after cleanup" >&2 + exit 1 + } + fi - name: Upload the qualification case evidence if: success() diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 5a0aff2e0ee..898be4eeea5 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -261,6 +261,11 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); expect(boundary.run).toContain('sudo -u "$account" env -i'); + expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); + expect(boundary.run).toContain("rootless_storage_path"); + expect(boundary.run).toContain("${home}/.local/share/containers/storage"); + expect(boundary.run).toContain('mount_program = "/usr/bin/fuse-overlayfs"'); + expect(boundary.run).toContain("0:0:444"); expect(boundary.run).toContain("podman info --format json"); expect(boundary.run).toContain("Credential-free rootless Podman readiness failed"); expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); @@ -294,6 +299,8 @@ describe("native runtime qualification producer workflow", () => { expect(execute.run).not.toContain("GH_TOKEN"); expect(execute.run).not.toContain("chown -R"); expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(execute.env?.STORAGE_CONFIG).toBe("${{ steps.boundary.outputs.storage_config }}"); + expect(execute.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); expect(execute.run).toContain( 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', ); @@ -309,6 +316,10 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.if).toBe("always()"); expect(cleanup.run).toContain('account="${ACCOUNT:-nemoclawq}"'); expect(cleanup.run).toContain("pkill -KILL -u"); + expect(cleanup.run).not.toContain("rm -rf"); + expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); + expect(cleanup.run).toContain('sudo rmdir "$runtime_dir/podman"'); + expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); }); From e40e2f0177e4f1a2ca9f6098d9f79cca551dacd3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 00:50:07 -0500 Subject: [PATCH 19/71] fix(e2e): harden rootless qualification host lifecycle Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 158 +++++++-- ...me-qualification-account-lifecycle.test.ts | 308 ++++++++++++++++++ ...me-qualification-producer-workflow.test.ts | 41 ++- 3 files changed, 479 insertions(+), 28 deletions(-) create mode 100644 test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bf4eb406750..b3d0341d6bb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1362,7 +1362,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in awk git jq node npm pgrep podman setfacl sha256sum stat systemctl tee usermod; do + for command in apparmor_parser awk cat getent git grep id jq node npm pgrep podman setfacl sha256sum stat systemctl tee useradd userdel usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1381,8 +1381,47 @@ jobs: ! pgrep -x dockerd >/dev/null [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] account="nemoclawq" - printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" + ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if getent passwd "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Qualification account identity or subordinate-ID authorization already exists" >&2 + exit 1 + fi + [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { + echo "::error::Qualification account ownership marker already exists" >&2 + exit 1 + } + account_created_without_marker=0 + rollback_unmarked_account() { + local result="$?" + trap - EXIT + if ((result != 0 && account_created_without_marker == 1)); then + sudo userdel --remove "$account" 2>/dev/null || true + sudo rm -f -- "$ownership_marker" + if getent passwd "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Partially created qualification account could not be rolled back" >&2 + exit 1 + fi + fi + exit "$result" + } + trap rollback_unmarked_account EXIT sudo useradd --create-home --shell /usr/sbin/nologin "$account" + account_created_without_marker=1 + uid="$(id -u "$account")" + home="$(getent passwd "$account" | cut -d: -f6)" + [[ "$uid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { + echo "::error::Qualification account identity is missing or invalid" >&2 + exit 1 + } + printf '%s:%s\n' "$account" "$uid" | sudo tee "$ownership_marker" >/dev/null + sudo chown root:root "$ownership_marker" + sudo chmod 0400 "$ownership_marker" + [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { + echo "::error::Qualification account ownership marker is invalid" >&2 + exit 1 + } + account_created_without_marker=0 + trap - EXIT ensure_subordinate_range() { local file="$1" local option="$2" @@ -1427,16 +1466,19 @@ jobs: } ensure_subordinate_range /etc/subuid --add-subuids ensure_subordinate_range /etc/subgid --add-subgids - uid="$(id -u "$account")" - home="$(getent passwd "$account" | cut -d: -f6)" - [[ "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { - echo "::error::Qualification account home is missing or invalid" >&2 + printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" + printf 'account_created=true\n' >>"$GITHUB_OUTPUT" + runtime_dir="/run/user/${uid}" + systemctl cat user-runtime-dir@.service >/dev/null + sudo systemctl start "user-runtime-dir@${uid}.service" + [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${uid}:700" ]] || { + echo "::error::Qualification runtime directory is missing or invalid" >&2 exit 1 } - runtime_dir="/run/user/${uid}" - sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" + apparmor_profile="${storage_config_directory}/podman.apparmor" + apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration directory already exists" >&2 exit 1 @@ -1457,6 +1499,24 @@ jobs: echo "::error::Qualification storage configuration is not root-owned and read-only" >&2 exit 1 } + if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then + printf '%s\n' \ + '# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \ + '' \ + 'abi ,' \ + 'include ' \ + '' \ + "profile ${apparmor_profile_name} /usr/local/bin/podman flags=(unconfined) {" \ + ' userns,' \ + '}' | sudo tee "$apparmor_profile" >/dev/null + sudo chown root:root "$apparmor_profile" + sudo chmod 0444 "$apparmor_profile" + [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification AppArmor profile is not root-owned and read-only" >&2 + exit 1 + } + sudo apparmor_parser -r "$apparmor_profile" + fi ancestor="$(dirname "$CANDIDATE_DIRECTORY")" while [[ "$ancestor" != "/home" ]]; do [[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || { @@ -1619,39 +1679,79 @@ jobs: if: always() env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_CREATED: ${{ steps.boundary.outputs.account_created }} shell: bash run: | set -euo pipefail - account="${ACCOUNT:-nemoclawq}" + reported_account="${ACCOUNT:-}" + reported_created="${ACCOUNT_CREATED:-}" + ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + account="" uid="" - if id "$account" >/dev/null 2>&1; then - uid="$(id -u "$account")" + if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then + [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { + echo "::error::Qualification account ownership marker cleanup target is invalid" >&2 + exit 1 + } + ownership="$(sudo cat "$ownership_marker")" + [[ "$ownership" =~ ^nemoclawq:([0-9]+)$ ]] || { + echo "::error::Qualification account ownership marker content is invalid" >&2 + exit 1 + } + account="nemoclawq" + uid="${BASH_REMATCH[1]}" + [[ -z "$reported_account" || "$reported_account" == "$account" ]] || { + echo "::error::Qualification account output does not match its ownership marker" >&2 + exit 1 + } + if getent passwd "$uid" >/dev/null && ! getent passwd "$account" >/dev/null; then + echo "::error::Qualification account UID belongs to a different host account" >&2 + exit 1 + fi + if getent passwd "$account" >/dev/null; then + [[ "$(id -u "$account")" == "$uid" ]] || { + echo "::error::Qualification account UID changed before cleanup" >&2 + exit 1 + } + fi runtime_dir="/run/user/${uid}" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" - sudo pkill -KILL -u "$uid" 2>/dev/null || true + apparmor_profile="${storage_config_directory}/podman.apparmor" + if getent passwd "$account" >/dev/null; then + sudo pkill -KILL -u "$uid" 2>/dev/null || true + fi + sudo systemctl stop "user-runtime-dir@${uid}.service" if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration cleanup target is invalid" >&2 exit 1 } + if [[ -e "$apparmor_profile" || -L "$apparmor_profile" ]]; then + [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification AppArmor profile cleanup target is invalid" >&2 + exit 1 + } + sudo apparmor_parser -R "$apparmor_profile" + sudo rm -f -- "$apparmor_profile" + fi sudo rm -f -- "$storage_config_directory/storage.conf" sudo rmdir "$storage_config_directory" fi if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then - [[ -d "$runtime_dir" && ! -L "$runtime_dir" ]] || { - echo "::error::Qualification runtime cleanup target is invalid" >&2 - exit 1 - } - sudo rmdir "$runtime_dir/podman" 2>/dev/null || true - sudo rmdir "$runtime_dir" + echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2 + exit 1 + fi + if getent passwd "$account" >/dev/null; then + sudo userdel --remove "$account" + fi + if getent passwd "$account" >/dev/null; then + echo "::error::Qualification account still exists after cleanup" >&2 + exit 1 + fi + if grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2 + exit 1 fi - sudo userdel --remove "$account" - fi - if id "$account" >/dev/null 2>&1; then - echo "::error::Qualification account still exists after cleanup" >&2 - exit 1 - fi - if [[ -n "$uid" ]]; then [[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || { echo "::error::Qualification runtime directory remains after cleanup" >&2 exit 1 @@ -1661,7 +1761,15 @@ jobs: echo "::error::Qualification storage configuration remains after cleanup" >&2 exit 1 } + sudo rm -f -- "$ownership_marker" + elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then + echo "::error::Qualification account output exists without its ownership marker" >&2 + exit 1 fi + [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { + echo "::error::Qualification account ownership marker remains after cleanup" >&2 + exit 1 + } - name: Upload the qualification case evidence if: success() diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts new file mode 100644 index 00000000000..466f1e05f78 --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { readYaml, type Workflow } from "../../helpers/e2e-workflow-contract"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +function workflowScripts(): { boundary: string; cleanup: string } { + const workflow = readYaml(".github/workflows/e2e.yaml") as Workflow; + const steps = workflow.jobs["native-runtime-qualification-producer"]?.steps ?? []; + const run = (name: string): string => { + const source = steps.find((entry) => entry.name === name)?.run; + if (!source) throw new Error(`Missing workflow step ${name}`); + return source; + }; + return { + boundary: run("Prepare the credential-free execution account and disable Docker"), + cleanup: run("Remove qualification resources"), + }; +} + +function extractFunction(source: string, name: string): string { + const match = source.match(new RegExp(`${name}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); + if (!match) throw new Error(`Missing shell function ${name}`); + return `${name}() {${match[1]}\n}`; +} + +function fixtureSource(source: string): string { + return source + .replaceAll("/etc/subuid", "${FIXTURE_ROOT}/etc/subuid") + .replaceAll("/etc/subgid", "${FIXTURE_ROOT}/etc/subgid") + .replaceAll( + 'ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + 'ownership_marker="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + ) + .replaceAll('"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"') + .replaceAll('runtime_dir="/run/user/${uid}"', 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"') + .replaceAll( + 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ) + .replaceAll('"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'); +} + +function writeExecutable(file: string, source: string): void { + fs.writeFileSync(file, `#!/usr/bin/env bash\nset -euo pipefail\n${source}\n`, { mode: 0o700 }); +} + +function createFixture(): { + root: string; + bin: string; + calls: string; + passwd: string; + subuid: string; + subgid: string; + home: string; + marker: string; +} { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-account-fixture-")); + roots.push(root); + const bin = path.join(root, "bin"); + const etc = path.join(root, "etc"); + const run = path.join(root, "run"); + const home = path.join(root, "home", "nemoclawq"); + const calls = path.join(root, "calls.log"); + const passwd = path.join(etc, "passwd"); + const subuid = path.join(etc, "subuid"); + const subgid = path.join(etc, "subgid"); + fs.mkdirSync(bin, { recursive: true }); + fs.mkdirSync(etc, { recursive: true }); + fs.mkdirSync(run, { recursive: true }); + for (const file of [calls, passwd, subuid, subgid]) fs.writeFileSync(file, ""); + + writeExecutable(path.join(bin, "sudo"), `printf 'sudo:%s\\n' "$*" >>"$FIXTURE_CALLS"\nexec "$@"`); + writeExecutable( + path.join(bin, "getent"), + `[[ "$1" == passwd ]]\nawk -F: -v key="$2" '$1 == key || $3 == key { print; found = 1 } END { exit found ? 0 : 2 }' "$FIXTURE_ROOT/etc/passwd"`, + ); + writeExecutable( + path.join(bin, "id"), + `[[ "$1" == -u ]]\n[[ "\${FAIL_ID:-0}" != 1 ]] || exit 26\nawk -F: -v account="$2" '$1 == account { print $3; found = 1 } END { exit found ? 0 : 1 }' "$FIXTURE_ROOT/etc/passwd"`, + ); + writeExecutable( + path.join(bin, "useradd"), + `printf 'useradd:%s\\n' "$*" >>"$FIXTURE_CALLS" +[[ "\${FAIL_USERADD:-0}" != 1 ]] || exit 23 +account="\${!#}" +mkdir -p "$FIXTURE_HOME" +printf '%s:x:1002:1002::%s:/usr/sbin/nologin\\n' "$account" "$FIXTURE_HOME" >>"$FIXTURE_ROOT/etc/passwd"`, + ); + writeExecutable( + path.join(bin, "usermod"), + `printf 'usermod:%s\\n' "$*" >>"$FIXTURE_CALLS" +case "$1" in + --add-subuids) file="$FIXTURE_ROOT/etc/subuid" ;; + --add-subgids) file="$FIXTURE_ROOT/etc/subgid" ;; + *) exit 24 ;; +esac +start="\${2%-*}" +end="\${2#*-}" +printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, + ); + writeExecutable(path.join(bin, "chown"), ":"); + writeExecutable(path.join(bin, "chmod"), `exec /bin/chmod "$@"`); + writeExecutable( + path.join(bin, "stat"), + `case "$3" in + *native-runtime-owner-*) printf '0:0:400\\n' ;; + *podman.apparmor|*storage.conf) printf '0:0:444\\n' ;; + *) exit 25 ;; +esac`, + ); + writeExecutable( + path.join(bin, "systemctl"), + `printf 'systemctl:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == stop && "$2" =~ user-runtime-dir@([0-9]+)\\.service ]]; then + /bin/rm -rf -- "$FIXTURE_ROOT/run/user/\${BASH_REMATCH[1]}" +fi`, + ); + writeExecutable(path.join(bin, "pkill"), `printf 'pkill:%s\\n' "$*" >>"$FIXTURE_CALLS"`); + writeExecutable( + path.join(bin, "apparmor_parser"), + `printf 'apparmor:%s\\n' "$*" >>"$FIXTURE_CALLS"`, + ); + writeExecutable( + path.join(bin, "userdel"), + `printf 'userdel:%s\\n' "$*" >>"$FIXTURE_CALLS" +account="\${!#}" +for file in "$FIXTURE_ROOT/etc/passwd" "$FIXTURE_ROOT/etc/subuid" "$FIXTURE_ROOT/etc/subgid"; do + awk -F: -v account="$account" '$1 != account' "$file" >"$file.next" + mv "$file.next" "$file" +done +rmdir "$FIXTURE_HOME" 2>/dev/null || true`, + ); + + return { + root, + bin, + calls, + passwd, + subuid, + subgid, + home, + marker: path.join(run, "nemoclaw-native-runtime-owner-42-1"), + }; +} + +function runFixture( + fixture: ReturnType, + source: string, + extraEnv: Record = {}, +) { + return spawnSync("bash", ["-c", fixtureSource(source)], { + encoding: "utf8", + timeout: 5000, + env: { + ...process.env, + ACCOUNT: "nemoclawq", + ACCOUNT_CREATED: "true", + FIXTURE_CALLS: fixture.calls, + FIXTURE_HOME: fixture.home, + FIXTURE_ROOT: fixture.root, + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "42", + PATH: `${fixture.bin}:${process.env.PATH ?? ""}`, + ...extraEnv, + }, + }); +} + +function provisionBlock(): string { + const source = workflowScripts().boundary; + const start = source.indexOf('account="nemoclawq"'); + const end = source.indexOf("ensure_subordinate_range()", start); + if (start < 0 || end < 0) throw new Error("Missing qualification account provision block"); + return `set -euo pipefail\n${source.slice(start, end)}`; +} + +describe("native runtime qualification account lifecycle", () => { + it("rejects pre-existing accounts and stale subordinate-ID authorization before mutation", () => { + for (const state of ["passwd", "subuid", "subgid"] as const) { + const fixture = createFixture(); + const file = fixture[state]; + fs.appendFileSync( + file, + state === "passwd" + ? `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n` + : "nemoclawq:200000:65536\n", + ); + const result = runFixture(fixture, provisionBlock()); + expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("useradd:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + } + }); + + it("does not publish ownership when account creation fails", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock(), { FAIL_USERADD: "1" }); + expect(result.status).toBe(23); + expect(fs.existsSync(fixture.marker)).toBe(false); + expect(fs.readFileSync(fixture.passwd, "utf8")).toBe(""); + }); + + it("rolls back an account when identity validation fails before marker publication", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock(), { FAIL_ID: "1" }); + expect(result.status).not.toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); + expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + }); + + it("keeps an existing valid range and advances past every overlapping range", () => { + const rangeFunction = extractFunction(workflowScripts().boundary, "ensure_subordinate_range"); + + const valid = createFixture(); + fs.writeFileSync(valid.subuid, "nemoclawq:200000:65536\n"); + const validResult = runFixture( + valid, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + ); + expect(validResult.status, validResult.stderr).toBe(0); + expect(fs.readFileSync(valid.calls, "utf8")).not.toContain("usermod:"); + + const overlapping = createFixture(); + fs.writeFileSync(overlapping.subuid, "runner-a:100000:65536\nrunner-b:165536:65536\n"); + const overlappingResult = runFixture( + overlapping, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + ); + expect(overlappingResult.status, overlappingResult.stderr).toBe(0); + expect(fs.readFileSync(overlapping.subuid, "utf8")).toContain("nemoclawq:231072:65536"); + }); + + it("fails closed when no complete subordinate-ID range remains", () => { + const fixture = createFixture(); + fs.writeFileSync(fixture.subgid, "runner:100000:4294867296\n"); + const rangeFunction = extractFunction(workflowScripts().boundary, "ensure_subordinate_range"); + const result = runFixture( + fixture, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subgid --add-subgids`, + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("no free subordinate-ID range"); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("usermod:"); + }); + + it("removes partial account setup and verifies subordinate-ID revocation", () => { + const fixture = createFixture(); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); + fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); + expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.subuid, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.subgid, "utf8")).not.toContain("nemoclawq:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + }); + + it("removes the run-owned runtime, storage configuration, and AppArmor profile", () => { + const fixture = createFixture(); + const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); + const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.mkdirSync(runtime, { recursive: true }); + fs.mkdirSync(storage, { recursive: true }); + fs.writeFileSync(path.join(runtime, "alive"), "fixture"); + fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); + fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); + fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup); + expect(result.status, result.stderr).toBe(0); + const calls = fs.readFileSync(fixture.calls, "utf8"); + expect(calls).toContain("systemctl:stop user-runtime-dir@1002.service"); + expect(calls).toContain("apparmor:-R"); + expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); + expect(fs.existsSync(storage)).toBe(false); + }); + + it("does not run destructive cleanup when the run-owned marker is absent", () => { + const fixture = createFixture(); + const result = runFixture(fixture, workflowScripts().cleanup, { ACCOUNT_CREATED: "" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("output exists without its ownership marker"); + const calls = fs.readFileSync(fixture.calls, "utf8"); + expect(calls).not.toMatch(/pkill:|systemctl:|userdel:|apparmor:/u); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 898be4eeea5..9c0c13562d1 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -257,22 +257,46 @@ describe("native runtime qualification producer workflow", () => { expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); + expect(boundary.run).toContain('getent passwd "$account"'); + expect(boundary.run).toContain('grep -q "^${account}:" /etc/subuid /etc/subgid'); + expect(boundary.run).toContain("Qualification account identity or subordinate-ID authorization already exists"); + expect(boundary.run).toContain('ownership_marker="/run/nemoclaw-native-runtime-owner-'); + expect(boundary.run).toContain("0:0:400"); + expect(boundary.run).toContain("Qualification account ownership marker is invalid"); + expect(boundary.run).toContain("rollback_unmarked_account"); + expect(boundary.run).toContain("Partially created qualification account could not be rolled back"); expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); + expect(boundary.run).toContain("systemctl cat user-runtime-dir@.service"); + expect(boundary.run).toContain('systemctl start "user-runtime-dir@${uid}.service"'); + expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); expect(boundary.run).toContain("rootless_storage_path"); expect(boundary.run).toContain("${home}/.local/share/containers/storage"); expect(boundary.run).toContain('mount_program = "/usr/bin/fuse-overlayfs"'); expect(boundary.run).toContain("0:0:444"); + expect(boundary.run).toContain("/sys/module/apparmor/parameters/enabled"); + expect(boundary.run).toContain( + 'profile ${apparmor_profile_name} /usr/local/bin/podman flags=(unconfined)', + ); + expect(boundary.run).toContain("userns,"); + expect(boundary.run).toContain('apparmor_parser -r "$apparmor_profile"'); + expect(boundary.run).not.toContain("apparmor_restrict_unprivileged_userns="); expect(boundary.run).toContain("podman info --format json"); expect(boundary.run).toContain("Credential-free rootless Podman readiness failed"); expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); expect(boundary.run).toContain('setfacl --modify "u:${account}:--x"'); expect(boundary.run).not.toContain("chmod o+x"); - expect(boundaryRun.indexOf("printf 'account=%s")).toBeLessThan( + expect(boundaryRun.indexOf("useradd --create-home")).toBeLessThan( + boundaryRun.indexOf("printf 'account=%s"), + ); + expect(boundaryRun.indexOf("Qualification account ownership marker is invalid")).toBeLessThan( + boundaryRun.indexOf("printf 'account=%s"), + ); + expect(boundaryRun.indexOf("printf 'account=%s")).toBeGreaterThan( boundaryRun.indexOf("useradd --create-home"), ); expect(dependencies.run).toContain('sudo -u "$ACCOUNT" env -i'); @@ -314,14 +338,25 @@ describe("native runtime qualification producer workflow", () => { path: "${{ runner.temp }}/native-runtime-evidence/", }); expect(cleanup.if).toBe("always()"); - expect(cleanup.run).toContain('account="${ACCOUNT:-nemoclawq}"'); + expect(cleanup.env?.ACCOUNT_CREATED).toBe("${{ steps.boundary.outputs.account_created }}"); + expect(cleanup.run).toContain('reported_account="${ACCOUNT:-}"'); + expect(cleanup.run).not.toContain('ACCOUNT:-nemoclawq'); + expect(cleanup.run).toContain("Qualification account ownership marker cleanup target is invalid"); + expect(cleanup.run).toContain('ownership="$(sudo cat "$ownership_marker")"'); expect(cleanup.run).toContain("pkill -KILL -u"); expect(cleanup.run).not.toContain("rm -rf"); + expect(cleanup.run).not.toContain("find "); + expect(cleanup.run).toContain('systemctl stop "user-runtime-dir@${uid}.service"'); + expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); + expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); - expect(cleanup.run).toContain('sudo rmdir "$runtime_dir/podman"'); + expect(cleanup.run).toContain("Qualification runtime directory remains after its systemd cleanup"); expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); + expect(cleanup.run).toContain("Qualification subordinate-ID authorization remains after cleanup"); + expect(cleanup.run).toContain("Qualification account output exists without its ownership marker"); + expect(cleanup.run).toContain('sudo rm -f -- "$ownership_marker"'); }); it("aggregates the exact successful 24-case cohort in a separate trusted job", () => { From 73a2c2c14b522d000fa2a3f9ee073b864bab69e9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 00:56:15 -0500 Subject: [PATCH 20/71] test(e2e): keep lifecycle fixtures linear Signed-off-by: Aaron Erickson --- ...ve-runtime-qualification-account-lifecycle.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 466f1e05f78..29fc9420c13 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -20,8 +20,8 @@ function workflowScripts(): { boundary: string; cleanup: string } { const steps = workflow.jobs["native-runtime-qualification-producer"]?.steps ?? []; const run = (name: string): string => { const source = steps.find((entry) => entry.name === name)?.run; - if (!source) throw new Error(`Missing workflow step ${name}`); - return source; + expect(source, `Missing workflow step ${name}`).toBeTruthy(); + return source!; }; return { boundary: run("Prepare the credential-free execution account and disable Docker"), @@ -31,8 +31,8 @@ function workflowScripts(): { boundary: string; cleanup: string } { function extractFunction(source: string, name: string): string { const match = source.match(new RegExp(`${name}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); - if (!match) throw new Error(`Missing shell function ${name}`); - return `${name}() {${match[1]}\n}`; + expect(match, `Missing shell function ${name}`).toBeTruthy(); + return `${name}() {${match![1]}\n}`; } function fixtureSource(source: string): string { @@ -182,7 +182,8 @@ function provisionBlock(): string { const source = workflowScripts().boundary; const start = source.indexOf('account="nemoclawq"'); const end = source.indexOf("ensure_subordinate_range()", start); - if (start < 0 || end < 0) throw new Error("Missing qualification account provision block"); + expect(start, "Missing qualification account provision start").toBeGreaterThanOrEqual(0); + expect(end, "Missing qualification account provision end").toBeGreaterThan(start); return `set -euo pipefail\n${source.slice(start, end)}`; } From ac68c6ed0e7df9cf4dff6d6e3def106e7721f5ab Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 01:08:03 -0500 Subject: [PATCH 21/71] fix(e2e): preserve qualification user runtime directory Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 9 ++++++--- ...ntime-qualification-account-lifecycle.test.ts | 16 +++++++++------- ...ntime-qualification-producer-workflow.test.ts | 6 +++--- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b3d0341d6bb..81a01353bca 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1469,8 +1469,11 @@ jobs: printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" printf 'account_created=true\n' >>"$GITHUB_OUTPUT" runtime_dir="/run/user/${uid}" - systemctl cat user-runtime-dir@.service >/dev/null - sudo systemctl start "user-runtime-dir@${uid}.service" + [[ -x /usr/lib/systemd/systemd-user-runtime-dir ]] || { + echo "::error::Protected runner is missing the signed systemd user-runtime helper" >&2 + exit 1 + } + sudo /usr/lib/systemd/systemd-user-runtime-dir start "$uid" [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${uid}:700" ]] || { echo "::error::Qualification runtime directory is missing or invalid" >&2 exit 1 @@ -1720,7 +1723,7 @@ jobs: if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true fi - sudo systemctl stop "user-runtime-dir@${uid}.service" + sudo /usr/lib/systemd/systemd-user-runtime-dir stop "$uid" if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration cleanup target is invalid" >&2 diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 29fc9420c13..0886332dd8d 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -49,6 +49,10 @@ function fixtureSource(source: string): string { 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) + .replaceAll( + "/usr/lib/systemd/systemd-user-runtime-dir", + "${FIXTURE_ROOT}/bin/systemd-user-runtime-dir", + ) .replaceAll('"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'); } @@ -121,11 +125,9 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, esac`, ); writeExecutable( - path.join(bin, "systemctl"), - `printf 'systemctl:%s\\n' "$*" >>"$FIXTURE_CALLS" -if [[ "$1" == stop && "$2" =~ user-runtime-dir@([0-9]+)\\.service ]]; then - /bin/rm -rf -- "$FIXTURE_ROOT/run/user/\${BASH_REMATCH[1]}" -fi`, + path.join(bin, "systemd-user-runtime-dir"), + `printf 'systemd-user-runtime-dir:%s\\n' "$*" >>"$FIXTURE_CALLS" +[[ "$1" != stop ]] || /bin/rm -rf -- "$FIXTURE_ROOT/run/user/$2"`, ); writeExecutable(path.join(bin, "pkill"), `printf 'pkill:%s\\n' "$*" >>"$FIXTURE_CALLS"`); writeExecutable( @@ -292,7 +294,7 @@ describe("native runtime qualification account lifecycle", () => { const result = runFixture(fixture, workflowScripts().cleanup); expect(result.status, result.stderr).toBe(0); const calls = fs.readFileSync(fixture.calls, "utf8"); - expect(calls).toContain("systemctl:stop user-runtime-dir@1002.service"); + expect(calls).toContain("systemd-user-runtime-dir:stop 1002"); expect(calls).toContain("apparmor:-R"); expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); expect(fs.existsSync(storage)).toBe(false); @@ -304,6 +306,6 @@ describe("native runtime qualification account lifecycle", () => { expect(result.status).not.toBe(0); expect(result.stderr).toContain("output exists without its ownership marker"); const calls = fs.readFileSync(fixture.calls, "utf8"); - expect(calls).not.toMatch(/pkill:|systemctl:|userdel:|apparmor:/u); + expect(calls).not.toMatch(/pkill:|systemd-user-runtime-dir:|userdel:|apparmor:/u); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 9c0c13562d1..9cfa2a81519 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -268,8 +268,8 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); - expect(boundary.run).toContain("systemctl cat user-runtime-dir@.service"); - expect(boundary.run).toContain('systemctl start "user-runtime-dir@${uid}.service"'); + expect(boundary.run).toContain("/usr/lib/systemd/systemd-user-runtime-dir"); + expect(boundary.run).toContain('systemd-user-runtime-dir start "$uid"'); expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); @@ -346,7 +346,7 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain("pkill -KILL -u"); expect(cleanup.run).not.toContain("rm -rf"); expect(cleanup.run).not.toContain("find "); - expect(cleanup.run).toContain('systemctl stop "user-runtime-dir@${uid}.service"'); + expect(cleanup.run).toContain('systemd-user-runtime-dir stop "$uid"'); expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); From 921efe4767755d9cffa11bbfe3d9a39a47507964 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 01:24:18 -0500 Subject: [PATCH 22/71] fix(e2e): bind qualified Podman executable authority --- test/e2e/live/native-runtime-qualification-case-executor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 5b3b5ecca82..15f8347df2c 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -44,6 +44,7 @@ const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; const COMMAND_TIMEOUT = 60_000; const INFERENCE_TIMEOUT = 900_000; const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE = "/usr/local/bin/podman"; export const NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES = [ "validate credential-free Docker-unavailable isolation", "bind the rootless Podman engine", @@ -215,7 +216,7 @@ function startPodmanQualificationService( ): PodmanQualificationService { let diagnostic = ""; const child = spawnObservedChild( - "podman", + NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, ["system", "service", "--time=0", `unix://${socket}`], { activityLabel: "command: rootless Podman qualification service", @@ -571,14 +572,17 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre await waitForSocket(socket, service); const socketAuthority = capturePodmanSocketAuthority(socket); hostEngine = createPodmanContainerEngine({ + executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, operation: "host-doctor", socketAuthority, }); inferenceEngine = createPodmanContainerEngine({ + executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, operation: "host-local-inference", socketAuthority, }); lifecycleEngine = createPodmanContainerEngine({ + executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, operation: "sandbox-lifecycle", socketAuthority, }); From 8548a374120e815109ddf3009d36c48742d99abd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 01:44:00 -0500 Subject: [PATCH 23/71] fix(e2e): isolate qualified Podman executable --- .github/workflows/e2e.yaml | 45 ++++++++++++++++++- ...ive-runtime-qualification-case-executor.ts | 13 +++--- ...tive-runtime-qualification-case-helpers.ts | 15 +++++++ ...me-qualification-account-lifecycle.test.ts | 11 ++++- ...runtime-qualification-case-helpers.test.ts | 24 ++++++++++ ...me-qualification-producer-workflow.test.ts | 21 ++++++++- 6 files changed, 119 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 81a01353bca..c286f6cd8ca 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1359,6 +1359,7 @@ jobs: id: boundary env: CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail @@ -1480,6 +1481,7 @@ jobs: } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" + podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { @@ -1487,6 +1489,30 @@ jobs: exit 1 } sudo install -d -o root -g root -m 0755 "$storage_config_directory" + [[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || { + echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2 + exit 1 + } + [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { + echo "::error::Run-owned qualification Podman executable already exists" >&2 + exit 1 + } + opt_mode="$(stat -c '%a' /opt)" + [[ -d /opt && ! -L /opt && "$(stat -c '%u:%g' /opt)" == "0:0" && "$opt_mode" =~ ^[0-7]{3,4}$ ]] && + (( (8#$opt_mode & 8#022) == 0 )) || { + echo "::error::Qualification Podman executable parent is not root-owned and non-writable" >&2 + exit 1 + } + sudo install --owner=root --group=root --mode=0555 \ + "$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable" + [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification Podman executable is invalid" >&2 + exit 1 + } + [[ "$(sha256sum "$podman_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/podman" | cut -d' ' -f1)" ]] || { + echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2 + exit 1 + } printf '%s\n' \ '[storage]' \ 'driver = "overlay"' \ @@ -1509,7 +1535,7 @@ jobs: 'abi ,' \ 'include ' \ '' \ - "profile ${apparmor_profile_name} /usr/local/bin/podman flags=(unconfined) {" \ + "profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined) {" \ ' userns,' \ '}' | sudo tee "$apparmor_profile" >/dev/null sudo chown root:root "$apparmor_profile" @@ -1545,7 +1571,7 @@ jobs: LANG=C.UTF-8 \ PATH="$guard_dir:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$runtime_dir" \ - podman info --format json >/dev/null || { + "$podman_executable" info --format json >/dev/null || { echo "::error::Credential-free rootless Podman readiness failed" >&2 exit 1 } @@ -1553,6 +1579,7 @@ jobs: printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" + printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" - name: Install locked candidate test dependencies without scripts @@ -1622,6 +1649,7 @@ jobs: CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} + PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} @@ -1646,6 +1674,7 @@ jobs: LANG=C.UTF-8 \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \ + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \ NEMOCLAW_RUN_LIVE_E2E=1 \ PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ @@ -1719,6 +1748,7 @@ jobs: fi runtime_dir="/run/user/${uid}" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true @@ -1740,6 +1770,13 @@ jobs: sudo rm -f -- "$storage_config_directory/storage.conf" sudo rmdir "$storage_config_directory" fi + if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then + [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { + echo "::error::Qualification Podman executable cleanup target is invalid" >&2 + exit 1 + } + sudo rm -f -- "$podman_executable" + fi if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2 exit 1 @@ -1764,6 +1801,10 @@ jobs: echo "::error::Qualification storage configuration remains after cleanup" >&2 exit 1 } + [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { + echo "::error::Qualification Podman executable remains after cleanup" >&2 + exit 1 + } sudo rm -f -- "$ownership_marker" elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then echo "::error::Qualification account output exists without its ownership marker" >&2 diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 15f8347df2c..937aad284b2 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -35,6 +35,7 @@ import { digestFromImageReference, nativeRuntimeQualificationAgentImage, nativeRuntimeQualificationInferenceImage, + nativeRuntimeQualificationPodmanExecutable, parseNativeRuntimeQualificationRow, readNativeRuntimeQualificationRunnerContract, } from "./native-runtime-qualification-case-helpers.ts"; @@ -44,7 +45,6 @@ const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; const COMMAND_TIMEOUT = 60_000; const INFERENCE_TIMEOUT = 900_000; const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; -const NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE = "/usr/local/bin/podman"; export const NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES = [ "validate credential-free Docker-unavailable isolation", "bind the rootless Podman engine", @@ -212,11 +212,12 @@ async function waitForSocket(socket: string, service: PodmanQualificationService function startPodmanQualificationService( socket: string, + podmanExecutable: string, progress: TestProgress, ): PodmanQualificationService { let diagnostic = ""; const child = spawnObservedChild( - NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + podmanExecutable, ["system", "service", "--time=0", `unix://${socket}`], { activityLabel: "command: rootless Podman qualification service", @@ -548,11 +549,13 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const dockerBefore = assertDockerUnavailable(); const runtimeDirectory = process.env.XDG_RUNTIME_DIR ?? ""; expect(runtimeDirectory).toBe(`/run/user/${String(uid)}`); + const podmanExecutable = nativeRuntimeQualificationPodmanExecutable(process.env, uid); const socket = path.join(runtimeDirectory, "podman", "podman.sock"); fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); let service: PodmanQualificationService | null = startPodmanQualificationService( socket, + podmanExecutable, progress, ); let hostEngine: PodmanBoundContainerEngine | null = null; @@ -572,17 +575,17 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre await waitForSocket(socket, service); const socketAuthority = capturePodmanSocketAuthority(socket); hostEngine = createPodmanContainerEngine({ - executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + executable: podmanExecutable, operation: "host-doctor", socketAuthority, }); inferenceEngine = createPodmanContainerEngine({ - executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + executable: podmanExecutable, operation: "host-local-inference", socketAuthority, }); lifecycleEngine = createPodmanContainerEngine({ - executable: NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + executable: podmanExecutable, operation: "sandbox-lifecycle", socketAuthority, }); diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts index a3283649b51..60d6c464f3c 100644 --- a/test/e2e/live/native-runtime-qualification-case-helpers.ts +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -206,6 +206,21 @@ export function assertCredentialFreeQualificationEnvironment(environment: NodeJS } } +export function nativeRuntimeQualificationPodmanExecutable( + environment: NodeJS.ProcessEnv, + uid: number, +): string { + const executable = environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE ?? ""; + const expected = new RegExp( + `^/opt/nemoclaw-native-runtime-podman-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}$`, + "u", + ); + if (!Number.isSafeInteger(uid) || uid <= 0 || !expected.test(executable)) { + throw new Error("Native runtime qualification Podman executable path is invalid"); + } + return executable; +} + function exactRunnerRuntime( value: unknown, label: "NIM" | "vLLM", diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 0886332dd8d..44011418aeb 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -49,6 +49,10 @@ function fixtureSource(source: string): string { 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) + .replaceAll( + 'podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="${FIXTURE_ROOT}/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ) .replaceAll( "/usr/lib/systemd/systemd-user-runtime-dir", "${FIXTURE_ROOT}/bin/systemd-user-runtime-dir", @@ -120,6 +124,7 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, path.join(bin, "stat"), `case "$3" in *native-runtime-owner-*) printf '0:0:400\\n' ;; + *native-runtime-podman-*) printf '0:0:555\\n' ;; *podman.apparmor|*storage.conf) printf '0:0:444\\n' ;; *) exit 25 ;; esac`, @@ -276,16 +281,19 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.existsSync(fixture.marker)).toBe(false); }); - it("removes the run-owned runtime, storage configuration, and AppArmor profile", () => { + it("removes the run-owned runtime, Podman executable, storage, and AppArmor profile", () => { const fixture = createFixture(); const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); + const podman = path.join(fixture.root, "opt", "nemoclaw-native-runtime-podman-42-1-1002"); fs.mkdirSync(fixture.home, { recursive: true }); fs.mkdirSync(runtime, { recursive: true }); fs.mkdirSync(storage, { recursive: true }); + fs.mkdirSync(path.dirname(podman), { recursive: true }); fs.writeFileSync(path.join(runtime, "alive"), "fixture"); fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); + fs.writeFileSync(podman, "fixture", { mode: 0o555 }); fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); @@ -298,6 +306,7 @@ describe("native runtime qualification account lifecycle", () => { expect(calls).toContain("apparmor:-R"); expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); expect(fs.existsSync(storage)).toBe(false); + expect(fs.existsSync(podman)).toBe(false); }); it("does not run destructive cleanup when the run-owned marker is absent", () => { diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index d27346f7dd0..f27e985303b 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -12,6 +12,7 @@ import { digestFromImageReference, nativeRuntimeQualificationAgentImage, nativeRuntimeQualificationInferenceImage, + nativeRuntimeQualificationPodmanExecutable, parseNativeRuntimeQualificationRow, parseNativeRuntimeQualificationRunnerContract, } from "../live/native-runtime-qualification-case-helpers.ts"; @@ -86,6 +87,29 @@ describe("native runtime qualification case boundaries", () => { expect(parseNativeRuntimeQualificationRow(JSON.stringify(candidateRow))).toEqual(candidateRow); }); + it("accepts only the run-owned rootless Podman executable path for the current uid", () => { + const environment = { + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: + "/opt/nemoclaw-native-runtime-podman-123456-1-1002", + }; + expect(nativeRuntimeQualificationPodmanExecutable(environment, 1002)).toBe( + environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + ); + for (const executable of [ + "/usr/local/bin/podman", + "/opt/nemoclaw-native-runtime-podman-123456-1-0", + "/opt/nemoclaw-native-runtime-podman-123456-1-1003", + "/opt/nemoclaw-native-runtime-podman-123456-1-1002/../podman", + ]) { + expect(() => + nativeRuntimeQualificationPodmanExecutable( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: executable }, + 1002, + ), + ).toThrow("Podman executable path is invalid"); + } + }); + it("rejects credential and alternate runtime authority environment names", () => { expect(() => assertCredentialFreeQualificationEnvironment({ diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 9cfa2a81519..24cc294cc9b 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -279,12 +279,21 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("0:0:444"); expect(boundary.run).toContain("/sys/module/apparmor/parameters/enabled"); expect(boundary.run).toContain( - 'profile ${apparmor_profile_name} /usr/local/bin/podman flags=(unconfined)', + 'profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined)', ); + expect(boundary.env?.TOOLCHAIN_DIRECTORY).toBe( + "${{ runner.temp }}/native-runtime-podman-toolchain", + ); + expect(boundary.run).toContain( + 'podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ); + expect(boundary.run).toContain('sudo install --owner=root --group=root --mode=0555'); + expect(boundary.run).toContain("Qualification Podman executable parent is not root-owned"); + expect(boundary.run).toContain("0:0:555"); + expect(boundary.run).toContain('"$podman_executable" info --format json'); expect(boundary.run).toContain("userns,"); expect(boundary.run).toContain('apparmor_parser -r "$apparmor_profile"'); expect(boundary.run).not.toContain("apparmor_restrict_unprivileged_userns="); - expect(boundary.run).toContain("podman info --format json"); expect(boundary.run).toContain("Credential-free rootless Podman readiness failed"); expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); @@ -323,8 +332,14 @@ describe("native runtime qualification producer workflow", () => { expect(execute.run).not.toContain("GH_TOKEN"); expect(execute.run).not.toContain("chown -R"); expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(execute.env?.PODMAN_EXECUTABLE).toBe( + "${{ steps.boundary.outputs.podman_executable }}", + ); expect(execute.env?.STORAGE_CONFIG).toBe("${{ steps.boundary.outputs.storage_config }}"); expect(execute.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); + expect(execute.run).toContain( + 'NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE"', + ); expect(execute.run).toContain( 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', ); @@ -350,6 +365,8 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); + expect(cleanup.run).toContain('sudo rm -f -- "$podman_executable"'); + expect(cleanup.run).toContain("Qualification Podman executable remains after cleanup"); expect(cleanup.run).toContain("Qualification runtime directory remains after its systemd cleanup"); expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); From da1e619fc31fec65c200bb0abceb64417f4db7b2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 01:55:45 -0500 Subject: [PATCH 24/71] fix(e2e): use stable qualified executable parent --- .github/workflows/e2e.yaml | 10 ++-------- .../live/native-runtime-qualification-case-helpers.ts | 2 +- ...ive-runtime-qualification-account-lifecycle.test.ts | 7 +++---- .../native-runtime-qualification-case-helpers.test.ts | 8 ++++---- ...ive-runtime-qualification-producer-workflow.test.ts | 3 +-- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c286f6cd8ca..0031d277b5c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1481,7 +1481,7 @@ jobs: } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" - podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { @@ -1497,12 +1497,6 @@ jobs: echo "::error::Run-owned qualification Podman executable already exists" >&2 exit 1 } - opt_mode="$(stat -c '%a' /opt)" - [[ -d /opt && ! -L /opt && "$(stat -c '%u:%g' /opt)" == "0:0" && "$opt_mode" =~ ^[0-7]{3,4}$ ]] && - (( (8#$opt_mode & 8#022) == 0 )) || { - echo "::error::Qualification Podman executable parent is not root-owned and non-writable" >&2 - exit 1 - } sudo install --owner=root --group=root --mode=0555 \ "$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable" [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { @@ -1748,7 +1742,7 @@ jobs: fi runtime_dir="/run/user/${uid}" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" - podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts index 60d6c464f3c..ff5c79cc8b6 100644 --- a/test/e2e/live/native-runtime-qualification-case-helpers.ts +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -212,7 +212,7 @@ export function nativeRuntimeQualificationPodmanExecutable( ): string { const executable = environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE ?? ""; const expected = new RegExp( - `^/opt/nemoclaw-native-runtime-podman-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}$`, + `^/nemoclaw-native-runtime-podman-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}$`, "u", ); if (!Number.isSafeInteger(uid) || uid <= 0 || !expected.test(executable)) { diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 44011418aeb..63c64db026d 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -50,8 +50,8 @@ function fixtureSource(source: string): string { 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) .replaceAll( - 'podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - 'podman_executable="${FIXTURE_ROOT}/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) .replaceAll( "/usr/lib/systemd/systemd-user-runtime-dir", @@ -285,11 +285,10 @@ describe("native runtime qualification account lifecycle", () => { const fixture = createFixture(); const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); - const podman = path.join(fixture.root, "opt", "nemoclaw-native-runtime-podman-42-1-1002"); + const podman = path.join(fixture.root, "nemoclaw-native-runtime-podman-42-1-1002"); fs.mkdirSync(fixture.home, { recursive: true }); fs.mkdirSync(runtime, { recursive: true }); fs.mkdirSync(storage, { recursive: true }); - fs.mkdirSync(path.dirname(podman), { recursive: true }); fs.writeFileSync(path.join(runtime, "alive"), "fixture"); fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index f27e985303b..ebc7d76dc11 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -90,16 +90,16 @@ describe("native runtime qualification case boundaries", () => { it("accepts only the run-owned rootless Podman executable path for the current uid", () => { const environment = { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: - "/opt/nemoclaw-native-runtime-podman-123456-1-1002", + "/nemoclaw-native-runtime-podman-123456-1-1002", }; expect(nativeRuntimeQualificationPodmanExecutable(environment, 1002)).toBe( environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, ); for (const executable of [ "/usr/local/bin/podman", - "/opt/nemoclaw-native-runtime-podman-123456-1-0", - "/opt/nemoclaw-native-runtime-podman-123456-1-1003", - "/opt/nemoclaw-native-runtime-podman-123456-1-1002/../podman", + "/nemoclaw-native-runtime-podman-123456-1-0", + "/nemoclaw-native-runtime-podman-123456-1-1003", + "/nemoclaw-native-runtime-podman-123456-1-1002/../podman", ]) { expect(() => nativeRuntimeQualificationPodmanExecutable( diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 24cc294cc9b..f7b1a9beb16 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -285,10 +285,9 @@ describe("native runtime qualification producer workflow", () => { "${{ runner.temp }}/native-runtime-podman-toolchain", ); expect(boundary.run).toContain( - 'podman_executable="/opt/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ); expect(boundary.run).toContain('sudo install --owner=root --group=root --mode=0555'); - expect(boundary.run).toContain("Qualification Podman executable parent is not root-owned"); expect(boundary.run).toContain("0:0:555"); expect(boundary.run).toContain('"$podman_executable" info --format json'); expect(boundary.run).toContain("userns,"); From cb7b609af75f2a2099943ca4ddafc84a01fcc7d2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 02:18:35 -0500 Subject: [PATCH 25/71] fix(e2e): bind native Podman network authority --- ...ive-runtime-qualification-case-executor.ts | 94 +++++++++---- ...untime-qualification-case-executor.test.ts | 124 ++++++++++++++++++ 2 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 test/e2e/support/native-runtime-qualification-case-executor.test.ts diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 937aad284b2..9dbb9bc2141 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -259,30 +259,67 @@ function createProviderNetwork( name: string, caseId: string, ): PodmanNetworkAuthority { - const id = capture( - engine, - ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], - "provider network creation", - ); - if (!FULL_ID.test(id)) throw new Error("Provider network did not return a full immutable ID"); - const inspected = JSON.parse( - capture(engine, ["network", "inspect", id], "provider network inspection"), - ) as Array<{ - id?: unknown; - name?: unknown; - subnets?: Array<{ gateway?: unknown }>; - }>; - const entry = inspected[0]; - const gateway = entry?.subnets?.[0]?.gateway; - if ( - inspected.length !== 1 || - entry?.id !== id || - entry.name !== name || - typeof gateway !== "string" - ) { - throw new Error("Provider network inspection lacks exact identity"); + let created = false; + try { + const createdIdentity = capture( + engine, + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], + "provider network creation", + ); + created = true; + if (createdIdentity !== name && !FULL_ID.test(createdIdentity)) { + throw new Error("Provider network creation returned an unexpected identity"); + } + type NetworkInspection = { + id?: unknown; + labels?: unknown; + name?: unknown; + subnets?: Array<{ gateway?: unknown }>; + }; + const inspect = (identity: string, label: string): NetworkInspection => { + const inspected = JSON.parse(capture(engine, ["network", "inspect", identity], label)) as + | NetworkInspection[] + | unknown; + if (!Array.isArray(inspected) || inspected.length !== 1) { + throw new Error("Provider network inspection lacks one exact identity"); + } + return inspected[0] as NetworkInspection; + }; + const entry = inspect(createdIdentity, "provider network creation inspection"); + const id = typeof entry.id === "string" ? entry.id : ""; + const gateway = entry?.subnets?.[0]?.gateway; + const labels = + typeof entry.labels === "object" && entry.labels !== null && !Array.isArray(entry.labels) + ? (entry.labels as Record) + : null; + if ( + !FULL_ID.test(id) || + (FULL_ID.test(createdIdentity) && createdIdentity !== id) || + entry.name !== name || + typeof gateway !== "string" || + labels?.[QUALIFICATION_LABEL] !== caseId + ) { + throw new Error("Provider network inspection lacks exact identity"); + } + const immutable = inspect(id, "provider network immutable-ID inspection"); + if ( + immutable.id !== id || + immutable.name !== name || + immutable.subnets?.[0]?.gateway !== gateway || + typeof immutable.labels !== "object" || + immutable.labels === null || + Array.isArray(immutable.labels) || + (immutable.labels as Record)[QUALIFICATION_LABEL] !== caseId + ) { + throw new Error("Provider network identity changed after immutable-ID resolution"); + } + return Object.freeze({ id, name, gateway }); + } catch (error) { + if (created) { + engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); + } + throw error; } - return Object.freeze({ id, name, gateway }); } function pullPublicImage(engine: PodmanBoundContainerEngine, imageRef: string): void { @@ -597,9 +634,10 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre }); expect(bundle.identity.id).toBe("podman"); expect(bundle.workload.profile.support).toBeNull(); - expect(bundle.preflightDoctor.inspectHost()).toMatchObject({ - status: "ok", - }); + const hostInspection = bundle.preflightDoctor.inspectHost(); + if (hostInspection.status !== "ok") { + throw new Error(`Podman host qualification failed: ${bounded(hostInspection.detail)}`); + } const caseSuffix = sha256(row.id).slice(0, 12); const networkName = `nemoclaw-q-${caseSuffix}`; const network = createProviderNetwork(inferenceEngine, networkName, row.id); @@ -1093,3 +1131,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre service = null; } } + +export const nativeRuntimeQualificationCaseInternals = Object.freeze({ + createProviderNetwork, +}); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts new file mode 100644 index 00000000000..c029bd44ecb --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { PodmanBoundContainerEngine } from "../../../src/lib/adapters/podman/index.ts"; +import { nativeRuntimeQualificationCaseInternals } from "../live/native-runtime-qualification-case-executor.ts"; + +const NETWORK_ID = "a".repeat(64); +const NETWORK_NAME = "nemoclaw-q-0123456789ab"; +const CASE_ID = "podman-openclaw-linux-amd64-cpu-ollama"; +const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; + +function inspection(overrides: Record = {}): string { + return JSON.stringify([ + { + id: NETWORK_ID, + labels: { [QUALIFICATION_LABEL]: CASE_ID }, + name: NETWORK_NAME, + subnets: [{ gateway: "10.89.0.1" }], + ...overrides, + }, + ]); +} + +function engine(outputs: readonly string[]): { + readonly capture: ReturnType; + readonly value: PodmanBoundContainerEngine; +} { + let index = 0; + const capture = vi.fn(() => ({ + status: 0, + stdout: outputs[index++] ?? "", + stderr: "", + })); + return { + capture, + value: { + operation: "host-local-inference", + engineId: "podman", + displayName: "Podman", + authorityId: `podman-sha256:${"b".repeat(64)}`, + endpointAuthorityId: `podman-sha256:${"c".repeat(64)}`, + capture, + captureHost: vi.fn(), + assertAuthority: vi.fn(), + } as unknown as PodmanBoundContainerEngine, + }; +} + +describe("native runtime provider-network authority", () => { + it("resolves Podman 6.1 name output to one immutable labeled network ID", () => { + const runtime = engine([NETWORK_NAME, inspection(), inspection()]); + + expect( + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toEqual({ id: NETWORK_ID, name: NETWORK_NAME, gateway: "10.89.0.1" }); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "inspect", NETWORK_NAME], + ["network", "inspect", NETWORK_ID], + ]); + }); + + it("also binds an implementation that returns the immutable network ID", () => { + const runtime = engine([NETWORK_ID, inspection(), inspection()]); + + expect( + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ).id, + ).toBe(NETWORK_ID); + }); + + it("rejects creation output outside the requested name or immutable-ID forms", () => { + const runtime = engine(["unexpected-network"]); + + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network creation returned an unexpected identity"); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "rm", "--force", NETWORK_NAME], + ]); + }); + + it("rejects label or immutable re-inspection drift", () => { + const missingLabel = engine([NETWORK_NAME, inspection({ labels: {} }), inspection()]); + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + missingLabel.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network inspection lacks exact identity"); + + const changedGateway = engine([ + NETWORK_NAME, + inspection(), + inspection({ subnets: [{ gateway: "10.90.0.1" }] }), + ]); + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + changedGateway.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network identity changed after immutable-ID resolution"); + expect(changedGateway.capture).toHaveBeenLastCalledWith( + ["network", "rm", "--force", NETWORK_NAME], + 60_000, + ); + }); +}); From 9711d7ec9bab2b3f722e2c0ccd25ff024b09995a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 02:36:26 -0500 Subject: [PATCH 26/71] fix(runtime): bind Podman mapping preflight to endpoint --- .../runtime-provider/podman-preflight.test.ts | 58 ++++++++++++++----- .../runtime-provider/podman-preflight.ts | 53 +++++++++++------ .../onboard/runtime-provider/podman.test.ts | 12 +++- 3 files changed, 89 insertions(+), 34 deletions(-) diff --git a/src/lib/onboard/runtime-provider/podman-preflight.test.ts b/src/lib/onboard/runtime-provider/podman-preflight.test.ts index 80d8b253068..c37a9280a39 100644 --- a/src/lib/onboard/runtime-provider/podman-preflight.test.ts +++ b/src/lib/onboard/runtime-provider/podman-preflight.test.ts @@ -19,6 +19,16 @@ const INFO = JSON.stringify({ arch: "amd64", os: "linux", cgroupVersion: "v2", + idMappings: { + uidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + gidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + }, networkBackend: "netavark", security: { rootless: true }, discoveredDevices: [ @@ -33,7 +43,6 @@ function engine( readonly info?: string; readonly serverVersion?: string; readonly version?: string; - readonly idMap?: string; } = {}, ): ContainerEngine { const capture = vi.fn((args: readonly string[]) => { @@ -55,10 +64,7 @@ function engine( }); const captureHost = vi.fn((args: readonly string[]) => ({ status: 0, - stdout: - args[0] === "--version" - ? (overrides.version ?? "podman version 5.6.2\n") - : (overrides.idMap ?? "0 1000 1\n1 100000 65536\n"), + stdout: args[0] === "--version" ? (overrides.version ?? "podman version 5.6.2\n") : "", stderr: "", })); return { @@ -98,14 +104,7 @@ describe("Podman host preflight", () => { expect(runtime.capture).toHaveBeenCalledWith(["info", "--format", "json"], 15_000); expect(runtime.capture).toHaveBeenCalledWith(["version", "--format", "json"], 10_000); expect(runtime.captureHost).toHaveBeenCalledWith(["--version"], 10_000); - expect(runtime.captureHost).toHaveBeenCalledWith( - ["unshare", "cat", "/proc/self/uid_map"], - 10_000, - ); - expect(runtime.captureHost).toHaveBeenCalledWith( - ["unshare", "cat", "/proc/self/gid_map"], - 10_000, - ); + expect(runtime.captureHost).toHaveBeenCalledTimes(1); }); it("keeps the CPU receipt server version canonical while preserving exact inference authority", () => { @@ -196,12 +195,41 @@ describe("Podman host preflight", () => { }); it("rejects missing subordinate user mappings", () => { + const info = JSON.stringify({ + ...JSON.parse(INFO), + host: { + ...JSON.parse(INFO).host, + idMappings: { + ...JSON.parse(INFO).host.idMappings, + uidmap: [{ container_id: 0, host_id: 1000, size: 1 }], + }, + }, + }); + expect(() => + qualifyPodmanHost(engine({ info }), { + platform: "linux", + architecture: "x64", + }), + ).toThrow("subordinate UID range for the API service user"); + }); + + it("rejects malformed API-service ID mappings", () => { + const info = JSON.stringify({ + ...JSON.parse(INFO), + host: { + ...JSON.parse(INFO).host, + idMappings: { + ...JSON.parse(INFO).host.idMappings, + gidmap: [{ container_id: 0, host_id: 1000, size: "65536" }], + }, + }, + }); expect(() => - qualifyPodmanHost(engine({ idMap: "0 1000 1\n" }), { + qualifyPodmanHost(engine({ info }), { platform: "linux", architecture: "x64", }), - ).toThrow("subordinate UID range"); + ).toThrow("Podman API returned malformed gidmap"); }); it("fails before commands for another engine scope or unsupported host platform", () => { diff --git a/src/lib/onboard/runtime-provider/podman-preflight.ts b/src/lib/onboard/runtime-provider/podman-preflight.ts index 97741cb5e1f..e393cf0d95a 100644 --- a/src/lib/onboard/runtime-provider/podman-preflight.ts +++ b/src/lib/onboard/runtime-provider/podman-preflight.ts @@ -177,25 +177,42 @@ function normalizeArchitecture(value: string): "amd64" | "arm64" | null { return null; } -function hasSubordinateIdMapping(output: string): boolean { - return output - .trim() - .split(/\r?\n/u) - .some((line) => { - const values = line.trim().split(/\s+/u).map(Number); - return values.length === 3 && values.every(Number.isFinite) && (values[2] ?? 0) > 1; - }); -} - -function requireSubordinateIdMappings(engine: ContainerEngine): void { - for (const mapping of ["uid_map", "gid_map"] as const) { - const result = requireSuccessful( - `${mapping} inspection`, - engine.captureHost(["unshare", "cat", `/proc/self/${mapping}`], 10_000), +function requireSubordinateIdMappings(host: unknown): void { + // Bind this check to the same rootless API service as the rest of the + // preflight. A local `podman unshare` can resolve different storage and user + // authority than an explicitly bound service endpoint. + const mappings = field(host, "idMappings", "IDMappings"); + for (const mapping of ["uidmap", "gidmap"] as const) { + const entries = field( + mappings, + mapping, + mapping === "uidmap" ? "UIDMap" : "GIDMap", ); - if (!hasSubordinateIdMapping(result.stdout)) { + if (!Array.isArray(entries) || entries.length === 0 || entries.length > 1_024) { + throw new PodmanHostPreflightError(`the Podman API returned malformed ${mapping}`); + } + let hasSubordinateRange = false; + for (const value of entries) { + const entry = record(value); + const containerId = field(entry, "container_id", "containerID", "ContainerID"); + const hostId = field(entry, "host_id", "hostID", "HostID"); + const size = field(entry, "size", "Size"); + if ( + !entry || + !Number.isSafeInteger(containerId) || + !Number.isSafeInteger(hostId) || + !Number.isSafeInteger(size) || + (containerId as number) < 0 || + (hostId as number) < 0 || + (size as number) <= 0 + ) { + throw new PodmanHostPreflightError(`the Podman API returned malformed ${mapping}`); + } + if ((size as number) > 1) hasSubordinateRange = true; + } + if (!hasSubordinateRange) { throw new PodmanHostPreflightError( - `rootless Podman requires a subordinate ${mapping === "uid_map" ? "UID" : "GID"} range for the current user`, + `rootless Podman requires a subordinate ${mapping === "uidmap" ? "UID" : "GID"} range for the API service user`, ); } } @@ -437,7 +454,7 @@ export function qualifyPodmanHost( `the Podman service architecture '${normalizedArchitecture}' does not match host '${expectedArchitecture}'`, ); } - requireSubordinateIdMappings(engine); + requireSubordinateIdMappings(host); return Object.freeze({ providerId: "podman", diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 2d99815ba69..9710ea2b099 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -128,6 +128,16 @@ function hostDoctorEngine(authorityId = AUTHORITY_ID): PodmanContainerEngine { arch: "amd64", os: "linux", cgroupVersion: "v2", + idMappings: { + uidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + gidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + }, networkBackend: "netavark", security: { rootless: true }, }, @@ -140,7 +150,7 @@ function hostDoctorEngine(authorityId = AUTHORITY_ID): PodmanContainerEngine { }), captureHost: vi.fn((args: readonly string[]) => ({ status: 0, - stdout: args[0] === "--version" ? "podman version 5.6.2\n" : "0 1000 1\n1 100000 65536\n", + stdout: args[0] === "--version" ? "podman version 5.6.2\n" : "", stderr: "", })), }; From bddd60f6eefa711c9748658eac278d0d6f38bcb9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 03:04:13 -0500 Subject: [PATCH 27/71] fix(e2e): require native rootless overlay Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 19 ++++++++++++------- ...me-qualification-producer-workflow.test.ts | 7 +++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 0031d277b5c..46d0575450d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1270,7 +1270,7 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ - acl apparmor btrfs-progs conmon fuse-overlayfs \ + acl apparmor btrfs-progs conmon \ golang-github-containers-common iptables nftables passt runc slirp4netns uidmap [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] @@ -1512,10 +1512,7 @@ jobs: 'driver = "overlay"' \ "runroot = \"${home}/.local/share/containers/runroot\"" \ "graphroot = \"${home}/.local/share/containers/storage\"" \ - "rootless_storage_path = \"${home}/.local/share/containers/storage\"" \ - '' \ - '[storage.options.overlay]' \ - 'mount_program = "/usr/bin/fuse-overlayfs"' | sudo tee "$storage_config" >/dev/null + "rootless_storage_path = \"${home}/.local/share/containers/storage\"" | sudo tee "$storage_config" >/dev/null sudo chown root:root "$storage_config" sudo chmod 0444 "$storage_config" [[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || { @@ -1559,16 +1556,24 @@ jobs: install -d -m 0755 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" chmod 0555 "$guard_dir/docker" - sudo -u "$account" env -i \ + podman_info="$(sudo -u "$account" env -i \ CONTAINERS_STORAGE_CONF="$storage_config" \ HOME="$home" \ LANG=C.UTF-8 \ PATH="$guard_dir:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$runtime_dir" \ - "$podman_executable" info --format json >/dev/null || { + "$podman_executable" info --format json)" || { echo "::error::Credential-free rootless Podman readiness failed" >&2 exit 1 } + jq -e ' + .host.security.rootless == true and + .store.graphDriverName == "overlay" and + (((.store.graphOptions // {})["overlay.mount_program"].Executable? // "") == "") + ' <<<"$podman_info" >/dev/null || { + echo "::error::Qualification requires native rootless overlay storage" >&2 + exit 1 + } printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index f7b1a9beb16..2cc98046da5 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -235,7 +235,6 @@ describe("native runtime qualification producer workflow", () => { "acl", "apparmor", "conmon", - "fuse-overlayfs", "golang-github-containers-common", "passt", "runc", @@ -244,6 +243,7 @@ describe("native runtime qualification producer workflow", () => { ]) { expect(podman.run).toContain(requiredPackage); } + expect(podman.run).not.toContain("fuse-overlayfs"); expect(podman.run).toContain("find -P"); expect(podman.run).toContain("sha256sum --check --strict SHA256SUMS"); expect(podman.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); @@ -275,7 +275,10 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); expect(boundary.run).toContain("rootless_storage_path"); expect(boundary.run).toContain("${home}/.local/share/containers/storage"); - expect(boundary.run).toContain('mount_program = "/usr/bin/fuse-overlayfs"'); + expect(boundary.run).not.toContain('mount_program = "/usr/bin/fuse-overlayfs"'); + expect(boundary.run).toContain('.store.graphDriverName == "overlay"'); + expect(boundary.run).toContain('["overlay.mount_program"].Executable?'); + expect(boundary.run).toContain("Qualification requires native rootless overlay storage"); expect(boundary.run).toContain("0:0:444"); expect(boundary.run).toContain("/sys/module/apparmor/parameters/enabled"); expect(boundary.run).toContain( From 02132c28ba5d0e8e5bbfdcad5c1b8cd1ea1a9637 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 03:53:21 -0500 Subject: [PATCH 28/71] fix(e2e): harden native qualification resources Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 343 +++++++++++++++++- ...ive-runtime-qualification-case-executor.ts | 54 ++- ...tive-runtime-qualification-case-helpers.ts | 224 ++++++++++-- ...me-qualification-account-lifecycle.test.ts | 60 ++- ...runtime-qualification-case-helpers.test.ts | 31 +- ...me-qualification-producer-workflow.test.ts | 73 +++- 6 files changed, 703 insertions(+), 82 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 46d0575450d..2bde02b695e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1363,7 +1363,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in apparmor_parser awk cat getent git grep id jq node npm pgrep podman setfacl sha256sum stat systemctl tee useradd userdel usermod; do + for command in apparmor_parser awk cat curl getent git grep id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1484,10 +1484,18 @@ jobs: podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + pasta_executable="${helper_directory}/pasta" + pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" + pasta_apparmor_profile_name="nemoclaw-native-pasta-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration directory already exists" >&2 exit 1 } + [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { + echo "::error::Qualification helper directory already exists" >&2 + exit 1 + } sudo install -d -o root -g root -m 0755 "$storage_config_directory" [[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || { echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2 @@ -1507,6 +1515,24 @@ jobs: echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2 exit 1 } + [[ -f /usr/bin/pasta && ! -L /usr/bin/pasta && "$(stat -c '%u:%g:%a' /usr/bin/pasta)" == "0:0:755" ]] || { + echo "::error::Signed-OS pasta helper is missing or writable" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0555 "$helper_directory" + sudo install --owner=root --group=root --mode=0555 /usr/bin/pasta "$pasta_executable" + [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification helper directory is invalid" >&2 + exit 1 + } + [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification pasta executable is invalid" >&2 + exit 1 + } + [[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum /usr/bin/pasta | cut -d' ' -f1)" ]] || { + echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2 + exit 1 + } printf '%s\n' \ '[storage]' \ 'driver = "overlay"' \ @@ -1536,6 +1562,22 @@ jobs: exit 1 } sudo apparmor_parser -r "$apparmor_profile" + printf '%s\n' \ + '# This ephemeral profile is limited to the immutable run-owned pasta helper.' \ + '' \ + 'abi ,' \ + 'include ' \ + '' \ + "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined) {" \ + ' userns,' \ + '}' | sudo tee "$pasta_apparmor_profile" >/dev/null + sudo chown root:root "$pasta_apparmor_profile" + sudo chmod 0444 "$pasta_apparmor_profile" + [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification pasta AppArmor profile is not root-owned and read-only" >&2 + exit 1 + } + sudo apparmor_parser -r "$pasta_apparmor_profile" fi ancestor="$(dirname "$CANDIDATE_DIRECTORY")" while [[ "$ancestor" != "/home" ]]; do @@ -1560,7 +1602,7 @@ jobs: CONTAINERS_STORAGE_CONF="$storage_config" \ HOME="$home" \ LANG=C.UTF-8 \ - PATH="$guard_dir:/usr/local/bin:/usr/bin:/bin" \ + PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$runtime_dir" \ "$podman_executable" info --format json)" || { echo "::error::Credential-free rootless Podman readiness failed" >&2 @@ -1577,15 +1619,213 @@ jobs: printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" + printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT" printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" + - name: Materialize exact credential-free GPU resources + id: gpu_resources + env: + ACCOUNT: ${{ steps.boundary.outputs.account }} + ARCHITECTURE: ${{ matrix.case.architecture }} + GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} + INFERENCE: ${{ matrix.case.inference }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} + QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + shell: bash + run: | + set -euo pipefail + umask 077 + if [[ "${{ matrix.case.acceleration }}" != "nvidia-gpu" ]]; then + printf 'runner_contract=\n' >>"$GITHUB_OUTPUT" + exit 0 + fi + [[ -n "$NVIDIA_API_KEY" ]] || { + echo "::error::Protected GPU resource preparation requires the existing NVIDIA registry credential" >&2 + exit 1 + } + uid="$(id -u "$ACCOUNT")" + storage_config_directory="$(dirname "$STORAGE_CONFIG")" + runner_contract="${storage_config_directory}/runner-contract.json" + registry_auth_directory="${storage_config_directory}/registry-auth" + registry_auth_file="${registry_auth_directory}/auth.json" + resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + model_directory="${resource_directory}/model" + model_revision="7ae557604adf67be50417f59c2c2f167def9a775" + model_name="Qwen/Qwen2.5-0.5B-Instruct" + case "$ARCHITECTURE" in + amd64) + probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:9855f4c8500addf185360474184b9efdaf4384284779aa9173dcd70164a4ae6f" + nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:a0fdbecdf51792dadc48d284fde3199a58d5a2067007ad5b80319975fe81ce93" + vllm_image="nvcr.io/nvidia/vllm@sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773" + ;; + arm64) + probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:a54fdceac3bc2a8d177f07db942defc2f7237e18d07fca2ff00718ba5aee4940" + nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:8342257b9744e9bc23a02e0f45badad6b88474727473965b4dc83e8fee45956a" + vllm_image="nvcr.io/nvidia/vllm@sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4" + ;; + *) + echo "::error::GPU resource architecture is unsupported" >&2 + exit 1 + ;; + esac + [[ ! -e "$runner_contract" && ! -L "$runner_contract" ]] || { + echo "::error::Run-owned GPU runner contract already exists" >&2 + exit 1 + } + [[ ! -e "$registry_auth_directory" && ! -L "$registry_auth_directory" ]] || { + echo "::error::Run-owned registry authentication directory already exists" >&2 + exit 1 + } + sudo install -d --owner="$uid" --group="$uid" --mode=0700 "$registry_auth_directory" + cleanup_registry_auth() { + local result="$?" + trap - EXIT + sudo -u "$ACCOUNT" env -i \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + REGISTRY_AUTH_FILE="$registry_auth_file" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + "$PODMAN_EXECUTABLE" logout --all >/dev/null 2>&1 || true + if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a' "$registry_auth_file")" == "${uid}:${uid}:600" ]] || { + echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 + return 1 + } + sudo unlink "$registry_auth_file" + fi + sudo rmdir "$registry_auth_directory" || { + echo "::error::Run-owned registry authentication directory was not empty" >&2 + return 1 + } + return "$result" + } + trap cleanup_registry_auth EXIT + printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + REGISTRY_AUTH_FILE="$registry_auth_file" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null + image_to_pull="$probe_image" + if [[ "$INFERENCE" == "nim" ]]; then + image_to_pull="$nim_image" + elif [[ "$INFERENCE" == "vllm" ]]; then + image_to_pull="$vllm_image" + fi + for image in "$probe_image" "$image_to_pull"; do + sudo -u "$ACCOUNT" env -i \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + REGISTRY_AUTH_FILE="$registry_auth_file" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + "$PODMAN_EXECUTABLE" pull "$image" + done + cleanup_registry_auth + trap - EXIT + unset NVIDIA_API_KEY + + if [[ "$INFERENCE" == "nim" || "$INFERENCE" == "vllm" ]]; then + [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { + echo "::error::Run-owned GPU model resource already exists" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0711 "$resource_directory" + sudo install -d --owner="$uid" --group="$uid" --mode=0700 "$model_directory" + download_model_file() { + local file="$1" + local size="$2" + local algorithm="$3" + local digest="$4" + local target="${model_directory}/${file}" + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + /usr/bin/curl \ + --fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \ + --output "$target" \ + "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" + [[ -f "$target" && ! -L "$target" && "$(stat -c '%u:%g:%a:%h:%s' "$target")" == "${uid}:${uid}:600:1:${size}" ]] || { + echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 + exit 1 + } + if [[ "$algorithm" == "sha256" ]]; then + [[ "$(sha256sum "$target" | cut -d' ' -f1)" == "$digest" ]] + else + [[ "$(git hash-object --no-filters "$target")" == "$digest" ]] + fi || { + echo "::error::Downloaded GPU model file digest is invalid: $file" >&2 + exit 1 + } + } + download_model_file config.json 659 sha1 0dbb161213629a23f0fc00ef286e6b1e366d180f + download_model_file generation_config.json 242 sha1 dfc11073787daf1b0f9c0f1499487ab5f4c93738 + download_model_file merges.txt 1671839 sha1 20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0 + download_model_file model.safetensors 988097824 sha256 fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe + download_model_file tokenizer.json 7031645 sha1 443909a61d429dff23010e5bddd28ff530edda00 + download_model_file tokenizer_config.json 7305 sha1 07bfe0640cb5a0037f9322287fbfc682806cf672 + download_model_file vocab.json 2776833 sha1 4783fe10ac3adce15ac8f358ef5462739852c569 + for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do + sudo chown root:root "${model_directory}/${file}" + sudo chmod 0444 "${model_directory}/${file}" + done + sudo chown root:root "$model_directory" + sudo chmod 0555 "$model_directory" + sudo chmod 0555 "$resource_directory" + fi + + jq -n \ + --arg architecture "$ARCHITECTURE" \ + --arg gpuProbeImageRef "$probe_image" \ + --arg model "$model_name" \ + --arg modelPath "$model_directory" \ + --arg modelRevision "$model_revision" \ + --arg nimImageRef "$nim_image" \ + --arg vllmImageRef "$vllm_image" ' + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture: $architecture, + gpuProbeImageRef: $gpuProbeImageRef, + nim: { + imageRef: $nimImageRef, + model: $model, + modelPath: $modelPath, + modelRevision: $modelRevision + }, + vllm: { + imageRef: $vllmImageRef, + model: $model, + modelPath: $modelPath, + modelRevision: $modelRevision + } + } + ' | sudo tee "$runner_contract" >/dev/null + sudo chown root:root "$runner_contract" + sudo chmod 0444 "$runner_contract" + [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { + echo "::error::Run-owned GPU runner contract is invalid" >&2 + exit 1 + } + printf 'runner_contract=%s\n' "$runner_contract" >>"$GITHUB_OUTPUT" + - name: Install locked candidate test dependencies without scripts env: ACCOUNT: ${{ steps.boundary.outputs.account }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} shell: bash @@ -1594,7 +1834,7 @@ jobs: sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ - PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ /bin/bash --noprofile --norc -c ' set -euo pipefail cd "$1" @@ -1614,6 +1854,7 @@ jobs: CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CANDIDATE_SHA: ${{ matrix.source.candidateSha }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer INSTALLER_SHA256: ${{ matrix.installerSha256 }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} @@ -1625,7 +1866,7 @@ jobs: sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ - PATH="$GUARD_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ bash .trusted-qualification/scripts/checks/run-native-runtime-installer-qualification.sh \ --candidate-checkout "$CANDIDATE_DIRECTORY" \ --candidate-sha "$CANDIDATE_SHA" \ @@ -1647,9 +1888,11 @@ jobs: ACCOUNT: ${{ steps.boundary.outputs.account }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} shell: bash @@ -1674,8 +1917,9 @@ jobs: NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \ + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT="$RUNNER_CONTRACT" \ NEMOCLAW_RUN_LIVE_E2E=1 \ - PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \ --config "$CANDIDATE_DIRECTORY/vitest.config.ts" \ @@ -1749,6 +1993,14 @@ jobs: storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" + pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" + helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + pasta_executable="${helper_directory}/pasta" + registry_auth_directory="${storage_config_directory}/registry-auth" + registry_auth_file="${registry_auth_directory}/auth.json" + runner_contract="${storage_config_directory}/runner-contract.json" + resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + model_directory="${resource_directory}/model" if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true fi @@ -1766,9 +2018,70 @@ jobs: sudo apparmor_parser -R "$apparmor_profile" sudo rm -f -- "$apparmor_profile" fi + if [[ -e "$pasta_apparmor_profile" || -L "$pasta_apparmor_profile" ]]; then + [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification pasta AppArmor profile cleanup target is invalid" >&2 + exit 1 + } + sudo apparmor_parser -R "$pasta_apparmor_profile" + sudo rm -f -- "$pasta_apparmor_profile" + fi + if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then + [[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${uid}:700" ]] || { + echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2 + exit 1 + } + if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a' "$registry_auth_file")" == "${uid}:${uid}:600" ]] || { + echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$registry_auth_file" + fi + sudo rmdir "$registry_auth_directory" + fi + if [[ -e "$runner_contract" || -L "$runner_contract" ]]; then + [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { + echo "::error::Qualification GPU runner contract cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$runner_contract" + fi sudo rm -f -- "$storage_config_directory/storage.conf" sudo rmdir "$storage_config_directory" fi + if [[ -e "$resource_directory" || -L "$resource_directory" ]]; then + [[ -d "$resource_directory" && ! -L "$resource_directory" && ("$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:711" || "$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:555") ]] || { + echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2 + exit 1 + } + if [[ -e "$model_directory" || -L "$model_directory" ]]; then + model_mode="$(stat -c '%u:%g:%a' "$model_directory")" + [[ -d "$model_directory" && ! -L "$model_directory" && ("$model_mode" == "${uid}:${uid}:700" || "$model_mode" == "0:0:555") ]] || { + echo "::error::Qualification GPU model directory cleanup target is invalid" >&2 + exit 1 + } + for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do + target="${model_directory}/${file}" + if [[ -e "$target" || -L "$target" ]]; then + file_mode="$(stat -c '%u:%g:%a:%h' "$target")" + [[ -f "$target" && ! -L "$target" && ("$file_mode" == "${uid}:${uid}:600:1" || "$file_mode" == "0:0:444:1") ]] || { + echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2 + exit 1 + } + sudo unlink "$target" + fi + done + sudo rmdir "$model_directory" || { + echo "::error::Qualification GPU model directory contains unexpected entries" >&2 + exit 1 + } + fi + sudo rmdir "$resource_directory" || { + echo "::error::Qualification GPU resource directory contains unexpected entries" >&2 + exit 1 + } + fi if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { echo "::error::Qualification Podman executable cleanup target is invalid" >&2 @@ -1776,6 +2089,18 @@ jobs: } sudo rm -f -- "$podman_executable" fi + if [[ -e "$helper_directory" || -L "$helper_directory" ]]; then + [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { + echo "::error::Qualification helper directory cleanup target is invalid" >&2 + exit 1 + } + [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { + echo "::error::Qualification pasta executable cleanup target is invalid" >&2 + exit 1 + } + sudo rm -f -- "$pasta_executable" + sudo rmdir "$helper_directory" + fi if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2 exit 1 @@ -1804,6 +2129,14 @@ jobs: echo "::error::Qualification Podman executable remains after cleanup" >&2 exit 1 } + [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { + echo "::error::Qualification helper directory remains after cleanup" >&2 + exit 1 + } + [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { + echo "::error::Qualification GPU resource directory remains after cleanup" >&2 + exit 1 + } sudo rm -f -- "$ownership_marker" elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then echo "::error::Qualification account output exists without its ownership marker" >&2 diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 9dbb9bc2141..0d9b0a53b47 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -32,10 +32,12 @@ import type { NativeRuntimeQualificationObligation } from "../registry/native-ru import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; import { assertCredentialFreeQualificationEnvironment, + assertNativeRuntimeQualificationModelResource, digestFromImageReference, nativeRuntimeQualificationAgentImage, nativeRuntimeQualificationInferenceImage, nativeRuntimeQualificationPodmanExecutable, + nativeRuntimeQualificationRunnerContractPath, parseNativeRuntimeQualificationRow, readNativeRuntimeQualificationRunnerContract, } from "./native-runtime-qualification-case-helpers.ts"; @@ -331,31 +333,6 @@ function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: str capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); } -function rootOwnedReadOnlyDirectory(directory: string): void { - const canonical = fs.realpathSync(directory); - if (canonical !== directory) { - throw new Error(`Runner model resource is not canonical: ${directory}`); - } - const boundary = "/var/lib/nemoclaw/native-runtime-qualification"; - let current = directory; - while (current.startsWith(`${boundary}/`) || current === boundary) { - const metadata = fs.lstatSync(current); - if ( - !metadata.isDirectory() || - metadata.isSymbolicLink() || - metadata.uid !== 0 || - (metadata.mode & 0o022) !== 0 - ) { - throw new Error( - `Runner model resource is not an exact root-owned read-only directory: ${current}`, - ); - } - if (current === boundary) return; - current = path.dirname(current); - } - throw new Error(`Runner model resource escapes its reviewed root: ${directory}`); -} - function proveGpuDevices( engine: PodmanBoundContainerEngine, probeImageRef: string, @@ -643,10 +620,13 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const network = createProviderNetwork(inferenceEngine, networkName, row.id); ownedNetworks.add(network.id); const hostPort = 20_000 + (Number.parseInt(caseSuffix.slice(0, 4), 16) % 20_000); - const runnerContract = + const runnerContractFile = row.case.acceleration === "nvidia-gpu" - ? readNativeRuntimeQualificationRunnerContract(row.case.architecture) + ? nativeRuntimeQualificationRunnerContractPath(process.env, uid) : undefined; + const runnerContract = runnerContractFile + ? readNativeRuntimeQualificationRunnerContract(row.case.architecture, runnerContractFile) + : undefined; const agentImage = nativeRuntimeQualificationAgentImage(row.case.architecture, row.case.agent); const inference = nativeRuntimeQualificationInferenceImage({ architecture: row.case.architecture, @@ -657,11 +637,11 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre pullPublicImage(inferenceEngine, agentImage); if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); else { - if (!inference.cachePath || !path.isAbsolute(inference.cachePath)) { - throw new Error("Native runtime qualification GPU cache path must be absolute"); + if (!inference.modelPath || !runnerContractFile || !path.isAbsolute(inference.modelPath)) { + throw new Error("Native runtime qualification GPU model path must be absolute"); } requirePreloadedImage(inferenceEngine, inference.imageRef); - rootOwnedReadOnlyDirectory(inference.cachePath); + assertNativeRuntimeQualificationModelResource(inference.modelPath, uid, runnerContractFile); } if (runnerContract) { requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); @@ -687,10 +667,19 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre `${QUALIFICATION_LABEL}=${row.id}`, ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), ...(row.case.inference === "nim" - ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/opt/nim/.cache:ro`] + ? [ + "--shm-size", + "16g", + "--env", + "NIM_MODEL_PATH=/models", + "--env", + `NIM_SERVED_MODEL_NAME=${inference.model}`, + "--volume", + `${inference.modelPath}:/models:ro`, + ] : []), ...(row.case.inference === "vllm" - ? ["--shm-size", "16g", "--volume", `${inference.cachePath}:/models:ro`] + ? ["--shm-size", "16g", "--volume", `${inference.modelPath}:/models:ro`] : []), inference.imageRef, ...(row.case.inference === "vllm" @@ -1034,6 +1023,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre rootMode: "rootless", podmanVersion, inferenceService: row.case.inference, + modelRevision: inference.modelRevision ?? null, focusedOperations: focusedResults, dockerBefore, dockerAfter, diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts index ff5c79cc8b6..c50d928f784 100644 --- a/test/e2e/live/native-runtime-qualification-case-helpers.ts +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -1,7 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { constants, closeSync, fstatSync, openSync, readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { + constants, + closeSync, + fstatSync, + openSync, + readFileSync, + readdirSync, + readSync, +} from "node:fs"; +import path from "node:path"; import { NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, @@ -16,15 +26,59 @@ import { type NativeRuntimeQualificationInference, } from "../registry/native-runtime-qualification.ts"; -export const NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT = - "/etc/nemoclaw/native-runtime-qualification-v1.json"; +export const NATIVE_RUNTIME_QUALIFICATION_MODEL_REVISION = + "7ae557604adf67be50417f59c2c2f167def9a775"; const SHA = /^[a-f0-9]{40}$/u; const SHA256 = /^[a-f0-9]{64}$/u; const OCI_DIGEST = /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,511}$/u; -const ABSOLUTE_CACHE = - /^\/var\/lib\/nemoclaw\/native-runtime-qualification\/[A-Za-z0-9._/-]{1,384}$/u; +const ABSOLUTE_MODEL = + /^\/var\/tmp\/nemoclaw-native-runtime-resources-[1-9][0-9]*-[1-9][0-9]*-[1-9][0-9]*\/model$/u; +const MODEL_FILES = Object.freeze([ + Object.freeze({ + path: "config.json", + size: 659, + algorithm: "sha1" as const, + digest: "0dbb161213629a23f0fc00ef286e6b1e366d180f", + }), + Object.freeze({ + path: "generation_config.json", + size: 242, + algorithm: "sha1" as const, + digest: "dfc11073787daf1b0f9c0f1499487ab5f4c93738", + }), + Object.freeze({ + path: "merges.txt", + size: 1_671_839, + algorithm: "sha1" as const, + digest: "20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0", + }), + Object.freeze({ + path: "model.safetensors", + size: 988_097_824, + algorithm: "sha256" as const, + digest: "fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe", + }), + Object.freeze({ + path: "tokenizer.json", + size: 7_031_645, + algorithm: "sha1" as const, + digest: "443909a61d429dff23010e5bddd28ff530edda00", + }), + Object.freeze({ + path: "tokenizer_config.json", + size: 7_305, + algorithm: "sha1" as const, + digest: "07bfe0640cb5a0037f9322287fbfc682806cf672", + }), + Object.freeze({ + path: "vocab.json", + size: 2_776_833, + algorithm: "sha1" as const, + digest: "4783fe10ac3adce15ac8f358ef5462739852c569", + }), +]); const SENSITIVE_ENVIRONMENT = /^(?:GH_TOKEN|GITHUB_TOKEN|NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|NGC_API_KEY|NIM_NGC_API_KEY|HF_TOKEN|HUGGING_FACE_HUB_TOKEN|SSH_AUTH_SOCK|DOCKER_CERT_PATH|DOCKER_CONFIG|DOCKER_CONTEXT|DOCKER_HOST|DOCKER_TLS_VERIFY|CONTAINER_HOST|AWS_.+|AZURE_.+|GOOGLE_.+|.*(?:_API_KEY|_ACCESS_TOKEN|_AUTH_TOKEN|_PASSWORD|_PRIVATE_KEY|_SECRET|_SECRET_KEY))$/u; @@ -67,12 +121,14 @@ export interface NativeRuntimeQualificationRunnerContract { readonly nim: { readonly imageRef: string; readonly model: string; - readonly cachePath: string; + readonly modelPath: string; + readonly modelRevision: string; }; readonly vllm: { readonly imageRef: string; readonly model: string; readonly modelPath: string; + readonly modelRevision: string; }; } @@ -224,31 +280,34 @@ export function nativeRuntimeQualificationPodmanExecutable( function exactRunnerRuntime( value: unknown, label: "NIM" | "vLLM", - cacheField: "cachePath" | "modelPath", ): { readonly imageRef: string; readonly model: string; - readonly cachePath: string; + readonly modelPath: string; + readonly modelRevision: string; } { const runtime = record(value, `${label} runner contract`); - exactKeys(runtime, ["imageRef", "model", cacheField], `${label} runner contract`); - const cachePath = runtime[cacheField]; + exactKeys( + runtime, + ["imageRef", "model", "modelPath", "modelRevision"], + `${label} runner contract`, + ); if ( typeof runtime.imageRef !== "string" || !OCI_DIGEST.test(runtime.imageRef) || typeof runtime.model !== "string" || !MODEL.test(runtime.model) || - typeof cachePath !== "string" || - !ABSOLUTE_CACHE.test(cachePath) || - cachePath.includes("//") || - cachePath.split("/").some((segment) => segment === "." || segment === "..") + typeof runtime.modelPath !== "string" || + !ABSOLUTE_MODEL.test(runtime.modelPath) || + runtime.modelRevision !== NATIVE_RUNTIME_QUALIFICATION_MODEL_REVISION ) { throw new Error(`${label} runner contract is invalid`); } return Object.freeze({ imageRef: runtime.imageRef, model: runtime.model, - cachePath, + modelPath: runtime.modelPath, + modelRevision: runtime.modelRevision, }); } @@ -271,8 +330,8 @@ export function parseNativeRuntimeQualificationRunnerContract( ) { throw new Error("Native runtime qualification runner contract identity is invalid"); } - const nim = exactRunnerRuntime(contract.nim, "NIM", "cachePath"); - const vllm = exactRunnerRuntime(contract.vllm, "vLLM", "modelPath"); + const nim = exactRunnerRuntime(contract.nim, "NIM"); + const vllm = exactRunnerRuntime(contract.vllm, "vLLM"); return Object.freeze({ schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-runner-v1", @@ -281,19 +340,21 @@ export function parseNativeRuntimeQualificationRunnerContract( nim: Object.freeze({ imageRef: nim.imageRef, model: nim.model, - cachePath: nim.cachePath, + modelPath: nim.modelPath, + modelRevision: nim.modelRevision, }), vllm: Object.freeze({ imageRef: vllm.imageRef, model: vllm.model, - modelPath: vllm.cachePath, + modelPath: vllm.modelPath, + modelRevision: vllm.modelRevision, }), }); } export function readNativeRuntimeQualificationRunnerContract( architecture: NativeRuntimeQualificationArchitecture, - file = NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT, + file: string, ): NativeRuntimeQualificationRunnerContract { let descriptor: number | undefined; try { @@ -303,7 +364,8 @@ export function readNativeRuntimeQualificationRunnerContract( !before.isFile() || before.nlink !== 1n || before.uid !== 0n || - (before.mode & 0o022n) !== 0n || + before.gid !== 0n || + (before.mode & 0o777n) !== 0o444n || before.size < 1n || before.size > 65_536n ) { @@ -336,6 +398,117 @@ export function readNativeRuntimeQualificationRunnerContract( } } +export function nativeRuntimeQualificationRunnerContractPath( + environment: NodeJS.ProcessEnv, + uid: number, +): string { + const file = environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT ?? ""; + const expected = new RegExp( + `^/run/nemoclaw-native-runtime-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}/runner-contract\\.json$`, + "u", + ); + if (!Number.isSafeInteger(uid) || uid <= 0 || !expected.test(file)) { + throw new Error("Native runtime qualification runner contract path is invalid"); + } + return file; +} + +function stableModelFileDigest(file: string, expected: (typeof MODEL_FILES)[number]): string { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== 0n || + before.gid !== 0n || + (before.mode & 0o777n) !== 0o444n || + before.size !== BigInt(expected.size) + ) { + throw new Error(`model file metadata is invalid: ${expected.path}`); + } + const digest = createHash(expected.algorithm); + if (expected.algorithm === "sha1") digest.update(`blob ${String(expected.size)}\0`); + const buffer = Buffer.allocUnsafe(1024 * 1024); + for (;;) { + const count = readSync(descriptor, buffer, 0, buffer.length, null); + if (count === 0) break; + digest.update(buffer.subarray(0, count)); + } + const after = fstatSync(descriptor, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mode !== after.mode || + before.uid !== after.uid || + before.gid !== after.gid || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) { + throw new Error(`model file changed during its stable read: ${expected.path}`); + } + return digest.digest("hex"); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function assertStableRootOwnedDirectory(directory: string): void { + let descriptor: number | undefined; + try { + descriptor = openSync( + directory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + const metadata = fstatSync(descriptor); + if ( + !metadata.isDirectory() || + metadata.uid !== 0 || + metadata.gid !== 0 || + (metadata.mode & 0o777) !== 0o555 + ) { + throw new Error(`Runner model resource is not root-owned and read-only: ${directory}`); + } + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +export function assertNativeRuntimeQualificationModelResource( + directory: string, + uid: number, + contractFile: string, +): void { + const contractMatch = contractFile.match( + /^\/run\/nemoclaw-native-runtime-([1-9][0-9]*)-([1-9][0-9]*)-([1-9][0-9]*)\/runner-contract\.json$/u, + ); + const modelMatch = directory.match( + /^\/var\/tmp\/nemoclaw-native-runtime-resources-([1-9][0-9]*)-([1-9][0-9]*)-([1-9][0-9]*)\/model$/u, + ); + if ( + !contractMatch || + !modelMatch || + contractMatch.slice(1).join(":") !== modelMatch.slice(1).join(":") || + contractMatch[3] !== String(uid) + ) { + throw new Error("Runner model resource does not match the run-owned contract identity"); + } + assertStableRootOwnedDirectory(path.dirname(directory)); + assertStableRootOwnedDirectory(directory); + const actual = readdirSync(directory).sort(); + const expectedFiles = MODEL_FILES.map((entry) => entry.path).sort(); + if (actual.join("\n") !== expectedFiles.join("\n")) { + throw new Error("Runner model resource file set is invalid"); + } + for (const expected of MODEL_FILES) { + if (stableModelFileDigest(path.join(directory, expected.path), expected) !== expected.digest) { + throw new Error(`Runner model resource digest is invalid: ${expected.path}`); + } + } +} + export function nativeRuntimeQualificationAgentImage( architecture: NativeRuntimeQualificationArchitecture, agent: NativeRuntimeQualificationAgent, @@ -351,7 +524,8 @@ export function nativeRuntimeQualificationInferenceImage(input: { }): { readonly imageRef: string; readonly model: string; - readonly cachePath?: string; + readonly modelPath?: string; + readonly modelRevision?: string; } { if (input.inference === "ollama") { return Object.freeze({ @@ -366,13 +540,15 @@ export function nativeRuntimeQualificationInferenceImage(input: { return Object.freeze({ imageRef: input.runnerContract.nim.imageRef, model: input.runnerContract.nim.model, - cachePath: input.runnerContract.nim.cachePath, + modelPath: input.runnerContract.nim.modelPath, + modelRevision: input.runnerContract.nim.modelRevision, }); } return Object.freeze({ imageRef: input.runnerContract.vllm.imageRef, model: input.runnerContract.vllm.model, - cachePath: input.runnerContract.vllm.modelPath, + modelPath: input.runnerContract.vllm.modelPath, + modelRevision: input.runnerContract.vllm.modelRevision, }); } diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 63c64db026d..3e6f225d29e 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -53,6 +53,14 @@ function fixtureSource(source: string): string { 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) + .replaceAll( + 'helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'helper_directory="${FIXTURE_ROOT}/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ) + .replaceAll( + 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ) .replaceAll( "/usr/lib/systemd/systemd-user-runtime-dir", "${FIXTURE_ROOT}/bin/systemd-user-runtime-dir", @@ -120,12 +128,29 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, ); writeExecutable(path.join(bin, "chown"), ":"); writeExecutable(path.join(bin, "chmod"), `exec /bin/chmod "$@"`); + writeExecutable( + path.join(bin, "unlink"), + `/bin/chmod u+w "$(/usr/bin/dirname "$1")"\nexec /bin/unlink "$1"`, + ); + writeExecutable( + path.join(bin, "rmdir"), + `/bin/chmod u+w "$(/usr/bin/dirname "$1")"\nexec /bin/rmdir "$1"`, + ); + writeExecutable( + path.join(bin, "rm"), + `target="\${!#}"\n/bin/chmod u+w "$(/usr/bin/dirname "$target")"\nexec /bin/rm "$@"`, + ); writeExecutable( path.join(bin, "stat"), `case "$3" in *native-runtime-owner-*) printf '0:0:400\\n' ;; *native-runtime-podman-*) printf '0:0:555\\n' ;; - *podman.apparmor|*storage.conf) printf '0:0:444\\n' ;; + *native-runtime-helpers-*) printf '0:0:555\\n' ;; + *native-runtime-resources-*/model/*) + [[ "$2" == '%u:%g:%a:%h' ]] && printf '0:0:444:1\\n' || printf '0:0:444\\n' + ;; + *native-runtime-resources-*) printf '0:0:555\\n' ;; + *podman.apparmor|*pasta.apparmor|*runner-contract.json|*storage.conf) printf '0:0:444\\n' ;; *) exit 25 ;; esac`, ); @@ -169,7 +194,7 @@ function runFixture( ) { return spawnSync("bash", ["-c", fixtureSource(source)], { encoding: "utf8", - timeout: 5000, + timeout: 15_000, env: { ...process.env, ACCOUNT: "nemoclawq", @@ -281,18 +306,45 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.existsSync(fixture.marker)).toBe(false); }); - it("removes the run-owned runtime, Podman executable, storage, and AppArmor profile", () => { + it("removes the run-owned runtime, immutable helpers, GPU resources, and AppArmor profiles", () => { const fixture = createFixture(); const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); const podman = path.join(fixture.root, "nemoclaw-native-runtime-podman-42-1-1002"); + const helpers = path.join(fixture.root, "nemoclaw-native-runtime-helpers-42-1-1002"); + const resources = path.join( + fixture.root, + "var", + "tmp", + "nemoclaw-native-runtime-resources-42-1-1002", + ); + const model = path.join(resources, "model"); fs.mkdirSync(fixture.home, { recursive: true }); fs.mkdirSync(runtime, { recursive: true }); fs.mkdirSync(storage, { recursive: true }); fs.writeFileSync(path.join(runtime, "alive"), "fixture"); fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); + fs.writeFileSync(path.join(storage, "pasta.apparmor"), "fixture"); + fs.writeFileSync(path.join(storage, "runner-contract.json"), "fixture"); fs.writeFileSync(podman, "fixture", { mode: 0o555 }); + fs.mkdirSync(helpers, { recursive: true, mode: 0o755 }); + fs.writeFileSync(path.join(helpers, "pasta"), "fixture", { mode: 0o555 }); + fs.chmodSync(helpers, 0o555); + fs.mkdirSync(model, { recursive: true, mode: 0o755 }); + for (const file of [ + "config.json", + "generation_config.json", + "merges.txt", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + ]) { + fs.writeFileSync(path.join(model, file), "fixture", { mode: 0o444 }); + } + fs.chmodSync(model, 0o555); + fs.chmodSync(resources, 0o555); fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); @@ -306,6 +358,8 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); expect(fs.existsSync(storage)).toBe(false); expect(fs.existsSync(podman)).toBe(false); + expect(fs.existsSync(helpers)).toBe(false); + expect(fs.existsSync(resources)).toBe(false); }); it("does not run destructive cleanup when the run-owned marker is absent", () => { diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index ebc7d76dc11..8e339baef2d 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -13,6 +13,7 @@ import { nativeRuntimeQualificationAgentImage, nativeRuntimeQualificationInferenceImage, nativeRuntimeQualificationPodmanExecutable, + nativeRuntimeQualificationRunnerContractPath, parseNativeRuntimeQualificationRow, parseNativeRuntimeQualificationRunnerContract, } from "../live/native-runtime-qualification-case-helpers.ts"; @@ -54,12 +55,14 @@ function runnerContract() { nim: { imageRef: `nvcr.io/nim/nvidia/model@sha256:${"1".repeat(64)}`, model: "nvidia/model", - cachePath: "/var/lib/nemoclaw/native-runtime-qualification/nim/cache", + modelPath: "/var/tmp/nemoclaw-native-runtime-resources-123456-1-1002/model", + modelRevision: "7ae557604adf67be50417f59c2c2f167def9a775", }, vllm: { imageRef: `docker.io/vllm/vllm-openai@sha256:${"2".repeat(64)}`, model: "qualification", - modelPath: "/var/lib/nemoclaw/native-runtime-qualification/vllm/model", + modelPath: "/var/tmp/nemoclaw-native-runtime-resources-123456-1-1002/model", + modelRevision: "7ae557604adf67be50417f59c2c2f167def9a775", }, } as const; } @@ -136,7 +139,7 @@ describe("native runtime qualification case boundaries", () => { it("accepts only typed immutable GPU runner resources", () => { const parsed = parseNativeRuntimeQualificationRunnerContract(runnerContract(), "amd64"); expect(parsed.nim.imageRef).toContain("@sha256:"); - expect(parsed.vllm.modelPath).toMatch(/^\/var\/lib\/nemoclaw\/native-runtime-qualification\//u); + expect(parsed.vllm.modelPath).toMatch(/^\/var\/tmp\/nemoclaw-native-runtime-resources-/u); expect(() => parseNativeRuntimeQualificationRunnerContract( @@ -158,6 +161,28 @@ describe("native runtime qualification case boundaries", () => { ).toThrow("vLLM runner contract is invalid"); }); + it("accepts only the current uid's run-owned GPU contract path", () => { + const environment = { + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: + "/run/nemoclaw-native-runtime-123456-1-1002/runner-contract.json", + }; + expect(nativeRuntimeQualificationRunnerContractPath(environment, 1002)).toBe( + environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT, + ); + for (const file of [ + "/etc/nemoclaw/native-runtime-qualification-v1.json", + "/run/nemoclaw-native-runtime-123456-1-1003/runner-contract.json", + "/run/nemoclaw-native-runtime-123456-1-1002/../runner-contract.json", + ]) { + expect(() => + nativeRuntimeQualificationRunnerContractPath( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: file }, + 1002, + ), + ).toThrow("runner contract path is invalid"); + } + }); + it("pins every public case image to architecture-specific immutable digests", () => { for (const architecture of ["amd64", "arm64"] as const) { for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 2cc98046da5..9489975382c 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -197,12 +197,18 @@ describe("native runtime qualification producer workflow", () => { producer, "Install locked candidate test dependencies without scripts", ); + const gpuResources = step(producer, "Materialize exact credential-free GPU resources"); const installer = step(producer, "Run the authenticated installer qualification"); const execute = step(producer, "Execute the candidate qualification case without credentials"); const validate = step(producer, "Validate receipts and emit bounded evidence"); const upload = step(producer, "Upload the qualification case evidence"); const cleanup = step(producer, "Remove qualification resources"); - const source = JSON.stringify(producer); + const credentialFreeSource = JSON.stringify({ + ...producer, + steps: producer.steps?.filter( + (entry) => entry.name !== "Materialize exact credential-free GPU resources", + ), + }); const boundaryRun = boundary.run ?? ""; const executeRun = execute.run ?? ""; @@ -221,7 +227,24 @@ describe("native runtime qualification producer workflow", () => { expect(harness.with?.["sparse-checkout"]).toContain( "test/e2e/registry/native-runtime-qualification.ts", ); - expect(source).not.toMatch(/NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u); + expect(credentialFreeSource).not.toMatch( + /NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u, + ); + expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); + expect(gpuResources.run).toContain("existing NVIDIA registry credential"); + expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); + expect(gpuResources.run).toContain("logout --all"); + expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); + expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); + expect(gpuResources.run).toContain("git hash-object --no-filters"); + expect(gpuResources.run).toContain("sha256sum"); + expect(gpuResources.run).toContain("model-free-nim@sha256:"); + expect(gpuResources.run).toContain("nvcr.io/nvidia/vllm@sha256:"); + expect(gpuResources.run).toContain("runner-contract.json"); + expect(gpuResources.run).toContain( + 'install -d --owner=root --group=root --mode=0711 "$resource_directory"', + ); + expect(gpuResources.run).toContain('chmod 0555 "$resource_directory"'); expect(podmanHost.run).toContain('[[ "${ID:-}" == "ubuntu" ]]'); expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "24.04"'); expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "26.04"'); @@ -259,12 +282,16 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); expect(boundary.run).toContain('getent passwd "$account"'); expect(boundary.run).toContain('grep -q "^${account}:" /etc/subuid /etc/subgid'); - expect(boundary.run).toContain("Qualification account identity or subordinate-ID authorization already exists"); + expect(boundary.run).toContain( + "Qualification account identity or subordinate-ID authorization already exists", + ); expect(boundary.run).toContain('ownership_marker="/run/nemoclaw-native-runtime-owner-'); expect(boundary.run).toContain("0:0:400"); expect(boundary.run).toContain("Qualification account ownership marker is invalid"); expect(boundary.run).toContain("rollback_unmarked_account"); - expect(boundary.run).toContain("Partially created qualification account could not be rolled back"); + expect(boundary.run).toContain( + "Partially created qualification account could not be rolled back", + ); expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); @@ -282,7 +309,14 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("0:0:444"); expect(boundary.run).toContain("/sys/module/apparmor/parameters/enabled"); expect(boundary.run).toContain( - 'profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined)', + "profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined)", + ); + expect(boundary.run).toContain( + "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined)", + ); + expect(boundary.run).toContain("Run-owned qualification pasta executable digest changed"); + expect(boundary.run).toContain( + 'PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin"', ); expect(boundary.env?.TOOLCHAIN_DIRECTORY).toBe( "${{ runner.temp }}/native-runtime-podman-toolchain", @@ -290,7 +324,7 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain( 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ); - expect(boundary.run).toContain('sudo install --owner=root --group=root --mode=0555'); + expect(boundary.run).toContain("sudo install --owner=root --group=root --mode=0555"); expect(boundary.run).toContain("0:0:555"); expect(boundary.run).toContain('"$podman_executable" info --format json'); expect(boundary.run).toContain("userns,"); @@ -334,16 +368,15 @@ describe("native runtime qualification producer workflow", () => { expect(execute.run).not.toContain("GH_TOKEN"); expect(execute.run).not.toContain("chown -R"); expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); - expect(execute.env?.PODMAN_EXECUTABLE).toBe( - "${{ steps.boundary.outputs.podman_executable }}", - ); + expect(execute.env?.PODMAN_EXECUTABLE).toBe("${{ steps.boundary.outputs.podman_executable }}"); expect(execute.env?.STORAGE_CONFIG).toBe("${{ steps.boundary.outputs.storage_config }}"); + expect(execute.env?.RUNNER_CONTRACT).toBe("${{ steps.gpu_resources.outputs.runner_contract }}"); expect(execute.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); expect(execute.run).toContain( 'NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE"', ); expect(execute.run).toContain( - 'PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', + 'PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', ); expect(validate.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); expect(validate.run).not.toContain('chown -R -h "$(id -u):$(id -g)"'); @@ -357,24 +390,34 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.if).toBe("always()"); expect(cleanup.env?.ACCOUNT_CREATED).toBe("${{ steps.boundary.outputs.account_created }}"); expect(cleanup.run).toContain('reported_account="${ACCOUNT:-}"'); - expect(cleanup.run).not.toContain('ACCOUNT:-nemoclawq'); - expect(cleanup.run).toContain("Qualification account ownership marker cleanup target is invalid"); + expect(cleanup.run).not.toContain("ACCOUNT:-nemoclawq"); + expect(cleanup.run).toContain( + "Qualification account ownership marker cleanup target is invalid", + ); expect(cleanup.run).toContain('ownership="$(sudo cat "$ownership_marker")"'); expect(cleanup.run).toContain("pkill -KILL -u"); expect(cleanup.run).not.toContain("rm -rf"); expect(cleanup.run).not.toContain("find "); expect(cleanup.run).toContain('systemd-user-runtime-dir stop "$uid"'); expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); + expect(cleanup.run).toContain('apparmor_parser -R "$pasta_apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); expect(cleanup.run).toContain('sudo rm -f -- "$podman_executable"'); expect(cleanup.run).toContain("Qualification Podman executable remains after cleanup"); - expect(cleanup.run).toContain("Qualification runtime directory remains after its systemd cleanup"); + expect(cleanup.run).toContain("Qualification GPU resource directory remains after cleanup"); + expect(cleanup.run).toContain( + "Qualification runtime directory remains after its systemd cleanup", + ); expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); - expect(cleanup.run).toContain("Qualification subordinate-ID authorization remains after cleanup"); - expect(cleanup.run).toContain("Qualification account output exists without its ownership marker"); + expect(cleanup.run).toContain( + "Qualification subordinate-ID authorization remains after cleanup", + ); + expect(cleanup.run).toContain( + "Qualification account output exists without its ownership marker", + ); expect(cleanup.run).toContain('sudo rm -f -- "$ownership_marker"'); }); From 94c1162ade57571196c3780533bcee3d151c57a7 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 04:17:25 -0500 Subject: [PATCH 29/71] fix(e2e): isolate qualification network and auth Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 50 +++++++++++++++---- ...me-qualification-account-lifecycle.test.ts | 5 +- ...me-qualification-producer-workflow.test.ts | 18 ++++++- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2bde02b695e..85b15ab0e71 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1481,6 +1481,7 @@ jobs: } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" + containers_config="${storage_config_directory}/containers.conf" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" @@ -1545,6 +1546,15 @@ jobs: echo "::error::Qualification storage configuration is not root-owned and read-only" >&2 exit 1 } + printf '%s\n' \ + '[network]' \ + 'firewall_driver = "nftables"' | sudo tee "$containers_config" >/dev/null + sudo chown root:root "$containers_config" + sudo chmod 0444 "$containers_config" + [[ -f "$containers_config" && ! -L "$containers_config" && "$(stat -c '%u:%g:%a' "$containers_config")" == "0:0:444" ]] || { + echo "::error::Qualification containers configuration is not root-owned and read-only" >&2 + exit 1 + } if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then printf '%s\n' \ '# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \ @@ -1599,6 +1609,7 @@ jobs: printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" chmod 0555 "$guard_dir/docker" podman_info="$(sudo -u "$account" env -i \ + CONTAINERS_CONF="$containers_config" \ CONTAINERS_STORAGE_CONF="$storage_config" \ HOME="$home" \ LANG=C.UTF-8 \ @@ -1622,6 +1633,7 @@ jobs: printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT" printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" + printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT" printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" - name: Materialize exact credential-free GPU resources @@ -1629,6 +1641,7 @@ jobs: env: ACCOUNT: ${{ steps.boundary.outputs.account }} ARCHITECTURE: ${{ matrix.case.architecture }} + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INFERENCE: ${{ matrix.case.inference }} @@ -1686,16 +1699,8 @@ jobs: cleanup_registry_auth() { local result="$?" trap - EXIT - sudo -u "$ACCOUNT" env -i \ - CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ - HOME="$QUALIFICATION_HOME" \ - LANG=C.UTF-8 \ - PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ - REGISTRY_AUTH_FILE="$registry_auth_file" \ - XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ - "$PODMAN_EXECUTABLE" logout --all >/dev/null 2>&1 || true if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a' "$registry_auth_file")" == "${uid}:${uid}:600" ]] || { + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 return 1 } @@ -1709,6 +1714,7 @@ jobs: } trap cleanup_registry_auth EXIT printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ @@ -1716,14 +1722,21 @@ jobs: REGISTRY_AUTH_FILE="$registry_auth_file" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { + echo "::error::Run-owned registry authentication file is invalid" >&2 + exit 1 + } image_to_pull="$probe_image" if [[ "$INFERENCE" == "nim" ]]; then image_to_pull="$nim_image" elif [[ "$INFERENCE" == "vllm" ]]; then image_to_pull="$vllm_image" fi - for image in "$probe_image" "$image_to_pull"; do + images=("$probe_image") + [[ "$image_to_pull" == "$probe_image" ]] || images+=("$image_to_pull") + for image in "${images[@]}"; do sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ @@ -1853,20 +1866,26 @@ jobs: ARCHITECTURE: ${{ matrix.case.architecture }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CANDIDATE_SHA: ${{ matrix.source.candidateSha }} + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer INSTALLER_SHA256: ${{ matrix.installerSha256 }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} shell: bash run: | set -euo pipefail install -d -m 0700 "$INSTALLER_RECEIPT_PARENT" sudo chown "$ACCOUNT:$ACCOUNT" "$INSTALLER_RECEIPT_PARENT" sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ bash .trusted-qualification/scripts/checks/run-native-runtime-installer-qualification.sh \ --candidate-checkout "$CANDIDATE_DIRECTORY" \ --candidate-sha "$CANDIDATE_SHA" \ @@ -1887,6 +1906,7 @@ jobs: env: ACCOUNT: ${{ steps.boundary.outputs.account }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} @@ -1909,6 +1929,7 @@ jobs: cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ E2E_DEFAULT_ENABLED=0 \ E2E_JOB=1 \ @@ -2032,7 +2053,7 @@ jobs: exit 1 } if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a' "$registry_auth_file")" == "${uid}:${uid}:600" ]] || { + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 exit 1 } @@ -2047,6 +2068,13 @@ jobs: } sudo unlink "$runner_contract" fi + if [[ -e "$storage_config_directory/containers.conf" || -L "$storage_config_directory/containers.conf" ]]; then + [[ -f "$storage_config_directory/containers.conf" && ! -L "$storage_config_directory/containers.conf" && "$(stat -c '%u:%g:%a' "$storage_config_directory/containers.conf")" == "0:0:444" ]] || { + echo "::error::Qualification containers configuration cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$storage_config_directory/containers.conf" + fi sudo rm -f -- "$storage_config_directory/storage.conf" sudo rmdir "$storage_config_directory" fi diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 3e6f225d29e..ce56276d0c8 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -150,7 +150,7 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, [[ "$2" == '%u:%g:%a:%h' ]] && printf '0:0:444:1\\n' || printf '0:0:444\\n' ;; *native-runtime-resources-*) printf '0:0:555\\n' ;; - *podman.apparmor|*pasta.apparmor|*runner-contract.json|*storage.conf) printf '0:0:444\\n' ;; + *podman.apparmor|*pasta.apparmor|*runner-contract.json|*containers.conf|*storage.conf) printf '0:0:444\\n' ;; *) exit 25 ;; esac`, ); @@ -324,6 +324,7 @@ describe("native runtime qualification account lifecycle", () => { fs.mkdirSync(storage, { recursive: true }); fs.writeFileSync(path.join(runtime, "alive"), "fixture"); fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); + fs.writeFileSync(path.join(storage, "containers.conf"), "fixture"); fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); fs.writeFileSync(path.join(storage, "pasta.apparmor"), "fixture"); fs.writeFileSync(path.join(storage, "runner-contract.json"), "fixture"); @@ -360,7 +361,7 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.existsSync(podman)).toBe(false); expect(fs.existsSync(helpers)).toBe(false); expect(fs.existsSync(resources)).toBe(false); - }); + }, 15_000); it("does not run destructive cleanup when the run-owned marker is absent", () => { const fixture = createFixture(); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 9489975382c..9818e54e54e 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -233,7 +233,9 @@ describe("native runtime qualification producer workflow", () => { expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); expect(gpuResources.run).toContain("existing NVIDIA registry credential"); expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); - expect(gpuResources.run).toContain("logout --all"); + expect(gpuResources.run).not.toContain("logout --all"); + expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"'); + expect(gpuResources.run).toContain("${uid}:${uid}:600:1"); expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); expect(gpuResources.run).toContain("git hash-object --no-filters"); @@ -299,7 +301,12 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain('systemd-user-runtime-dir start "$uid"'); expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); + expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); + expect(boundary.run).toContain('firewall_driver = "nftables"'); + expect(boundary.run).toContain( + "Qualification containers configuration is not root-owned and read-only", + ); expect(boundary.run).toContain("rootless_storage_path"); expect(boundary.run).toContain("${home}/.local/share/containers/storage"); expect(boundary.run).not.toContain('mount_program = "/usr/bin/fuse-overlayfs"'); @@ -351,6 +358,12 @@ describe("native runtime qualification producer workflow", () => { expect(dependencies.run).toContain("npm --prefix"); expect(dependencies.run).toContain("ci --ignore-scripts"); expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(installer.env?.CONTAINERS_CONFIG).toBe( + "${{ steps.boundary.outputs.containers_config }}", + ); + expect(installer.run).toContain('CONTAINERS_CONF="$CONTAINERS_CONFIG"'); + expect(installer.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); + expect(installer.run).toContain('XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY"'); expect(installer.run).toContain("run-native-runtime-installer-qualification.sh"); expect(installer.run).not.toContain("chown -R"); expect(installer.run).toContain('sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts"'); @@ -368,10 +381,12 @@ describe("native runtime qualification producer workflow", () => { expect(execute.run).not.toContain("GH_TOKEN"); expect(execute.run).not.toContain("chown -R"); expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(execute.env?.CONTAINERS_CONFIG).toBe("${{ steps.boundary.outputs.containers_config }}"); expect(execute.env?.PODMAN_EXECUTABLE).toBe("${{ steps.boundary.outputs.podman_executable }}"); expect(execute.env?.STORAGE_CONFIG).toBe("${{ steps.boundary.outputs.storage_config }}"); expect(execute.env?.RUNNER_CONTRACT).toBe("${{ steps.gpu_resources.outputs.runner_contract }}"); expect(execute.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); + expect(execute.run).toContain('CONTAINERS_CONF="$CONTAINERS_CONFIG"'); expect(execute.run).toContain( 'NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE"', ); @@ -402,6 +417,7 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); expect(cleanup.run).toContain('apparmor_parser -R "$pasta_apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); + expect(cleanup.run).toContain('sudo unlink "$storage_config_directory/containers.conf"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); expect(cleanup.run).toContain('sudo rm -f -- "$podman_executable"'); expect(cleanup.run).toContain("Qualification Podman executable remains after cleanup"); From afbeaf577fc3e92027c0253324493f225f561ef8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 04:51:28 -0500 Subject: [PATCH 30/71] fix(e2e): stabilize native qualification identity Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 137 ++++++++++++++---- ...me-qualification-account-lifecycle.test.ts | 98 ++++++++++--- ...me-qualification-producer-workflow.test.ts | 44 +++++- 3 files changed, 223 insertions(+), 56 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 85b15ab0e71..5c70569693d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1363,7 +1363,7 @@ jobs: shell: bash run: | set -euo pipefail - for command in apparmor_parser awk cat curl getent git grep id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do + for command in apparmor_parser awk cat curl getent git grep groupdel id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 @@ -1383,8 +1383,8 @@ jobs: [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] account="nemoclawq" ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - if getent passwd "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then - echo "::error::Qualification account identity or subordinate-ID authorization already exists" >&2 + if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Qualification account, group, or subordinate-ID authorization already exists" >&2 exit 1 fi [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { @@ -1397,8 +1397,9 @@ jobs: trap - EXIT if ((result != 0 && account_created_without_marker == 1)); then sudo userdel --remove "$account" 2>/dev/null || true + sudo groupdel "$account" 2>/dev/null || true sudo rm -f -- "$ownership_marker" - if getent passwd "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then echo "::error::Partially created qualification account could not be rolled back" >&2 exit 1 fi @@ -1406,15 +1407,21 @@ jobs: exit "$result" } trap rollback_unmarked_account EXIT - sudo useradd --create-home --shell /usr/sbin/nologin "$account" + sudo useradd --create-home --shell /usr/sbin/nologin --user-group "$account" account_created_without_marker=1 uid="$(id -u "$account")" + gid="$(id -g "$account")" home="$(getent passwd "$account" | cut -d: -f6)" - [[ "$uid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { + group_entry="$(getent group "$account")" + [[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { echo "::error::Qualification account identity is missing or invalid" >&2 exit 1 } - printf '%s:%s\n' "$account" "$uid" | sudo tee "$ownership_marker" >/dev/null + [[ "$group_entry" == "${account}:x:${gid}:" ]] || { + echo "::error::Qualification private group identity is missing or invalid" >&2 + exit 1 + } + printf '%s:%s:%s\n' "$account" "$uid" "$gid" | sudo tee "$ownership_marker" >/dev/null sudo chown root:root "$ownership_marker" sudo chmod 0400 "$ownership_marker" [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { @@ -1469,16 +1476,28 @@ jobs: ensure_subordinate_range /etc/subgid --add-subgids printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" printf 'account_created=true\n' >>"$GITHUB_OUTPUT" + printf 'uid=%s\n' "$uid" >>"$GITHUB_OUTPUT" + printf 'gid=%s\n' "$gid" >>"$GITHUB_OUTPUT" runtime_dir="/run/user/${uid}" - [[ -x /usr/lib/systemd/systemd-user-runtime-dir ]] || { - echo "::error::Protected runner is missing the signed systemd user-runtime helper" >&2 + runtime_directory_unit="user-runtime-dir@${uid}.service" + user_manager_unit="user@${uid}.service" + sudo systemctl start "$user_manager_unit" + systemctl is-active --quiet "$runtime_directory_unit" || { + echo "::error::Qualification systemd runtime-directory unit is not active" >&2 exit 1 } - sudo /usr/lib/systemd/systemd-user-runtime-dir start "$uid" - [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${uid}:700" ]] || { + [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${gid}:700" ]] || { echo "::error::Qualification runtime directory is missing or invalid" >&2 exit 1 } + systemctl is-active --quiet "$user_manager_unit" || { + echo "::error::Qualification systemd user manager is not active" >&2 + exit 1 + } + [[ -S "$runtime_dir/bus" && ! -L "$runtime_dir/bus" && "$(stat -c '%u:%g' "$runtime_dir/bus")" == "${uid}:${gid}" ]] || { + echo "::error::Qualification systemd user bus is missing or invalid" >&2 + exit 1 + } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" containers_config="${storage_config_directory}/containers.conf" @@ -1598,7 +1617,7 @@ jobs: sudo setfacl --modify "u:${account}:--x" "$ancestor" ancestor="$(dirname "$ancestor")" done - sudo chown -R "$uid:$uid" "$CANDIDATE_DIRECTORY" + sudo chown -R "$uid:$gid" "$CANDIDATE_DIRECTORY" node_directory="$(dirname "$(command -v node)")" [[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || { echo "::error::Pinned Node toolchain path is invalid" >&2 @@ -1634,12 +1653,16 @@ jobs: printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT" + printf 'runtime_directory_unit=%s\n' "$runtime_directory_unit" >>"$GITHUB_OUTPUT" printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" + printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT" - name: Materialize exact credential-free GPU resources id: gpu_resources env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} ARCHITECTURE: ${{ matrix.case.architecture }} CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} @@ -1663,6 +1686,11 @@ jobs: exit 1 } uid="$(id -u "$ACCOUNT")" + gid="$(id -g "$ACCOUNT")" + [[ "$uid" == "$ACCOUNT_UID" && "$gid" == "$ACCOUNT_GID" ]] || { + echo "::error::Qualification account identity changed before GPU resource preparation" >&2 + exit 1 + } storage_config_directory="$(dirname "$STORAGE_CONFIG")" runner_contract="${storage_config_directory}/runner-contract.json" registry_auth_directory="${storage_config_directory}/registry-auth" @@ -1695,12 +1723,12 @@ jobs: echo "::error::Run-owned registry authentication directory already exists" >&2 exit 1 } - sudo install -d --owner="$uid" --group="$uid" --mode=0700 "$registry_auth_directory" + sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$registry_auth_directory" cleanup_registry_auth() { local result="$?" trap - EXIT if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 return 1 } @@ -1722,7 +1750,7 @@ jobs: REGISTRY_AUTH_FILE="$registry_auth_file" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Run-owned registry authentication file is invalid" >&2 exit 1 } @@ -1755,7 +1783,7 @@ jobs: exit 1 } sudo install -d --owner=root --group=root --mode=0711 "$resource_directory" - sudo install -d --owner="$uid" --group="$uid" --mode=0700 "$model_directory" + sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$model_directory" download_model_file() { local file="$1" local size="$2" @@ -1769,7 +1797,7 @@ jobs: --fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \ --output "$target" \ "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" - [[ -f "$target" && ! -L "$target" && "$(stat -c '%u:%g:%a:%h:%s' "$target")" == "${uid}:${uid}:600:1:${size}" ]] || { + [[ -f "$target" && ! -L "$target" && "$(stat -c '%u:%g:%a:%h:%s' "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 exit 1 } @@ -1863,6 +1891,8 @@ jobs: - name: Run the authenticated installer qualification env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} ARCHITECTURE: ${{ matrix.case.architecture }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CANDIDATE_SHA: ${{ matrix.source.candidateSha }} @@ -1873,12 +1903,14 @@ jobs: INSTALLER_SHA256: ${{ matrix.installerSha256 }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail install -d -m 0700 "$INSTALLER_RECEIPT_PARENT" - sudo chown "$ACCOUNT:$ACCOUNT" "$INSTALLER_RECEIPT_PARENT" + sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$INSTALLER_RECEIPT_PARENT" sudo -u "$ACCOUNT" env -i \ CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ @@ -1892,7 +1924,21 @@ jobs: --installer-sha256 "$INSTALLER_SHA256" \ --architecture "$ARCHITECTURE" \ --artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts" - sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true + sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" + ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle remained active after installer isolation" >&2 + exit 1 + } + sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true + sudo systemctl start "$USER_MANAGER_UNIT" + systemctl is-active --quiet "$USER_MANAGER_UNIT" && systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2 + exit 1 + } + [[ -S "$RUNTIME_DIRECTORY/bus" && ! -L "$RUNTIME_DIRECTORY/bus" && "$(stat -c '%u:%g' "$RUNTIME_DIRECTORY/bus")" == "${ACCOUNT_UID}:${ACCOUNT_GID}" ]] || { + echo "::error::Qualification systemd user bus did not restart after installer isolation" >&2 + exit 1 + } sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { echo "::error::Installer receipt directory is missing or invalid" >&2 exit 1 @@ -1905,6 +1951,8 @@ jobs: - name: Execute the candidate qualification case without credentials env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} @@ -1914,7 +1962,9 @@ jobs: QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail @@ -1925,7 +1975,7 @@ jobs: } receipt_directory="${RUNNER_TEMP}/native-runtime-case" install -d -m 0700 "$receipt_directory" - sudo chown "$ACCOUNT:$ACCOUNT" "$receipt_directory" + sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$receipt_directory" cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ @@ -1946,7 +1996,12 @@ jobs: --config "$CANDIDATE_DIRECTORY/vitest.config.ts" \ --project e2e-live \ "$live_test" - sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true + sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" + ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle remained active after candidate execution" >&2 + exit 1 + } + sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true - name: Verify Docker stayed unavailable shell: bash @@ -1984,18 +2039,20 @@ jobs: ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" account="" uid="" + gid="" if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { echo "::error::Qualification account ownership marker cleanup target is invalid" >&2 exit 1 } ownership="$(sudo cat "$ownership_marker")" - [[ "$ownership" =~ ^nemoclawq:([0-9]+)$ ]] || { + [[ "$ownership" =~ ^nemoclawq:([0-9]+):([0-9]+)$ ]] || { echo "::error::Qualification account ownership marker content is invalid" >&2 exit 1 } account="nemoclawq" uid="${BASH_REMATCH[1]}" + gid="${BASH_REMATCH[2]}" [[ -z "$reported_account" || "$reported_account" == "$account" ]] || { echo "::error::Qualification account output does not match its ownership marker" >&2 exit 1 @@ -2005,12 +2062,23 @@ jobs: exit 1 fi if getent passwd "$account" >/dev/null; then - [[ "$(id -u "$account")" == "$uid" ]] || { - echo "::error::Qualification account UID changed before cleanup" >&2 + [[ "$(id -u "$account")" == "$uid" && "$(id -g "$account")" == "$gid" ]] || { + echo "::error::Qualification account identity changed before cleanup" >&2 + exit 1 + } + fi + if getent group "$account" >/dev/null; then + [[ "$(getent group "$account")" == "${account}:x:${gid}:" ]] || { + echo "::error::Qualification private group identity changed before cleanup" >&2 exit 1 } + elif getent passwd "$account" >/dev/null; then + echo "::error::Qualification private group disappeared before cleanup" >&2 + exit 1 fi runtime_dir="/run/user/${uid}" + runtime_directory_unit="user-runtime-dir@${uid}.service" + user_manager_unit="user@${uid}.service" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" @@ -2022,10 +2090,14 @@ jobs: runner_contract="${storage_config_directory}/runner-contract.json" resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" model_directory="${resource_directory}/model" + sudo systemctl stop "$user_manager_unit" "$runtime_directory_unit" 2>/dev/null || true + ! systemctl is-active --quiet "$user_manager_unit" && ! systemctl is-active --quiet "$runtime_directory_unit" || { + echo "::error::Qualification systemd user lifecycle remained active during cleanup" >&2 + exit 1 + } if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true fi - sudo /usr/lib/systemd/systemd-user-runtime-dir stop "$uid" if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration cleanup target is invalid" >&2 @@ -2048,12 +2120,12 @@ jobs: sudo rm -f -- "$pasta_apparmor_profile" fi if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then - [[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${uid}:700" ]] || { + [[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${gid}:700" ]] || { echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2 exit 1 } if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${uid}:600:1" ]] || { + [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 exit 1 } @@ -2085,7 +2157,7 @@ jobs: } if [[ -e "$model_directory" || -L "$model_directory" ]]; then model_mode="$(stat -c '%u:%g:%a' "$model_directory")" - [[ -d "$model_directory" && ! -L "$model_directory" && ("$model_mode" == "${uid}:${uid}:700" || "$model_mode" == "0:0:555") ]] || { + [[ -d "$model_directory" && ! -L "$model_directory" && ("$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555") ]] || { echo "::error::Qualification GPU model directory cleanup target is invalid" >&2 exit 1 } @@ -2093,7 +2165,7 @@ jobs: target="${model_directory}/${file}" if [[ -e "$target" || -L "$target" ]]; then file_mode="$(stat -c '%u:%g:%a:%h' "$target")" - [[ -f "$target" && ! -L "$target" && ("$file_mode" == "${uid}:${uid}:600:1" || "$file_mode" == "0:0:444:1") ]] || { + [[ -f "$target" && ! -L "$target" && ("$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1") ]] || { echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2 exit 1 } @@ -2136,10 +2208,17 @@ jobs: if getent passwd "$account" >/dev/null; then sudo userdel --remove "$account" fi + if getent group "$account" >/dev/null; then + sudo groupdel "$account" + fi if getent passwd "$account" >/dev/null; then echo "::error::Qualification account still exists after cleanup" >&2 exit 1 fi + if getent group "$account" >/dev/null; then + echo "::error::Qualification private group still exists after cleanup" >&2 + exit 1 + fi if grep -q "^${account}:" /etc/subuid /etc/subgid; then echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2 exit 1 diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index ce56276d0c8..7b1388fde5b 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -61,10 +61,6 @@ function fixtureSource(source: string): string { 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', ) - .replaceAll( - "/usr/lib/systemd/systemd-user-runtime-dir", - "${FIXTURE_ROOT}/bin/systemd-user-runtime-dir", - ) .replaceAll('"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'); } @@ -77,6 +73,7 @@ function createFixture(): { bin: string; calls: string; passwd: string; + group: string; subuid: string; subgid: string; home: string; @@ -90,21 +87,31 @@ function createFixture(): { const home = path.join(root, "home", "nemoclawq"); const calls = path.join(root, "calls.log"); const passwd = path.join(etc, "passwd"); + const group = path.join(etc, "group"); const subuid = path.join(etc, "subuid"); const subgid = path.join(etc, "subgid"); fs.mkdirSync(bin, { recursive: true }); fs.mkdirSync(etc, { recursive: true }); fs.mkdirSync(run, { recursive: true }); - for (const file of [calls, passwd, subuid, subgid]) fs.writeFileSync(file, ""); + for (const file of [calls, passwd, group, subuid, subgid]) fs.writeFileSync(file, ""); writeExecutable(path.join(bin, "sudo"), `printf 'sudo:%s\\n' "$*" >>"$FIXTURE_CALLS"\nexec "$@"`); writeExecutable( path.join(bin, "getent"), - `[[ "$1" == passwd ]]\nawk -F: -v key="$2" '$1 == key || $3 == key { print; found = 1 } END { exit found ? 0 : 2 }' "$FIXTURE_ROOT/etc/passwd"`, + `case "$1" in + passwd) file="$FIXTURE_ROOT/etc/passwd" ;; + group) file="$FIXTURE_ROOT/etc/group" ;; + *) exit 2 ;; +esac +awk -F: -v key="$2" '$1 == key || $3 == key { print; found = 1 } END { exit found ? 0 : 2 }' "$file"`, ); writeExecutable( path.join(bin, "id"), - `[[ "$1" == -u ]]\n[[ "\${FAIL_ID:-0}" != 1 ]] || exit 26\nawk -F: -v account="$2" '$1 == account { print $3; found = 1 } END { exit found ? 0 : 1 }' "$FIXTURE_ROOT/etc/passwd"`, + `[[ "$1" == -u || "$1" == -g ]] +[[ "\${FAIL_ID:-0}" != 1 ]] || exit 26 +field=3 +[[ "$1" == -u ]] || field=4 +awk -F: -v account="$2" -v field="$field" '$1 == account { print $field; found = 1 } END { exit found ? 0 : 1 }' "$FIXTURE_ROOT/etc/passwd"`, ); writeExecutable( path.join(bin, "useradd"), @@ -112,7 +119,8 @@ function createFixture(): { [[ "\${FAIL_USERADD:-0}" != 1 ]] || exit 23 account="\${!#}" mkdir -p "$FIXTURE_HOME" -printf '%s:x:1002:1002::%s:/usr/sbin/nologin\\n' "$account" "$FIXTURE_HOME" >>"$FIXTURE_ROOT/etc/passwd"`, +printf '%s:x:1002:1007::%s:/usr/sbin/nologin\\n' "$account" "$FIXTURE_HOME" >>"$FIXTURE_ROOT/etc/passwd" +printf '%s:x:1007:\\n' "$account" >>"$FIXTURE_ROOT/etc/group"`, ); writeExecutable( path.join(bin, "usermod"), @@ -154,12 +162,21 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, *) exit 25 ;; esac`, ); + writeExecutable(path.join(bin, "pkill"), `printf 'pkill:%s\\n' "$*" >>"$FIXTURE_CALLS"`); writeExecutable( - path.join(bin, "systemd-user-runtime-dir"), - `printf 'systemd-user-runtime-dir:%s\\n' "$*" >>"$FIXTURE_CALLS" -[[ "$1" != stop ]] || /bin/rm -rf -- "$FIXTURE_ROOT/run/user/$2"`, + path.join(bin, "systemctl"), + `printf 'systemctl:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == stop ]]; then + shift + for unit in "$@"; do + if [[ "$unit" =~ ^user-runtime-dir@([0-9]+)\\.service$ ]]; then + /bin/rm -rf -- "$FIXTURE_ROOT/run/user/\${BASH_REMATCH[1]}" + fi + done +elif [[ "$1" == is-active ]]; then + exit 3 +fi`, ); - writeExecutable(path.join(bin, "pkill"), `printf 'pkill:%s\\n' "$*" >>"$FIXTURE_CALLS"`); writeExecutable( path.join(bin, "apparmor_parser"), `printf 'apparmor:%s\\n' "$*" >>"$FIXTURE_CALLS"`, @@ -174,12 +191,19 @@ for file in "$FIXTURE_ROOT/etc/passwd" "$FIXTURE_ROOT/etc/subuid" "$FIXTURE_ROOT done rmdir "$FIXTURE_HOME" 2>/dev/null || true`, ); + writeExecutable( + path.join(bin, "groupdel"), + `printf 'groupdel:%s\\n' "$*" >>"$FIXTURE_CALLS" +awk -F: -v account="$1" '$1 != account' "$FIXTURE_ROOT/etc/group" >"$FIXTURE_ROOT/etc/group.next" +mv "$FIXTURE_ROOT/etc/group.next" "$FIXTURE_ROOT/etc/group"`, + ); return { root, bin, calls, passwd, + group, subuid, subgid, home, @@ -221,14 +245,16 @@ function provisionBlock(): string { describe("native runtime qualification account lifecycle", () => { it("rejects pre-existing accounts and stale subordinate-ID authorization before mutation", () => { - for (const state of ["passwd", "subuid", "subgid"] as const) { + for (const state of ["passwd", "group", "subuid", "subgid"] as const) { const fixture = createFixture(); const file = fixture[state]; fs.appendFileSync( file, state === "passwd" - ? `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n` - : "nemoclawq:200000:65536\n", + ? `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n` + : state === "group" + ? "nemoclawq:x:1007:\n" + : "nemoclawq:200000:65536\n", ); const result = runFixture(fixture, provisionBlock()); expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); @@ -245,12 +271,23 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.readFileSync(fixture.passwd, "utf8")).toBe(""); }); + it("records the exact private group when its numeric GID differs from the UID", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock()); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(fixture.marker, "utf8")).toBe("nemoclawq:1002:1007\n"); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain( + "useradd:--create-home --shell /usr/sbin/nologin --user-group nemoclawq", + ); + }); + it("rolls back an account when identity validation fails before marker publication", () => { const fixture = createFixture(); const result = runFixture(fixture, provisionBlock(), { FAIL_ID: "1" }); expect(result.status).not.toBe(0); expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.group, "utf8")).not.toContain("nemoclawq:"); expect(fs.existsSync(fixture.marker)).toBe(false); }); @@ -292,15 +329,17 @@ describe("native runtime qualification account lifecycle", () => { it("removes partial account setup and verifies subordinate-ID revocation", () => { const fixture = createFixture(); fs.mkdirSync(fixture.home, { recursive: true }); - fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1007:\n"); fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); - fs.writeFileSync(fixture.marker, "nemoclawq:1002\n", { mode: 0o400 }); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); const result = runFixture(fixture, workflowScripts().cleanup); expect(result.status, result.stderr).toBe(0); expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.group, "utf8")).not.toContain("nemoclawq:"); expect(fs.readFileSync(fixture.subuid, "utf8")).not.toContain("nemoclawq:"); expect(fs.readFileSync(fixture.subgid, "utf8")).not.toContain("nemoclawq:"); expect(fs.existsSync(fixture.marker)).toBe(false); @@ -346,15 +385,17 @@ describe("native runtime qualification account lifecycle", () => { } fs.chmodSync(model, 0o555); fs.chmodSync(resources, 0o555); - fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1002::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1007:\n"); fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); - fs.writeFileSync(fixture.marker, "nemoclawq:1002\n", { mode: 0o400 }); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); const result = runFixture(fixture, workflowScripts().cleanup); expect(result.status, result.stderr).toBe(0); const calls = fs.readFileSync(fixture.calls, "utf8"); - expect(calls).toContain("systemd-user-runtime-dir:stop 1002"); + expect(calls).toContain("systemctl:stop user@1002.service user-runtime-dir@1002.service"); + expect(calls).toContain("groupdel:nemoclawq"); expect(calls).toContain("apparmor:-R"); expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); expect(fs.existsSync(storage)).toBe(false); @@ -369,6 +410,21 @@ describe("native runtime qualification account lifecycle", () => { expect(result.status).not.toBe(0); expect(result.stderr).toContain("output exists without its ownership marker"); const calls = fs.readFileSync(fixture.calls, "utf8"); - expect(calls).not.toMatch(/pkill:|systemd-user-runtime-dir:|userdel:|apparmor:/u); + expect(calls).not.toMatch(/pkill:|systemctl:|userdel:|groupdel:|apparmor:/u); + }); + + it("fails closed before destructive cleanup when the private group identity changes", () => { + const fixture = createFixture(); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1008:\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("private group identity changed before cleanup"); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toMatch( + /pkill:|systemctl:|userdel:|groupdel:|apparmor:/u, + ); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 9818e54e54e..3807cfc4252 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -235,7 +235,9 @@ describe("native runtime qualification producer workflow", () => { expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); expect(gpuResources.run).not.toContain("logout --all"); expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"'); - expect(gpuResources.run).toContain("${uid}:${uid}:600:1"); + expect(gpuResources.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(gpuResources.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(gpuResources.run).toContain("${uid}:${gid}:600:1"); expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); expect(gpuResources.run).toContain("git hash-object --no-filters"); @@ -281,11 +283,12 @@ describe("native runtime qualification producer workflow", () => { expect(podman.run).toContain('[[ "$version" == "podman version 6.1.0" ]]'); expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); - expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); + expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin --user-group"); expect(boundary.run).toContain('getent passwd "$account"'); + expect(boundary.run).toContain('getent group "$account"'); expect(boundary.run).toContain('grep -q "^${account}:" /etc/subuid /etc/subgid'); expect(boundary.run).toContain( - "Qualification account identity or subordinate-ID authorization already exists", + "Qualification account, group, or subordinate-ID authorization already exists", ); expect(boundary.run).toContain('ownership_marker="/run/nemoclaw-native-runtime-owner-'); expect(boundary.run).toContain("0:0:400"); @@ -297,9 +300,12 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); - expect(boundary.run).toContain("/usr/lib/systemd/systemd-user-runtime-dir"); - expect(boundary.run).toContain('systemd-user-runtime-dir start "$uid"'); + expect(boundary.run).toContain('runtime_directory_unit="user-runtime-dir@${uid}.service"'); expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); + expect(boundary.run).toContain('user_manager_unit="user@${uid}.service"'); + expect(boundary.run).toContain('systemctl start "$user_manager_unit"'); + expect(boundary.run).toContain("stat -c '%u:%g' \"$runtime_dir/bus\""); + expect(boundary.run).toContain("Qualification systemd user bus is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); @@ -358,6 +364,12 @@ describe("native runtime qualification producer workflow", () => { expect(dependencies.run).toContain("npm --prefix"); expect(dependencies.run).toContain("ci --ignore-scripts"); expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(installer.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(installer.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(installer.env?.RUNTIME_DIRECTORY_UNIT).toBe( + "${{ steps.boundary.outputs.runtime_directory_unit }}", + ); + expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", ); @@ -369,6 +381,12 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(installer.run).toContain('sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(execute.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(execute.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(execute.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(execute.env?.RUNTIME_DIRECTORY_UNIT).toBe( + "${{ steps.boundary.outputs.runtime_directory_unit }}", + ); + expect(execute.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); expect(execute.run).toContain( 'live_test="test/e2e/live/native-runtime-qualification-case.test.ts"', ); @@ -413,7 +431,10 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain("pkill -KILL -u"); expect(cleanup.run).not.toContain("rm -rf"); expect(cleanup.run).not.toContain("find "); - expect(cleanup.run).toContain('systemd-user-runtime-dir stop "$uid"'); + expect(cleanup.run).toContain('systemctl stop "$user_manager_unit" "$runtime_directory_unit"'); + expect(cleanup.run).toContain( + "Qualification systemd user lifecycle remained active during cleanup", + ); expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); expect(cleanup.run).toContain('apparmor_parser -R "$pasta_apparmor_profile"'); expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); @@ -428,6 +449,7 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); + expect(cleanup.run).toContain("Qualification private group still exists after cleanup"); expect(cleanup.run).toContain( "Qualification subordinate-ID authorization remains after cleanup", ); @@ -435,6 +457,16 @@ describe("native runtime qualification producer workflow", () => { "Qualification account output exists without its ownership marker", ); expect(cleanup.run).toContain('sudo rm -f -- "$ownership_marker"'); + const accountOwnershipSource = [ + boundary.run, + gpuResources.run, + installer.run, + execute.run, + cleanup.run, + ].join("\n"); + expect(accountOwnershipSource).not.toContain("${uid}:${uid}"); + expect(accountOwnershipSource).not.toContain("$uid:$uid"); + expect(accountOwnershipSource).not.toContain("$ACCOUNT:$ACCOUNT"); }); it("aggregates the exact successful 24-case cohort in a separate trusted job", () => { From 94efdf032db9c5ca8a306abf8f46293402f6a000 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 05:01:08 -0500 Subject: [PATCH 31/71] fix(e2e): prove failed network cleanup Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 27 +++++++++- ...untime-qualification-case-executor.test.ts | 53 ++++++++++++++++--- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 0d9b0a53b47..170169aeb7e 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -318,7 +318,32 @@ function createProviderNetwork( return Object.freeze({ id, name, gateway }); } catch (error) { if (created) { - engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); + let removalOutcome = "not attempted"; + try { + const removal = engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); + removalOutcome = `exit ${String(removal.status)}`; + } catch (cleanupError) { + removalOutcome = `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`; + } + let existenceOutcome = "not attempted"; + let removalProven = false; + try { + const existence = engine.capture(["network", "exists", name], COMMAND_TIMEOUT); + existenceOutcome = `exit ${String(existence.status)}`; + removalProven = existence.status === 1; + } catch (cleanupError) { + existenceOutcome = `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`; + } + if (!removalProven) { + const validationFailure = bounded(error instanceof Error ? error.message : String(error)); + throw new Error( + `${validationFailure}; provider network cleanup could not prove removal (remove ${removalOutcome}; exists ${existenceOutcome})`, + ); + } } throw error; } diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index c029bd44ecb..415c8d48e5b 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -23,16 +23,26 @@ function inspection(overrides: Record = {}): string { ]); } -function engine(outputs: readonly string[]): { +type EngineOutput = + | string + | { readonly status: number; readonly stdout?: string; readonly stderr?: string }; + +function engine(outputs: readonly EngineOutput[]): { readonly capture: ReturnType; readonly value: PodmanBoundContainerEngine; } { let index = 0; - const capture = vi.fn(() => ({ - status: 0, - stdout: outputs[index++] ?? "", - stderr: "", - })); + const capture = vi.fn((args: readonly string[]) => { + const output = outputs[index++]; + if (typeof output === "object") { + return { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" }; + } + return { + status: output === undefined && args[0] === "network" && args[1] === "exists" ? 1 : 0, + stdout: output ?? "", + stderr: "", + }; + }); return { capture, value: { @@ -91,6 +101,7 @@ describe("native runtime provider-network authority", () => { expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], ["network", "rm", "--force", NETWORK_NAME], + ["network", "exists", NETWORK_NAME], ]); }); @@ -116,9 +127,37 @@ describe("native runtime provider-network authority", () => { CASE_ID, ), ).toThrow("Provider network identity changed after immutable-ID resolution"); - expect(changedGateway.capture).toHaveBeenLastCalledWith( + expect(changedGateway.capture).toHaveBeenNthCalledWith( + 4, ["network", "rm", "--force", NETWORK_NAME], 60_000, ); + expect(changedGateway.capture).toHaveBeenLastCalledWith( + ["network", "exists", NETWORK_NAME], + 60_000, + ); + }); + + it("reports validation and cleanup together when network removal cannot be proven", () => { + const runtime = engine([ + "unexpected-network", + { status: 1, stderr: "remove failed" }, + { status: 0 }, + ]); + + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow( + "Provider network creation returned an unexpected identity; provider network cleanup could not prove removal (remove exit 1; exists exit 0)", + ); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "rm", "--force", NETWORK_NAME], + ["network", "exists", NETWORK_NAME], + ]); }); }); From 5ef5ee72953a1940d8d3d41fb9c48e60f84c5f03 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 05:05:06 -0500 Subject: [PATCH 32/71] test(e2e): keep network fixture branchless Signed-off-by: Aaron Erickson --- ...ve-runtime-qualification-case-executor.test.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 415c8d48e5b..c490ffb9355 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -34,14 +34,13 @@ function engine(outputs: readonly EngineOutput[]): { let index = 0; const capture = vi.fn((args: readonly string[]) => { const output = outputs[index++]; - if (typeof output === "object") { - return { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" }; - } - return { - status: output === undefined && args[0] === "network" && args[1] === "exists" ? 1 : 0, - stdout: output ?? "", - stderr: "", - }; + return typeof output === "object" + ? { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" } + : { + status: output === undefined && args[0] === "network" && args[1] === "exists" ? 1 : 0, + stdout: output ?? "", + stderr: "", + }; }); return { capture, From 6c5103815f88f11be98579bc5dfe2e5bdfc51205 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 05:18:26 -0500 Subject: [PATCH 33/71] fix(e2e): activate qualification user bus Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 30 +++++++++++++++++++ ...me-qualification-producer-workflow.test.ts | 5 ++++ 2 files changed, 35 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5c70569693d..6881052c41e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1494,6 +1494,21 @@ jobs: echo "::error::Qualification systemd user manager is not active" >&2 exit 1 } + sudo -u "$account" env -i \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$runtime_dir" \ + /usr/bin/systemctl --user start dbus.socket + sudo -u "$account" env -i \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$runtime_dir" \ + /usr/bin/systemctl --user is-active --quiet dbus.socket || { + echo "::error::Qualification systemd user bus socket unit is not active" >&2 + exit 1 + } [[ -S "$runtime_dir/bus" && ! -L "$runtime_dir/bus" && "$(stat -c '%u:%g' "$runtime_dir/bus")" == "${uid}:${gid}" ]] || { echo "::error::Qualification systemd user bus is missing or invalid" >&2 exit 1 @@ -1935,6 +1950,21 @@ jobs: echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2 exit 1 } + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user start dbus.socket + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user is-active --quiet dbus.socket || { + echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 + exit 1 + } [[ -S "$RUNTIME_DIRECTORY/bus" && ! -L "$RUNTIME_DIRECTORY/bus" && "$(stat -c '%u:%g' "$RUNTIME_DIRECTORY/bus")" == "${ACCOUNT_UID}:${ACCOUNT_GID}" ]] || { echo "::error::Qualification systemd user bus did not restart after installer isolation" >&2 exit 1 diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 3807cfc4252..fd1baedef66 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -304,6 +304,9 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); expect(boundary.run).toContain('user_manager_unit="user@${uid}.service"'); expect(boundary.run).toContain('systemctl start "$user_manager_unit"'); + expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); + expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); + expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); expect(boundary.run).toContain("stat -c '%u:%g' \"$runtime_dir/bus\""); expect(boundary.run).toContain("Qualification systemd user bus is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); @@ -370,6 +373,8 @@ describe("native runtime qualification producer workflow", () => { "${{ steps.boundary.outputs.runtime_directory_unit }}", ); expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); + expect(installer.run).toContain("/usr/bin/systemctl --user start dbus.socket"); + expect(installer.run).toContain("Qualification systemd user bus socket unit did not restart"); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", ); From db20d5f563e61b10a1e1c522da4f1ff7910c19b1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 05:39:17 -0500 Subject: [PATCH 34/71] fix(e2e): accept systemd bus group ownership Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 4 ++-- .../native-runtime-qualification-producer-workflow.test.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6881052c41e..144ed87eca3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1509,7 +1509,7 @@ jobs: echo "::error::Qualification systemd user bus socket unit is not active" >&2 exit 1 } - [[ -S "$runtime_dir/bus" && ! -L "$runtime_dir/bus" && "$(stat -c '%u:%g' "$runtime_dir/bus")" == "${uid}:${gid}" ]] || { + [[ -S "$runtime_dir/bus" && ! -L "$runtime_dir/bus" && "$(stat -c '%u' "$runtime_dir/bus")" == "$uid" ]] || { echo "::error::Qualification systemd user bus is missing or invalid" >&2 exit 1 } @@ -1965,7 +1965,7 @@ jobs: echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 exit 1 } - [[ -S "$RUNTIME_DIRECTORY/bus" && ! -L "$RUNTIME_DIRECTORY/bus" && "$(stat -c '%u:%g' "$RUNTIME_DIRECTORY/bus")" == "${ACCOUNT_UID}:${ACCOUNT_GID}" ]] || { + [[ -S "$RUNTIME_DIRECTORY/bus" && ! -L "$RUNTIME_DIRECTORY/bus" && "$(stat -c '%u' "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] || { echo "::error::Qualification systemd user bus did not restart after installer isolation" >&2 exit 1 } diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index fd1baedef66..7fdda2a608a 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -307,7 +307,8 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); - expect(boundary.run).toContain("stat -c '%u:%g' \"$runtime_dir/bus\""); + expect(boundary.run).toContain("stat -c '%u' \"$runtime_dir/bus\""); + expect(boundary.run).not.toContain("stat -c '%u:%g' \"$runtime_dir/bus\""); expect(boundary.run).toContain("Qualification systemd user bus is missing or invalid"); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); @@ -375,6 +376,8 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); expect(installer.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(installer.run).toContain("Qualification systemd user bus socket unit did not restart"); + expect(installer.run).toContain("stat -c '%u' \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).not.toContain("stat -c '%u:%g' \"$RUNTIME_DIRECTORY/bus\""); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", ); From a20254ea06b73968731b9f8d9c1b39a22de46c35 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 05:51:34 -0500 Subject: [PATCH 35/71] fix(e2e): verify user bus access Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 8 ++++---- ...time-qualification-producer-workflow.test.ts | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 144ed87eca3..a2d7b790276 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1509,8 +1509,8 @@ jobs: echo "::error::Qualification systemd user bus socket unit is not active" >&2 exit 1 } - [[ -S "$runtime_dir/bus" && ! -L "$runtime_dir/bus" && "$(stat -c '%u' "$runtime_dir/bus")" == "$uid" ]] || { - echo "::error::Qualification systemd user bus is missing or invalid" >&2 + sudo -u "$account" /usr/bin/test -S "$runtime_dir/bus" || { + echo "::error::Qualification systemd user bus is not accessible to the execution account" >&2 exit 1 } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" @@ -1965,8 +1965,8 @@ jobs: echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 exit 1 } - [[ -S "$RUNTIME_DIRECTORY/bus" && ! -L "$RUNTIME_DIRECTORY/bus" && "$(stat -c '%u' "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] || { - echo "::error::Qualification systemd user bus did not restart after installer isolation" >&2 + sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || { + echo "::error::Qualification systemd user bus is not accessible after installer isolation" >&2 exit 1 } sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 7fdda2a608a..6d442158bda 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -307,9 +307,11 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); - expect(boundary.run).toContain("stat -c '%u' \"$runtime_dir/bus\""); - expect(boundary.run).not.toContain("stat -c '%u:%g' \"$runtime_dir/bus\""); - expect(boundary.run).toContain("Qualification systemd user bus is missing or invalid"); + expect(boundary.run).toContain('sudo -u "$account" /usr/bin/test -S "$runtime_dir/bus"'); + expect(boundary.run).not.toContain("stat -c '%u' \"$runtime_dir/bus\""); + expect(boundary.run).toContain( + "Qualification systemd user bus is not accessible to the execution account", + ); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); @@ -376,8 +378,13 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); expect(installer.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(installer.run).toContain("Qualification systemd user bus socket unit did not restart"); - expect(installer.run).toContain("stat -c '%u' \"$RUNTIME_DIRECTORY/bus\""); - expect(installer.run).not.toContain("stat -c '%u:%g' \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).toContain( + 'sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus"', + ); + expect(installer.run).not.toContain("stat -c '%u' \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).toContain( + "Qualification systemd user bus is not accessible after installer isolation", + ); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", ); From e0d9f2f0810905fe0036d6d4994332df0d98a84b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 06:10:39 -0500 Subject: [PATCH 36/71] fix(e2e): secure registry auth inspection Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 21 ++++++++++++++----- ...me-qualification-producer-workflow.test.ts | 11 ++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a2d7b790276..daef7b96c92 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1742,8 +1742,10 @@ jobs: cleanup_registry_auth() { local result="$?" trap - EXIT - if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 return 1 } @@ -1765,10 +1767,17 @@ jobs: REGISTRY_AUTH_FILE="$registry_auth_file" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%h' -- "$registry_auth_file")" == "${uid}:${gid}:1" ]] || { echo "::error::Run-owned registry authentication file is invalid" >&2 exit 1 } + sudo chmod 0600 -- "$registry_auth_file" + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + echo "::error::Run-owned registry authentication file permissions are invalid" >&2 + exit 1 + } image_to_pull="$probe_image" if [[ "$INFERENCE" == "nim" ]]; then image_to_pull="$nim_image" @@ -2154,8 +2163,10 @@ jobs: echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2 exit 1 } - if [[ -e "$registry_auth_file" || -L "$registry_auth_file" ]]; then - [[ -f "$registry_auth_file" && ! -L "$registry_auth_file" && "$(stat -c '%u:%g:%a:%h' "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 exit 1 } diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 6d442158bda..28e283ed192 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -211,6 +211,7 @@ describe("native runtime qualification producer workflow", () => { }); const boundaryRun = boundary.run ?? ""; const executeRun = execute.run ?? ""; + const gpuResourcesRun = gpuResources.run ?? ""; expect(producer.name).toBe("${{ matrix.jobName }}"); expect(producer.needs).toEqual([ @@ -235,9 +236,19 @@ describe("native runtime qualification producer workflow", () => { expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); expect(gpuResources.run).not.toContain("logout --all"); expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"'); + expect(gpuResources.run).toContain( + "$(sudo stat -c '%u:%g:%h' -- \"$registry_auth_file\")", + ); + expect(gpuResources.run).toContain('sudo chmod 0600 -- "$registry_auth_file"'); expect(gpuResources.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); expect(gpuResources.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); expect(gpuResources.run).toContain("${uid}:${gid}:600:1"); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$registry_auth_file"')).toBeGreaterThan( + gpuResourcesRun.indexOf("login nvcr.io --username '$oauthtoken' --password-stdin"), + ); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$registry_auth_file"')).toBeLessThan( + gpuResourcesRun.indexOf('"$PODMAN_EXECUTABLE" pull "$image"'), + ); expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); expect(gpuResources.run).toContain("git hash-object --no-filters"); From e91044e505c3e130305522ddc490a63f728e3245 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 06:28:33 -0500 Subject: [PATCH 37/71] fix(e2e): harden qualification isolation Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 89 +++++++++++++++-- ...ive-runtime-qualification-case-executor.ts | 82 +++++++++++----- ...me-qualification-account-lifecycle.test.ts | 97 ++++++++++++++++++- ...untime-qualification-case-executor.test.ts | 23 +++++ ...me-qualification-producer-workflow.test.ts | 28 +++++- 5 files changed, 281 insertions(+), 38 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index daef7b96c92..f0a5ae8c107 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1481,6 +1481,49 @@ jobs: runtime_dir="/run/user/${uid}" runtime_directory_unit="user-runtime-dir@${uid}.service" user_manager_unit="user@${uid}.service" + user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" + user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" + trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" + [[ ! -e "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" ]] || { + echo "::error::Qualification systemd user-manager drop-in directory already exists" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0755 "$user_manager_dropin_directory" + printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"\n' "$trusted_user_unit_path" | + sudo tee "$user_manager_dropin" >/dev/null + sudo chown root:root "$user_manager_dropin" + sudo chmod 0444 "$user_manager_dropin" + [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { + echo "::error::Qualification systemd user-manager drop-in is invalid" >&2 + exit 1 + } + sudo systemctl daemon-reload + verify_user_manager_unit_path() { + local environment + environment="$(sudo -u "$1" env -i \ + HOME="$2" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$3" \ + /usr/bin/systemctl --user show-environment)" + tr ' ' '\n' <<<"$environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$trusted_user_unit_path" >/dev/null || { + echo "::error::Qualification systemd user manager did not inherit the trusted unit path" >&2 + return 1 + } + } + verify_user_bus() { + local execution_account="$1" + local expected_uid="$2" + local bus="$3" + local context="$4" + sudo /usr/bin/test -S "$bus" && + sudo /usr/bin/test ! -L "$bus" && + [[ "$(sudo stat -c '%u' -- "$bus")" == "$expected_uid" ]] && + sudo -u "$execution_account" /usr/bin/test -S "$bus" || { + echo "::error::Qualification systemd user bus $context" >&2 + return 1 + } + } sudo systemctl start "$user_manager_unit" systemctl is-active --quiet "$runtime_directory_unit" || { echo "::error::Qualification systemd runtime-directory unit is not active" >&2 @@ -1509,10 +1552,9 @@ jobs: echo "::error::Qualification systemd user bus socket unit is not active" >&2 exit 1 } - sudo -u "$account" /usr/bin/test -S "$runtime_dir/bus" || { - echo "::error::Qualification systemd user bus is not accessible to the execution account" >&2 - exit 1 - } + verify_user_bus "$account" "$uid" "$runtime_dir/bus" \ + "is invalid or inaccessible to the execution account" + verify_user_manager_unit_path "$account" "$home" "$runtime_dir" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" containers_config="${storage_config_directory}/containers.conf" @@ -1929,6 +1971,7 @@ jobs: RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + TRUSTED_USER_UNIT_PATH: /usr/lib/systemd/user:/lib/systemd/user USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | @@ -1974,8 +2017,21 @@ jobs: echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 exit 1 } - sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || { - echo "::error::Qualification systemd user bus is not accessible after installer isolation" >&2 + sudo /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" && + sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus" && + [[ "$(sudo stat -c '%u' -- "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] && + sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || { + echo "::error::Qualification systemd user bus is invalid or inaccessible after installer isolation" >&2 + exit 1 + } + manager_environment="$(sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user show-environment)" + tr ' ' '\n' <<<"$manager_environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$TRUSTED_USER_UNIT_PATH" >/dev/null || { + echo "::error::Qualification systemd user manager lost the trusted unit path after installer isolation" >&2 exit 1 } sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { @@ -2118,6 +2174,9 @@ jobs: runtime_dir="/run/user/${uid}" runtime_directory_unit="user-runtime-dir@${uid}.service" user_manager_unit="user@${uid}.service" + user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" + user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" + trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" @@ -2137,6 +2196,24 @@ jobs: if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true fi + if [[ -e "$user_manager_dropin_directory" || -L "$user_manager_dropin_directory" ]]; then + [[ -d "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" && "$(stat -c '%u:%g:%a' "$user_manager_dropin_directory")" == "0:0:755" ]] || { + echo "::error::Qualification systemd user-manager drop-in directory cleanup target is invalid" >&2 + exit 1 + } + [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { + echo "::error::Qualification systemd user-manager drop-in cleanup target is invalid" >&2 + exit 1 + } + expected_user_manager_dropin="$(printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"' "$trusted_user_unit_path")" + [[ "$(cat "$user_manager_dropin")" == "$expected_user_manager_dropin" ]] || { + echo "::error::Qualification systemd user-manager drop-in content changed before cleanup" >&2 + exit 1 + } + sudo unlink "$user_manager_dropin" + sudo rmdir "$user_manager_dropin_directory" + sudo systemctl daemon-reload + fi if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration cleanup target is invalid" >&2 diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 170169aeb7e..d8e9dc626d5 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -143,6 +143,10 @@ function exactDirectory(directory: string): void { } } +function removeQualificationSnapshot(snapshot: string | null): void { + if (snapshot !== null) fs.rmSync(snapshot, { force: true }); +} + function writeJson(directory: string, file: string, value: unknown): void { const target = path.join(directory, file); const temporary = `${target}.tmp`; @@ -318,30 +322,33 @@ function createProviderNetwork( return Object.freeze({ id, name, gateway }); } catch (error) { if (created) { - let removalOutcome = "not attempted"; - try { - const removal = engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); - removalOutcome = `exit ${String(removal.status)}`; - } catch (cleanupError) { - removalOutcome = `threw ${bounded( - cleanupError instanceof Error ? cleanupError.message : String(cleanupError), - )}`; - } - let existenceOutcome = "not attempted"; - let removalProven = false; - try { - const existence = engine.capture(["network", "exists", name], COMMAND_TIMEOUT); - existenceOutcome = `exit ${String(existence.status)}`; - removalProven = existence.status === 1; - } catch (cleanupError) { - existenceOutcome = `threw ${bounded( - cleanupError instanceof Error ? cleanupError.message : String(cleanupError), - )}`; - } - if (!removalProven) { + const removalOutcome = (() => { + try { + const removal = engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); + return `exit ${String(removal.status)}`; + } catch (cleanupError) { + return `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`; + } + })(); + const existence = (() => { + try { + const result = engine.capture(["network", "exists", name], COMMAND_TIMEOUT); + return { outcome: `exit ${String(result.status)}`, removalProven: result.status === 1 }; + } catch (cleanupError) { + return { + outcome: `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`, + removalProven: false, + }; + } + })(); + if (!existence.removalProven) { const validationFailure = bounded(error instanceof Error ? error.message : String(error)); throw new Error( - `${validationFailure}; provider network cleanup could not prove removal (remove ${removalOutcome}; exists ${existenceOutcome})`, + `${validationFailure}; provider network cleanup could not prove removal (remove ${removalOutcome}; exists ${existence.outcome})`, ); } } @@ -607,6 +614,8 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre let gpuDevices: readonly string[] = []; let gpuComputeProcesses: readonly GpuComputeProcess[] = []; let completed = false; + let qualificationFailure: unknown; + let snapshot: string | null = null; const operationDetails = new Map>(); try { @@ -813,7 +822,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre stoppedAndStarted: true, }); - const snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); + snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); expect(lifecycle.stop(input, { beforeStop: () => undefined })).toMatchObject({ exitCode: 0 }); capture( lifecycleEngine, @@ -995,7 +1004,8 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre ownedContainers.delete(inferenceContainerId); capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); ownedNetworks.delete(network.id); - fs.rmSync(snapshot, { force: true }); + removeQualificationSnapshot(snapshot); + snapshot = null; assertNoQualificationResidue(lifecycleEngine, row.id); operationDetails.set("cleanup.exact", { containersRemaining: 0, @@ -1126,7 +1136,16 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre result: "passed", }); completed = true; + } catch (error) { + qualificationFailure = error; + throw error; } finally { + const cleanupFailures: unknown[] = []; + try { + removeQualificationSnapshot(snapshot); + } catch (error) { + cleanupFailures.push(error); + } if (!completed) { if (lifecycleEngine) { for (const containerId of ownedContainers) { @@ -1142,11 +1161,24 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre } } } - await stopService(service?.child ?? null, socket); + try { + await stopService(service?.child ?? null, socket); + } catch (error) { + cleanupFailures.push(error); + } service = null; + if (cleanupFailures.length > 0) { + throw new AggregateError( + qualificationFailure === undefined + ? cleanupFailures + : [qualificationFailure, ...cleanupFailures], + "Native runtime qualification cleanup failed", + ); + } } } export const nativeRuntimeQualificationCaseInternals = Object.freeze({ createProviderNetwork, + removeQualificationSnapshot, }); diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 7b1388fde5b..0731c75a4ef 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import * as fs from "node:fs"; +import * as net from "node:net"; import * as os from "node:os"; import * as path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -37,6 +38,9 @@ function extractFunction(source: string, name: string): string { function fixtureSource(source: string): string { return source + .replaceAll("/usr/bin/test", "/bin/test") + .replaceAll("/usr/bin/systemctl", "systemctl") + .replaceAll("PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"') .replaceAll("/etc/subuid", "${FIXTURE_ROOT}/etc/subuid") .replaceAll("/etc/subgid", "${FIXTURE_ROOT}/etc/subgid") .replaceAll( @@ -45,6 +49,10 @@ function fixtureSource(source: string): string { ) .replaceAll('"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"') .replaceAll('runtime_dir="/run/user/${uid}"', 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"') + .replaceAll( + 'user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"', + 'user_manager_dropin_directory="${FIXTURE_ROOT}/run/systemd/system/${user_manager_unit}.d"', + ) .replaceAll( 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', @@ -95,7 +103,12 @@ function createFixture(): { fs.mkdirSync(run, { recursive: true }); for (const file of [calls, passwd, group, subuid, subgid]) fs.writeFileSync(file, ""); - writeExecutable(path.join(bin, "sudo"), `printf 'sudo:%s\\n' "$*" >>"$FIXTURE_CALLS"\nexec "$@"`); + writeExecutable( + path.join(bin, "sudo"), + `printf 'sudo:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == -u ]]; then shift 2; fi +exec "$@"`, + ); writeExecutable( path.join(bin, "getent"), `case "$1" in @@ -150,7 +163,11 @@ printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, ); writeExecutable( path.join(bin, "stat"), - `case "$3" in + `target="\${!#}" +case "$target" in + *bus) printf '%s\\n' "\${BUS_UID:-1002}" ;; + *user@*.service.d) printf '0:0:755\\n' ;; + *50-nemoclaw-native-runtime.conf) printf '0:0:444:1\\n' ;; *native-runtime-owner-*) printf '0:0:400\\n' ;; *native-runtime-podman-*) printf '0:0:555\\n' ;; *native-runtime-helpers-*) printf '0:0:555\\n' ;; @@ -166,6 +183,7 @@ esac`, writeExecutable( path.join(bin, "systemctl"), `printf 'systemctl:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == --user ]]; then shift; fi if [[ "$1" == stop ]]; then shift for unit in "$@"; do @@ -175,6 +193,8 @@ if [[ "$1" == stop ]]; then done elif [[ "$1" == is-active ]]; then exit 3 +elif [[ "$1" == show || "$1" == show-environment ]]; then + printf '%s\\n' "\${SYSTEMD_ENVIRONMENT:-}" fi`, ); writeExecutable( @@ -224,6 +244,7 @@ function runFixture( ACCOUNT: "nemoclawq", ACCOUNT_CREATED: "true", FIXTURE_CALLS: fixture.calls, + FIXTURE_BIN: fixture.bin, FIXTURE_HOME: fixture.home, FIXTURE_ROOT: fixture.root, GITHUB_RUN_ATTEMPT: "1", @@ -244,6 +265,64 @@ function provisionBlock(): string { } describe("native runtime qualification account lifecycle", () => { + it("rejects a symlinked or wrong-owner user bus while accepting the exact account socket", async () => { + const fixture = createFixture(); + const socketRoot = fs.mkdtempSync("/tmp/nrq-bus-"); + roots.push(socketRoot); + const socket = path.join(socketRoot, "bus"); + const socketLink = path.join(socketRoot, "bus-link"); + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socket, resolve); + }); + fs.symlinkSync(socket, socketLink); + const verifyUserBus = extractFunction(workflowScripts().boundary, "verify_user_bus"); + const source = (candidate: string) => + `set -euo pipefail\n${verifyUserBus}\nverify_user_bus nemoclawq 1002 ${JSON.stringify(candidate)} fixture`; + + try { + const exact = runFixture(fixture, source(socket)); + expect(exact.status, exact.stderr).toBe(0); + + const symlink = runFixture(fixture, source(socketLink)); + expect(symlink.status).not.toBe(0); + expect(symlink.stderr).toContain("Qualification systemd user bus fixture"); + + const wrongOwner = runFixture(fixture, source(socket), { BUS_UID: "1003" }); + expect(wrongOwner.status).not.toBe(0); + expect(wrongOwner.stderr).toContain("Qualification systemd user bus fixture"); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + + it("rejects a user manager whose unit path includes candidate-writable authority", () => { + const fixture = createFixture(); + const verifyUnitPath = extractFunction( + workflowScripts().boundary, + "verify_user_manager_unit_path", + ).replace("env -i", "env"); + const source = `set -euo pipefail +trusted_user_unit_path=/usr/lib/systemd/user:/lib/systemd/user +${verifyUnitPath} +verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/1002"`; + + const trusted = runFixture(fixture, source, { + SYSTEMD_ENVIRONMENT: "SYSTEMD_UNIT_PATH=/usr/lib/systemd/user:/lib/systemd/user", + }); + expect(trusted.status, trusted.stderr).toBe(0); + + const candidateWritable = runFixture(fixture, source, { + SYSTEMD_ENVIRONMENT: + "SYSTEMD_UNIT_PATH=/home/nemoclawq/.config/systemd/user:/usr/lib/systemd/user:/lib/systemd/user", + }); + expect(candidateWritable.status).not.toBe(0); + expect(candidateWritable.stderr).toContain("did not inherit the trusted unit path"); + }); + it("rejects pre-existing accounts and stale subordinate-ID authorization before mutation", () => { for (const state of ["passwd", "group", "subuid", "subgid"] as const) { const fixture = createFixture(); @@ -351,6 +430,13 @@ describe("native runtime qualification account lifecycle", () => { const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); const podman = path.join(fixture.root, "nemoclaw-native-runtime-podman-42-1-1002"); const helpers = path.join(fixture.root, "nemoclaw-native-runtime-helpers-42-1-1002"); + const userManagerDropinDirectory = path.join( + fixture.root, + "run", + "systemd", + "system", + "user@1002.service.d", + ); const resources = path.join( fixture.root, "var", @@ -361,6 +447,12 @@ describe("native runtime qualification account lifecycle", () => { fs.mkdirSync(fixture.home, { recursive: true }); fs.mkdirSync(runtime, { recursive: true }); fs.mkdirSync(storage, { recursive: true }); + fs.mkdirSync(userManagerDropinDirectory, { recursive: true, mode: 0o755 }); + fs.writeFileSync( + path.join(userManagerDropinDirectory, "50-nemoclaw-native-runtime.conf"), + '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=/usr/lib/systemd/user:/lib/systemd/user"\n', + { mode: 0o444 }, + ); fs.writeFileSync(path.join(runtime, "alive"), "fixture"); fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); fs.writeFileSync(path.join(storage, "containers.conf"), "fixture"); @@ -402,6 +494,7 @@ describe("native runtime qualification account lifecycle", () => { expect(fs.existsSync(podman)).toBe(false); expect(fs.existsSync(helpers)).toBe(false); expect(fs.existsSync(resources)).toBe(false); + expect(fs.existsSync(userManagerDropinDirectory)).toBe(false); }, 15_000); it("does not run destructive cleanup when the run-owned marker is absent", () => { diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index c490ffb9355..23631361a92 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it, vi } from "vitest"; import type { PodmanBoundContainerEngine } from "../../../src/lib/adapters/podman/index.ts"; @@ -159,4 +163,23 @@ describe("native runtime provider-network authority", () => { ["network", "exists", NETWORK_NAME], ]); }); + + it("removes an exported snapshot without replacing the case failure", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-snapshot-cleanup-")); + const snapshot = path.join(root, "nemoclaw-q-fixture.tar"); + const original = new Error("failure after export"); + fs.writeFileSync(snapshot, "snapshot"); + + const failAfterExport = () => { + try { + throw original; + } finally { + nativeRuntimeQualificationCaseInternals.removeQualificationSnapshot(snapshot); + } + }; + + expect(failAfterExport).toThrow(original); + expect(fs.existsSync(snapshot)).toBe(false); + fs.rmSync(root, { force: true, recursive: true }); + }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 28e283ed192..af3d0bbd7c0 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -318,11 +318,19 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); - expect(boundary.run).toContain('sudo -u "$account" /usr/bin/test -S "$runtime_dir/bus"'); - expect(boundary.run).not.toContain("stat -c '%u' \"$runtime_dir/bus\""); expect(boundary.run).toContain( - "Qualification systemd user bus is not accessible to the execution account", + 'sudo -u "$execution_account" /usr/bin/test -S "$bus"', ); + expect(boundary.run).toContain('sudo /usr/bin/test ! -L "$bus"'); + expect(boundary.run).toContain("sudo stat -c '%u' -- \"$bus\""); + expect(boundary.run).toContain( + "Qualification systemd user bus $context", + ); + expect(boundary.run).toContain( + 'trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"', + ); + expect(boundary.run).toContain('Environment="SYSTEMD_UNIT_PATH=%s"'); + expect(boundary.run).toContain("/usr/bin/systemctl --user show-environment"); expect(boundary.run).toContain('sudo -u "$account" env -i'); expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); @@ -392,9 +400,16 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain( 'sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus"', ); - expect(installer.run).not.toContain("stat -c '%u' \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).toContain('sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus"'); + expect(installer.run).toContain("sudo stat -c '%u' -- \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).toContain( + "Qualification systemd user bus is invalid or inaccessible after installer isolation", + ); + expect(installer.env?.TRUSTED_USER_UNIT_PATH).toBe( + "/usr/lib/systemd/user:/lib/systemd/user", + ); expect(installer.run).toContain( - "Qualification systemd user bus is not accessible after installer isolation", + "/usr/bin/systemctl --user show-environment", ); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", @@ -458,6 +473,9 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).not.toContain("rm -rf"); expect(cleanup.run).not.toContain("find "); expect(cleanup.run).toContain('systemctl stop "$user_manager_unit" "$runtime_directory_unit"'); + expect(cleanup.run).toContain('sudo unlink "$user_manager_dropin"'); + expect(cleanup.run).toContain('sudo rmdir "$user_manager_dropin_directory"'); + expect(cleanup.run).toContain("sudo systemctl daemon-reload"); expect(cleanup.run).toContain( "Qualification systemd user lifecycle remained active during cleanup", ); From deb483bdf14f95272c3fe84c46e3cd135f406254 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 06:47:09 -0500 Subject: [PATCH 38/71] fix(e2e): route inference within provider network Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 21 ++++++++----------- ...untime-qualification-case-executor.test.ts | 11 ++++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index d8e9dc626d5..6745fd11773 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -653,7 +653,6 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const networkName = `nemoclaw-q-${caseSuffix}`; const network = createProviderNetwork(inferenceEngine, networkName, row.id); ownedNetworks.add(network.id); - const hostPort = 20_000 + (Number.parseInt(caseSuffix.slice(0, 4), 16) % 20_000); const runnerContractFile = row.case.acceleration === "nvidia-gpu" ? nativeRuntimeQualificationRunnerContractPath(process.env, uid) @@ -682,9 +681,9 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre } const inferenceName = `nemoclaw-inference-${caseSuffix}`; - const endpoint = `http://${network.gateway}:${String(hostPort)}`; progress.phase("launch exact local inference"); const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; + const endpoint = `http://${inferenceName}:${String(inferencePort)}`; const inferenceArguments = [ "run", "--detach", @@ -693,10 +692,6 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre inferenceName, "--network", network.name, - "--publish", - `127.0.0.1:${String(hostPort)}:${String(inferencePort)}`, - "--publish", - `${network.gateway}:${String(hostPort)}:${String(inferencePort)}`, "--label", `${QUALIFICATION_LABEL}=${row.id}`, ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), @@ -799,7 +794,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre protocol: "openai-chat-completions", model: inference.model, responseSha256: turnSha256, - route: "provider-network-gateway", + route: "provider-network-dns", }); if (!bundle.lifecycle.supported) throw new Error("Podman lifecycle surface is unavailable"); @@ -1168,11 +1163,13 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre } service = null; if (cleanupFailures.length > 0) { - throw new AggregateError( - qualificationFailure === undefined - ? cleanupFailures - : [qualificationFailure, ...cleanupFailures], - "Native runtime qualification cleanup failed", + if (qualificationFailure === undefined) { + throw new AggregateError(cleanupFailures, "Native runtime qualification cleanup failed"); + } + console.error( + `Native runtime qualification also encountered cleanup failures: ${cleanupFailures + .map((error) => bounded(error instanceof Error ? error.message : String(error))) + .join("; ")}`, ); } } diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 23631361a92..fe478a0eed3 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -182,4 +182,15 @@ describe("native runtime provider-network authority", () => { expect(fs.existsSync(snapshot)).toBe(false); fs.rmSync(root, { force: true, recursive: true }); }); + + it("routes inference inside the provider network without a host port publication", () => { + const source = fs.readFileSync( + path.join(process.cwd(), "test/e2e/live/native-runtime-qualification-case-executor.ts"), + "utf8", + ); + + expect(source).not.toContain('"--publish"'); + expect(source).toContain('route: "provider-network-dns"'); + expect(source).toContain("http://${inferenceName}:${String(inferencePort)}"); + }); }); From be19dc790859940272d024b83ddb9e4a56ee6b51 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 07:04:02 -0500 Subject: [PATCH 39/71] fix(e2e): complete native inference setup Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 8 ++-- ...ive-runtime-qualification-case-executor.ts | 39 +++++++++++++++---- ...untime-qualification-case-executor.test.ts | 2 + ...me-qualification-producer-workflow.test.ts | 5 ++- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f0a5ae8c107..fe9ba3784ad 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1863,14 +1863,16 @@ jobs: --fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \ --output "$target" \ "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" - [[ -f "$target" && ! -L "$target" && "$(stat -c '%u:%g:%a:%h:%s' "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { + sudo test -f "$target" && + sudo test ! -L "$target" && + [[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 exit 1 } if [[ "$algorithm" == "sha256" ]]; then - [[ "$(sha256sum "$target" | cut -d' ' -f1)" == "$digest" ]] + [[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]] else - [[ "$(git hash-object --no-filters "$target")" == "$digest" ]] + [[ "$(sudo git hash-object --no-filters -- "$target")" == "$digest" ]] fi || { echo "::error::Downloaded GPU model file digest is invalid: $file" >&2 exit 1 diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 6745fd11773..1f84b9f0b0f 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -28,7 +28,10 @@ import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; -import type { NativeRuntimeQualificationObligation } from "../registry/native-runtime-qualification.ts"; +import type { + NativeRuntimeQualificationInference, + NativeRuntimeQualificationObligation, +} from "../registry/native-runtime-qualification.ts"; import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; import { assertCredentialFreeQualificationEnvironment, @@ -492,11 +495,20 @@ async function agentTurn( containerId: string, endpoint: string, model: string, + inference: NativeRuntimeQualificationInference, ): Promise { const body = JSON.stringify({ model, - messages: [{ role: "user", content: "Reply with the single word qualified." }], - max_tokens: 32, + messages: [ + { + role: "user", + content: + inference === "ollama" + ? "/no_think\nReply with the single word qualified." + : "Reply with the single word qualified.", + }, + ], + max_tokens: 128, stream: false, }); const args = [ @@ -533,17 +545,23 @@ async function agentTurn( model?: unknown; choices?: Array<{ finish_reason?: unknown; - message?: { content?: unknown; tool_calls?: unknown }; + message?: { content?: unknown; reasoning?: unknown; tool_calls?: unknown }; }>; }; const first = response.choices?.[0]; + const completeMessage = + typeof first?.message?.content === "string" || + typeof first?.message?.reasoning === "string" || + Array.isArray(first?.message?.tool_calls); if ( response.model !== model || typeof first?.finish_reason !== "string" || first.finish_reason === "length" || - (typeof first.message?.content !== "string" && !Array.isArray(first.message?.tool_calls)) + !completeMessage ) { - throw new Error("Agent turn did not return a complete exact-model inference response"); + throw new Error( + `Agent turn did not return a complete exact-model inference response (modelMatch=${String(response.model === model)}; finishReason=${typeof first?.finish_reason === "string" ? bounded(first.finish_reason) : typeof first?.finish_reason}; contentType=${typeof first?.message?.content}; reasoningType=${typeof first?.message?.reasoning}; toolCalls=${String(Array.isArray(first?.message?.tool_calls))})`, + ); } return sha256(output); } @@ -782,7 +800,13 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre imageDigest: digestFromImageReference(agentImage), }); - const turnSha256 = await agentTurn(lifecycleEngine, agentId, endpoint, inference.model); + const turnSha256 = await agentTurn( + lifecycleEngine, + agentId, + endpoint, + inference.model, + row.case.inference, + ); if (row.case.acceleration === "nvidia-gpu") { gpuComputeProcesses = proveGpuBackedInference( inferenceEngine, @@ -951,6 +975,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre agentId, endpoint, inference.model, + row.case.inference, ); const reconciledGpuComputeProcesses = row.case.acceleration === "nvidia-gpu" diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index fe478a0eed3..e959790d04d 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -192,5 +192,7 @@ describe("native runtime provider-network authority", () => { expect(source).not.toContain('"--publish"'); expect(source).toContain('route: "provider-network-dns"'); expect(source).toContain("http://${inferenceName}:${String(inferencePort)}"); + expect(source).toContain("/no_think\\nReply with the single word qualified."); + expect(source).toContain("max_tokens: 128"); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index af3d0bbd7c0..e038dedacc8 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -251,8 +251,9 @@ describe("native runtime qualification producer workflow", () => { ); expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); - expect(gpuResources.run).toContain("git hash-object --no-filters"); - expect(gpuResources.run).toContain("sha256sum"); + expect(gpuResources.run).toContain("sudo stat -c '%u:%g:%a:%h:%s' -- \"$target\""); + expect(gpuResources.run).toContain('sudo git hash-object --no-filters -- "$target"'); + expect(gpuResources.run).toContain('sudo sha256sum -- "$target"'); expect(gpuResources.run).toContain("model-free-nim@sha256:"); expect(gpuResources.run).toContain("nvcr.io/nvidia/vllm@sha256:"); expect(gpuResources.run).toContain("runner-contract.json"); From 5031cebdb1e87a78a08c30aec30c6bedaf8fbd6e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 07:46:09 -0500 Subject: [PATCH 40/71] fix(e2e): harden native qualification execution Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 76 +++++++++++++++---- ...ive-runtime-qualification-case-executor.ts | 26 ++++--- ...untime-qualification-case-executor.test.ts | 4 + ...me-qualification-producer-workflow.test.ts | 44 ++++++++++- 4 files changed, 127 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fe9ba3784ad..cba03b10783 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1076,7 +1076,7 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ - gcc git libapparmor-dev libbtrfs-dev libc6-dev \ + curl gcc git libapparmor-dev libbtrfs-dev libc6-dev \ libdevmapper-dev libglib2.0-dev \ libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler @@ -1086,6 +1086,9 @@ jobs: AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08 EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab + PASTA_SOURCE_ARCHIVE_SHA256: 54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958 + PASTA_SOURCE_SHA: f8df3f1b228fe19a74a269334fdfe6cc7d0605ce + PASTA_VERSION: 2026_07_28.f8df3f1 PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074 TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash @@ -1095,6 +1098,24 @@ jobs: [[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]] [[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]] [[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]] + [[ ! -e .passt-source && ! -L .passt-source ]] + pasta_source_archive="${RUNNER_TEMP}/passt-${PASTA_SOURCE_SHA}.tar.gz" + [[ ! -e "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] + /usr/bin/curl \ + --fail --location --proto '=https' --proto-redir '=https' \ + --retry 3 --show-error --silent --tlsv1.2 \ + --output "$pasta_source_archive" \ + "https://passt.top/passt/snapshot/passt-${PASTA_SOURCE_SHA}.tar.gz" + [[ -f "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] + [[ "$(sha256sum "$pasta_source_archive" | cut -d' ' -f1)" == "$PASTA_SOURCE_ARCHIVE_SHA256" ]] + mkdir .passt-source + tar \ + --extract --gzip --file="$pasta_source_archive" \ + --directory=.passt-source --strip-components=1 \ + --no-same-owner --no-same-permissions + [[ -f .passt-source/Makefile && ! -L .passt-source/Makefile ]] + [[ -f .passt-source/passt.c && ! -L .passt-source/passt.c ]] + [[ ! -e .passt-source/passt && ! -L .passt-source/passt ]] for source in .podman-source .netavark-source .aardvark-source; do [[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]] done @@ -1110,6 +1131,8 @@ jobs: make --directory=.netavark-source --jobs=2 build SOURCE_DATE_EPOCH=1785940850 CI=1 \ make --directory=.aardvark-source --jobs=2 build + SOURCE_DATE_EPOCH=1785255008 \ + make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt podman_dependencies="$(ldd .podman-source/bin/podman)" printf '%s\n' "$podman_dependencies" @@ -1123,6 +1146,8 @@ jobs: fi install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" + [[ -f .passt-source/passt && ! -L .passt-source/passt ]] + install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta" install -D -m 0755 .podman-source/bin/rootlessport \ "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" install -D -m 0755 .netavark-source/bin/netavark \ @@ -1134,12 +1159,16 @@ jobs: "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]] + [[ "$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)" == "pasta $PASTA_VERSION" ]] [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]] [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]] jq -n \ --arg architecture "$EXPECTED_ARCHITECTURE" \ --arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \ --arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \ + --arg pastaSourceArchiveSha256 "$PASTA_SOURCE_ARCHIVE_SHA256" \ + --arg pastaSourceSha "$PASTA_SOURCE_SHA" \ + --arg pastaVersion "$PASTA_VERSION" \ --arg podmanSourceSha "$PODMAN_SOURCE_SHA" ' { schemaVersion: 1, @@ -1151,6 +1180,9 @@ jobs: netavarkSourceSha: $netavarkSourceSha, aardvarkDnsVersion: "2.1.0", aardvarkDnsSourceSha: $aardvarkDnsSourceSha, + pastaVersion: $pastaVersion, + pastaSourceArchiveSha256: $pastaSourceArchiveSha256, + pastaSourceSha: $pastaSourceSha, goVersion: "1.25.9", rustVersion: "1.88.0" } @@ -1158,6 +1190,7 @@ jobs: ( cd "$TOOLCHAIN_DIRECTORY" sha256sum \ + bin/pasta \ bin/podman \ libexec/podman/aardvark-dns \ libexec/podman/netavark \ @@ -1271,7 +1304,7 @@ jobs: sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ acl apparmor btrfs-progs conmon \ - golang-github-containers-common iptables nftables passt runc slirp4netns uidmap + golang-github-containers-common iptables nftables runc slirp4netns uidmap [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] @@ -1281,6 +1314,7 @@ jobs: ) expected_files=( ./SHA256SUMS + ./bin/pasta ./bin/podman ./libexec/podman/aardvark-dns ./libexec/podman/netavark @@ -1307,6 +1341,9 @@ jobs: "kind", "netavarkSourceSha", "netavarkVersion", + "pastaSourceArchiveSha256", + "pastaSourceSha", + "pastaVersion", "podmanSourceSha", "podmanVersion", "rustVersion", @@ -1321,6 +1358,9 @@ jobs: .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and .aardvarkDnsVersion == "2.1.0" and .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and + .pastaVersion == "2026_07_28.f8df3f1" and + .pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and + .pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and .goVersion == "1.25.9" and .rustVersion == "1.88.0" ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null @@ -1592,12 +1632,13 @@ jobs: echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2 exit 1 } - [[ -f /usr/bin/pasta && ! -L /usr/bin/pasta && "$(stat -c '%u:%g:%a' /usr/bin/pasta)" == "0:0:755" ]] || { - echo "::error::Signed-OS pasta helper is missing or writable" >&2 + [[ -f "$TOOLCHAIN_DIRECTORY/bin/pasta" && ! -L "$TOOLCHAIN_DIRECTORY/bin/pasta" ]] || { + echo "::error::Pinned qualification pasta executable source is missing or invalid" >&2 exit 1 } sudo install -d --owner=root --group=root --mode=0555 "$helper_directory" - sudo install --owner=root --group=root --mode=0555 /usr/bin/pasta "$pasta_executable" + sudo install --owner=root --group=root --mode=0555 \ + "$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable" [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { echo "::error::Run-owned qualification helper directory is invalid" >&2 exit 1 @@ -1606,7 +1647,7 @@ jobs: echo "::error::Run-owned qualification pasta executable is invalid" >&2 exit 1 } - [[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum /usr/bin/pasta | cut -d' ' -f1)" ]] || { + [[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/pasta" | cut -d' ' -f1)" ]] || { echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2 exit 1 } @@ -1865,10 +1906,15 @@ jobs: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" sudo test -f "$target" && sudo test ! -L "$target" && - [[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { + [[ "$(sudo stat -c '%u:%g:%h:%s' -- "$target")" == "${uid}:${gid}:1:${size}" ]] || { echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 exit 1 } + sudo chmod 0600 -- "$target" + [[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { + echo "::error::Downloaded GPU model file permissions are invalid: $file" >&2 + exit 1 + } if [[ "$algorithm" == "sha256" ]]; then [[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]] else @@ -2275,17 +2321,21 @@ jobs: echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2 exit 1 } - if [[ -e "$model_directory" || -L "$model_directory" ]]; then - model_mode="$(stat -c '%u:%g:%a' "$model_directory")" - [[ -d "$model_directory" && ! -L "$model_directory" && ("$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555") ]] || { + if sudo test -e "$model_directory" || sudo test -L "$model_directory"; then + model_mode="$(sudo stat -c '%u:%g:%a' -- "$model_directory")" + sudo test -d "$model_directory" && + sudo test ! -L "$model_directory" && + [[ "$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555" ]] || { echo "::error::Qualification GPU model directory cleanup target is invalid" >&2 exit 1 } for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do target="${model_directory}/${file}" - if [[ -e "$target" || -L "$target" ]]; then - file_mode="$(stat -c '%u:%g:%a:%h' "$target")" - [[ -f "$target" && ! -L "$target" && ("$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1") ]] || { + if sudo test -e "$target" || sudo test -L "$target"; then + file_mode="$(sudo stat -c '%u:%g:%a:%h' -- "$target")" + sudo test -f "$target" && + sudo test ! -L "$target" && + [[ "$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1" ]] || { echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2 exit 1 } diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 1f84b9f0b0f..8118a083dbe 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -481,7 +481,7 @@ function createAgentContainer(input: { "/bin/sh", input.imageRef, "-c", - "while :; do sleep 3600; done", + "trap 'exit 0' TERM INT; while :; do sleep 3600 & wait $!; done", ], "agent container creation", INFERENCE_TIMEOUT, @@ -499,6 +499,7 @@ async function agentTurn( ): Promise { const body = JSON.stringify({ model, + ...(inference === "ollama" ? { reasoning_effort: "none" } : {}), messages: [ { role: "user", @@ -826,13 +827,15 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const lifecycle = bundle.lifecycle; const input = lifecycleInput(row.case.agent, sandboxName); let beforeStopCalled = false; - expect( - lifecycle.stop(input, { - beforeStop: () => { - beforeStopCalled = true; - }, - }), - ).toMatchObject({ exitCode: 0, state: "stopped" }); + const firstStop = lifecycle.stop(input, { + beforeStop: () => { + beforeStopCalled = true; + }, + }); + if (firstStop.exitCode !== 0) { + throw new Error(`Initial sandbox stop failed: ${bounded(firstStop.message ?? "unknown")}`); + } + expect(firstStop.state).toBe("stopped"); expect(beforeStopCalled).toBe(true); expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); operationDetails.set("sandbox.stop-start", { @@ -842,7 +845,12 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre }); snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); - expect(lifecycle.stop(input, { beforeStop: () => undefined })).toMatchObject({ exitCode: 0 }); + const snapshotStop = lifecycle.stop(input, { beforeStop: () => undefined }); + if (snapshotStop.exitCode !== 0) { + throw new Error( + `Snapshot sandbox stop failed: ${bounded(snapshotStop.message ?? "unknown")}`, + ); + } capture( lifecycleEngine, ["volume", "export", "--output", snapshot, volumeName], diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index e959790d04d..766f5624669 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -193,6 +193,10 @@ describe("native runtime provider-network authority", () => { expect(source).toContain('route: "provider-network-dns"'); expect(source).toContain("http://${inferenceName}:${String(inferencePort)}"); expect(source).toContain("/no_think\\nReply with the single word qualified."); + expect(source).toContain('reasoning_effort: "none"'); expect(source).toContain("max_tokens: 128"); + expect(source).toContain("trap 'exit 0' TERM INT; while :; do sleep 3600 & wait $!; done"); + expect(source).toContain("Initial sandbox stop failed:"); + expect(source).toContain("Snapshot sandbox stop failed:"); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index e038dedacc8..f809cf6b9d9 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -159,11 +159,31 @@ describe("native runtime qualification producer workflow", () => { expect(buildDependencies.run).not.toContain("libgpgme-dev"); expect(buildDependencies.run).not.toContain("libassuan-dev"); expect(buildDependencies.run).not.toContain("libgpg-error-dev"); + expect(buildDependencies.run).toContain("curl"); expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); expect(build.run).toContain("EXTRA_BUILDTAGS=containers_image_openpgp"); expect(build.run).not.toContain("quadlet"); expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); expect(build.run).toContain("make --directory=.aardvark-source --jobs=2 build"); + expect(build.run).toContain("https://passt.top/passt"); + expect(build.run).toContain("/usr/bin/curl"); + expect(build.env?.PASTA_SOURCE_ARCHIVE_SHA256).toBe( + "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958", + ); + expect(build.env?.PASTA_SOURCE_SHA).toBe("f8df3f1b228fe19a74a269334fdfe6cc7d0605ce"); + expect(build.env?.PASTA_VERSION).toBe("2026_07_28.f8df3f1"); + expect(build.run).toContain("sha256sum \"$pasta_source_archive\""); + expect(build.run).toContain("--no-same-owner --no-same-permissions"); + expect(build.run).toContain("[[ ! -e .passt-source/passt && ! -L .passt-source/passt ]]"); + expect(build.run).not.toMatch(/\bgit\s+fetch\b/u); + expect(build.run).toContain( + 'make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt', + ); + expect(build.run).toContain( + 'install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta"', + ); + expect(build.run).toContain('"pasta $PASTA_VERSION"'); + expect(build.run).toMatch(/sha256sum[\s\S]+bin\/pasta[\s\S]+manifest\.json/u); expect(build.run).toContain("sha256sum"); expect(build.run).toContain("Pinned Podman build has an unresolved runtime dependency"); expect(build.run).toContain("Pinned Podman build must not require an optional host ABI"); @@ -252,6 +272,14 @@ describe("native runtime qualification producer workflow", () => { expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); expect(gpuResources.run).toContain("sudo stat -c '%u:%g:%a:%h:%s' -- \"$target\""); + expect(gpuResources.run).toContain("sudo stat -c '%u:%g:%h:%s' -- \"$target\""); + expect(gpuResources.run).toContain('sudo chmod 0600 -- "$target"'); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$target"')).toBeGreaterThan( + gpuResourcesRun.indexOf("sudo stat -c '%u:%g:%h:%s'"), + ); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$target"')).toBeLessThan( + gpuResourcesRun.indexOf("sudo stat -c '%u:%g:%a:%h:%s'"), + ); expect(gpuResources.run).toContain('sudo git hash-object --no-filters -- "$target"'); expect(gpuResources.run).toContain('sudo sha256sum -- "$target"'); expect(gpuResources.run).toContain("model-free-nim@sha256:"); @@ -275,16 +303,17 @@ describe("native runtime qualification producer workflow", () => { "apparmor", "conmon", "golang-github-containers-common", - "passt", "runc", "slirp4netns", "uidmap", ]) { expect(podman.run).toContain(requiredPackage); } + expect(podman.run).not.toMatch(/\s+passt(?:\s|$)/u); expect(podman.run).not.toContain("fuse-overlayfs"); expect(podman.run).toContain("find -P"); expect(podman.run).toContain("sha256sum --check --strict SHA256SUMS"); + expect(podman.run).toContain("./bin/pasta"); expect(podman.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); expect(podman.run).toContain("Downloaded native Podman toolchain contains unexpected files"); expect(podman.run).toContain("Native Podman toolchain target must not be a symlink"); @@ -292,6 +321,11 @@ describe("native runtime qualification producer workflow", () => { expect(podman.run).toContain('dpkg --compare-versions "$runc_version" ge 1.1.11'); expect(podman.run).toContain('"netavark 2.1.0"'); expect(podman.run).toContain('"aardvark-dns 2.1.0"'); + expect(podman.run).toContain('"2026_07_28.f8df3f1"'); + expect(podman.run).toContain( + '"54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958"', + ); + expect(podman.run).toContain('"f8df3f1b228fe19a74a269334fdfe6cc7d0605ce"'); expect(podman.run).toContain('[[ "$version" == "podman version 6.1.0" ]]'); expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); @@ -354,6 +388,10 @@ describe("native runtime qualification producer workflow", () => { "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined)", ); expect(boundary.run).toContain("Run-owned qualification pasta executable digest changed"); + expect(boundary.run).toContain( + '"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"', + ); + expect(boundary.run).not.toContain("/usr/bin/pasta"); expect(boundary.run).toContain( 'PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin"', ); @@ -486,6 +524,10 @@ describe("native runtime qualification producer workflow", () => { expect(cleanup.run).toContain('sudo unlink "$storage_config_directory/containers.conf"'); expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); expect(cleanup.run).toContain('sudo rm -f -- "$podman_executable"'); + expect(cleanup.run).toContain('sudo test -e "$model_directory"'); + expect(cleanup.run).toContain("sudo stat -c '%u:%g:%a' -- \"$model_directory\""); + expect(cleanup.run).toContain('sudo test -f "$target"'); + expect(cleanup.run).toContain('sudo test ! -L "$target"'); expect(cleanup.run).toContain("Qualification Podman executable remains after cleanup"); expect(cleanup.run).toContain("Qualification GPU resource directory remains after cleanup"); expect(cleanup.run).toContain( From 65d16ceebd9ddf49917f6e0ebb3db38a40beba1b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 07:58:54 -0500 Subject: [PATCH 41/71] fix(e2e): validate pasta version line Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 6 +++++- .../native-runtime-qualification-producer-workflow.test.ts | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index cba03b10783..f4b67373170 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1159,7 +1159,11 @@ jobs: "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]] - [[ "$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)" == "pasta $PASTA_VERSION" ]] + pasta_version_output="$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)" + [[ "${pasta_version_output%%$'\n'*}" == "pasta $PASTA_VERSION" ]] || { + echo "::error::Pinned qualification pasta version is invalid" >&2 + exit 1 + } [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]] [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]] jq -n \ diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index f809cf6b9d9..da85946dffa 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -182,7 +182,10 @@ describe("native runtime qualification producer workflow", () => { expect(build.run).toContain( 'install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta"', ); + expect(build.run).toContain("pasta_version_output="); + expect(build.run).toContain("pasta_version_output%%$'\\n'*"); expect(build.run).toContain('"pasta $PASTA_VERSION"'); + expect(build.run).toContain("Pinned qualification pasta version is invalid"); expect(build.run).toMatch(/sha256sum[\s\S]+bin\/pasta[\s\S]+manifest\.json/u); expect(build.run).toContain("sha256sum"); expect(build.run).toContain("Pinned Podman build has an unresolved runtime dependency"); From c1e73a9834d76c715ff690918e128893d9dcc5ca Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 08:15:58 -0500 Subject: [PATCH 42/71] fix(e2e): use valid lifecycle sandbox names Signed-off-by: Aaron Erickson --- .../native-runtime-qualification-case-executor.ts | 13 ++++++++++++- ...tive-runtime-qualification-case-executor.test.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 8118a083dbe..918c63e9009 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -29,6 +29,7 @@ import { expect } from "../fixtures/e2e-test.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import type { + NativeRuntimeQualificationAgent, NativeRuntimeQualificationInference, NativeRuntimeQualificationObligation, } from "../registry/native-runtime-qualification.ts"; @@ -50,6 +51,11 @@ const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; const COMMAND_TIMEOUT = 60_000; const INFERENCE_TIMEOUT = 900_000; const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const LIFECYCLE_SANDBOX_NAMES = Object.freeze({ + hermes: "q-hermes", + "langchain-deepagents-code": "q-deepagents", + openclaw: "q-openclaw", +} as const satisfies Record); export const NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES = [ "validate credential-free Docker-unavailable isolation", "bind the rootless Podman engine", @@ -581,6 +587,10 @@ function lifecycleInput(agent: string, sandboxName: string): RuntimeProviderLife }; } +function lifecycleSandboxName(agent: NativeRuntimeQualificationAgent): string { + return LIFECYCLE_SANDBOX_NAMES[agent]; +} + function assertNoQualificationResidue(engine: PodmanBoundContainerEngine, caseId: string): void { for (const [resource, args] of [ ["container", ["ps", "--all", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], @@ -770,7 +780,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const sandboxId = caseSuffix; progress.phase("onboard the managed agent image"); - const sandboxName = `qualification-${row.case.agent}`; + const sandboxName = lifecycleSandboxName(row.case.agent); const agentName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; const volumeName = `nemoclaw-q-state-${caseSuffix}`; capture( @@ -1210,5 +1220,6 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre export const nativeRuntimeQualificationCaseInternals = Object.freeze({ createProviderNetwork, + lifecycleSandboxName, removeQualificationSnapshot, }); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 766f5624669..39e63509b47 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -8,7 +8,9 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { PodmanBoundContainerEngine } from "../../../src/lib/adapters/podman/index.ts"; +import { isValidName } from "../../../src/lib/name-validation.ts"; import { nativeRuntimeQualificationCaseInternals } from "../live/native-runtime-qualification-case-executor.ts"; +import { NATIVE_RUNTIME_QUALIFICATION_AGENTS } from "../registry/native-runtime-qualification.ts"; const NETWORK_ID = "a".repeat(64); const NETWORK_NAME = "nemoclaw-q-0123456789ab"; @@ -62,6 +64,16 @@ function engine(outputs: readonly EngineOutput[]): { } describe("native runtime provider-network authority", () => { + it("uses distinct canonical sandbox names for every qualified agent", () => { + const names = NATIVE_RUNTIME_QUALIFICATION_AGENTS.map((agent) => + nativeRuntimeQualificationCaseInternals.lifecycleSandboxName(agent), + ); + + expect(names).toEqual(["q-openclaw", "q-hermes", "q-deepagents"]); + expect(names.every((name) => isValidName(name))).toBe(true); + expect(new Set(names).size).toBe(names.length); + }); + it("resolves Podman 6.1 name output to one immutable labeled network ID", () => { const runtime = engine([NETWORK_NAME, inspection(), inspection()]); From 0e9d559cc4be8f3f3e0f6c8bbfbdfd908359021c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 08:35:45 -0500 Subject: [PATCH 43/71] fix(e2e): accept safe physical GPU identities Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 50 +++++++++++-------- ...untime-qualification-case-executor.test.ts | 32 ++++++++++++ 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 918c63e9009..57e829cc92b 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -48,6 +48,9 @@ import { const FULL_ID = /^[a-f0-9]{64}$/u; const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; +// nvidia-smi defines the immutable UUID as alphanumeric; retain the repository's +// bounded physical GPU identifier envelope while excluding MIG device names. +const PHYSICAL_GPU_UUID = /^GPU-[A-Za-z0-9][A-Za-z0-9-]{6,121}[A-Za-z0-9]$/u; const COMMAND_TIMEOUT = 60_000; const INFERENCE_TIMEOUT = 900_000; const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; @@ -374,25 +377,8 @@ function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: str capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); } -function proveGpuDevices( - engine: PodmanBoundContainerEngine, - probeImageRef: string, -): readonly string[] { - const devices = capture( - engine, - [ - "run", - "--rm", - "--pull=never", - "--device", - "nvidia.com/gpu=all", - probeImageRef, - "nvidia-smi", - "--query-gpu=uuid", - "--format=csv,noheader", - ], - "NVIDIA CDI runtime proof", - ) +function parsePhysicalGpuDevices(output: string): readonly string[] { + const devices = output .split(/\r?\n/u) .map((entry) => entry.trim()) .filter(Boolean) @@ -400,13 +386,36 @@ function proveGpuDevices( if ( devices.length === 0 || new Set(devices).size !== devices.length || - devices.some((device) => !/^GPU-[0-9A-Fa-f-]{36}$/u.test(device)) + devices.some((device) => !PHYSICAL_GPU_UUID.test(device)) ) { throw new Error("NVIDIA CDI runtime proof did not return exact physical GPU UUIDs"); } return Object.freeze(devices); } +function proveGpuDevices( + engine: PodmanBoundContainerEngine, + probeImageRef: string, +): readonly string[] { + return parsePhysicalGpuDevices( + capture( + engine, + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + probeImageRef, + "nvidia-smi", + "--query-gpu=uuid", + "--format=csv,noheader", + ], + "NVIDIA CDI runtime proof", + ), + ); +} + function proveGpuBackedInference( engine: PodmanBoundContainerEngine, containerId: string, @@ -1221,5 +1230,6 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre export const nativeRuntimeQualificationCaseInternals = Object.freeze({ createProviderNetwork, lifecycleSandboxName, + parsePhysicalGpuDevices, removeQualificationSnapshot, }); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 39e63509b47..c205c0ef6c0 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -63,6 +63,38 @@ function engine(outputs: readonly EngineOutput[]): { }; } +describe("native runtime GPU evidence", () => { + it("accepts bounded NVIDIA physical GPU identities and sorts them exactly", () => { + expect( + nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices( + [ + "GPU-z9Y8x7W6-v5U4-t3S2-r1Q0-p9O8n7M6l5K4", + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + ].join("\n"), + ), + ).toEqual([ + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + "GPU-z9Y8x7W6-v5U4-t3S2-r1Q0-p9O8n7M6l5K4", + ]); + }); + + it.each([ + ["empty output", ""], + [ + "duplicate identities", + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6\nGPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + ], + ["MIG identities", "MIG-8932f937-d72c-4106-c12f-20bd9faed9f6"], + ["leading hyphens", "GPU--932f937"], + ["trailing hyphens", "GPU-8932f937-"], + ["control characters", "GPU-8932f937\u0000"], + ])("rejects %s as physical GPU proof", (_label, output) => { + expect(() => nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices(output)).toThrow( + "NVIDIA CDI runtime proof did not return exact physical GPU UUIDs", + ); + }); +}); + describe("native runtime provider-network authority", () => { it("uses distinct canonical sandbox names for every qualified agent", () => { const names = NATIVE_RUNTIME_QUALIFICATION_AGENTS.map((agent) => From eca6e683c57105722d3e68d739d70f75111f5a01 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 08:52:30 -0500 Subject: [PATCH 44/71] test(e2e): report rejected GPU identity rows Signed-off-by: Aaron Erickson --- .../live/native-runtime-qualification-case-executor.ts | 4 +++- .../native-runtime-qualification-case-executor.test.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 57e829cc92b..9902cf8f0a7 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -388,7 +388,9 @@ function parsePhysicalGpuDevices(output: string): readonly string[] { new Set(devices).size !== devices.length || devices.some((device) => !PHYSICAL_GPU_UUID.test(device)) ) { - throw new Error("NVIDIA CDI runtime proof did not return exact physical GPU UUIDs"); + throw new Error( + `NVIDIA CDI runtime proof did not return exact physical GPU UUIDs: ${bounded(JSON.stringify(devices))}`, + ); } return Object.freeze(devices); } diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index c205c0ef6c0..3a620e4c411 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -93,6 +93,14 @@ describe("native runtime GPU evidence", () => { "NVIDIA CDI runtime proof did not return exact physical GPU UUIDs", ); }); + + it("reports the bounded rejected rows for protected-run diagnosis", () => { + expect(() => + nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices("unexpected-row"), + ).toThrow( + 'NVIDIA CDI runtime proof did not return exact physical GPU UUIDs: ["unexpected-row"]', + ); + }); }); describe("native runtime provider-network authority", () => { From 6ef48ce27aa3ec151d64d98aa793195f184f8462 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 09:07:55 -0500 Subject: [PATCH 45/71] fix(e2e): override GPU probe entrypoint Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 4 +++- ...untime-qualification-case-executor.test.ts | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 9902cf8f0a7..27f18d8d595 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -408,8 +408,9 @@ function proveGpuDevices( "--pull=never", "--device", "nvidia.com/gpu=all", - probeImageRef, + "--entrypoint", "nvidia-smi", + probeImageRef, "--query-gpu=uuid", "--format=csv,noheader", ], @@ -1233,5 +1234,6 @@ export const nativeRuntimeQualificationCaseInternals = Object.freeze({ createProviderNetwork, lifecycleSandboxName, parsePhysicalGpuDevices, + proveGpuDevices, removeQualificationSnapshot, }); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 3a620e4c411..54f52980bf2 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -16,6 +16,7 @@ const NETWORK_ID = "a".repeat(64); const NETWORK_NAME = "nemoclaw-q-0123456789ab"; const CASE_ID = "podman-openclaw-linux-amd64-cpu-ollama"; const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const GPU_PROBE_IMAGE = `nvcr.io/nvidia/k8s/cuda-sample@sha256:${"d".repeat(64)}`; function inspection(overrides: Record = {}): string { return JSON.stringify([ @@ -64,6 +65,29 @@ function engine(outputs: readonly EngineOutput[]): { } describe("native runtime GPU evidence", () => { + it("overrides the probe image entrypoint with the exact nvidia-smi UUID query", () => { + const gpuUuid = "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6"; + const runtime = engine([gpuUuid]); + + expect( + nativeRuntimeQualificationCaseInternals.proveGpuDevices(runtime.value, GPU_PROBE_IMAGE), + ).toEqual([gpuUuid]); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + "--entrypoint", + "nvidia-smi", + GPU_PROBE_IMAGE, + "--query-gpu=uuid", + "--format=csv,noheader", + ], + ]); + }); + it("accepts bounded NVIDIA physical GPU identities and sorts them exactly", () => { expect( nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices( From 22cee86549d77b11fe7bc7522bd3e89c3b18c887 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 09:38:17 -0500 Subject: [PATCH 46/71] test(e2e): report inference exit diagnostics Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 27 ++++++++++++++++++- ...untime-qualification-case-executor.test.ts | 23 ++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 27f18d8d595..ab669b4e65e 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -461,6 +461,25 @@ function proveGpuBackedInference( return Object.freeze(processes.map((entry) => Object.freeze(entry))); } +function inferenceFailureDiagnostic( + engine: PodmanBoundContainerEngine, + containerId: string, +): string { + const inspect = (args: readonly string[]): string => { + try { + const result = engine.capture(args, COMMAND_TIMEOUT); + return bounded( + result.stderr || result.stdout || `command returned exit ${String(result.status)}`, + ); + } catch (error) { + return bounded(error instanceof Error ? error.message : String(error)); + } + }; + const state = inspect(["inspect", "--format", "{{json .State}}", containerId]); + const logs = inspect(["logs", "--tail", "50", containerId]); + return `state=${state}; logs=${logs}`; +} + function createAgentContainer(input: { readonly engine: PodmanBoundContainerEngine; readonly imageRef: string; @@ -651,7 +670,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const ownedContainers = new Set(); const ownedVolumes = new Set(); const ownedNetworks = new Set(); - let inferenceContainerId: string; + let inferenceContainerId = ""; let gpuDevices: readonly string[] = []; let gpuComputeProcesses: readonly GpuComputeProcess[] = []; let completed = false; @@ -1197,6 +1216,11 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre cleanupFailures.push(error); } if (!completed) { + if (inferenceEngine && FULL_ID.test(inferenceContainerId)) { + console.error( + `Native runtime qualification inference failure diagnostic: ${inferenceFailureDiagnostic(inferenceEngine, inferenceContainerId)}`, + ); + } if (lifecycleEngine) { for (const containerId of ownedContainers) { lifecycleEngine.capture(["rm", "--force", containerId], COMMAND_TIMEOUT); @@ -1232,6 +1256,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre export const nativeRuntimeQualificationCaseInternals = Object.freeze({ createProviderNetwork, + inferenceFailureDiagnostic, lifecycleSandboxName, parsePhysicalGpuDevices, proveGpuDevices, diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index 54f52980bf2..e486b4e7825 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -127,6 +127,29 @@ describe("native runtime GPU evidence", () => { }); }); +describe("native runtime failure diagnosis", () => { + it("captures bounded inference state and logs without changing the case result", () => { + const containerId = "e".repeat(64); + const runtime = engine([ + '{"Status":"exited","ExitCode":1}', + "runtime failed after credential-free startup", + ]); + + expect( + nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( + runtime.value, + containerId, + ), + ).toBe( + 'state={"Status":"exited","ExitCode":1}; logs=runtime failed after credential-free startup', + ); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["inspect", "--format", "{{json .State}}", containerId], + ["logs", "--tail", "50", containerId], + ]); + }); +}); + describe("native runtime provider-network authority", () => { it("uses distinct canonical sandbox names for every qualified agent", () => { const names = NATIVE_RUNTIME_QUALIFICATION_AGENTS.map((agent) => From f0bac8db9ed1f935ca50a22a640569021e5dc554 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 09:57:24 -0500 Subject: [PATCH 47/71] fix(e2e): launch vllm serve explicitly Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 32 +++++++++++-------- ...untime-qualification-case-executor.test.ts | 23 +++++++++++++ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index ab669b4e65e..88b530fbe56 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -480,6 +480,22 @@ function inferenceFailureDiagnostic( return `state=${state}; logs=${logs}`; } +function vllmServeArguments(model: string, port: number): readonly string[] { + return [ + "vllm", + "serve", + "/models", + "--served-model-name", + model, + "--host", + "0.0.0.0", + "--port", + String(port), + "--max-model-len", + "2048", + ]; +} + function createAgentContainer(input: { readonly engine: PodmanBoundContainerEngine; readonly imageRef: string; @@ -771,20 +787,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre ? ["--shm-size", "16g", "--volume", `${inference.modelPath}:/models:ro`] : []), inference.imageRef, - ...(row.case.inference === "vllm" - ? [ - "--model", - "/models", - "--served-model-name", - inference.model, - "--host", - "0.0.0.0", - "--port", - String(inferencePort), - "--max-model-len", - "2048", - ] - : []), + ...(row.case.inference === "vllm" ? vllmServeArguments(inference.model, inferencePort) : []), ]; inferenceContainerId = capture( inferenceEngine, @@ -1261,4 +1264,5 @@ export const nativeRuntimeQualificationCaseInternals = Object.freeze({ parsePhysicalGpuDevices, proveGpuDevices, removeQualificationSnapshot, + vllmServeArguments, }); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index e486b4e7825..f21112b79a9 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -150,6 +150,29 @@ describe("native runtime failure diagnosis", () => { }); }); +describe("native runtime vLLM launch", () => { + it("provides the pinned NGC image entrypoint with an exact serve command", () => { + expect( + nativeRuntimeQualificationCaseInternals.vllmServeArguments( + "Qwen/Qwen2.5-0.5B-Instruct", + 8000, + ), + ).toEqual([ + "vllm", + "serve", + "/models", + "--served-model-name", + "Qwen/Qwen2.5-0.5B-Instruct", + "--host", + "0.0.0.0", + "--port", + "8000", + "--max-model-len", + "2048", + ]); + }); +}); + describe("native runtime provider-network authority", () => { it("uses distinct canonical sandbox names for every qualified agent", () => { const names = NATIVE_RUNTIME_QUALIFICATION_AGENTS.map((agent) => From 25acda5a29d4374438577ff9dbb67fda29454e13 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 10:09:13 -0500 Subject: [PATCH 48/71] fix(e2e): harden failed-case cleanup Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 139 ++++++++++++++---- ...untime-qualification-case-executor.test.ts | 96 ++++++++++-- 2 files changed, 195 insertions(+), 40 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 88b530fbe56..4d6a648d711 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -465,19 +465,86 @@ function inferenceFailureDiagnostic( engine: PodmanBoundContainerEngine, containerId: string, ): string { - const inspect = (args: readonly string[]): string => { - try { - const result = engine.capture(args, COMMAND_TIMEOUT); - return bounded( - result.stderr || result.stdout || `command returned exit ${String(result.status)}`, - ); - } catch (error) { - return bounded(error instanceof Error ? error.message : String(error)); + let result: ReturnType; + try { + result = engine.capture( + ["inspect", "--format", "{{json .State}}", containerId], + COMMAND_TIMEOUT, + ); + } catch { + return "state=unavailable; inspect=threw"; + } + if (result.status !== 0) { + return `state=unavailable; inspectExit=${String(result.status)}`; + } + try { + const parsed = JSON.parse(result.stdout) as Record; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return "state=unparseable"; } - }; - const state = inspect(["inspect", "--format", "{{json .State}}", containerId]); - const logs = inspect(["logs", "--tail", "50", containerId]); - return `state=${state}; logs=${logs}`; + const knownStatuses = new Set([ + "configured", + "created", + "exited", + "initialized", + "paused", + "removing", + "running", + "stopped", + "stopping", + "unknown", + ]); + return `state=${JSON.stringify({ + status: + typeof parsed.Status === "string" && knownStatuses.has(parsed.Status) + ? parsed.Status + : "unknown", + exitCode: Number.isSafeInteger(parsed.ExitCode) ? parsed.ExitCode : null, + oomKilled: parsed.OOMKilled === true, + running: parsed.Running === true, + })}`; + } catch { + return "state=unparseable"; + } +} + +type OwnedResourceKind = "container" | "network" | "volume"; + +type OwnedResourceGroup = { + readonly engine: PodmanBoundContainerEngine; + readonly identities: readonly string[]; + readonly kind: OwnedResourceKind; +}; + +function collectOwnedResourceCleanupFailures(groups: readonly OwnedResourceGroup[]): Error[] { + const failures: Error[] = []; + for (const { engine, identities, kind } of groups) { + for (const identity of identities) { + const outcomes: string[] = []; + const removeArgs = + kind === "container" ? ["rm", "--force", identity] : [kind, "rm", "--force", identity]; + try { + const removal = engine.capture(removeArgs, COMMAND_TIMEOUT); + if (removal.status !== 0) outcomes.push(`remove exit ${String(removal.status)}`); + } catch { + outcomes.push("remove threw"); + } + try { + const existence = engine.capture([kind, "exists", identity], COMMAND_TIMEOUT); + if (existence.status !== 1) outcomes.push(`exists exit ${String(existence.status)}`); + } catch { + outcomes.push("exists threw"); + } + if (outcomes.length > 0) { + failures.push( + new Error( + `Native runtime qualification ${kind} cleanup failed for ${identity} (${outcomes.join("; ")})`, + ), + ); + } + } + } + return failures; } function vllmServeArguments(model: string, port: number): readonly string[] { @@ -1224,19 +1291,33 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre `Native runtime qualification inference failure diagnostic: ${inferenceFailureDiagnostic(inferenceEngine, inferenceContainerId)}`, ); } - if (lifecycleEngine) { - for (const containerId of ownedContainers) { - lifecycleEngine.capture(["rm", "--force", containerId], COMMAND_TIMEOUT); - } - for (const volume of ownedVolumes) { - lifecycleEngine.capture(["volume", "rm", "--force", volume], COMMAND_TIMEOUT); - } - } - if (inferenceEngine) { - for (const networkId of ownedNetworks) { - inferenceEngine.capture(["network", "rm", "--force", networkId], COMMAND_TIMEOUT); - } - } + cleanupFailures.push( + ...collectOwnedResourceCleanupFailures([ + ...(lifecycleEngine + ? [ + { + engine: lifecycleEngine, + identities: [...ownedContainers], + kind: "container" as const, + }, + { + engine: lifecycleEngine, + identities: [...ownedVolumes], + kind: "volume" as const, + }, + ] + : []), + ...(inferenceEngine + ? [ + { + engine: inferenceEngine, + identities: [...ownedNetworks], + kind: "network" as const, + }, + ] + : []), + ]), + ); } try { await stopService(service?.child ?? null, socket); @@ -1248,16 +1329,16 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre if (qualificationFailure === undefined) { throw new AggregateError(cleanupFailures, "Native runtime qualification cleanup failed"); } - console.error( - `Native runtime qualification also encountered cleanup failures: ${cleanupFailures - .map((error) => bounded(error instanceof Error ? error.message : String(error))) - .join("; ")}`, + throw new AggregateError( + [qualificationFailure, ...cleanupFailures], + "Native runtime qualification failed and cleanup could not be proven", ); } } } export const nativeRuntimeQualificationCaseInternals = Object.freeze({ + collectOwnedResourceCleanupFailures, createProviderNetwork, inferenceFailureDiagnostic, lifecycleSandboxName, diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index f21112b79a9..a0939e22a9e 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -32,6 +32,7 @@ function inspection(overrides: Record = {}): string { type EngineOutput = | string + | Error | { readonly status: number; readonly stdout?: string; readonly stderr?: string }; function engine(outputs: readonly EngineOutput[]): { @@ -41,6 +42,7 @@ function engine(outputs: readonly EngineOutput[]): { let index = 0; const capture = vi.fn((args: readonly string[]) => { const output = outputs[index++]; + if (output instanceof Error) throw output; return typeof output === "object" ? { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" } : { @@ -128,24 +130,96 @@ describe("native runtime GPU evidence", () => { }); describe("native runtime failure diagnosis", () => { - it("captures bounded inference state and logs without changing the case result", () => { + it("emits only allowlisted inference state without logs or child output", () => { const containerId = "e".repeat(64); const runtime = engine([ - '{"Status":"exited","ExitCode":1}', - "runtime failed after credential-free startup", + JSON.stringify({ + Status: "exited", + ExitCode: 1, + OOMKilled: false, + Running: false, + Error: "Authorization: Bearer credential-like-value", + Request: "private request content", + }), ]); - expect( - nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( - runtime.value, - containerId, - ), - ).toBe( - 'state={"Status":"exited","ExitCode":1}; logs=runtime failed after credential-free startup', + const diagnostic = nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( + runtime.value, + containerId, + ); + + expect(diagnostic).toBe( + 'state={"status":"exited","exitCode":1,"oomKilled":false,"running":false}', ); + expect(diagnostic).not.toContain("credential-like-value"); + expect(diagnostic).not.toContain("private request content"); expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ ["inspect", "--format", "{{json .State}}", containerId], - ["logs", "--tail", "50", containerId], + ]); + }); + + it("does not echo an unexpected state string", () => { + const diagnostic = nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( + engine([JSON.stringify({ Status: "credential-like-value" })]).value, + "e".repeat(64), + ); + + expect(diagnostic).toBe( + 'state={"status":"unknown","exitCode":null,"oomKilled":false,"running":false}', + ); + expect(diagnostic).not.toContain("credential-like-value"); + }); +}); + +describe("native runtime failed-case cleanup", () => { + it.each([ + ["container", ["rm", "--force", "container-id"], ["container", "exists", "container-id"]], + ["volume", ["volume", "rm", "--force", "volume-id"], ["volume", "exists", "volume-id"]], + ["network", ["network", "rm", "--force", "network-id"], ["network", "exists", "network-id"]], + ] as const)("reports an unproven %s removal without child output", (kind, remove, exists) => { + const runtime = engine([ + { status: 1, stderr: "Authorization: Bearer cleanup-secret" }, + { status: 0 }, + ]); + + const failures = nativeRuntimeQualificationCaseInternals.collectOwnedResourceCleanupFailures([ + { engine: runtime.value, identities: [`${kind}-id`], kind }, + ]); + + expect(failures.map((failure) => failure.message)).toEqual([ + `Native runtime qualification ${kind} cleanup failed for ${kind}-id (remove exit 1; exists exit 0)`, + ]); + expect(failures[0]?.message).not.toContain("cleanup-secret"); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([remove, exists]); + }); + + it("continues every resource class after one removal throws", () => { + const lifecycle = engine([ + new Error("container removal failed"), + { status: 0 }, + { status: 0 }, + { status: 1 }, + ]); + const inference = engine([{ status: 0 }, { status: 1 }]); + + const failures = nativeRuntimeQualificationCaseInternals.collectOwnedResourceCleanupFailures([ + { engine: lifecycle.value, identities: ["container-id"], kind: "container" }, + { engine: lifecycle.value, identities: ["volume-id"], kind: "volume" }, + { engine: inference.value, identities: ["network-id"], kind: "network" }, + ]); + + expect(failures.map((failure) => failure.message)).toEqual([ + "Native runtime qualification container cleanup failed for container-id (remove threw; exists exit 0)", + ]); + expect(lifecycle.capture.mock.calls.map(([args]) => args)).toEqual([ + ["rm", "--force", "container-id"], + ["container", "exists", "container-id"], + ["volume", "rm", "--force", "volume-id"], + ["volume", "exists", "volume-id"], + ]); + expect(inference.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "rm", "--force", "network-id"], + ["network", "exists", "network-id"], ]); }); }); From a56b5c704f6b9df5086bb4d56ae158c4d00a275b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 10:15:38 -0500 Subject: [PATCH 49/71] test(e2e): keep cleanup fixtures linear Signed-off-by: Aaron Erickson --- .../native-runtime-qualification-case-executor.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index a0939e22a9e..cdf7544516a 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -32,7 +32,7 @@ function inspection(overrides: Record = {}): string { type EngineOutput = | string - | Error + | (() => never) | { readonly status: number; readonly stdout?: string; readonly stderr?: string }; function engine(outputs: readonly EngineOutput[]): { @@ -41,8 +41,8 @@ function engine(outputs: readonly EngineOutput[]): { } { let index = 0; const capture = vi.fn((args: readonly string[]) => { - const output = outputs[index++]; - if (output instanceof Error) throw output; + const configured = outputs[index++]; + const output = typeof configured === "function" ? configured() : configured; return typeof output === "object" ? { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" } : { @@ -195,7 +195,9 @@ describe("native runtime failed-case cleanup", () => { it("continues every resource class after one removal throws", () => { const lifecycle = engine([ - new Error("container removal failed"), + () => { + throw new Error("container removal failed"); + }, { status: 0 }, { status: 0 }, { status: 1 }, From 1b3ba2badc9ba2210ea3ec8d36104e5c68fab311 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 10:29:37 -0500 Subject: [PATCH 50/71] fix(e2e): preserve cleanup engine ownership Signed-off-by: Aaron Erickson --- ...ive-runtime-qualification-case-executor.ts | 96 ++++++++++++------- ...untime-qualification-case-executor.test.ts | 22 +++-- 2 files changed, 74 insertions(+), 44 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 4d6a648d711..515150b1a2b 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -547,6 +547,46 @@ function collectOwnedResourceCleanupFailures(groups: readonly OwnedResourceGroup return failures; } +function collectQualificationResourceCleanupFailures(input: { + readonly inferenceContainers: readonly string[]; + readonly inferenceEngine: PodmanBoundContainerEngine | null; + readonly lifecycleContainers: readonly string[]; + readonly lifecycleEngine: PodmanBoundContainerEngine | null; + readonly networks: readonly string[]; + readonly volumes: readonly string[]; +}): Error[] { + return collectOwnedResourceCleanupFailures([ + ...(input.lifecycleEngine + ? [ + { + engine: input.lifecycleEngine, + identities: input.lifecycleContainers, + kind: "container" as const, + }, + { + engine: input.lifecycleEngine, + identities: input.volumes, + kind: "volume" as const, + }, + ] + : []), + ...(input.inferenceEngine + ? [ + { + engine: input.inferenceEngine, + identities: input.inferenceContainers, + kind: "container" as const, + }, + { + engine: input.inferenceEngine, + identities: input.networks, + kind: "network" as const, + }, + ] + : []), + ]); +} + function vllmServeArguments(model: string, port: number): readonly string[] { return [ "vllm", @@ -750,7 +790,8 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre let hostEngine: PodmanBoundContainerEngine | null = null; let inferenceEngine: PodmanBoundContainerEngine | null = null; let lifecycleEngine: PodmanBoundContainerEngine | null = null; - const ownedContainers = new Set(); + const ownedInferenceContainers = new Set(); + const ownedLifecycleContainers = new Set(); const ownedVolumes = new Set(); const ownedNetworks = new Set(); let inferenceContainerId = ""; @@ -865,7 +906,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre if (!FULL_ID.test(inferenceContainerId)) { throw new Error("Inference container did not return a full immutable ID"); } - ownedContainers.add(inferenceContainerId); + ownedInferenceContainers.add(inferenceContainerId); if (row.case.inference === "ollama") { capture( inferenceEngine, @@ -900,7 +941,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre sandboxName, volume: volumeName, }); - ownedContainers.add(agentId); + ownedLifecycleContainers.add(agentId); capture( lifecycleEngine, ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' qualified >/qualification/state"], @@ -977,7 +1018,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre "sandbox state mutation", ); capture(lifecycleEngine, ["rm", "--force", agentId], "remove sandbox before rebuild"); - ownedContainers.delete(agentId); + ownedLifecycleContainers.delete(agentId); capture(lifecycleEngine, ["volume", "rm", volumeName], "remove sandbox volume"); ownedVolumes.delete(volumeName); capture( @@ -1002,7 +1043,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre sandboxName, volume: volumeName, }); - ownedContainers.add(agentId); + ownedLifecycleContainers.add(agentId); expect( capture(lifecycleEngine, ["exec", agentId, "cat", "/qualification/state"], "restored state"), ).toBe("qualified"); @@ -1042,7 +1083,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre sandboxName: `${sandboxName}-clone`, volume: cloneVolume, }); - ownedContainers.add(cloneId); + ownedLifecycleContainers.add(cloneId); expect( capture(lifecycleEngine, ["exec", cloneId, "cat", "/qualification/state"], "clone state"), ).toBe("qualified"); @@ -1129,18 +1170,17 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre progress.phase("prove exact cleanup"); capture(lifecycleEngine, ["rm", "--force", agentId], "agent cleanup"); - ownedContainers.delete(agentId); - for (const containerId of [...ownedContainers]) { - if (containerId === inferenceContainerId) continue; + ownedLifecycleContainers.delete(agentId); + for (const containerId of [...ownedLifecycleContainers]) { capture(lifecycleEngine, ["rm", "--force", containerId], "focused container cleanup"); - ownedContainers.delete(containerId); + ownedLifecycleContainers.delete(containerId); } for (const volume of [...ownedVolumes]) { capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); ownedVolumes.delete(volume); } capture(inferenceEngine, ["rm", "--force", inferenceContainerId], "inference runtime cleanup"); - ownedContainers.delete(inferenceContainerId); + ownedInferenceContainers.delete(inferenceContainerId); capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); ownedNetworks.delete(network.id); removeQualificationSnapshot(snapshot); @@ -1292,31 +1332,14 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre ); } cleanupFailures.push( - ...collectOwnedResourceCleanupFailures([ - ...(lifecycleEngine - ? [ - { - engine: lifecycleEngine, - identities: [...ownedContainers], - kind: "container" as const, - }, - { - engine: lifecycleEngine, - identities: [...ownedVolumes], - kind: "volume" as const, - }, - ] - : []), - ...(inferenceEngine - ? [ - { - engine: inferenceEngine, - identities: [...ownedNetworks], - kind: "network" as const, - }, - ] - : []), - ]), + ...collectQualificationResourceCleanupFailures({ + inferenceContainers: [...ownedInferenceContainers], + inferenceEngine, + lifecycleContainers: [...ownedLifecycleContainers], + lifecycleEngine, + networks: [...ownedNetworks], + volumes: [...ownedVolumes], + }), ); } try { @@ -1339,6 +1362,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre export const nativeRuntimeQualificationCaseInternals = Object.freeze({ collectOwnedResourceCleanupFailures, + collectQualificationResourceCleanupFailures, createProviderNetwork, inferenceFailureDiagnostic, lifecycleSandboxName, diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index cdf7544516a..fb4b9c6058a 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -193,7 +193,7 @@ describe("native runtime failed-case cleanup", () => { expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([remove, exists]); }); - it("continues every resource class after one removal throws", () => { + it("routes each resource to its owning engine and continues after one removal throws", () => { const lifecycle = engine([ () => { throw new Error("container removal failed"); @@ -202,13 +202,17 @@ describe("native runtime failed-case cleanup", () => { { status: 0 }, { status: 1 }, ]); - const inference = engine([{ status: 0 }, { status: 1 }]); - - const failures = nativeRuntimeQualificationCaseInternals.collectOwnedResourceCleanupFailures([ - { engine: lifecycle.value, identities: ["container-id"], kind: "container" }, - { engine: lifecycle.value, identities: ["volume-id"], kind: "volume" }, - { engine: inference.value, identities: ["network-id"], kind: "network" }, - ]); + const inference = engine([{ status: 0 }, { status: 1 }, { status: 0 }, { status: 1 }]); + + const failures = + nativeRuntimeQualificationCaseInternals.collectQualificationResourceCleanupFailures({ + inferenceContainers: ["inference-id"], + inferenceEngine: inference.value, + lifecycleContainers: ["container-id"], + lifecycleEngine: lifecycle.value, + networks: ["network-id"], + volumes: ["volume-id"], + }); expect(failures.map((failure) => failure.message)).toEqual([ "Native runtime qualification container cleanup failed for container-id (remove threw; exists exit 0)", @@ -220,6 +224,8 @@ describe("native runtime failed-case cleanup", () => { ["volume", "exists", "volume-id"], ]); expect(inference.capture.mock.calls.map(([args]) => args)).toEqual([ + ["rm", "--force", "inference-id"], + ["container", "exists", "inference-id"], ["network", "rm", "--force", "network-id"], ["network", "exists", "network-id"], ]); From 240240cb41ce72421d28b0fe5a7a96f8984a1468 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 11:51:26 -0500 Subject: [PATCH 51/71] fix(e2e): close qualification review gaps --- .github/workflows/e2e.yaml | 3 +- ...ive-runtime-qualification-case-executor.ts | 121 +++++++++++------- ...me-qualification-account-lifecycle.test.ts | 56 +++++--- ...untime-qualification-case-executor.test.ts | 29 +++-- ...e-qualification-producer-aggregate.test.ts | 17 +++ 5 files changed, 146 insertions(+), 80 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f4b67373170..64dfc85de63 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1765,6 +1765,7 @@ jobs: ACCOUNT: ${{ steps.boundary.outputs.account }} ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} + ACCELERATION: ${{ matrix.case.acceleration }} ARCHITECTURE: ${{ matrix.case.architecture }} CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} @@ -1779,7 +1780,7 @@ jobs: run: | set -euo pipefail umask 077 - if [[ "${{ matrix.case.acceleration }}" != "nvidia-gpu" ]]; then + if [[ "$ACCELERATION" != "nvidia-gpu" ]]; then printf 'runner_contract=\n' >>"$GITHUB_OUTPUT" exit 0 fi diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index 515150b1a2b..daedbc420c8 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -29,6 +29,7 @@ import { expect } from "../fixtures/e2e-test.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import type { + NativeRuntimeQualificationAcceleration, NativeRuntimeQualificationAgent, NativeRuntimeQualificationInference, NativeRuntimeQualificationObligation, @@ -603,6 +604,54 @@ function vllmServeArguments(model: string, port: number): readonly string[] { ]; } +function inferenceContainerPlan(input: { + readonly acceleration: NativeRuntimeQualificationAcceleration; + readonly caseId: string; + readonly imageRef: string; + readonly inference: NativeRuntimeQualificationInference; + readonly model: string; + readonly modelPath?: string; + readonly name: string; + readonly network: string; + readonly port: number; +}): { readonly arguments: readonly string[]; readonly endpoint: string } { + if ((input.inference === "nim" || input.inference === "vllm") && !input.modelPath) { + throw new Error("Native runtime qualification GPU inference requires a model path"); + } + return { + arguments: [ + "run", + "--detach", + "--pull=never", + "--name", + input.name, + "--network", + input.network, + "--label", + `${QUALIFICATION_LABEL}=${input.caseId}`, + ...(input.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), + ...(input.inference === "nim" + ? [ + "--shm-size", + "16g", + "--env", + "NIM_MODEL_PATH=/models", + "--env", + `NIM_SERVED_MODEL_NAME=${input.model}`, + "--volume", + `${input.modelPath}:/models:ro`, + ] + : []), + ...(input.inference === "vllm" + ? ["--shm-size", "16g", "--volume", `${input.modelPath}:/models:ro`] + : []), + input.imageRef, + ...(input.inference === "vllm" ? vllmServeArguments(input.model, input.port) : []), + ], + endpoint: `http://${input.name}:${String(input.port)}`, + }; +} + function createAgentContainer(input: { readonly engine: PodmanBoundContainerEngine; readonly imageRef: string; @@ -799,6 +848,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre let gpuComputeProcesses: readonly GpuComputeProcess[] = []; let completed = false; let qualificationFailure: unknown; + const cleanupFailures: unknown[] = []; let snapshot: string | null = null; const operationDetails = new Map>(); @@ -867,39 +917,20 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const inferenceName = `nemoclaw-inference-${caseSuffix}`; progress.phase("launch exact local inference"); const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; - const endpoint = `http://${inferenceName}:${String(inferencePort)}`; - const inferenceArguments = [ - "run", - "--detach", - "--pull=never", - "--name", - inferenceName, - "--network", - network.name, - "--label", - `${QUALIFICATION_LABEL}=${row.id}`, - ...(row.case.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), - ...(row.case.inference === "nim" - ? [ - "--shm-size", - "16g", - "--env", - "NIM_MODEL_PATH=/models", - "--env", - `NIM_SERVED_MODEL_NAME=${inference.model}`, - "--volume", - `${inference.modelPath}:/models:ro`, - ] - : []), - ...(row.case.inference === "vllm" - ? ["--shm-size", "16g", "--volume", `${inference.modelPath}:/models:ro`] - : []), - inference.imageRef, - ...(row.case.inference === "vllm" ? vllmServeArguments(inference.model, inferencePort) : []), - ]; + const inferencePlan = inferenceContainerPlan({ + acceleration: row.case.acceleration, + caseId: row.id, + imageRef: inference.imageRef, + inference: row.case.inference, + model: inference.model, + ...(inference.modelPath ? { modelPath: inference.modelPath } : {}), + name: inferenceName, + network: network.name, + port: inferencePort, + }); inferenceContainerId = capture( inferenceEngine, - inferenceArguments, + inferencePlan.arguments, `${row.case.inference} container start`, INFERENCE_TIMEOUT, ); @@ -956,7 +987,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const turnSha256 = await agentTurn( lifecycleEngine, agentId, - endpoint, + inferencePlan.endpoint, inference.model, row.case.inference, ); @@ -1133,7 +1164,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre const reconciledTurnSha256 = await agentTurn( lifecycleEngine, agentId, - endpoint, + inferencePlan.endpoint, inference.model, row.case.inference, ); @@ -1317,9 +1348,7 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre completed = true; } catch (error) { qualificationFailure = error; - throw error; } finally { - const cleanupFailures: unknown[] = []; try { removeQualificationSnapshot(snapshot); } catch (error) { @@ -1348,15 +1377,18 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre cleanupFailures.push(error); } service = null; - if (cleanupFailures.length > 0) { - if (qualificationFailure === undefined) { - throw new AggregateError(cleanupFailures, "Native runtime qualification cleanup failed"); - } - throw new AggregateError( - [qualificationFailure, ...cleanupFailures], - "Native runtime qualification failed and cleanup could not be proven", - ); - } + } + if (cleanupFailures.length > 0 && qualificationFailure === undefined) { + throw new AggregateError(cleanupFailures, "Native runtime qualification cleanup failed"); + } + if (cleanupFailures.length > 0) { + throw new AggregateError( + [qualificationFailure, ...cleanupFailures], + "Native runtime qualification failed and cleanup could not be proven", + ); + } + if (qualificationFailure !== undefined) { + throw qualificationFailure; } } @@ -1364,6 +1396,7 @@ export const nativeRuntimeQualificationCaseInternals = Object.freeze({ collectOwnedResourceCleanupFailures, collectQualificationResourceCleanupFailures, createProviderNetwork, + inferenceContainerPlan, inferenceFailureDiagnostic, lifecycleSandboxName, parsePhysicalGpuDevices, diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 0731c75a4ef..63be9ad16bc 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -36,40 +36,54 @@ function extractFunction(source: string, name: string): string { return `${name}() {${match![1]}\n}`; } +function rewriteFixtureTarget(source: string, from: string, to: string): string { + if (!source.includes(from)) return source; + const rewritten = source.replaceAll(from, to); + expect(rewritten, `Fixture rewrite did not match host target: ${from}`).not.toBe(source); + return rewritten; +} + function fixtureSource(source: string): string { - return source - .replaceAll("/usr/bin/test", "/bin/test") - .replaceAll("/usr/bin/systemctl", "systemctl") - .replaceAll("PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"') - .replaceAll("/etc/subuid", "${FIXTURE_ROOT}/etc/subuid") - .replaceAll("/etc/subgid", "${FIXTURE_ROOT}/etc/subgid") - .replaceAll( + const rewrites = [ + ["/usr/bin/test", "/bin/test"], + ["/usr/bin/systemctl", "systemctl"], + ["PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"'], + ["/etc/subuid", "${FIXTURE_ROOT}/etc/subuid"], + ["/etc/subgid", "${FIXTURE_ROOT}/etc/subgid"], + [ 'ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', 'ownership_marker="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', - ) - .replaceAll('"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"') - .replaceAll('runtime_dir="/run/user/${uid}"', 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"') - .replaceAll( + ], + ['"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"'], + ['runtime_dir="/run/user/${uid}"', 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"'], + [ 'user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"', 'user_manager_dropin_directory="${FIXTURE_ROOT}/run/systemd/system/${user_manager_unit}.d"', - ) - .replaceAll( + ], + [ 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ) - .replaceAll( + ], + [ 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ) - .replaceAll( + ], + [ 'helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'helper_directory="${FIXTURE_ROOT}/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ) - .replaceAll( + ], + [ 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ) - .replaceAll('"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'); + ], + ['"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'], + ] as const; + let rewritten = source; + for (const [from, to] of rewrites) rewritten = rewriteFixtureTarget(rewritten, from, to); + expect(rewritten, "Fixture source retains an unredirected destructive host path").not.toMatch( + /(?:^|[\s"'=])\/(?:etc\/sub(?:uid|gid)|run\/(?:nemoclaw-native-runtime|systemd\/system|user\/)|(?:nemoclaw-native-runtime-(?:podman|helpers)|var\/tmp\/nemoclaw-native-runtime-resources)-)/mu, + ); + return rewritten; } function writeExecutable(file: string, source: string): void { diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts index fb4b9c6058a..4a41e0076f7 100644 --- a/test/e2e/support/native-runtime-qualification-case-executor.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -388,19 +388,20 @@ describe("native runtime provider-network authority", () => { }); it("routes inference inside the provider network without a host port publication", () => { - const source = fs.readFileSync( - path.join(process.cwd(), "test/e2e/live/native-runtime-qualification-case-executor.ts"), - "utf8", - ); - - expect(source).not.toContain('"--publish"'); - expect(source).toContain('route: "provider-network-dns"'); - expect(source).toContain("http://${inferenceName}:${String(inferencePort)}"); - expect(source).toContain("/no_think\\nReply with the single word qualified."); - expect(source).toContain('reasoning_effort: "none"'); - expect(source).toContain("max_tokens: 128"); - expect(source).toContain("trap 'exit 0' TERM INT; while :; do sleep 3600 & wait $!; done"); - expect(source).toContain("Initial sandbox stop failed:"); - expect(source).toContain("Snapshot sandbox stop failed:"); + const plan = nativeRuntimeQualificationCaseInternals.inferenceContainerPlan({ + acceleration: "cpu", + caseId: CASE_ID, + imageRef: `docker.io/ollama/ollama@sha256:${"e".repeat(64)}`, + inference: "ollama", + model: "qwen2.5:0.5b", + name: "nemoclaw-inference-fixture", + network: NETWORK_NAME, + port: 11434, + }); + + expect(plan.endpoint).toBe("http://nemoclaw-inference-fixture:11434"); + expect(plan.arguments).toContain("--network"); + expect(plan.arguments).toContain(NETWORK_NAME); + expect(plan.arguments).not.toContain("--publish"); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts index 5da9ddc123f..16eb99e01de 100644 --- a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -241,6 +241,23 @@ describe("native runtime qualification producer aggregate", () => { }, ); + it("preserves an existing immutable aggregate output", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const sentinel = path.join(value.evidenceDirectory, "sentinel.txt"); + fs.mkdirSync(value.evidenceDirectory); + fs.writeFileSync(sentinel, "preserve me"); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("output must not already exist"); + expect(fs.readFileSync(sentinel, "utf8")).toBe("preserve me"); + }); + it("rejects an omitted case artifact", AGGREGATE_TEST_OPTIONS, () => { const value = fixture(); fs.renameSync( From b60bb48d2317e669f413ba3a723c0d9d83b9a9d1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 11:57:39 -0500 Subject: [PATCH 52/71] test(e2e): keep lifecycle fixture branchless Signed-off-by: Aaron Erickson --- .../native-runtime-qualification-account-lifecycle.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 63be9ad16bc..5e3e2cc85fa 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -37,10 +37,7 @@ function extractFunction(source: string, name: string): string { } function rewriteFixtureTarget(source: string, from: string, to: string): string { - if (!source.includes(from)) return source; - const rewritten = source.replaceAll(from, to); - expect(rewritten, `Fixture rewrite did not match host target: ${from}`).not.toBe(source); - return rewritten; + return source.replaceAll(from, to); } function fixtureSource(source: string): string { From d00ffec61db086c38e839f8cfc181ebe9abc6848 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 12:11:37 -0500 Subject: [PATCH 53/71] test(e2e): enforce lifecycle fixture rewrites Signed-off-by: Aaron Erickson --- ...me-qualification-account-lifecycle.test.ts | 163 ++++++++++++------ ...e-qualification-producer-aggregate.test.ts | 2 +- 2 files changed, 111 insertions(+), 54 deletions(-) diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index 5e3e2cc85fa..b9b88ae4310 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -36,49 +36,94 @@ function extractFunction(source: string, name: string): string { return `${name}() {${match![1]}\n}`; } -function rewriteFixtureTarget(source: string, from: string, to: string): string { +const fixtureRewrites = { + testExecutable: ["/usr/bin/test", "/bin/test"], + systemctlExecutable: ["/usr/bin/systemctl", "systemctl"], + commandPath: ["PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"'], + subuid: ["/etc/subuid", "${FIXTURE_ROOT}/etc/subuid"], + subgid: ["/etc/subgid", "${FIXTURE_ROOT}/etc/subgid"], + ownershipMarker: [ + 'ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + 'ownership_marker="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + ], + homeIdentity: ['"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"'], + runtimeDirectory: [ + 'runtime_dir="/run/user/${uid}"', + 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"', + ], + userManagerDropinDirectory: [ + 'user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"', + 'user_manager_dropin_directory="${FIXTURE_ROOT}/run/systemd/system/${user_manager_unit}.d"', + ], + storageConfigDirectory: [ + 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + podmanExecutable: [ + 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + helperDirectory: [ + 'helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'helper_directory="${FIXTURE_ROOT}/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + resourceDirectory: [ + 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + userRuntimePath: ['"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'], +} as const; + +type FixtureRewrite = keyof typeof fixtureRewrites; +type FixtureRewriteProfile = "bus" | "cleanup" | "provision" | "subgid" | "subuid" | "unitPath"; +type FixtureRewriteContract = { + readonly required: readonly FixtureRewrite[]; + readonly optional: readonly FixtureRewrite[]; +}; + +const fixtureRewriteProfiles: Record = { + bus: { required: ["testExecutable"], optional: [] }, + unitPath: { required: ["systemctlExecutable", "commandPath"], optional: [] }, + provision: { + required: ["subuid", "subgid", "ownershipMarker", "homeIdentity"], + optional: [], + }, + subuid: { required: ["subuid"], optional: [] }, + subgid: { required: ["subgid"], optional: [] }, + cleanup: { + required: [ + "subuid", + "subgid", + "ownershipMarker", + "runtimeDirectory", + "userManagerDropinDirectory", + "storageConfigDirectory", + "podmanExecutable", + "helperDirectory", + "resourceDirectory", + "userRuntimePath", + ], + optional: [], + }, +}; + +function rewriteRequiredFixtureTarget(source: string, target: FixtureRewrite): string { + const [from, to] = fixtureRewrites[target]; + expect(source, `Missing mandatory fixture rewrite '${target}'`).toContain(from); return source.replaceAll(from, to); } -function fixtureSource(source: string): string { - const rewrites = [ - ["/usr/bin/test", "/bin/test"], - ["/usr/bin/systemctl", "systemctl"], - ["PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"'], - ["/etc/subuid", "${FIXTURE_ROOT}/etc/subuid"], - ["/etc/subgid", "${FIXTURE_ROOT}/etc/subgid"], - [ - 'ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', - 'ownership_marker="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', - ], - ['"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"'], - ['runtime_dir="/run/user/${uid}"', 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"'], - [ - 'user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"', - 'user_manager_dropin_directory="${FIXTURE_ROOT}/run/systemd/system/${user_manager_unit}.d"', - ], - [ - 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ], - [ - 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ], - [ - 'helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - 'helper_directory="${FIXTURE_ROOT}/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ], - [ - 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', - ], - ['"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'], - ] as const; +function fixtureSource(source: string, profile: FixtureRewriteProfile): string { let rewritten = source; - for (const [from, to] of rewrites) rewritten = rewriteFixtureTarget(rewritten, from, to); + for (const target of fixtureRewriteProfiles[profile].required) { + rewritten = rewriteRequiredFixtureTarget(rewritten, target); + } + for (const target of fixtureRewriteProfiles[profile].optional) { + const [from, to] = fixtureRewrites[target]; + rewritten = rewritten.replaceAll(from, to); + } expect(rewritten, "Fixture source retains an unredirected destructive host path").not.toMatch( - /(?:^|[\s"'=])\/(?:etc\/sub(?:uid|gid)|run\/(?:nemoclaw-native-runtime|systemd\/system|user\/)|(?:nemoclaw-native-runtime-(?:podman|helpers)|var\/tmp\/nemoclaw-native-runtime-resources)-)/mu, + /(?:\/usr\/bin\/(?:systemctl|test)|PATH=\/usr\/bin:\/bin|"\$home" == "\/home\/\$\{account\}"|(?:^|[\s"'=])\/(?:etc\/sub(?:uid|gid)|run\/(?:nemoclaw-native-runtime|systemd\/system|user\/)|(?:nemoclaw-native-runtime-(?:podman|helpers)|var\/tmp\/nemoclaw-native-runtime-resources)-))/mu, ); return rewritten; } @@ -245,9 +290,10 @@ mv "$FIXTURE_ROOT/etc/group.next" "$FIXTURE_ROOT/etc/group"`, function runFixture( fixture: ReturnType, source: string, + profile: FixtureRewriteProfile, extraEnv: Record = {}, ) { - return spawnSync("bash", ["-c", fixtureSource(source)], { + return spawnSync("bash", ["-c", fixtureSource(source, profile)], { encoding: "utf8", timeout: 15_000, env: { @@ -276,6 +322,12 @@ function provisionBlock(): string { } describe("native runtime qualification account lifecycle", () => { + it("fails closed when a mandatory fixture rewrite no longer matches", () => { + expect(() => fixtureSource("set -euo pipefail", "bus")).toThrow( + "Missing mandatory fixture rewrite 'testExecutable'", + ); + }); + it("rejects a symlinked or wrong-owner user bus while accepting the exact account socket", async () => { const fixture = createFixture(); const socketRoot = fs.mkdtempSync("/tmp/nrq-bus-"); @@ -293,14 +345,14 @@ describe("native runtime qualification account lifecycle", () => { `set -euo pipefail\n${verifyUserBus}\nverify_user_bus nemoclawq 1002 ${JSON.stringify(candidate)} fixture`; try { - const exact = runFixture(fixture, source(socket)); + const exact = runFixture(fixture, source(socket), "bus"); expect(exact.status, exact.stderr).toBe(0); - const symlink = runFixture(fixture, source(socketLink)); + const symlink = runFixture(fixture, source(socketLink), "bus"); expect(symlink.status).not.toBe(0); expect(symlink.stderr).toContain("Qualification systemd user bus fixture"); - const wrongOwner = runFixture(fixture, source(socket), { BUS_UID: "1003" }); + const wrongOwner = runFixture(fixture, source(socket), "bus", { BUS_UID: "1003" }); expect(wrongOwner.status).not.toBe(0); expect(wrongOwner.stderr).toContain("Qualification systemd user bus fixture"); } finally { @@ -321,12 +373,12 @@ trusted_user_unit_path=/usr/lib/systemd/user:/lib/systemd/user ${verifyUnitPath} verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/1002"`; - const trusted = runFixture(fixture, source, { + const trusted = runFixture(fixture, source, "unitPath", { SYSTEMD_ENVIRONMENT: "SYSTEMD_UNIT_PATH=/usr/lib/systemd/user:/lib/systemd/user", }); expect(trusted.status, trusted.stderr).toBe(0); - const candidateWritable = runFixture(fixture, source, { + const candidateWritable = runFixture(fixture, source, "unitPath", { SYSTEMD_ENVIRONMENT: "SYSTEMD_UNIT_PATH=/home/nemoclawq/.config/systemd/user:/usr/lib/systemd/user:/lib/systemd/user", }); @@ -346,7 +398,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ ? "nemoclawq:x:1007:\n" : "nemoclawq:200000:65536\n", ); - const result = runFixture(fixture, provisionBlock()); + const result = runFixture(fixture, provisionBlock(), "provision"); expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("useradd:"); expect(fs.existsSync(fixture.marker)).toBe(false); @@ -355,7 +407,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ it("does not publish ownership when account creation fails", () => { const fixture = createFixture(); - const result = runFixture(fixture, provisionBlock(), { FAIL_USERADD: "1" }); + const result = runFixture(fixture, provisionBlock(), "provision", { FAIL_USERADD: "1" }); expect(result.status).toBe(23); expect(fs.existsSync(fixture.marker)).toBe(false); expect(fs.readFileSync(fixture.passwd, "utf8")).toBe(""); @@ -363,7 +415,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ it("records the exact private group when its numeric GID differs from the UID", () => { const fixture = createFixture(); - const result = runFixture(fixture, provisionBlock()); + const result = runFixture(fixture, provisionBlock(), "provision"); expect(result.status, result.stderr).toBe(0); expect(fs.readFileSync(fixture.marker, "utf8")).toBe("nemoclawq:1002:1007\n"); expect(fs.readFileSync(fixture.calls, "utf8")).toContain( @@ -373,7 +425,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ it("rolls back an account when identity validation fails before marker publication", () => { const fixture = createFixture(); - const result = runFixture(fixture, provisionBlock(), { FAIL_ID: "1" }); + const result = runFixture(fixture, provisionBlock(), "provision", { FAIL_ID: "1" }); expect(result.status).not.toBe(0); expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); @@ -389,6 +441,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ const validResult = runFixture( valid, `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + "subuid", ); expect(validResult.status, validResult.stderr).toBe(0); expect(fs.readFileSync(valid.calls, "utf8")).not.toContain("usermod:"); @@ -398,6 +451,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ const overlappingResult = runFixture( overlapping, `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + "subuid", ); expect(overlappingResult.status, overlappingResult.stderr).toBe(0); expect(fs.readFileSync(overlapping.subuid, "utf8")).toContain("nemoclawq:231072:65536"); @@ -410,6 +464,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ const result = runFixture( fixture, `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subgid --add-subgids`, + "subgid", ); expect(result.status).not.toBe(0); expect(result.stderr).toContain("no free subordinate-ID range"); @@ -425,7 +480,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); - const result = runFixture(fixture, workflowScripts().cleanup); + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); expect(result.status, result.stderr).toBe(0); expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); @@ -435,7 +490,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ expect(fs.existsSync(fixture.marker)).toBe(false); }); - it("removes the run-owned runtime, immutable helpers, GPU resources, and AppArmor profiles", () => { + it("removes the run-owned runtime, helper copies, GPU resources, and AppArmor profiles", () => { const fixture = createFixture(); const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); @@ -494,7 +549,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); - const result = runFixture(fixture, workflowScripts().cleanup); + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); expect(result.status, result.stderr).toBe(0); const calls = fs.readFileSync(fixture.calls, "utf8"); expect(calls).toContain("systemctl:stop user@1002.service user-runtime-dir@1002.service"); @@ -510,7 +565,9 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ it("does not run destructive cleanup when the run-owned marker is absent", () => { const fixture = createFixture(); - const result = runFixture(fixture, workflowScripts().cleanup, { ACCOUNT_CREATED: "" }); + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup", { + ACCOUNT_CREATED: "", + }); expect(result.status).not.toBe(0); expect(result.stderr).toContain("output exists without its ownership marker"); const calls = fs.readFileSync(fixture.calls, "utf8"); @@ -524,7 +581,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ fs.writeFileSync(fixture.group, "nemoclawq:x:1008:\n"); fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); - const result = runFixture(fixture, workflowScripts().cleanup); + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); expect(result.status).not.toBe(0); expect(result.stderr).toContain("private group identity changed before cleanup"); expect(fs.readFileSync(fixture.calls, "utf8")).not.toMatch( diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts index 16eb99e01de..e16649ac1bf 100644 --- a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -241,7 +241,7 @@ describe("native runtime qualification producer aggregate", () => { }, ); - it("preserves an existing immutable aggregate output", AGGREGATE_TEST_OPTIONS, () => { + it("rejects replacing an existing aggregate output", AGGREGATE_TEST_OPTIONS, () => { const value = fixture(); const sentinel = path.join(value.evidenceDirectory, "sentinel.txt"); fs.mkdirSync(value.evidenceDirectory); From f78d0028bd37cfa7cf27951f94517aad51d10e5c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 14:01:53 -0500 Subject: [PATCH 54/71] fix(uninstall): retire portable config directories Signed-off-by: Aaron Erickson --- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 394 +++++++++++++++++- .github/workflows/e2e.yaml | 2 +- .../portable-uninstall-retirement.test.ts | 150 ++++++- .../state/portable-uninstall-retirement.ts | 139 +++++- test/e2e/README.md | 10 +- test/e2e/docs/README.md | 22 +- ...me-qualification-producer-workflow.test.ts | 2 +- ...ad-e2e-artifacts-workflow-boundary.test.ts | 2 +- ...upload-e2e-artifacts-workflow-boundary.mts | 2 +- 9 files changed, 690 insertions(+), 33 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 7ad7c422274..d8f0f9c588e 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-e2e -description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the current PR head commit. Use for requests such as run E2E for PR #123, run the E2E suite, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. +description: Dispatches and verifies trusted or administrator-authorized GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the current PR head commit. Use for requests such as run E2E for PR #123, run native runtime qualification, run the E2E suite, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. --- @@ -8,7 +8,9 @@ description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw mai # Run Maintainer E2E -Use `.github/workflows/e2e.yaml` from trusted `main`. +Use `.github/workflows/e2e.yaml` from trusted `main` except for the narrow +administrator-only native runtime PR source-branch workflow path documented +below. Each push to `main` selects catalogue targets and retained workflow jobs that own changed files. Each trusted push also selects the CPU-only `jetson-nvmap-gpu` proof. Push runs skip `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` because push events cannot set the required workflow dispatch flag. @@ -20,7 +22,10 @@ Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly ## Manual PR E2E Use this mode when the maintainer requests E2E for a pull request. -It runs an authorized E2E selection against the current PR head commit while the workflow definition remains on `main`. +It normally runs an authorized E2E selection against the current PR head commit +while the workflow definition remains on `main`. The native runtime producer +also accepts the administrator-only source-branch workflow path below for an +open same-repository PR. It is advisory and does not create a required PR check. An empty-selector manual run exposes these values to candidate-controlled job processes: @@ -51,15 +56,19 @@ Resolve the current PR and trusted workflow identities: set -euo pipefail PR_NUMBER=123 git fetch --prune origin main -WORKFLOW_SHA="$(git rev-parse origin/main)" +MAIN_WORKFLOW_SHA="$(git rev-parse origin/main)" PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" test "$(jq -r .state <<<"$PR_JSON")" = open HEAD_SHA="$(jq -r .head.sha <<<"$PR_JSON")" BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" HEAD_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" +HEAD_REF="$(jq -r .head.ref <<<"$PR_JSON")" +test "$(jq -r .base.ref <<<"$PR_JSON")" = main +test "$(jq -r .base.repo.full_name <<<"$PR_JSON")" = NVIDIA/NemoClaw [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] -[[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$MAIN_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] +test -n "$HEAD_REF" ``` Require a review reason containing 10 to 500 printable characters. @@ -73,7 +82,17 @@ Choose exactly one mode: The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. - For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. The trusted workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. +- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. Choose either the trusted `main` workflow at the exact PR-recorded base commit or, after a repository administrator authorizes the exact candidate workflow commit, the PR source-branch workflow at the exact candidate commit. The workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. + +For the administrator-authorized source-branch path, record the authorization and +select it explicitly before running the dispatch block: + +```bash +NATIVE_RUNTIME_WORKFLOW_MODE=administrator-source-branch +``` + +Use `NATIVE_RUNTIME_WORKFLOW_MODE=trusted-main` for the trusted `main` native +runtime path. Do not set either value for another job selector. Leave `targets` empty and keep Launchable disabled: @@ -83,6 +102,28 @@ case "$E2E_JOBS" in "" | managed-image-protected-runtime | native-runtime-qualification-producer) ;; *) echo "Unsupported manual PR E2E job selector" >&2; exit 1 ;; esac +WORKFLOW_REF=main +WORKFLOW_SHA="$MAIN_WORKFLOW_SHA" +if [[ "$E2E_JOBS" == "native-runtime-qualification-producer" ]]; then + case "${NATIVE_RUNTIME_WORKFLOW_MODE:-trusted-main}" in + trusted-main) + test "$MAIN_WORKFLOW_SHA" = "$BASE_SHA" || { + echo "Trusted-main native runtime qualification requires origin/main to equal the PR-recorded base SHA" >&2 + exit 1 + } + WORKFLOW_SHA="$BASE_SHA" + ;; + administrator-source-branch) + test "$HEAD_REPOSITORY" = "NVIDIA/NemoClaw" || { + echo "Administrator source-branch qualification requires a same-repository PR" >&2 + exit 1 + } + WORKFLOW_REF="$HEAD_REF" + WORKFLOW_SHA="$HEAD_SHA" + ;; + *) echo "Unsupported native runtime workflow mode" >&2; exit 1 ;; + esac +fi REVIEW_REASON='Reviewed the commit under review and selected E2E boundary.' CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" INFERENCE_MODE=mock @@ -90,7 +131,7 @@ ALLOW_JETSON_DISPATCH=false ALLOW_DGX_SPARK_RUNNER_QUEUE=false gh workflow run .github/workflows/e2e.yaml \ --repo NVIDIA/NemoClaw \ - --ref main \ + --ref "$WORKFLOW_REF" \ -f targets= \ -f "jobs=${E2E_JOBS}" \ -f "inference_mode=${INFERENCE_MODE}" \ @@ -106,11 +147,26 @@ gh workflow run .github/workflows/e2e.yaml \ -f "correlation_id=${CORRELATION_ID}" ``` -The trusted pre-checkout step requires current `maintain` or `admin` permission. -It validates the actor, open PR, repository, head SHA, base SHA, workflow SHA, review reason, and allowed jobs, targets, and Launchable combination. +The trusted `main` pre-checkout path requires current `maintain` or `admin` +permission. The native runtime PR source-branch workflow path requires `admin` +permission for the actor and, when different, the triggering actor. Both paths +validate the actor, open PR, repository, head SHA, base SHA, workflow SHA, review +reason, and allowed jobs, targets, and Launchable combination. A second validation after checkout rejects a changed PR identity before preparation. -The native-runtime producer binds the open PR, candidate commit, base commit, trusted workflow commit, and first workflow attempt. It runs the trusted plan from `main` and passes no GitHub, model-provider, API, or messaging credentials to candidate code. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU case also requires `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. +The native-runtime producer binds the open PR, candidate commit, base commit, +executing workflow commit, and first workflow attempt. The administrator-only +path executes candidate workflow code, so review and authorize that exact commit +before dispatch. Runner-only privileged preparation receives `NVIDIA_API_KEY` +only to pull pinned GPU images, then deletes its registry authentication file and +unsets the key before downloading pinned public model files, installing candidate +dependencies, or executing tests. The unprivileged installer and live-test +processes run with `env -i`, receive no GitHub, model-provider, API, or messaging +credential, and run with Docker unavailable. Configure +`NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU +cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides +no fallback runner. The qualification neither registers nor selects production +Podman and does not establish public Podman support. The producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command before candidate execution. It runs the candidate case under a temporary unprivileged account and uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. @@ -121,7 +177,7 @@ RUN_TITLE="E2E PR #${PR_NUMBER} (${CORRELATION_ID})" MATCHES='[]' for POLL_INDEX in $(seq 1 30); do RUNS="$(gh run list --repo NVIDIA/NemoClaw --workflow e2e.yaml \ - --event workflow_dispatch --branch main --limit 50 \ + --event workflow_dispatch --branch "$WORKFLOW_REF" --limit 50 \ --json databaseId,displayTitle,url)" MATCHES="$(jq -c --arg title "$RUN_TITLE" \ '[.[] | select(.displayTitle == $title)]' <<<"$RUNS")" @@ -133,13 +189,21 @@ if test "$(jq 'length' <<<"$MATCHES")" -ne 1; then echo 'The dispatched run was not visible after bounded polling. Do not dispatch again. Inspect the E2E Actions runs for the recorded correlation ID and clean up any resources from a matching run.' >&2 exit 1 fi -RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES") +RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES")" RUN_URL="$(jq -r '.[0].url' <<<"$MATCHES")" gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status RUN_JSON="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" -jq -e --arg sha "$WORKFLOW_SHA" ' - .run_attempt == 1 and +jq -e \ + --argjson runId "$RUN_ID" \ + --arg branch "$WORKFLOW_REF" \ + --arg sha "$WORKFLOW_SHA" ' + .id == $runId and + .event == "workflow_dispatch" and .head_sha == $sha and + .head_branch == $branch and + .path == ".github/workflows/e2e.yaml" and + .repository.full_name == "NVIDIA/NemoClaw" and + .run_attempt == 1 and .status == "completed" and .conclusion == "success" ' <<<"$RUN_JSON" >/dev/null @@ -148,9 +212,309 @@ test "$(jq -r .state <<<"$CURRENT_PR")" = open test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" test "$(jq -r .base.sha <<<"$CURRENT_PR")" = "$BASE_SHA" test "$(jq -r .head.repo.full_name <<<"$CURRENT_PR")" = "$HEAD_REPOSITORY" +test "$(jq -r .head.ref <<<"$CURRENT_PR")" = "$HEAD_REF" +test "$(jq -r .base.ref <<<"$CURRENT_PR")" = main +test "$(jq -r .base.repo.full_name <<<"$CURRENT_PR")" = NVIDIA/NemoClaw +``` + +For native runtime qualification, workflow success is not sufficient. Resolve +the exact aggregate job and artifact, verify the downloaded archive digest and +safe file inventory, then run the canonical evidence consumer from the exact +workflow checkout. This validates all 24 case identities and every declared +installer, runtime, operation, and NVIDIA CDI receipt digest. The four additional +installer identity receipts are required in each case and reported with their +downloaded SHA-256 digests. This is underlying evidence validation; it is not a +substitute for any separately required collector authority receipt. + +```bash +if [[ "$E2E_JOBS" == "native-runtime-qualification-producer" ]]; then + RUN_ATTEMPT="$(jq -er '.run_attempt | select(. == 1)' <<<"$RUN_JSON")" + JOBS_JSON="$(gh api --method GET \ + "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs" \ + -f per_page=100)" + jq -e ' + .total_count == (.jobs | length) and + .total_count >= 1 and + .total_count <= 100 + ' <<<"$JOBS_JSON" >/dev/null + AGGREGATE_JOB_ID="$(jq -er \ + --arg name 'Aggregate native runtime qualification evidence' \ + --argjson runId "$RUN_ID" \ + --argjson attempt "$RUN_ATTEMPT" \ + --arg workflowSha "$WORKFLOW_SHA" ' + [.jobs[] | select( + .name == $name and + .run_id == $runId and + .run_attempt == $attempt and + .head_sha == $workflowSha and + .status == "completed" and + .conclusion == "success" + )] | + select(length == 1) | + .[0].id + ' <<<"$JOBS_JSON")" + + ARTIFACT_NAME="native-runtime-qualification-${HEAD_SHA}" + ARTIFACTS_JSON="$(gh api --method GET \ + "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/artifacts" -f per_page=100)" + jq -e ' + .total_count == (.artifacts | length) and + .total_count >= 1 and + .total_count <= 100 + ' <<<"$ARTIFACTS_JSON" >/dev/null + ARTIFACT_JSON="$(jq -ec \ + --arg name "$ARTIFACT_NAME" \ + --argjson runId "$RUN_ID" \ + --arg workflowSha "$WORKFLOW_SHA" ' + [.artifacts[] | select( + .name == $name and + .expired == false and + (.size_in_bytes | type) == "number" and + .size_in_bytes >= 1 and + .size_in_bytes <= 4194304 and + .workflow_run.id == $runId and + .workflow_run.head_sha == $workflowSha and + (.digest | test("^sha256:[a-f0-9]{64}$")) + )] | + select(length == 1) | + .[0] + ' <<<"$ARTIFACTS_JSON")" + ARTIFACT_ID="$(jq -er '.id | select(type == "number" and . >= 1)' <<<"$ARTIFACT_JSON")" + ARTIFACT_DIGEST="$(jq -er '.digest' <<<"$ARTIFACT_JSON")" + ARTIFACT_SIZE="$(jq -er '.size_in_bytes' <<<"$ARTIFACT_JSON")" + + EVIDENCE_DIR="$(mktemp -d)" + chmod 700 "$EVIDENCE_DIR" + trap 'rm -rf "$EVIDENCE_DIR"' EXIT + ARCHIVE_PATH="$EVIDENCE_DIR/native-runtime-qualification.zip" + export ARCHIVE_PATH ARTIFACT_ID + node --input-type=module <<'DOWNLOAD' +import fs from "node:fs"; +import { spawnSync } from "node:child_process"; + +const limit = 4 * 1024 * 1024; +const artifactId = process.env.ARTIFACT_ID; +const archivePath = process.env.ARCHIVE_PATH; +if (!artifactId || !archivePath) throw new Error("Artifact download identity is missing"); +const result = spawnSync( + "gh", + ["api", `repos/NVIDIA/NemoClaw/actions/artifacts/${artifactId}/zip`], + { encoding: null, maxBuffer: limit, timeout: 120_000 }, +); +if ( + result.error || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.length < 1 || + result.stdout.length > limit +) { + throw new Error("Bounded aggregate artifact download failed"); +} +fs.writeFileSync(archivePath, result.stdout, { flag: "wx", mode: 0o600 }); +DOWNLOAD + DOWNLOADED_SIZE="$(wc -c <"$ARCHIVE_PATH" | tr -d '[:space:]')" + [[ "$DOWNLOADED_SIZE" =~ ^[1-9][0-9]*$ ]] && + (( DOWNLOADED_SIZE <= 4194304 )) + ACTUAL_ARCHIVE_DIGEST="sha256:$(shasum -a 256 "$ARCHIVE_PATH" | awk '{print $1}')" + test "$ACTUAL_ARCHIVE_DIGEST" = "$ARTIFACT_DIGEST" + + CONFIRMED_ARTIFACT="$(gh api "repos/NVIDIA/NemoClaw/actions/artifacts/${ARTIFACT_ID}")" + jq -e \ + --argjson id "$ARTIFACT_ID" \ + --arg name "$ARTIFACT_NAME" \ + --arg digest "$ARTIFACT_DIGEST" \ + --argjson size "$ARTIFACT_SIZE" \ + --argjson runId "$RUN_ID" \ + --arg workflowSha "$WORKFLOW_SHA" ' + .id == $id and + .name == $name and + .digest == $digest and + .size_in_bytes == $size and + .expired == false and + .workflow_run.id == $runId and + .workflow_run.head_sha == $workflowSha + ' <<<"$CONFIRMED_ARTIFACT" >/dev/null + + test "$(git rev-parse HEAD)" = "$WORKFLOW_SHA" + test -z "$(git status --porcelain=v1 --untracked-files=all)" + export ARCHIVE_PATH ARTIFACT_DIGEST ARTIFACT_ID ARTIFACT_NAME ARTIFACT_SIZE + export AGGREGATE_JOB_ID BASE_SHA HEAD_REPOSITORY HEAD_SHA PR_NUMBER + export RUN_ATTEMPT RUN_ID WORKFLOW_SHA + node --experimental-strip-types --no-warnings --input-type=module <<'NODE' +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import { + listValidatedArtifactZipEntries, + readValidatedArtifactZipEntryBytes, +} from "./scripts/scorecard/read-artifact-zip.mts"; +import { + consumeNativeRuntimeQualificationEvidence, + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, +} from "./test/e2e/registry/native-runtime-qualification.ts"; + +const required = (name) => { + const value = process.env[name]; + if (!value) throw new Error(`Missing ${name}`); + return value; +}; +const descriptor = fs.openSync( + required("ARCHIVE_PATH"), + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, +); +let archive; +try { + const before = fs.fstatSync(descriptor); + if (!before.isFile() || before.size < 1 || before.size > 4 * 1024 * 1024) { + throw new Error("Aggregate artifact archive is oversized or invalid"); + } + archive = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + archive.length !== before.size + ) { + throw new Error("Aggregate artifact archive changed while reading"); + } +} finally { + fs.closeSync(descriptor); +} +const actualArchiveDigest = + `sha256:${createHash("sha256").update(archive).digest("hex")}`; +if (actualArchiveDigest !== required("ARTIFACT_DIGEST")) { + throw new Error("Aggregate artifact digest does not match the consumed bytes"); +} +const entries = listValidatedArtifactZipEntries(archive, { maxEntries: 512 }); +if (!entries) throw new Error("Aggregate artifact ZIP structure is invalid"); +const readReceipt = (receiptPath) => + readValidatedArtifactZipEntryBytes(archive, receiptPath, { + maxBytes: 524_288, + maxEntries: 512, + }); +const evidencePath = "native-runtime-qualification-evidence.json"; +const evidenceBytes = readReceipt(evidencePath); +if (!evidenceBytes) throw new Error("Aggregate evidence envelope is missing"); +const evidence = JSON.parse(evidenceBytes.toString("utf8")); +const expectedSource = { + repository: "NVIDIA/NemoClaw", + workflow: ".github/workflows/e2e.yaml", + pullRequestNumber: Number(required("PR_NUMBER")), + candidateRepository: required("HEAD_REPOSITORY"), + headSha: required("HEAD_SHA"), + baseRef: "main", + baseSha: required("BASE_SHA"), + runId: Number(required("RUN_ID")), + attempt: Number(required("RUN_ATTEMPT")), + jobId: Number(required("AGGREGATE_JOB_ID")), + artifact: { + id: Number(required("ARTIFACT_ID")), + name: required("ARTIFACT_NAME"), + digest: required("ARTIFACT_DIGEST"), + }, +}; +const authority = consumeNativeRuntimeQualificationEvidence( + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, + evidence, + expectedSource, + (receiptPath) => readReceipt(receiptPath), +); +const installerNames = [ + "architecture.json", + "candidate-source.json", + "docker-absence.json", + "installed-source.json", + "installer.sh", + "invocation.json", +]; +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const receipt = (receiptPath) => { + const bytes = readReceipt(receiptPath); + if (!bytes) throw new Error(`Missing receipt ${receiptPath}`); + return { path: receiptPath, sha256: digest(bytes) }; +}; +const cases = evidence.cases.map((entry) => ({ + caseId: entry.caseId, + installerReceipts: installerNames.map((name) => + receipt(`receipts/${entry.caseId}/installer/${name}`), + ), + executionReceipts: [ + receipt(entry.runtime.result.path), + ...entry.operations.map(({ artifact }) => receipt(artifact.path)), + ...(entry.nvidiaCdi ? [receipt(entry.nvidiaCdi.artifact.path)] : []), + ], +})); +const expectedEntries = [ + evidencePath, + ...cases.flatMap((entry) => [ + ...entry.installerReceipts.map(({ path }) => path), + ...entry.executionReceipts.map(({ path }) => path), + ]), +].sort(); +if ( + cases.length !== 24 || + new Set(cases.map(({ caseId }) => caseId)).size !== 24 || + JSON.stringify(entries) !== JSON.stringify(expectedEntries) +) { + throw new Error("Aggregate artifact does not contain the exact 24-case receipt cohort"); +} +console.log(JSON.stringify({ + caseCount: cases.length, + workflowSha: required("WORKFLOW_SHA"), + authority: authority.source, + cases, +}, null, 2)); +NODE + + CONFIRMED_PR="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" + test "$(jq -r .state <<<"$CONFIRMED_PR")" = open + test "$(jq -r .head.sha <<<"$CONFIRMED_PR")" = "$HEAD_SHA" + test "$(jq -r .base.sha <<<"$CONFIRMED_PR")" = "$BASE_SHA" + test "$(jq -r .head.repo.full_name <<<"$CONFIRMED_PR")" = "$HEAD_REPOSITORY" + test "$(jq -r .head.ref <<<"$CONFIRMED_PR")" = "$HEAD_REF" + test "$(jq -r .base.ref <<<"$CONFIRMED_PR")" = main + test "$(jq -r .base.repo.full_name <<<"$CONFIRMED_PR")" = NVIDIA/NemoClaw + CONFIRMED_RUN="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" + jq -e \ + --argjson runId "$RUN_ID" \ + --argjson attempt "$RUN_ATTEMPT" \ + --arg branch "$WORKFLOW_REF" \ + --arg sha "$WORKFLOW_SHA" ' + .id == $runId and + .event == "workflow_dispatch" and + .head_sha == $sha and + .head_branch == $branch and + .path == ".github/workflows/e2e.yaml" and + .repository.full_name == "NVIDIA/NemoClaw" and + .run_attempt == $attempt and + .status == "completed" and + .conclusion == "success" + ' <<<"$CONFIRMED_RUN" >/dev/null + CONFIRMED_ARTIFACT="$(gh api "repos/NVIDIA/NemoClaw/actions/artifacts/${ARTIFACT_ID}")" + jq -e \ + --argjson id "$ARTIFACT_ID" \ + --arg name "$ARTIFACT_NAME" \ + --arg digest "$ARTIFACT_DIGEST" \ + --argjson size "$ARTIFACT_SIZE" \ + --argjson runId "$RUN_ID" \ + --arg workflowSha "$WORKFLOW_SHA" ' + .id == $id and + .name == $name and + .digest == $digest and + .size_in_bytes == $size and + .expired == false and + .workflow_run.id == $runId and + .workflow_run.head_sha == $workflowSha + ' <<<"$CONFIRMED_ARTIFACT" >/dev/null +fi ``` -Return the PR number, head repository, head SHA, base SHA, workflow SHA, correlation ID, workflow URL, and result. +Return the PR number, head repository, head SHA, base SHA, workflow ref and SHA, +correlation ID, run ID, run attempt, workflow URL, and result. For native runtime +qualification, also return the aggregate job ID, artifact ID/name/digest, the +24 case IDs, and each case's installer and execution receipt paths and SHA-256 +digests. A changed head repository, head SHA, or base SHA invalidates the evidence and requires a new run. ## Select the Main Mode diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 712dd80dc4a..dcf17c9540a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2531,7 +2531,7 @@ jobs: node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-aggregate.mts - - name: Upload the immutable aggregate evidence + - name: Upload aggregate evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: native-runtime-qualification-${{ inputs.checkout_sha }} diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts index 448d4338804..bb856eae790 100644 --- a/src/lib/state/portable-uninstall-retirement.test.ts +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -173,6 +173,8 @@ describe("portable uninstall retirement state", () => { expect( [test.config, test.receipt, test.registryFile].every((target) => !fs.existsSync(target)), ).toBe(true); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(false); expect(inspectPortableRetirementRecovery(test.homeDir)).toEqual({ artifacts: [], fixedState: "1000", @@ -182,6 +184,144 @@ describe("portable uninstall retirement state", () => { expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); }); + it.each([ + [ + "portable configuration", + (test: Fixture) => path.join(path.dirname(test.config), "kept.conf"), + true, + ], + [ + "NemoClaw configuration", + (test: Fixture) => path.join(path.dirname(path.dirname(test.config)), "kept.conf"), + false, + ], + ])( + "preserves unrelated %s content during retirement (#9189)", + (_label, markerPath, portableDirectoryRemains) => { + const test = fixture(); + const marker = markerPath(test); + fs.writeFileSync(marker, "operator-owned\n", { mode: 0o600 }); + + publishAndRetirePortableEvidence(prepareFixture(test)); + + expect(fs.existsSync(test.config)).toBe(false); + expect(fs.readFileSync(marker, "utf8")).toBe("operator-owned\n"); + expect(fs.existsSync(path.dirname(test.config))).toBe(portableDirectoryRemains); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(true); + }, + ); + + it("rejects a symlink that replaces the portable configuration directory (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const outside = path.join(test.homeDir, "outside"); + fs.mkdirSync(outside, { mode: 0o700 }); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + unlink(target); + if (String(target).includes(".containers.conf.portable-uninstall-")) { + fs.rmdirSync(portableDir); + fs.symlinkSync(outside, portableDir, "dir"); + } + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); + expect(fs.lstatSync(portableDir).isSymbolicLink()).toBe(true); + expect(fs.statSync(outside).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects a symlink that replaces the NemoClaw configuration directory (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const configDir = path.dirname(portableDir); + const outside = path.join(test.homeDir, "outside"); + fs.mkdirSync(path.join(outside, "portable"), { mode: 0o700, recursive: true }); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + unlink(target); + if (String(target).includes(".containers.conf.portable-uninstall-")) { + fs.rmdirSync(portableDir); + fs.rmdirSync(configDir); + fs.symlinkSync(outside, configDir, "dir"); + } + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); + expect(fs.lstatSync(configDir).isSymbolicLink()).toBe(true); + expect(fs.statSync(path.join(outside, "portable")).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects group-writable portable configuration authority (#9189)", () => { + const test = fixture(); + fs.chmodSync(path.dirname(test.config), 0o770); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/Unsafe/); + expect(fs.existsSync(path.dirname(test.config))).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects portable configuration ownership drift (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const lstat = fs.lstatSync.bind(fs); + vi.spyOn(fs, "lstatSync").mockImplementation(((target, options) => { + const stat = lstat(target, options as never); + if (String(target) !== portableDir || typeof stat.uid !== "bigint") return stat; + return new Proxy(stat, { + get(current, property) { + if (property === "uid") return current.uid + 1n; + const value = Reflect.get(current, property, current) as unknown; + return typeof value === "function" ? value.bind(current) : value; + }, + }); + }) as typeof fs.lstatSync); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/Unsafe/); + expect(fs.existsSync(portableDir)).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("preserves an entry inserted before empty-directory removal (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const marker = path.join(portableDir, "concurrent.conf"); + const rmdir = fs.rmdirSync.bind(fs); + let inserted = false; + vi.spyOn(fs, "rmdirSync").mockImplementation((target) => { + if (!inserted && String(target) === portableDir) { + inserted = true; + fs.writeFileSync(marker, "concurrent\n", { mode: 0o600 }); + } + return rmdir(target); + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).not.toThrow(); + expect(fs.readFileSync(marker, "utf8")).toBe("concurrent\n"); + expect(fs.existsSync(portableDir)).toBe(true); + }); + + it("rejects portable configuration directory replacement between identity checks (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const readdir = fs.readdirSync.bind(fs); + let replaced = false; + vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => { + const entries = readdir(target, options as never); + if (!replaced && String(target) === portableDir) { + replaced = true; + fs.rmdirSync(portableDir); + fs.mkdirSync(portableDir, { mode: 0o700 }); + } + return entries; + }) as typeof fs.readdirSync); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/changed/); + expect(fs.statSync(portableDir).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + it("rejects non-UTF-8 retirement and authority JSON before cleanup (#9189)", () => { const malformedRecord = fixture(); fs.writeFileSync(retirementRecordPath(malformedRecord), Buffer.from([0xff]), { mode: 0o600 }); @@ -385,7 +525,13 @@ describe("portable uninstall retirement state", () => { }; retirement.publishAndRetirePortableEvidence(prepared); `; - const boundaries = { fsyncSync: 10, linkSync: 1, renameSync: 4, unlinkSync: 4 } as const; + const boundaries = { + fsyncSync: 12, + linkSync: 1, + renameSync: 4, + rmdirSync: 2, + unlinkSync: 4, + } as const; const cases = Object.entries(boundaries).flatMap(([operation, count]) => Array.from({ length: count }, (_value, index) => [operation, index + 1] as const), ); @@ -413,6 +559,8 @@ describe("portable uninstall retirement state", () => { const assertRecovered = () => { resumePortableEvidenceRetirement(test.homeDir); expect(targets.every((target) => !fs.existsSync(target))).toBe(true); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(false); }; const assertPrior = () => expect(targets.every(fs.existsSync)).toBe(true); (hasPortableRetirementRecord(test.homeDir) ? assertRecovered : assertPrior)(); diff --git a/src/lib/state/portable-uninstall-retirement.ts b/src/lib/state/portable-uninstall-retirement.ts index 11d46d4029c..81d763be97f 100644 --- a/src/lib/state/portable-uninstall-retirement.ts +++ b/src/lib/state/portable-uninstall-retirement.ts @@ -180,7 +180,10 @@ const paths = (homeDir: string) => [Key in keyof typeof NAMES]: string; }; function fsyncDirectory(directory: string): void { - const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + const descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_DIRECTORY, + ); try { fs.fsyncSync(descriptor); } finally { @@ -593,6 +596,127 @@ function detachDelete( throw new Error(`Portable uninstall survivor changed: ${survivor.path}`); } } +interface OwnedDirectoryHandle { + readonly descriptor: number; + readonly identity: fs.BigIntStats; + readonly path: string; +} +function sameDirectoryIdentity(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.isDirectory() && + right.isDirectory() && + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.uid === right.uid + ); +} +function assertOwnedDirectory(handle: OwnedDirectoryHandle): void { + const descriptorStat = fs.fstatSync(handle.descriptor, { bigint: true }); + const namedStat = fs.lstatSync(handle.path, { bigint: true }); + if ( + namedStat.isSymbolicLink() || + !sameDirectoryIdentity(handle.identity, descriptorStat) || + !sameDirectoryIdentity(handle.identity, namedStat) + ) + throw new Error(`Portable uninstall directory changed: ${handle.path}`); +} +function openOwnedDirectory( + directory: string, + required: boolean, + requirePrivateMode = false, +): OwnedDirectoryHandle | null { + let descriptor: number | null = null; + try { + descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_DIRECTORY, + ); + const identity = fs.fstatSync(descriptor, { bigint: true }); + const named = fs.lstatSync(directory, { bigint: true }); + const uid = process.getuid?.(); + const permissions = identity.mode & 0o777n; + if ( + uid === undefined || + named.isSymbolicLink() || + !sameStat(identity, named) || + identity.uid !== BigInt(uid) || + identity.nlink < 1n || + (requirePrivateMode ? permissions !== 0o700n : (permissions & 0o022n) !== 0n) + ) + throw new Error(`Unsafe portable uninstall directory: ${directory}`); + return { descriptor, identity, path: directory }; + } catch (error) { + if (descriptor !== null) fs.closeSync(descriptor); + if (isErrnoException(error) && error.code === "ENOENT" && !required) return null; + throw error; + } +} +function ownedDirectoryEntries(handle: OwnedDirectoryHandle): string[] { + assertOwnedDirectory(handle); + const entries = fs.readdirSync(handle.path).sort(); + if (entries.length > MAX_DIRECTORY_ENTRIES) + throw new Error(`Portable uninstall directory has too many entries: ${handle.path}`); + assertOwnedDirectory(handle); + return entries; +} +function removeOwnedEmptyDirectory( + handle: OwnedDirectoryHandle, + parent: OwnedDirectoryHandle, +): boolean { + if (ownedDirectoryEntries(handle).length > 0) return false; + assertOwnedDirectory(parent); + assertOwnedDirectory(handle); + try { + fs.rmdirSync(handle.path); + } catch (error) { + if (isErrnoException(error) && (error.code === "ENOTEMPTY" || error.code === "EEXIST")) { + assertOwnedDirectory(parent); + assertOwnedDirectory(handle); + return false; + } + if (!(isErrnoException(error) && error.code === "ENOENT")) throw error; + } + fs.fsyncSync(parent.descriptor); + assertOwnedDirectory(parent); + if (entryExists(handle.path)) + throw new Error(`Portable uninstall directory was replaced: ${handle.path}`); + return true; +} +function removeRetiredPortableConfigDirectories(homeDir: string): void { + const home = openOwnedDirectory(homeDir, true)!; + const configHomePath = path.join(homeDir, ".config"); + const configDir = path.join(homeDir, ".config", "nemoclaw"); + let configHome: OwnedDirectoryHandle | null = null; + let nemoclawConfig: OwnedDirectoryHandle | null = null; + let portableConfig: OwnedDirectoryHandle | null = null; + try { + configHome = openOwnedDirectory(configHomePath, true)!; + assertOwnedDirectory(home); + nemoclawConfig = openOwnedDirectory(configDir, false); + assertOwnedDirectory(configHome); + assertOwnedDirectory(home); + if (!nemoclawConfig) { + fs.fsyncSync(configHome.descriptor); + assertOwnedDirectory(configHome); + return; + } + portableConfig = openOwnedDirectory(path.join(configDir, "portable"), false, true); + assertOwnedDirectory(nemoclawConfig); + assertOwnedDirectory(configHome); + if (portableConfig && !removeOwnedEmptyDirectory(portableConfig, nemoclawConfig)) return; + assertOwnedDirectory(nemoclawConfig); + if (ownedDirectoryEntries(nemoclawConfig).length > 0) return; + removeOwnedEmptyDirectory(nemoclawConfig, configHome); + assertOwnedDirectory(configHome); + assertOwnedDirectory(home); + } finally { + if (portableConfig) fs.closeSync(portableConfig.descriptor); + if (nemoclawConfig) fs.closeSync(nemoclawConfig.descriptor); + if (configHome) fs.closeSync(configHome.descriptor); + fs.closeSync(home.descriptor); + } +} function recoverTemp(homeDir: string): ExactFile | null { const state = paths(homeDir); const pendingPaths = [state.T, state.TC].filter(entryExists); @@ -654,7 +778,10 @@ function retireTargets( const target = record.targets[index]!; const state = states[index]!; if (state === "retired") { - fsyncDirectory(path.dirname(canonical(homeDir, target))); + const parent = path.dirname(canonical(homeDir, target)); + if (target[0] === "config" && !entryExists(parent)) + fsyncDirectory(path.join(homeDir, ".config")); + else fsyncDirectory(parent); continue; } const source = canonical(homeDir, target); @@ -669,11 +796,15 @@ function retireTargets( detachDelete(replacement ? null : source, staged, exact, roleLimit(target[0]), undefined, true); } } +function finishPortableEvidenceRetirement(homeDir: string, record: RecordState): void { + retireTargets(homeDir, record); + removeRetiredPortableConfigDirectories(homeDir); +} export function publishAndRetirePortableEvidence(prepared: PreparedPortableRetirement): void { if (!allTargetsExact(prepared.homeDir, prepared.record)) throw new Error("Portable uninstall authority changed before publication"); publish(prepared); - retireTargets(prepared.homeDir, prepared.record); + finishPortableEvidenceRetirement(prepared.homeDir, prepared.record); } function load(homeDir: string): { file: ExactFile; record: RecordState } | null { @@ -727,7 +858,7 @@ export function resumePortableOnboardReplacementEvidence(homeDir: string): void export function resumePortableEvidenceRetirement(homeDir: string): void { const recorded = load(homeDir); if (!recorded) throw new Error("Portable uninstall retirement record is missing"); - retireTargets(homeDir, recorded.record); + finishPortableEvidenceRetirement(homeDir, recorded.record); } function durableFile(target: string): ExactFile { diff --git a/test/e2e/README.md b/test/e2e/README.md index 21a64236126..91f92d03b20 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1068,11 +1068,13 @@ After a failure, inspect the workflow artifacts and remove resources that target For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary, cohort-owned NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. Before starting NIM or vLLM, the live fixture rejects a pre-existing cohort container name. It records the full container ID, requested image, immutable image ID, cohort owner, and provider label, then removes only that exact container after revalidating every field. Missing, ambiguous, name-reused, drifted, or indeterminate cleanup evidence fails the test, as does any retained exact ID or name. A fail-closed refusal can leave the secret-bearing NIM container alive until runner teardown; inspect the redacted artifacts and remove only the verified container. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. -For `native-runtime-qualification-producer`, use a same-repository open PR and the first workflow attempt. The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, and plan from `main`. It passes no GitHub, model-provider, API, or messaging credentials to candidate code. Candidate execution uses `env -i` under a temporary unprivileged account on a reviewed ephemeral runner. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU case also requires `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Until that executor and the required runner capacity exist, the producer fails closed instead of claiming qualification. +For `native-runtime-qualification-producer`, use a same-repository open PR and the first workflow attempt. A trusted `main` workflow dispatch requires the executing workflow commit to equal the exact PR-recorded base commit. The administrator-only PR source-branch workflow path instead requires the PR source branch as the workflow ref and the same latest PR commit SHA for the candidate and workflow. The actor must have repository `admin` permission. If `github.triggering_actor` differs from the actor, it must also have repository `admin` permission. This path executes candidate workflow code; review and authorize that exact commit before dispatch. + +Both paths bind the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The runner-only privileged preparation step receives the repository `NVIDIA_API_KEY` secret only to pull pinned GPU images, then deletes its registry authentication file and unsets the key before downloading pinned public model files, installing candidate dependencies, or executing tests. The unprivileged installer and live-test processes run with `env -i` under a temporary account, receive no GitHub, model-provider, API, or messaging credential, and run with Docker unavailable. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Each case uploads installer and execution receipts. The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. This qualification does not register or select Podman in production and does not establish public Podman support. Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. It uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. Dispatch a new run after recovery. -For a manual PR run, provide the current PR number, lowercase 40-character candidate commit SHA, PR source repository, lowercase 40-character base commit SHA, trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. +For a manual PR run, provide the current PR number, lowercase 40-character candidate commit SHA, PR source repository, lowercase 40-character base commit SHA, exact executing workflow SHA, and a review reason containing 10 to 500 printable characters. For a trusted `main` native runtime producer run, the executing workflow SHA and `workflow_sha` input must both equal the PR-recorded base SHA. For an administrator-only native runtime PR source-branch workflow run, the executing workflow SHA and `workflow_sha` input must both equal the candidate commit SHA, and the dispatch must use the PR source branch. Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. @@ -1084,11 +1086,11 @@ The exact candidate must contain `ci/protected-managed-image-multiarch-activatio To select native-runtime qualification evidence production, set `jobs=native-runtime-qualification-producer`. Leave `targets` empty and keep `include_staging_brev_launchable=false`. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -The trusted pre-checkout step requires current `maintain` or `admin` permission and validates the exact open PR and selected mode before candidate code runs. +A trusted `main` workflow pre-checkout step requires current `maintain` or `admin` permission. The native runtime PR source-branch workflow path requires `admin` permission for the actor and any different triggering actor. Both paths validate the exact open PR and selected mode before candidate code runs. A second validation after checkout rejects a changed candidate commit, base commit, or PR source repository before preparation. The Actions run is advisory for the pull request and is not a required merge context. -Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and trusted workflow SHA. +Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and executing workflow SHA. A changed PR source repository, candidate commit SHA, or base commit SHA invalidates the evidence and requires a new manual run. The platform-evidence workflow runs on configured pushes to `main` and supports manual dispatch for branch diagnosis. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 52c58e83ff2..150d019421d 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -257,8 +257,17 @@ test/e2e/ A maintainer can also dispatch the trusted `main` workflow against the latest commit from an open internal or fork PR. The manual path validates the actor, PR number, PR source repository, candidate commit SHA, base commit SHA, - workflow SHA, review reason, and - allowed jobs, targets, and Launchable combination before candidate checkout. + workflow SHA, review reason, and allowed jobs, targets, and Launchable + combination before candidate checkout. + A trusted `main` native runtime producer run requires the executing workflow + commit and `workflow_sha` input to equal the exact PR-recorded base commit. + The `native-runtime-qualification-producer` job also accepts an + administrator-only PR source-branch workflow dispatch for a same-repository + PR. That path requires the first workflow attempt, the PR source branch as + the workflow ref, and the same latest PR commit SHA for `checkout_sha` and + `workflow_sha`. + It executes candidate workflow code, not trusted `main` workflow code. Review + and authorize that exact candidate commit before dispatch. For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`, every catalogue target in the @@ -271,9 +280,12 @@ test/e2e/ this default selection. If the DGX Spark flag is `true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. - Accepted nonempty `jobs` values are `inference-routing` and - `managed-image-protected-runtime`. The `jetson-nvmap-gpu` target is also - accepted when `allow_jetson_dispatch` is `true`. + Accepted nonempty `jobs` values are `inference-routing`, + `managed-image-protected-runtime`, and + `native-runtime-qualification-producer`. Only the native runtime producer + accepts the administrator-only PR source-branch workflow path. The + `jetson-nvmap-gpu` target is also accepted when + `allow_jetson_dispatch` is `true`. Refer to [NemoClaw E2E CI](../README.md). - [Jetson dispatch controller](jetson-dispatch.md) defines the NemoClaw-owned diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index da85946dffa..4b8024b5ee4 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -565,7 +565,7 @@ describe("native runtime qualification producer workflow", () => { const identity = step(aggregate, "Resolve this aggregate job identity"); const setupNode = step(aggregate, "Set up Node for qualification aggregation"); const collect = step(aggregate, "Validate and aggregate all 24 case receipts"); - const upload = step(aggregate, "Upload the immutable aggregate evidence"); + const upload = step(aggregate, "Upload aggregate evidence"); const aggregateCheckout = step(aggregate, "Check out the trusted qualification aggregator"); expect(aggregate.name).toBe("Aggregate native runtime qualification evidence"); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 8c6efb1cf60..1f8c128b35e 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -153,7 +153,7 @@ describe("E2E artifact uploads", () => { it("allows only the exact 30-day native runtime aggregate upload", () => { const workflow = mutableWorkflow(); const upload = workflow.jobs["native-runtime-qualification-producer-aggregate"].steps?.find( - (step) => step.name === "Upload the immutable aggregate evidence", + (step) => step.name === "Upload aggregate evidence", ); expect(upload).toBeDefined(); upload!.with!["retention-days"] = 14; diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 4879d3f4f67..c5b02b050f1 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -53,7 +53,7 @@ const RELEASE_QUALIFICATION_WAIVER_UPLOAD_CONTRACT: WorkflowStep = { }, }; const NATIVE_RUNTIME_AGGREGATE_UPLOAD_CONTRACT: WorkflowStep = { - name: "Upload the immutable aggregate evidence", + name: "Upload aggregate evidence", uses: UPLOAD_ARTIFACT_ACTION, with: { name: "native-runtime-qualification-${{ inputs.checkout_sha }}", From 8a7043138f38129609437cb886161c09a74e393b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 14:25:43 -0500 Subject: [PATCH 55/71] test(uninstall): satisfy conditional guardrail Signed-off-by: Aaron Erickson --- .../portable-uninstall-retirement.test.ts | 65 +++++++++++-------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts index bb856eae790..bd34cc4dfa2 100644 --- a/src/lib/state/portable-uninstall-retirement.test.ts +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -39,6 +39,7 @@ function fixture() { type Fixture = ReturnType; type TargetRole = "config" | "receipt" | "registry"; +const noMutation = (): void => undefined; function prepareFixture(test: Fixture) { return preparePortableRetirement(test.homeDir, [RECEIPT_BASENAME]); @@ -217,12 +218,15 @@ describe("portable uninstall retirement state", () => { const outside = path.join(test.homeDir, "outside"); fs.mkdirSync(outside, { mode: 0o700 }); const unlink = fs.unlinkSync.bind(fs); + const replacePortableDirectory = () => { + fs.rmdirSync(portableDir); + fs.symlinkSync(outside, portableDir, "dir"); + }; vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { unlink(target); - if (String(target).includes(".containers.conf.portable-uninstall-")) { - fs.rmdirSync(portableDir); - fs.symlinkSync(outside, portableDir, "dir"); - } + (String(target).includes(".containers.conf.portable-uninstall-") + ? replacePortableDirectory + : noMutation)(); }); expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); @@ -238,13 +242,16 @@ describe("portable uninstall retirement state", () => { const outside = path.join(test.homeDir, "outside"); fs.mkdirSync(path.join(outside, "portable"), { mode: 0o700, recursive: true }); const unlink = fs.unlinkSync.bind(fs); + const replaceConfigDirectory = () => { + fs.rmdirSync(portableDir); + fs.rmdirSync(configDir); + fs.symlinkSync(outside, configDir, "dir"); + }; vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { unlink(target); - if (String(target).includes(".containers.conf.portable-uninstall-")) { - fs.rmdirSync(portableDir); - fs.rmdirSync(configDir); - fs.symlinkSync(outside, configDir, "dir"); - } + (String(target).includes(".containers.conf.portable-uninstall-") + ? replaceConfigDirectory + : noMutation)(); }); expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); @@ -268,14 +275,18 @@ describe("portable uninstall retirement state", () => { const lstat = fs.lstatSync.bind(fs); vi.spyOn(fs, "lstatSync").mockImplementation(((target, options) => { const stat = lstat(target, options as never); - if (String(target) !== portableDir || typeof stat.uid !== "bigint") return stat; - return new Proxy(stat, { - get(current, property) { - if (property === "uid") return current.uid + 1n; - const value = Reflect.get(current, property, current) as unknown; - return typeof value === "function" ? value.bind(current) : value; - }, - }); + return String(target) === portableDir && typeof stat.uid === "bigint" + ? new Proxy(stat, { + get(current, property) { + const value = Reflect.get(current, property, current) as unknown; + return property === "uid" + ? current.uid + 1n + : typeof value === "function" + ? value.bind(current) + : value; + }, + }) + : stat; }) as typeof fs.lstatSync); expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/Unsafe/); @@ -289,11 +300,12 @@ describe("portable uninstall retirement state", () => { const marker = path.join(portableDir, "concurrent.conf"); const rmdir = fs.rmdirSync.bind(fs); let inserted = false; + const insertMarker = () => { + inserted = true; + fs.writeFileSync(marker, "concurrent\n", { mode: 0o600 }); + }; vi.spyOn(fs, "rmdirSync").mockImplementation((target) => { - if (!inserted && String(target) === portableDir) { - inserted = true; - fs.writeFileSync(marker, "concurrent\n", { mode: 0o600 }); - } + (!inserted && String(target) === portableDir ? insertMarker : noMutation)(); return rmdir(target); }); @@ -307,13 +319,14 @@ describe("portable uninstall retirement state", () => { const portableDir = path.dirname(test.config); const readdir = fs.readdirSync.bind(fs); let replaced = false; + const replacePortableDirectory = () => { + replaced = true; + fs.rmdirSync(portableDir); + fs.mkdirSync(portableDir, { mode: 0o700 }); + }; vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => { const entries = readdir(target, options as never); - if (!replaced && String(target) === portableDir) { - replaced = true; - fs.rmdirSync(portableDir); - fs.mkdirSync(portableDir, { mode: 0o700 }); - } + (!replaced && String(target) === portableDir ? replacePortableDirectory : noMutation)(); return entries; }) as typeof fs.readdirSync); From ce95d851f9a7dc0e6f581ec48d989ade7c5a1244 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 14:46:20 -0500 Subject: [PATCH 56/71] test(uninstall): align live directory retirement proof Signed-off-by: Aaron Erickson --- test/e2e/live/podman-portable-uninstall.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/podman-portable-uninstall.test.ts b/test/e2e/live/podman-portable-uninstall.test.ts index 62dd1b654c0..bb9a1f4de3f 100644 --- a/test/e2e/live/podman-portable-uninstall.test.ts +++ b/test/e2e/live/podman-portable-uninstall.test.ts @@ -334,8 +334,8 @@ test( .filter((entry) => entry.isFile() || entry.isSymbolicLink()) .map((entry) => path.join(entry.parentPath, entry.name)); expect(residualFiles).toEqual([retirementRecord]); - expect(fs.readdirSync(path.dirname(expectedContainersConf))).toEqual([]); - expect(fs.statSync(path.dirname(expectedContainersConf)).mode & 0o777).toBe(0o700); + expect(fs.existsSync(path.dirname(expectedContainersConf))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(expectedContainersConf)))).toBe(false); const managerEnvironment = await runCommand( shellProbe, "systemctl", From ce6248e43938316afb63ea4dbfde5702846d8b5c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 15:35:12 -0500 Subject: [PATCH 57/71] test(security): exercise YAML config boundary Signed-off-by: Aaron Erickson --- .../credential-filter-failure.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/lib/security/credential-filter-failure.test.ts b/src/lib/security/credential-filter-failure.test.ts index 3849b138471..672d4ae5d8e 100644 --- a/src/lib/security/credential-filter-failure.test.ts +++ b/src/lib/security/credential-filter-failure.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { parse as parseYaml } from "yaml"; const fsControl = vi.hoisted(() => ({ noFollowUnavailable: false, @@ -26,6 +27,7 @@ vi.mock("node:fs", async (importOriginal) => { import { sanitizeConfigFile, sanitizeEnvFile, + sanitizeYamlConfigContent, sanitizeYamlConfigFile, } from "./credential-filter.js"; @@ -66,3 +68,32 @@ describe("credential filter no-follow boundary", () => { expect(readFileSync(envPath, "utf-8")).toBe(envSource); }); }); + +describe("credential filter YAML boundary", () => { + it("sanitizes nested arrays and rejects non-object documents", () => { + const sanitized = sanitizeYamlConfigContent( + [ + "items:", + " - safe", + " - api_key: sk-secret-value-long-enough", + " enabled: true", + " count: 2", + " optional: null", + "", + ].join("\n"), + ); + + expect(parseYaml(sanitized as string)).toEqual({ + items: [ + "safe", + { + api_key: "[STRIPPED_BY_MIGRATION]", + enabled: true, + count: 2, + optional: null, + }, + ], + }); + expect(sanitizeYamlConfigContent("42\n")).toBeNull(); + }); +}); From 2dbe18bf56d6e48f44c50bce3a5bc9b304902ac3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 16:59:18 -0500 Subject: [PATCH 58/71] fix(uninstall): preserve large config directories Signed-off-by: Aaron Erickson --- .../state/portable-uninstall-retirement.test.ts | 16 ++++++++++++++++ src/lib/state/portable-uninstall-retirement.ts | 12 +++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts index bd34cc4dfa2..cc933aad846 100644 --- a/src/lib/state/portable-uninstall-retirement.test.ts +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -212,6 +212,22 @@ describe("portable uninstall retirement state", () => { }, ); + it("preserves a NemoClaw configuration directory with more than 1,024 entries (#9189)", () => { + const test = fixture(); + const prepared = prepareFixture(test); + const configDir = path.dirname(path.dirname(test.config)); + const readdir = fs.readdirSync.bind(fs); + vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => + String(target) === configDir + ? new Array(1_025).fill("operator-owned.conf") + : readdir(target, options as never)) as typeof fs.readdirSync); + + expect(() => publishAndRetirePortableEvidence(prepared)).not.toThrow(); + expect(fs.existsSync(test.config)).toBe(false); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(configDir)).toBe(true); + }); + it("rejects a symlink that replaces the portable configuration directory (#9189)", () => { const test = fixture(); const portableDir = path.dirname(test.config); diff --git a/src/lib/state/portable-uninstall-retirement.ts b/src/lib/state/portable-uninstall-retirement.ts index 81d763be97f..d0d8122443a 100644 --- a/src/lib/state/portable-uninstall-retirement.ts +++ b/src/lib/state/portable-uninstall-retirement.ts @@ -652,19 +652,17 @@ function openOwnedDirectory( throw error; } } -function ownedDirectoryEntries(handle: OwnedDirectoryHandle): string[] { +function ownedDirectoryIsEmpty(handle: OwnedDirectoryHandle): boolean { assertOwnedDirectory(handle); - const entries = fs.readdirSync(handle.path).sort(); - if (entries.length > MAX_DIRECTORY_ENTRIES) - throw new Error(`Portable uninstall directory has too many entries: ${handle.path}`); + const isEmpty = fs.readdirSync(handle.path).length === 0; assertOwnedDirectory(handle); - return entries; + return isEmpty; } function removeOwnedEmptyDirectory( handle: OwnedDirectoryHandle, parent: OwnedDirectoryHandle, ): boolean { - if (ownedDirectoryEntries(handle).length > 0) return false; + if (!ownedDirectoryIsEmpty(handle)) return false; assertOwnedDirectory(parent); assertOwnedDirectory(handle); try { @@ -706,7 +704,7 @@ function removeRetiredPortableConfigDirectories(homeDir: string): void { assertOwnedDirectory(configHome); if (portableConfig && !removeOwnedEmptyDirectory(portableConfig, nemoclawConfig)) return; assertOwnedDirectory(nemoclawConfig); - if (ownedDirectoryEntries(nemoclawConfig).length > 0) return; + if (!ownedDirectoryIsEmpty(nemoclawConfig)) return; removeOwnedEmptyDirectory(nemoclawConfig, configHome); assertOwnedDirectory(configHome); assertOwnedDirectory(home); From f16d40710c1b0a63a861d6130ff2c442cbff8912 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 16 Aug 2026 16:49:19 -0700 Subject: [PATCH 59/71] docs(uninstall): document empty directory cleanup --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 32c0f687604..6fd733af52d 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -180,6 +180,9 @@ It skips the generic Docker availability probe and all generic Docker container, It does not remove the user's Podman installation, storage, networks, or unrelated containers. It does not disable a user-managed socket. On every successful portable cleanup, the final retirement operation removes the exact portable lifecycle receipts, their matching `sandboxes.json` rows, and `~/.config/nemoclaw/portable/containers.conf`, regardless of `--destroy-user-data`. +After NemoClaw deletes `containers.conf`, it deletes `~/.config/nemoclaw/portable/` only when that directory is empty. +It then deletes `~/.config/nemoclaw/` only when that directory is empty. +It preserves unrelated entries in either directory. Other preserved user data follows the normal `--destroy-user-data` behavior. After NemoClaw releases the locks, later uninstall-plan cleanup never recursively revisits the canonical receipt, sandbox registry, or `~/.config/nemoclaw/portable/containers.conf` paths. This preserves any new lifecycle generation published after lock release. From 5b79e5aea1a451430d01ebc0a840540e9a81ef5c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 16 Aug 2026 16:49:20 -0700 Subject: [PATCH 60/71] docs(e2e): prepare exact workflow checkout --- .agents/skills/nemoclaw-maintainer-e2e/SKILL.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 59c17e1b0d0..00d4b1206de 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -22,7 +22,7 @@ Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly ## Manual PR E2E Use this mode when the maintainer requests E2E for a pull request. -It normally runs an authorized E2E selection against the current PR head commit +It normally runs an authorized E2E selection against the latest PR commit while the workflow definition remains on `main`. The native runtime producer also accepts the administrator-only source-branch workflow path below for an open same-repository PR. @@ -81,8 +81,8 @@ Choose exactly one mode: - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -- For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. Choose either the trusted `main` workflow at the exact PR-recorded base commit or, after a repository administrator authorizes the exact candidate workflow commit, the PR source-branch workflow at the exact candidate commit. The workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. +- For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The commit under review must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. +- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. Choose either the trusted `main` workflow at the PR-recorded base commit or, after a repository administrator authorizes the commit under review as the workflow commit, the PR source-branch workflow at that commit. The workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The commit under review must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. For the administrator-authorized source-branch path, record the authorization and select it explicitly before running the dispatch block: @@ -219,7 +219,7 @@ test "$(jq -r .base.repo.full_name <<<"$CURRENT_PR")" = NVIDIA/NemoClaw For native runtime qualification, workflow success is not sufficient. Resolve the exact aggregate job and artifact, verify the downloaded archive digest and -safe file inventory, then run the canonical evidence consumer from the exact +exact file inventory, then run the canonical evidence consumer from the exact workflow checkout. This validates all 24 case identities and every declared installer, runtime, operation, and NVIDIA CDI receipt digest. The four additional installer identity receipts are required in each case and reported with their @@ -335,6 +335,10 @@ DOWNLOAD .workflow_run.head_sha == $workflowSha ' <<<"$CONFIRMED_ARTIFACT" >/dev/null + test -z "$(git status --porcelain=v1 --untracked-files=all)" + git fetch --no-tags origin "$WORKFLOW_REF" + test "$(git rev-parse FETCH_HEAD)" = "$WORKFLOW_SHA" + git switch --detach "$WORKFLOW_SHA" test "$(git rev-parse HEAD)" = "$WORKFLOW_SHA" test -z "$(git status --porcelain=v1 --untracked-files=all)" export ARCHIVE_PATH ARTIFACT_DIGEST ARTIFACT_ID ARTIFACT_NAME ARTIFACT_SIZE @@ -510,12 +514,12 @@ NODE fi ``` -Return the PR number, head repository, head SHA, base SHA, workflow ref and SHA, +Return the PR number, PR source repository, latest PR commit SHA, base SHA, workflow ref and SHA, correlation ID, run ID, run attempt, workflow URL, and result. For native runtime qualification, also return the aggregate job ID, artifact ID/name/digest, the 24 case IDs, and each case's installer and execution receipt paths and SHA-256 digests. -A changed head repository, head SHA, or base SHA invalidates the evidence and requires a new run. +A changed PR source repository, latest PR commit SHA, or base SHA invalidates the evidence and requires a new run. ## Select the Main Mode From 6804f095d76b489e6a04d8af4c012f8d0971b227 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 16 Aug 2026 16:59:36 -0700 Subject: [PATCH 61/71] docs(e2e): clarify source-branch credential boundary --- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 33 +++++++++++++------ test/e2e/README.md | 10 ++++-- test/e2e/docs/README.md | 22 +++++++++++-- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 00d4b1206de..2388664f0ce 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -150,19 +150,32 @@ gh workflow run .github/workflows/e2e.yaml \ The trusted `main` pre-checkout path requires current `maintain` or `admin` permission. The native runtime PR source-branch workflow path requires `admin` permission for the actor and, when different, the triggering actor. Both paths -validate the actor, open PR, repository, head SHA, base SHA, workflow SHA, review -reason, and allowed jobs, targets, and Launchable combination. +validate the actor, open PR, repository, latest PR commit SHA, base SHA, workflow +SHA, review reason, and allowed jobs, targets, and Launchable combination. A second validation after checkout rejects a changed PR identity before preparation. The native-runtime producer binds the open PR, candidate commit, base commit, -executing workflow commit, and first workflow attempt. The administrator-only -path executes candidate workflow code, so review and authorize that exact commit -before dispatch. Runner-only privileged preparation receives `NVIDIA_API_KEY` -only to pull pinned GPU images, then deletes its registry authentication file and -unsets the key before downloading pinned public model files, installing candidate -dependencies, or executing tests. The unprivileged installer and live-test -processes run with `env -i`, receive no GitHub, model-provider, API, or messaging -credential, and run with Docker unavailable. Configure +executing workflow commit, and first workflow attempt. Candidate workflow code +controls the administrator check that the source-branch workflow runs. NemoClaw +repository policy permits only a repository administrator to dispatch this path. +The administrator check is defense in depth, not an independent authorization +boundary. Before dispatch, the administrator must review and authorize the exact +commit, including the workflow and every action or script that the commit loads. + +The candidate workflow commit can access each repository secret granted to the +workflow. The runner-only privileged preparation step receives the long-lived +`NVIDIA_API_KEY` repository secret in its environment. It uses the key +to create runner-local registry authentication and pull pinned GPU images. The +step then deletes the registry authentication and unsets the environment +variable before it downloads public model files or runs candidate code. These +actions remove runner-local access but do not revoke the key. The key remains +valid in the issuing NVIDIA service until it expires or that service revokes it. +If exposure occurs or cleanup cannot be confirmed, revoke or rotate the key in +the issuing NVIDIA service. Verify that the old value is invalid. + +The unprivileged installer and live-test processes run with `env -i`, receive +no GitHub, model-provider, API, or messaging credential, and run with Docker +unavailable. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The qualification neither registers nor selects production diff --git a/test/e2e/README.md b/test/e2e/README.md index 696f7fbb17a..81b4c692d1c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1116,7 +1116,7 @@ The PR selection does not forward an NVIDIA API key, `BRAVE_API_KEY`, or `GITHUB The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -The trusted workflow definition remains on `main` and binds the candidate head to the current PR base SHA. +The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. It does not run GitHub's synthetic merge commit. Before candidate execution, the workflow uploads a `nemoclaw-e2e-dispatch-v2` receipt for the trusted manual run. OpenShell PR qualification uses that receipt to bind the candidate repository, candidate commit SHA, base SHA, workflow SHA, run, and selectors. @@ -1148,9 +1148,13 @@ After a failure, inspect the workflow artifacts and remove resources that target For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary, cohort-owned NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. Before starting NIM or vLLM, the live fixture rejects a pre-existing cohort container name. It records the full container ID, requested image, immutable image ID, cohort owner, and provider label, then removes only that exact container after revalidating every field. Missing, ambiguous, name-reused, drifted, or indeterminate cleanup evidence fails the test, as does any retained exact ID or name. A fail-closed refusal can leave the secret-bearing NIM container alive until runner teardown; inspect the redacted artifacts and remove only the verified container. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. -For `native-runtime-qualification-producer`, use a same-repository open PR and the first workflow attempt. A trusted `main` workflow dispatch requires the executing workflow commit to equal the exact PR-recorded base commit. The administrator-only PR source-branch workflow path instead requires the PR source branch as the workflow ref and the same latest PR commit SHA for the candidate and workflow. The actor must have repository `admin` permission. If `github.triggering_actor` differs from the actor, it must also have repository `admin` permission. This path executes candidate workflow code; review and authorize that exact commit before dispatch. +For `native-runtime-qualification-producer`, use a same-repository open PR and the first workflow attempt. A trusted `main` workflow dispatch requires the executing workflow commit to equal the exact PR-recorded base commit. The administrator-only PR source-branch workflow path instead requires the PR source branch as the workflow ref and the same latest PR commit SHA for the candidate and workflow. The actor must have repository `admin` permission. If `github.triggering_actor` differs from the actor, it must also have repository `admin` permission. -Both paths bind the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The runner-only privileged preparation step receives the repository `NVIDIA_API_KEY` secret only to pull pinned GPU images, then deletes its registry authentication file and unsets the key before downloading pinned public model files, installing candidate dependencies, or executing tests. The unprivileged installer and live-test processes run with `env -i` under a temporary account, receive no GitHub, model-provider, API, or messaging credential, and run with Docker unavailable. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Each case uploads installer and execution receipts. The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. This qualification does not register or select Podman in production and does not establish public Podman support. +Candidate workflow code controls the administrator check that the source-branch workflow runs. NemoClaw repository policy permits only a repository administrator to dispatch this path. The administrator check is defense in depth, not an independent authorization boundary. Before dispatch, the administrator must review and authorize the exact commit, including the workflow and every action or script that the commit loads. + +Both paths bind the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The candidate workflow commit can access each repository secret granted to the workflow. The runner-only privileged preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in its environment. It uses the key to create runner-local registry authentication and pull pinned GPU images. The step then deletes the registry authentication and unsets the environment variable before it downloads public model files or runs candidate code. These actions remove runner-local access but do not revoke the key. The key remains valid in the issuing NVIDIA service until it expires or that service revokes it. If exposure occurs or cleanup cannot be confirmed, revoke or rotate the key in the issuing NVIDIA service. Verify that the old value is invalid. + +The unprivileged installer and live-test processes run with `env -i` under a temporary account, receive no GitHub, model-provider, API, or messaging credential, and run with Docker unavailable. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Each case uploads installer and execution receipts. The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. This qualification does not register or select Podman in production and does not establish public Podman support. Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. It uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. Dispatch a new run after recovery. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 65c14e3e765..65f47a0213f 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -294,9 +294,25 @@ test/e2e/ administrator-only PR source-branch workflow dispatch for a same-repository PR. That path requires the first workflow attempt, the PR source branch as the workflow ref, and the same latest PR commit SHA for `checkout_sha` and - `workflow_sha`. - It executes candidate workflow code, not trusted `main` workflow code. Review - and authorize that exact candidate commit before dispatch. + `workflow_sha`. Candidate workflow code controls the administrator check + that the source-branch workflow runs. NemoClaw repository policy permits only + a repository administrator to dispatch this path. The administrator check is + defense in depth, not an independent authorization boundary. Before dispatch, + the administrator must review and authorize the exact commit, including the + workflow and every action or script that the commit loads. + + The candidate workflow commit can access each repository secret granted to + the workflow. The runner-only privileged preparation step receives the + long-lived `NVIDIA_API_KEY` repository secret in its environment. It + uses the key to create runner-local registry authentication and pull pinned + GPU images. The step then deletes the registry authentication and unsets the + environment variable before it downloads public model files or runs candidate + code. These actions remove runner-local access but do not revoke the key. The + key remains valid in the issuing NVIDIA service until it expires or that + service revokes it. If exposure occurs or cleanup cannot be confirmed, revoke + or rotate the key in the issuing NVIDIA service. Verify that the old value is + invalid. + For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`, every catalogue target in the From 562528aec325136f123447a6dd10af9dd482264d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 16 Aug 2026 17:05:15 -0700 Subject: [PATCH 62/71] docs(e2e): distinguish candidate credential access --- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 32 +++++++++++-------- test/e2e/README.md | 26 +++++++++++++-- test/e2e/docs/README.md | 25 ++++++++------- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 2388664f0ce..6ad0de7b2e9 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -162,20 +162,24 @@ The administrator check is defense in depth, not an independent authorization boundary. Before dispatch, the administrator must review and authorize the exact commit, including the workflow and every action or script that the commit loads. -The candidate workflow commit can access each repository secret granted to the -workflow. The runner-only privileged preparation step receives the long-lived -`NVIDIA_API_KEY` repository secret in its environment. It uses the key -to create runner-local registry authentication and pull pinned GPU images. The -step then deletes the registry authentication and unsets the environment -variable before it downloads public model files or runs candidate code. These -actions remove runner-local access but do not revoke the key. The key remains -valid in the issuing NVIDIA service until it expires or that service revokes it. -If exposure occurs or cleanup cannot be confirmed, revoke or rotate the key in -the issuing NVIDIA service. Verify that the old value is invalid. - -The unprivileged installer and live-test processes run with `env -i`, receive -no GitHub, model-provider, API, or messaging credential, and run with Docker -unavailable. Configure +On the source-branch path, every repository secret received by the workflow +is accessible to candidate workflow code. In the reviewed workflow, the host-side +preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in +its environment. It creates runner-local registry authentication and pulls +pinned GPU images. It then deletes the registry authentication file and unsets +the variable before the separate candidate installer or live-test process +starts. The preparation step is itself candidate-controlled workflow code on +this path. It can read or copy the key before cleanup, so cleanup does not +prevent exposure. Cleanup removes runner-local registry authentication but does +not revoke the key. The key remains valid in the issuing NVIDIA service until +it expires or that service revokes it. If exposure occurs or cleanup cannot be +confirmed, revoke the key in the issuing NVIDIA service. Alternatively, rotate +the key and invalidate the old value. Verify that the old value is invalid. + +The unprivileged installer and live-test processes run with `env -i`. +They receive no GitHub, inference provider, API, or messaging credential. +Docker is unavailable to these processes. +Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The qualification neither registers nor selects production diff --git a/test/e2e/README.md b/test/e2e/README.md index 81b4c692d1c..d6f5558376d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1152,9 +1152,29 @@ For `native-runtime-qualification-producer`, use a same-repository open PR and t Candidate workflow code controls the administrator check that the source-branch workflow runs. NemoClaw repository policy permits only a repository administrator to dispatch this path. The administrator check is defense in depth, not an independent authorization boundary. Before dispatch, the administrator must review and authorize the exact commit, including the workflow and every action or script that the commit loads. -Both paths bind the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The candidate workflow commit can access each repository secret granted to the workflow. The runner-only privileged preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in its environment. It uses the key to create runner-local registry authentication and pull pinned GPU images. The step then deletes the registry authentication and unsets the environment variable before it downloads public model files or runs candidate code. These actions remove runner-local access but do not revoke the key. The key remains valid in the issuing NVIDIA service until it expires or that service revokes it. If exposure occurs or cleanup cannot be confirmed, revoke or rotate the key in the issuing NVIDIA service. Verify that the old value is invalid. - -The unprivileged installer and live-test processes run with `env -i` under a temporary account, receive no GitHub, model-provider, API, or messaging credential, and run with Docker unavailable. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Each case uploads installer and execution receipts. The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. This qualification does not register or select Podman in production and does not establish public Podman support. +Both paths bind the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. +On the source-branch path, every repository secret received by the workflow is accessible to candidate workflow code. +In the reviewed workflow, the host-side preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in its environment. +It creates runner-local registry authentication and pulls pinned GPU images. +It then deletes the registry authentication file and unsets the variable before the separate candidate installer or live-test process starts. +The preparation step is itself candidate-controlled workflow code on this path. +It can read or copy the key before cleanup, so cleanup does not prevent exposure. +Cleanup removes runner-local registry authentication but does not revoke the key. +The key remains valid in the issuing NVIDIA service until it expires or that service revokes it. +If exposure occurs or cleanup cannot be confirmed, revoke the key in the issuing NVIDIA service. +Alternatively, rotate the key and invalidate the old value. +Verify that the old value is invalid. + +The unprivileged installer and live-test processes run with `env -i` under a temporary account. +They receive no GitHub, inference provider, API, or messaging credential. +Docker is unavailable to these processes. +Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. +The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. +The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. +Each case uploads installer and execution receipts. +The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. +If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. +This qualification does not register or select Podman in production and does not establish public Podman support. Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. It uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. Dispatch a new run after recovery. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 65f47a0213f..efc7526d610 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -301,17 +301,20 @@ test/e2e/ the administrator must review and authorize the exact commit, including the workflow and every action or script that the commit loads. - The candidate workflow commit can access each repository secret granted to - the workflow. The runner-only privileged preparation step receives the - long-lived `NVIDIA_API_KEY` repository secret in its environment. It - uses the key to create runner-local registry authentication and pull pinned - GPU images. The step then deletes the registry authentication and unsets the - environment variable before it downloads public model files or runs candidate - code. These actions remove runner-local access but do not revoke the key. The - key remains valid in the issuing NVIDIA service until it expires or that - service revokes it. If exposure occurs or cleanup cannot be confirmed, revoke - or rotate the key in the issuing NVIDIA service. Verify that the old value is - invalid. + On the source-branch path, every repository secret received by the + workflow is accessible to candidate workflow code. In the reviewed workflow, + the host-side preparation step receives the long-lived `NVIDIA_API_KEY` + repository secret in its environment. It creates runner-local registry + authentication and pulls pinned GPU images. It then deletes the registry + authentication file and unsets the variable before the separate candidate + installer or live-test process starts. The preparation step is itself + candidate-controlled workflow code on this path. It can read or copy the key + before cleanup, so cleanup does not prevent exposure. Cleanup removes + runner-local registry authentication but does not revoke the key. The key + remains valid in the issuing NVIDIA service until it expires or that service + revokes it. If exposure occurs or cleanup cannot be confirmed, revoke the key + in the issuing NVIDIA service. Alternatively, rotate the key and invalidate + the old value. Verify that the old value is invalid. For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow From 2cd71fa44536e7e304f34bffb0b621c1183adda6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 18:57:36 -0500 Subject: [PATCH 63/71] test(e2e): satisfy growth guardrails Move required test iteration into named helpers. Restore the maintainer E2E skill to main so B4-G does not alter core skills. Signed-off-by: Aaron Erickson --- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 419 +----------------- ...me-qualification-account-lifecycle.test.ts | 62 +-- ...runtime-qualification-case-helpers.test.ts | 122 ++--- ...me-qualification-producer-workflow.test.ts | 56 +-- 4 files changed, 144 insertions(+), 515 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 6ad0de7b2e9..d4f9317e351 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-e2e -description: Dispatches and verifies trusted or administrator-authorized GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit and staging Launchable image publication. Use for requests such as run E2E for PR #123, run native runtime qualification, run the E2E suite, publish the Launchable image, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. +description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit and staging Launchable image publication. Use for requests such as run E2E for PR #123, run the E2E suite, publish the Launchable image, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. --- @@ -8,9 +8,7 @@ description: Dispatches and verifies trusted or administrator-authorized GitHub # Run Maintainer E2E -Use `.github/workflows/e2e.yaml` from trusted `main` except for the narrow -administrator-only native runtime PR source-branch workflow path documented -below. +Use `.github/workflows/e2e.yaml` from trusted `main`. Each push to `main` selects catalogue targets and retained workflow jobs that own changed files. Each trusted push also selects the CPU-only `jetson-nvmap-gpu` proof. Push runs skip `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` because push events cannot set the required workflow dispatch flag. @@ -22,10 +20,7 @@ Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly ## Manual PR E2E Use this mode when the maintainer requests E2E for a pull request. -It normally runs an authorized E2E selection against the latest PR commit -while the workflow definition remains on `main`. The native runtime producer -also accepts the administrator-only source-branch workflow path below for an -open same-repository PR. +It runs an authorized E2E selection against the current PR head commit while the workflow definition remains on `main`. It is advisory and does not create a required PR check. An empty-selector manual run exposes these values to candidate-controlled job processes: @@ -56,19 +51,15 @@ Resolve the current PR and trusted workflow identities: set -euo pipefail PR_NUMBER=123 git fetch --prune origin main -MAIN_WORKFLOW_SHA="$(git rev-parse origin/main)" +WORKFLOW_SHA="$(git rev-parse origin/main)" PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" test "$(jq -r .state <<<"$PR_JSON")" = open HEAD_SHA="$(jq -r .head.sha <<<"$PR_JSON")" BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" HEAD_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" -HEAD_REF="$(jq -r .head.ref <<<"$PR_JSON")" -test "$(jq -r .base.ref <<<"$PR_JSON")" = main -test "$(jq -r .base.repo.full_name <<<"$PR_JSON")" = NVIDIA/NemoClaw [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] -[[ "$MAIN_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] -test -n "$HEAD_REF" +[[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] ``` Require a review reason containing 10 to 500 printable characters. @@ -81,18 +72,8 @@ Choose exactly one mode: - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -- For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The commit under review must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. Choose either the trusted `main` workflow at the PR-recorded base commit or, after a repository administrator authorizes the commit under review as the workflow commit, the PR source-branch workflow at that commit. The workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The commit under review must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. - -For the administrator-authorized source-branch path, record the authorization and -select it explicitly before running the dispatch block: - -```bash -NATIVE_RUNTIME_WORKFLOW_MODE=administrator-source-branch -``` - -Use `NATIVE_RUNTIME_WORKFLOW_MODE=trusted-main` for the trusted `main` native -runtime path. Do not set either value for another job selector. +- For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. +- For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. The trusted workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. Leave `targets` empty and keep Launchable disabled: @@ -102,28 +83,6 @@ case "$E2E_JOBS" in "" | managed-image-protected-runtime | native-runtime-qualification-producer) ;; *) echo "Unsupported manual PR E2E job selector" >&2; exit 1 ;; esac -WORKFLOW_REF=main -WORKFLOW_SHA="$MAIN_WORKFLOW_SHA" -if [[ "$E2E_JOBS" == "native-runtime-qualification-producer" ]]; then - case "${NATIVE_RUNTIME_WORKFLOW_MODE:-trusted-main}" in - trusted-main) - test "$MAIN_WORKFLOW_SHA" = "$BASE_SHA" || { - echo "Trusted-main native runtime qualification requires origin/main to equal the PR-recorded base SHA" >&2 - exit 1 - } - WORKFLOW_SHA="$BASE_SHA" - ;; - administrator-source-branch) - test "$HEAD_REPOSITORY" = "NVIDIA/NemoClaw" || { - echo "Administrator source-branch qualification requires a same-repository PR" >&2 - exit 1 - } - WORKFLOW_REF="$HEAD_REF" - WORKFLOW_SHA="$HEAD_SHA" - ;; - *) echo "Unsupported native runtime workflow mode" >&2; exit 1 ;; - esac -fi REVIEW_REASON='Reviewed the commit under review and selected E2E boundary.' CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" INFERENCE_MODE=mock @@ -131,7 +90,7 @@ ALLOW_JETSON_DISPATCH=false ALLOW_DGX_SPARK_RUNNER_QUEUE=false gh workflow run .github/workflows/e2e.yaml \ --repo NVIDIA/NemoClaw \ - --ref "$WORKFLOW_REF" \ + --ref main \ -f targets= \ -f "jobs=${E2E_JOBS}" \ -f "inference_mode=${INFERENCE_MODE}" \ @@ -147,43 +106,11 @@ gh workflow run .github/workflows/e2e.yaml \ -f "correlation_id=${CORRELATION_ID}" ``` -The trusted `main` pre-checkout path requires current `maintain` or `admin` -permission. The native runtime PR source-branch workflow path requires `admin` -permission for the actor and, when different, the triggering actor. Both paths -validate the actor, open PR, repository, latest PR commit SHA, base SHA, workflow -SHA, review reason, and allowed jobs, targets, and Launchable combination. +The trusted pre-checkout step requires current `maintain` or `admin` permission. +It validates the actor, open PR, repository, head SHA, base SHA, workflow SHA, review reason, and allowed jobs, targets, and Launchable combination. A second validation after checkout rejects a changed PR identity before preparation. -The native-runtime producer binds the open PR, candidate commit, base commit, -executing workflow commit, and first workflow attempt. Candidate workflow code -controls the administrator check that the source-branch workflow runs. NemoClaw -repository policy permits only a repository administrator to dispatch this path. -The administrator check is defense in depth, not an independent authorization -boundary. Before dispatch, the administrator must review and authorize the exact -commit, including the workflow and every action or script that the commit loads. - -On the source-branch path, every repository secret received by the workflow -is accessible to candidate workflow code. In the reviewed workflow, the host-side -preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in -its environment. It creates runner-local registry authentication and pulls -pinned GPU images. It then deletes the registry authentication file and unsets -the variable before the separate candidate installer or live-test process -starts. The preparation step is itself candidate-controlled workflow code on -this path. It can read or copy the key before cleanup, so cleanup does not -prevent exposure. Cleanup removes runner-local registry authentication but does -not revoke the key. The key remains valid in the issuing NVIDIA service until -it expires or that service revokes it. If exposure occurs or cleanup cannot be -confirmed, revoke the key in the issuing NVIDIA service. Alternatively, rotate -the key and invalidate the old value. Verify that the old value is invalid. - -The unprivileged installer and live-test processes run with `env -i`. -They receive no GitHub, inference provider, API, or messaging credential. -Docker is unavailable to these processes. -Configure -`NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU -cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides -no fallback runner. The qualification neither registers nor selects production -Podman and does not establish public Podman support. +The native-runtime producer binds the open PR, candidate commit, base commit, trusted workflow commit, and first workflow attempt. It runs the trusted plan from `main` and passes no GitHub, model-provider, API, or messaging credentials to candidate code. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU case also requires `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command before candidate execution. It runs the candidate case under a temporary unprivileged account and uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. @@ -194,7 +121,7 @@ RUN_TITLE="E2E PR #${PR_NUMBER} (${CORRELATION_ID})" MATCHES='[]' for POLL_INDEX in $(seq 1 30); do RUNS="$(gh run list --repo NVIDIA/NemoClaw --workflow e2e.yaml \ - --event workflow_dispatch --branch "$WORKFLOW_REF" --limit 50 \ + --event workflow_dispatch --branch main --limit 50 \ --json databaseId,displayTitle,url)" MATCHES="$(jq -c --arg title "$RUN_TITLE" \ '[.[] | select(.displayTitle == $title)]' <<<"$RUNS")" @@ -206,21 +133,13 @@ if test "$(jq 'length' <<<"$MATCHES")" -ne 1; then echo 'The dispatched run was not visible after bounded polling. Do not dispatch again. Inspect the E2E Actions runs for the recorded correlation ID and clean up any resources from a matching run.' >&2 exit 1 fi -RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES")" +RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES") RUN_URL="$(jq -r '.[0].url' <<<"$MATCHES")" gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status RUN_JSON="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" -jq -e \ - --argjson runId "$RUN_ID" \ - --arg branch "$WORKFLOW_REF" \ - --arg sha "$WORKFLOW_SHA" ' - .id == $runId and - .event == "workflow_dispatch" and - .head_sha == $sha and - .head_branch == $branch and - .path == ".github/workflows/e2e.yaml" and - .repository.full_name == "NVIDIA/NemoClaw" and +jq -e --arg sha "$WORKFLOW_SHA" ' .run_attempt == 1 and + .head_sha == $sha and .status == "completed" and .conclusion == "success" ' <<<"$RUN_JSON" >/dev/null @@ -229,314 +148,10 @@ test "$(jq -r .state <<<"$CURRENT_PR")" = open test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" test "$(jq -r .base.sha <<<"$CURRENT_PR")" = "$BASE_SHA" test "$(jq -r .head.repo.full_name <<<"$CURRENT_PR")" = "$HEAD_REPOSITORY" -test "$(jq -r .head.ref <<<"$CURRENT_PR")" = "$HEAD_REF" -test "$(jq -r .base.ref <<<"$CURRENT_PR")" = main -test "$(jq -r .base.repo.full_name <<<"$CURRENT_PR")" = NVIDIA/NemoClaw -``` - -For native runtime qualification, workflow success is not sufficient. Resolve -the exact aggregate job and artifact, verify the downloaded archive digest and -exact file inventory, then run the canonical evidence consumer from the exact -workflow checkout. This validates all 24 case identities and every declared -installer, runtime, operation, and NVIDIA CDI receipt digest. The four additional -installer identity receipts are required in each case and reported with their -downloaded SHA-256 digests. This is underlying evidence validation; it is not a -substitute for any separately required collector authority receipt. - -```bash -if [[ "$E2E_JOBS" == "native-runtime-qualification-producer" ]]; then - RUN_ATTEMPT="$(jq -er '.run_attempt | select(. == 1)' <<<"$RUN_JSON")" - JOBS_JSON="$(gh api --method GET \ - "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs" \ - -f per_page=100)" - jq -e ' - .total_count == (.jobs | length) and - .total_count >= 1 and - .total_count <= 100 - ' <<<"$JOBS_JSON" >/dev/null - AGGREGATE_JOB_ID="$(jq -er \ - --arg name 'Aggregate native runtime qualification evidence' \ - --argjson runId "$RUN_ID" \ - --argjson attempt "$RUN_ATTEMPT" \ - --arg workflowSha "$WORKFLOW_SHA" ' - [.jobs[] | select( - .name == $name and - .run_id == $runId and - .run_attempt == $attempt and - .head_sha == $workflowSha and - .status == "completed" and - .conclusion == "success" - )] | - select(length == 1) | - .[0].id - ' <<<"$JOBS_JSON")" - - ARTIFACT_NAME="native-runtime-qualification-${HEAD_SHA}" - ARTIFACTS_JSON="$(gh api --method GET \ - "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/artifacts" -f per_page=100)" - jq -e ' - .total_count == (.artifacts | length) and - .total_count >= 1 and - .total_count <= 100 - ' <<<"$ARTIFACTS_JSON" >/dev/null - ARTIFACT_JSON="$(jq -ec \ - --arg name "$ARTIFACT_NAME" \ - --argjson runId "$RUN_ID" \ - --arg workflowSha "$WORKFLOW_SHA" ' - [.artifacts[] | select( - .name == $name and - .expired == false and - (.size_in_bytes | type) == "number" and - .size_in_bytes >= 1 and - .size_in_bytes <= 4194304 and - .workflow_run.id == $runId and - .workflow_run.head_sha == $workflowSha and - (.digest | test("^sha256:[a-f0-9]{64}$")) - )] | - select(length == 1) | - .[0] - ' <<<"$ARTIFACTS_JSON")" - ARTIFACT_ID="$(jq -er '.id | select(type == "number" and . >= 1)' <<<"$ARTIFACT_JSON")" - ARTIFACT_DIGEST="$(jq -er '.digest' <<<"$ARTIFACT_JSON")" - ARTIFACT_SIZE="$(jq -er '.size_in_bytes' <<<"$ARTIFACT_JSON")" - - EVIDENCE_DIR="$(mktemp -d)" - chmod 700 "$EVIDENCE_DIR" - trap 'rm -rf "$EVIDENCE_DIR"' EXIT - ARCHIVE_PATH="$EVIDENCE_DIR/native-runtime-qualification.zip" - export ARCHIVE_PATH ARTIFACT_ID - node --input-type=module <<'DOWNLOAD' -import fs from "node:fs"; -import { spawnSync } from "node:child_process"; - -const limit = 4 * 1024 * 1024; -const artifactId = process.env.ARTIFACT_ID; -const archivePath = process.env.ARCHIVE_PATH; -if (!artifactId || !archivePath) throw new Error("Artifact download identity is missing"); -const result = spawnSync( - "gh", - ["api", `repos/NVIDIA/NemoClaw/actions/artifacts/${artifactId}/zip`], - { encoding: null, maxBuffer: limit, timeout: 120_000 }, -); -if ( - result.error || - result.status !== 0 || - !Buffer.isBuffer(result.stdout) || - result.stdout.length < 1 || - result.stdout.length > limit -) { - throw new Error("Bounded aggregate artifact download failed"); -} -fs.writeFileSync(archivePath, result.stdout, { flag: "wx", mode: 0o600 }); -DOWNLOAD - DOWNLOADED_SIZE="$(wc -c <"$ARCHIVE_PATH" | tr -d '[:space:]')" - [[ "$DOWNLOADED_SIZE" =~ ^[1-9][0-9]*$ ]] && - (( DOWNLOADED_SIZE <= 4194304 )) - ACTUAL_ARCHIVE_DIGEST="sha256:$(shasum -a 256 "$ARCHIVE_PATH" | awk '{print $1}')" - test "$ACTUAL_ARCHIVE_DIGEST" = "$ARTIFACT_DIGEST" - - CONFIRMED_ARTIFACT="$(gh api "repos/NVIDIA/NemoClaw/actions/artifacts/${ARTIFACT_ID}")" - jq -e \ - --argjson id "$ARTIFACT_ID" \ - --arg name "$ARTIFACT_NAME" \ - --arg digest "$ARTIFACT_DIGEST" \ - --argjson size "$ARTIFACT_SIZE" \ - --argjson runId "$RUN_ID" \ - --arg workflowSha "$WORKFLOW_SHA" ' - .id == $id and - .name == $name and - .digest == $digest and - .size_in_bytes == $size and - .expired == false and - .workflow_run.id == $runId and - .workflow_run.head_sha == $workflowSha - ' <<<"$CONFIRMED_ARTIFACT" >/dev/null - - test -z "$(git status --porcelain=v1 --untracked-files=all)" - git fetch --no-tags origin "$WORKFLOW_REF" - test "$(git rev-parse FETCH_HEAD)" = "$WORKFLOW_SHA" - git switch --detach "$WORKFLOW_SHA" - test "$(git rev-parse HEAD)" = "$WORKFLOW_SHA" - test -z "$(git status --porcelain=v1 --untracked-files=all)" - export ARCHIVE_PATH ARTIFACT_DIGEST ARTIFACT_ID ARTIFACT_NAME ARTIFACT_SIZE - export AGGREGATE_JOB_ID BASE_SHA HEAD_REPOSITORY HEAD_SHA PR_NUMBER - export RUN_ATTEMPT RUN_ID WORKFLOW_SHA - node --experimental-strip-types --no-warnings --input-type=module <<'NODE' -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import { - listValidatedArtifactZipEntries, - readValidatedArtifactZipEntryBytes, -} from "./scripts/scorecard/read-artifact-zip.mts"; -import { - consumeNativeRuntimeQualificationEvidence, - PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, -} from "./test/e2e/registry/native-runtime-qualification.ts"; - -const required = (name) => { - const value = process.env[name]; - if (!value) throw new Error(`Missing ${name}`); - return value; -}; -const descriptor = fs.openSync( - required("ARCHIVE_PATH"), - fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, -); -let archive; -try { - const before = fs.fstatSync(descriptor); - if (!before.isFile() || before.size < 1 || before.size > 4 * 1024 * 1024) { - throw new Error("Aggregate artifact archive is oversized or invalid"); - } - archive = fs.readFileSync(descriptor); - const after = fs.fstatSync(descriptor); - if ( - before.dev !== after.dev || - before.ino !== after.ino || - before.size !== after.size || - before.mtimeMs !== after.mtimeMs || - archive.length !== before.size - ) { - throw new Error("Aggregate artifact archive changed while reading"); - } -} finally { - fs.closeSync(descriptor); -} -const actualArchiveDigest = - `sha256:${createHash("sha256").update(archive).digest("hex")}`; -if (actualArchiveDigest !== required("ARTIFACT_DIGEST")) { - throw new Error("Aggregate artifact digest does not match the consumed bytes"); -} -const entries = listValidatedArtifactZipEntries(archive, { maxEntries: 512 }); -if (!entries) throw new Error("Aggregate artifact ZIP structure is invalid"); -const readReceipt = (receiptPath) => - readValidatedArtifactZipEntryBytes(archive, receiptPath, { - maxBytes: 524_288, - maxEntries: 512, - }); -const evidencePath = "native-runtime-qualification-evidence.json"; -const evidenceBytes = readReceipt(evidencePath); -if (!evidenceBytes) throw new Error("Aggregate evidence envelope is missing"); -const evidence = JSON.parse(evidenceBytes.toString("utf8")); -const expectedSource = { - repository: "NVIDIA/NemoClaw", - workflow: ".github/workflows/e2e.yaml", - pullRequestNumber: Number(required("PR_NUMBER")), - candidateRepository: required("HEAD_REPOSITORY"), - headSha: required("HEAD_SHA"), - baseRef: "main", - baseSha: required("BASE_SHA"), - runId: Number(required("RUN_ID")), - attempt: Number(required("RUN_ATTEMPT")), - jobId: Number(required("AGGREGATE_JOB_ID")), - artifact: { - id: Number(required("ARTIFACT_ID")), - name: required("ARTIFACT_NAME"), - digest: required("ARTIFACT_DIGEST"), - }, -}; -const authority = consumeNativeRuntimeQualificationEvidence( - PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, - evidence, - expectedSource, - (receiptPath) => readReceipt(receiptPath), -); -const installerNames = [ - "architecture.json", - "candidate-source.json", - "docker-absence.json", - "installed-source.json", - "installer.sh", - "invocation.json", -]; -const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); -const receipt = (receiptPath) => { - const bytes = readReceipt(receiptPath); - if (!bytes) throw new Error(`Missing receipt ${receiptPath}`); - return { path: receiptPath, sha256: digest(bytes) }; -}; -const cases = evidence.cases.map((entry) => ({ - caseId: entry.caseId, - installerReceipts: installerNames.map((name) => - receipt(`receipts/${entry.caseId}/installer/${name}`), - ), - executionReceipts: [ - receipt(entry.runtime.result.path), - ...entry.operations.map(({ artifact }) => receipt(artifact.path)), - ...(entry.nvidiaCdi ? [receipt(entry.nvidiaCdi.artifact.path)] : []), - ], -})); -const expectedEntries = [ - evidencePath, - ...cases.flatMap((entry) => [ - ...entry.installerReceipts.map(({ path }) => path), - ...entry.executionReceipts.map(({ path }) => path), - ]), -].sort(); -if ( - cases.length !== 24 || - new Set(cases.map(({ caseId }) => caseId)).size !== 24 || - JSON.stringify(entries) !== JSON.stringify(expectedEntries) -) { - throw new Error("Aggregate artifact does not contain the exact 24-case receipt cohort"); -} -console.log(JSON.stringify({ - caseCount: cases.length, - workflowSha: required("WORKFLOW_SHA"), - authority: authority.source, - cases, -}, null, 2)); -NODE - - CONFIRMED_PR="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" - test "$(jq -r .state <<<"$CONFIRMED_PR")" = open - test "$(jq -r .head.sha <<<"$CONFIRMED_PR")" = "$HEAD_SHA" - test "$(jq -r .base.sha <<<"$CONFIRMED_PR")" = "$BASE_SHA" - test "$(jq -r .head.repo.full_name <<<"$CONFIRMED_PR")" = "$HEAD_REPOSITORY" - test "$(jq -r .head.ref <<<"$CONFIRMED_PR")" = "$HEAD_REF" - test "$(jq -r .base.ref <<<"$CONFIRMED_PR")" = main - test "$(jq -r .base.repo.full_name <<<"$CONFIRMED_PR")" = NVIDIA/NemoClaw - CONFIRMED_RUN="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" - jq -e \ - --argjson runId "$RUN_ID" \ - --argjson attempt "$RUN_ATTEMPT" \ - --arg branch "$WORKFLOW_REF" \ - --arg sha "$WORKFLOW_SHA" ' - .id == $runId and - .event == "workflow_dispatch" and - .head_sha == $sha and - .head_branch == $branch and - .path == ".github/workflows/e2e.yaml" and - .repository.full_name == "NVIDIA/NemoClaw" and - .run_attempt == $attempt and - .status == "completed" and - .conclusion == "success" - ' <<<"$CONFIRMED_RUN" >/dev/null - CONFIRMED_ARTIFACT="$(gh api "repos/NVIDIA/NemoClaw/actions/artifacts/${ARTIFACT_ID}")" - jq -e \ - --argjson id "$ARTIFACT_ID" \ - --arg name "$ARTIFACT_NAME" \ - --arg digest "$ARTIFACT_DIGEST" \ - --argjson size "$ARTIFACT_SIZE" \ - --argjson runId "$RUN_ID" \ - --arg workflowSha "$WORKFLOW_SHA" ' - .id == $id and - .name == $name and - .digest == $digest and - .size_in_bytes == $size and - .expired == false and - .workflow_run.id == $runId and - .workflow_run.head_sha == $workflowSha - ' <<<"$CONFIRMED_ARTIFACT" >/dev/null -fi ``` -Return the PR number, PR source repository, latest PR commit SHA, base SHA, workflow ref and SHA, -correlation ID, run ID, run attempt, workflow URL, and result. For native runtime -qualification, also return the aggregate job ID, artifact ID/name/digest, the -24 case IDs, and each case's installer and execution receipt paths and SHA-256 -digests. -A changed PR source repository, latest PR commit SHA, or base SHA invalidates the evidence and requires a new run. +Return the PR number, head repository, head SHA, base SHA, workflow SHA, correlation ID, workflow URL, and result. +A changed head repository, head SHA, or base SHA invalidates the evidence and requires a new run. ## Select the Main Mode diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts index b9b88ae4310..70d84f0014f 100644 --- a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -321,6 +321,39 @@ function provisionBlock(): string { return `set -euo pipefail\n${source.slice(start, end)}`; } +function expectPreExistingAccountStateRejected(): void { + for (const state of ["passwd", "group", "subuid", "subgid"] as const) { + const fixture = createFixture(); + const file = fixture[state]; + fs.appendFileSync( + file, + state === "passwd" + ? `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n` + : state === "group" + ? "nemoclawq:x:1007:\n" + : "nemoclawq:200000:65536\n", + ); + const result = runFixture(fixture, provisionBlock(), "provision"); + expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("useradd:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + } +} + +function writeModelFixtureFiles(model: string): void { + for (const file of [ + "config.json", + "generation_config.json", + "merges.txt", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + ]) { + fs.writeFileSync(path.join(model, file), "fixture", { mode: 0o444 }); + } +} + describe("native runtime qualification account lifecycle", () => { it("fails closed when a mandatory fixture rewrite no longer matches", () => { expect(() => fixtureSource("set -euo pipefail", "bus")).toThrow( @@ -387,22 +420,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ }); it("rejects pre-existing accounts and stale subordinate-ID authorization before mutation", () => { - for (const state of ["passwd", "group", "subuid", "subgid"] as const) { - const fixture = createFixture(); - const file = fixture[state]; - fs.appendFileSync( - file, - state === "passwd" - ? `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n` - : state === "group" - ? "nemoclawq:x:1007:\n" - : "nemoclawq:200000:65536\n", - ); - const result = runFixture(fixture, provisionBlock(), "provision"); - expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); - expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("useradd:"); - expect(fs.existsSync(fixture.marker)).toBe(false); - } + expectPreExistingAccountStateRejected(); }); it("does not publish ownership when account creation fails", () => { @@ -530,17 +548,7 @@ verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/ fs.writeFileSync(path.join(helpers, "pasta"), "fixture", { mode: 0o555 }); fs.chmodSync(helpers, 0o555); fs.mkdirSync(model, { recursive: true, mode: 0o755 }); - for (const file of [ - "config.json", - "generation_config.json", - "merges.txt", - "model.safetensors", - "tokenizer.json", - "tokenizer_config.json", - "vocab.json", - ]) { - fs.writeFileSync(path.join(model, file), "fixture", { mode: 0o444 }); - } + writeModelFixtureFiles(model); fs.chmodSync(model, 0o555); fs.chmodSync(resources, 0o555); fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index 8e339baef2d..5ef5c5df4e0 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -67,6 +67,71 @@ function runnerContract() { } as const; } +function expectInvalidPodmanExecutablesRejected(): void { + for (const executable of [ + "/usr/local/bin/podman", + "/nemoclaw-native-runtime-podman-123456-1-0", + "/nemoclaw-native-runtime-podman-123456-1-1003", + "/nemoclaw-native-runtime-podman-123456-1-1002/../podman", + ]) { + expect(() => + nativeRuntimeQualificationPodmanExecutable( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: executable }, + 1002, + ), + ).toThrow("Podman executable path is invalid"); + } +} + +function expectCredentialEnvironmentNamesRejected(): void { + for (const name of [ + "GITHUB_TOKEN", + "NGC_API_KEY", + "HF_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "SSH_AUTH_SOCK", + "DOCKER_CONFIG", + "DOCKER_HOST", + "CUSTOM_API_KEY", + ]) { + expect(() => assertCredentialFreeQualificationEnvironment({ [name]: "forbidden" })).toThrow( + name, + ); + } +} + +function expectInvalidRunnerContractPathsRejected(): void { + for (const file of [ + "/etc/nemoclaw/native-runtime-qualification-v1.json", + "/run/nemoclaw-native-runtime-123456-1-1003/runner-contract.json", + "/run/nemoclaw-native-runtime-123456-1-1002/../runner-contract.json", + ]) { + expect(() => + nativeRuntimeQualificationRunnerContractPath( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: file }, + 1002, + ), + ).toThrow("runner contract path is invalid"); + } +} + +function expectPublicCaseImagesPinned(): void { + for (const architecture of ["amd64", "arm64"] as const) { + for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + expect(nativeRuntimeQualificationAgentImage(architecture, agent)).toMatch( + /@sha256:[a-f0-9]{64}$/u, + ); + } + const ollama = nativeRuntimeQualificationInferenceImage({ + architecture, + acceleration: "cpu", + inference: "ollama", + }); + expect(ollama).toMatchObject({ model: "qwen3:0.6b" }); + expect(digestFromImageReference(ollama.imageRef)).toMatch(/^sha256:[a-f0-9]{64}$/u); + } +} + describe("native runtime qualification case boundaries", () => { it("accepts only an exact canonical trusted-plan row", () => { const expected = row(); @@ -98,19 +163,7 @@ describe("native runtime qualification case boundaries", () => { expect(nativeRuntimeQualificationPodmanExecutable(environment, 1002)).toBe( environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, ); - for (const executable of [ - "/usr/local/bin/podman", - "/nemoclaw-native-runtime-podman-123456-1-0", - "/nemoclaw-native-runtime-podman-123456-1-1003", - "/nemoclaw-native-runtime-podman-123456-1-1002/../podman", - ]) { - expect(() => - nativeRuntimeQualificationPodmanExecutable( - { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: executable }, - 1002, - ), - ).toThrow("Podman executable path is invalid"); - } + expectInvalidPodmanExecutablesRejected(); }); it("rejects credential and alternate runtime authority environment names", () => { @@ -120,20 +173,7 @@ describe("native runtime qualification case boundaries", () => { PATH: "/usr/bin", }), ).not.toThrow(); - for (const name of [ - "GITHUB_TOKEN", - "NGC_API_KEY", - "HF_TOKEN", - "AWS_SECRET_ACCESS_KEY", - "SSH_AUTH_SOCK", - "DOCKER_CONFIG", - "DOCKER_HOST", - "CUSTOM_API_KEY", - ]) { - expect(() => assertCredentialFreeQualificationEnvironment({ [name]: "forbidden" })).toThrow( - name, - ); - } + expectCredentialEnvironmentNamesRejected(); }); it("accepts only typed immutable GPU runner resources", () => { @@ -169,35 +209,11 @@ describe("native runtime qualification case boundaries", () => { expect(nativeRuntimeQualificationRunnerContractPath(environment, 1002)).toBe( environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT, ); - for (const file of [ - "/etc/nemoclaw/native-runtime-qualification-v1.json", - "/run/nemoclaw-native-runtime-123456-1-1003/runner-contract.json", - "/run/nemoclaw-native-runtime-123456-1-1002/../runner-contract.json", - ]) { - expect(() => - nativeRuntimeQualificationRunnerContractPath( - { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: file }, - 1002, - ), - ).toThrow("runner contract path is invalid"); - } + expectInvalidRunnerContractPathsRejected(); }); it("pins every public case image to architecture-specific immutable digests", () => { - for (const architecture of ["amd64", "arm64"] as const) { - for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { - expect(nativeRuntimeQualificationAgentImage(architecture, agent)).toMatch( - /@sha256:[a-f0-9]{64}$/u, - ); - } - const ollama = nativeRuntimeQualificationInferenceImage({ - architecture, - acceleration: "cpu", - inference: "ollama", - }); - expect(ollama).toMatchObject({ model: "qwen3:0.6b" }); - expect(digestFromImageReference(ollama.imageRef)).toMatch(/^sha256:[a-f0-9]{64}$/u); - } + expectPublicCaseImagesPinned(); }); it("requires the root-owned typed contract for NIM and vLLM", () => { diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 4b8024b5ee4..df4c26fbe1d 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -26,6 +26,20 @@ function step(owner: WorkflowJob, name: string): WorkflowStep { return value!; } +function expectRequiredPodmanPackages(run: string): void { + for (const requiredPackage of [ + "acl", + "apparmor", + "conmon", + "golang-github-containers-common", + "runc", + "slirp4netns", + "uidmap", + ]) { + expect(run).toContain(requiredPackage); + } +} + describe("native runtime qualification producer workflow", () => { it("keeps candidate execution out of the authenticated controller", () => { const generate = job("generate-matrix"); @@ -172,7 +186,7 @@ describe("native runtime qualification producer workflow", () => { ); expect(build.env?.PASTA_SOURCE_SHA).toBe("f8df3f1b228fe19a74a269334fdfe6cc7d0605ce"); expect(build.env?.PASTA_VERSION).toBe("2026_07_28.f8df3f1"); - expect(build.run).toContain("sha256sum \"$pasta_source_archive\""); + expect(build.run).toContain('sha256sum "$pasta_source_archive"'); expect(build.run).toContain("--no-same-owner --no-same-permissions"); expect(build.run).toContain("[[ ! -e .passt-source/passt && ! -L .passt-source/passt ]]"); expect(build.run).not.toMatch(/\bgit\s+fetch\b/u); @@ -259,9 +273,7 @@ describe("native runtime qualification producer workflow", () => { expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); expect(gpuResources.run).not.toContain("logout --all"); expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"'); - expect(gpuResources.run).toContain( - "$(sudo stat -c '%u:%g:%h' -- \"$registry_auth_file\")", - ); + expect(gpuResources.run).toContain("$(sudo stat -c '%u:%g:%h' -- \"$registry_auth_file\")"); expect(gpuResources.run).toContain('sudo chmod 0600 -- "$registry_auth_file"'); expect(gpuResources.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); expect(gpuResources.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); @@ -301,17 +313,7 @@ describe("native runtime qualification producer workflow", () => { path: "${{ runner.temp }}/native-runtime-podman-toolchain", }); expect(podman.run).toContain("/usr/bin/apt-get install"); - for (const requiredPackage of [ - "acl", - "apparmor", - "conmon", - "golang-github-containers-common", - "runc", - "slirp4netns", - "uidmap", - ]) { - expect(podman.run).toContain(requiredPackage); - } + expectRequiredPodmanPackages(podman.run ?? ""); expect(podman.run).not.toMatch(/\s+passt(?:\s|$)/u); expect(podman.run).not.toContain("fuse-overlayfs"); expect(podman.run).toContain("find -P"); @@ -356,14 +358,10 @@ describe("native runtime qualification producer workflow", () => { expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); - expect(boundary.run).toContain( - 'sudo -u "$execution_account" /usr/bin/test -S "$bus"', - ); + expect(boundary.run).toContain('sudo -u "$execution_account" /usr/bin/test -S "$bus"'); expect(boundary.run).toContain('sudo /usr/bin/test ! -L "$bus"'); expect(boundary.run).toContain("sudo stat -c '%u' -- \"$bus\""); - expect(boundary.run).toContain( - "Qualification systemd user bus $context", - ); + expect(boundary.run).toContain("Qualification systemd user bus $context"); expect(boundary.run).toContain( 'trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"', ); @@ -391,9 +389,7 @@ describe("native runtime qualification producer workflow", () => { "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined)", ); expect(boundary.run).toContain("Run-owned qualification pasta executable digest changed"); - expect(boundary.run).toContain( - '"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"', - ); + expect(boundary.run).toContain('"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"'); expect(boundary.run).not.toContain("/usr/bin/pasta"); expect(boundary.run).toContain( 'PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin"', @@ -439,20 +435,14 @@ describe("native runtime qualification producer workflow", () => { expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); expect(installer.run).toContain("/usr/bin/systemctl --user start dbus.socket"); expect(installer.run).toContain("Qualification systemd user bus socket unit did not restart"); - expect(installer.run).toContain( - 'sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus"', - ); + expect(installer.run).toContain('sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus"'); expect(installer.run).toContain('sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus"'); expect(installer.run).toContain("sudo stat -c '%u' -- \"$RUNTIME_DIRECTORY/bus\""); expect(installer.run).toContain( "Qualification systemd user bus is invalid or inaccessible after installer isolation", ); - expect(installer.env?.TRUSTED_USER_UNIT_PATH).toBe( - "/usr/lib/systemd/user:/lib/systemd/user", - ); - expect(installer.run).toContain( - "/usr/bin/systemctl --user show-environment", - ); + expect(installer.env?.TRUSTED_USER_UNIT_PATH).toBe("/usr/lib/systemd/user:/lib/systemd/user"); + expect(installer.run).toContain("/usr/bin/systemctl --user show-environment"); expect(installer.env?.CONTAINERS_CONFIG).toBe( "${{ steps.boundary.outputs.containers_config }}", ); From f256d07fc6f3e077fc7d610a73c0847b6fd7de2a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 19:10:02 -0500 Subject: [PATCH 64/71] test(e2e): clarify candidate workflow authority Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 22 +++++++++---------- .../e2e-operations-workflow-boundary.test.ts | 2 +- ...me-qualification-producer-workflow.test.ts | 16 +++++++------- tools/e2e/operations-workflow-boundary.mts | 10 ++++----- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c3ca23cd007..db5f1094a83 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -891,7 +891,7 @@ jobs: outputs: matrix: ${{ steps.plan.outputs.matrix }} steps: - - name: Check out the trusted qualification producer + - name: Check out the qualification producer uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} @@ -911,7 +911,7 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Set up Node for trusted qualification planning + - name: Set up Node for qualification planning uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 @@ -980,7 +980,7 @@ jobs: select((.workflow_run.id | tostring) == $runId) | select(.workflow_run.head_sha == $workflowSha) ' <<<"$artifacts")" || { - echo "::error::Trusted dispatch artifact is missing or ambiguous" >&2 + echo "::error::Authenticated dispatch artifact is missing or ambiguous" >&2 exit 1 } installer_sha256="$(sha256sum .candidate-source/scripts/install.sh | awk '{print $1}')" @@ -995,7 +995,7 @@ jobs: printf 'installer_sha256=%s\n' "$installer_sha256" >>"$GITHUB_OUTPUT" - id: plan - name: Compile the trusted qualification producer matrix + name: Compile the qualification producer matrix env: BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} @@ -1240,11 +1240,11 @@ jobs: exit 1 fi - - name: Check out the trusted qualification harness + - name: Check out the qualification harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.source.workflowSha }} - path: .trusted-qualification + path: .qualification-workflow persist-credentials: false sparse-checkout: | src/lib/onboard/runtime-provider/native-qualification-authority.ts @@ -2038,7 +2038,7 @@ jobs: LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ - bash .trusted-qualification/scripts/checks/run-native-runtime-installer-qualification.sh \ + bash .qualification-workflow/scripts/checks/run-native-runtime-installer-qualification.sh \ --candidate-checkout "$CANDIDATE_DIRECTORY" \ --candidate-sha "$CANDIDATE_SHA" \ --installer-sha256 "$INSTALLER_SHA256" \ @@ -2171,7 +2171,7 @@ jobs: set -euo pipefail sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ - .trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts + .qualification-workflow/tools/e2e/native-runtime-qualification-producer-evidence.mts sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources @@ -2452,12 +2452,12 @@ jobs: contents: read pull-requests: read steps: - - name: Check out the trusted qualification aggregator + - name: Check out the qualification aggregator uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} - path: .trusted-qualification-aggregate + path: .qualification-aggregate persist-credentials: false sparse-checkout: | src/lib/onboard/runtime-provider/native-qualification-authority.ts @@ -2521,7 +2521,7 @@ jobs: node-version: 22.19.0 - name: Validate and aggregate all 24 case receipts - working-directory: .trusted-qualification-aggregate + working-directory: .qualification-aggregate env: AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }} CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 54f46eb72a7..04761d3062d 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -497,7 +497,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ["maintain", "refs/heads/feat/native", 1, "requires a repository administrator"], ["admin", "refs/heads/feat/other", 1, "must match the PR source branch"], ])( - "requires the latest commit on an admin-controlled PR source branch for %s on %s", + "requires the latest commit on an administrator-authorized PR source branch for %s on %s", (role, workflowRef, expectedStatus, expectedStderr) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index df4c26fbe1d..a65af53ce5c 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -66,7 +66,7 @@ describe("native runtime qualification producer workflow", () => { it("compiles the matrix only from authenticated source and repository-owned runner policy", () => { const plan = job("native-runtime-qualification-producer-plan"); const authenticate = step(plan, "Authenticate the candidate and dispatch artifact"); - const compile = step(plan, "Compile the trusted qualification producer matrix"); + const compile = step(plan, "Compile the qualification producer matrix"); expect(plan.if).toContain("github.ref == 'refs/heads/main'"); expect(plan.if).toContain("inputs.jobs == 'native-runtime-qualification-producer'"); @@ -89,13 +89,13 @@ describe("native runtime qualification producer workflow", () => { ); expect(compile.run).toContain("native-runtime-qualification-producer-plan.mts --ci-output"); expect(JSON.stringify(plan)).not.toContain("linux-arm64-gpu-dgx-spark-gb10-protected-1"); - const trustedCheckout = step(plan, "Check out the trusted qualification producer"); - expect(trustedCheckout.with?.["sparse-checkout"]).toContain( + const producerCheckout = step(plan, "Check out the qualification producer"); + expect(producerCheckout.with?.["sparse-checkout"]).toContain( "src/lib/onboard/runtime-provider/native-qualification-authority.ts", ); }); - it("limits candidate-workflow protected execution to the latest commit on an administrator-controlled PR source branch", () => { + it("limits candidate-workflow protected execution to the latest commit on an administrator-authorized PR source branch", () => { const generate = job("generate-matrix"); const authenticate = step(generate, "Authenticate manual PR dispatch"); const source = authenticate.run ?? ""; @@ -217,9 +217,9 @@ describe("native runtime qualification producer workflow", () => { ); }); - it("runs each candidate case in an isolated account and emits one trusted artifact", () => { + it("runs each candidate case in an isolated account and emits one bounded evidence artifact", () => { const producer = job("native-runtime-qualification-producer"); - const harness = step(producer, "Check out the trusted qualification harness"); + const harness = step(producer, "Check out the qualification harness"); const podmanHost = step(producer, "Require a reviewed Ubuntu runtime host"); const podmanDownload = step(producer, "Download the pinned native Podman toolchain"); const podman = step( @@ -549,14 +549,14 @@ describe("native runtime qualification producer workflow", () => { expect(accountOwnershipSource).not.toContain("$ACCOUNT:$ACCOUNT"); }); - it("aggregates the exact successful 24-case cohort in a separate trusted job", () => { + it("aggregates the exact successful 24-case cohort in a separate workflow job", () => { const aggregate = job("native-runtime-qualification-producer-aggregate"); const download = step(aggregate, "Download the exact case evidence cohort"); const identity = step(aggregate, "Resolve this aggregate job identity"); const setupNode = step(aggregate, "Set up Node for qualification aggregation"); const collect = step(aggregate, "Validate and aggregate all 24 case receipts"); const upload = step(aggregate, "Upload aggregate evidence"); - const aggregateCheckout = step(aggregate, "Check out the trusted qualification aggregator"); + const aggregateCheckout = step(aggregate, "Check out the qualification aggregator"); expect(aggregate.name).toBe("Aggregate native runtime qualification evidence"); expect(aggregate.needs).toEqual([ diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 63de9ccc362..6609d90f52d 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -489,7 +489,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.repository === "${{ github.repository }}" && step.with?.ref === "${{ inputs.workflow_sha || github.workflow_sha }}" && step.with?.path === ".trusted-openshell-dev-artifact"; - const trustedNativeRuntimeCheckout = + const nativeRuntimeQualificationCheckout = (jobName === "native-runtime-qualification-podman-toolchain" && step.name === "Check out the pinned Podman source" && step.with?.repository === "podman-container-tools/podman" && @@ -512,17 +512,17 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.["fetch-depth"] === 1 && step.with?.["persist-credentials"] === false) || (jobName === "native-runtime-qualification-producer-plan" && - step.name === "Check out the trusted qualification producer" && + step.name === "Check out the qualification producer" && step.with?.ref === "${{ github.workflow_sha }}") || (jobName === "native-runtime-qualification-producer" && - step.name === "Check out the trusted qualification harness" && + step.name === "Check out the qualification harness" && step.with?.ref === "${{ matrix.source.workflowSha }}") || (jobName === "native-runtime-qualification-producer" && step.name === "Check out the candidate commit" && step.with?.repository === "${{ matrix.source.candidateRepository }}" && step.with?.ref === "${{ matrix.source.candidateSha }}") || (jobName === "native-runtime-qualification-producer-aggregate" && - step.name === "Check out the trusted qualification aggregator" && + step.name === "Check out the qualification aggregator" && step.with?.repository === "${{ github.repository }}" && step.with?.ref === "${{ github.workflow_sha }}"); const trustedCheckout = @@ -536,7 +536,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow trustedLlamaCppPlanCheckout || trustedLlamaCppQualificationCheckout || trustedJetsonControllerCheckout || - trustedNativeRuntimeCheckout || + nativeRuntimeQualificationCheckout || trustedOpenShellDevToolingCheckout; if ( step.uses?.startsWith("actions/checkout@") && From 504fcf718a8ece560c021c5ed4656851ef419e84 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 19:57:03 -0500 Subject: [PATCH 65/71] test(security): preserve credential coverage ratchet --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 3 --- src/lib/security/credential-filter-failure.test.ts | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 6fd733af52d..32c0f687604 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -180,9 +180,6 @@ It skips the generic Docker availability probe and all generic Docker container, It does not remove the user's Podman installation, storage, networks, or unrelated containers. It does not disable a user-managed socket. On every successful portable cleanup, the final retirement operation removes the exact portable lifecycle receipts, their matching `sandboxes.json` rows, and `~/.config/nemoclaw/portable/containers.conf`, regardless of `--destroy-user-data`. -After NemoClaw deletes `containers.conf`, it deletes `~/.config/nemoclaw/portable/` only when that directory is empty. -It then deletes `~/.config/nemoclaw/` only when that directory is empty. -It preserves unrelated entries in either directory. Other preserved user data follows the normal `--destroy-user-data` behavior. After NemoClaw releases the locks, later uninstall-plan cleanup never recursively revisits the canonical receipt, sandbox registry, or `~/.config/nemoclaw/portable/containers.conf` paths. This preserves any new lifecycle generation published after lock release. diff --git a/src/lib/security/credential-filter-failure.test.ts b/src/lib/security/credential-filter-failure.test.ts index 672d4ae5d8e..2d8362ac7f0 100644 --- a/src/lib/security/credential-filter-failure.test.ts +++ b/src/lib/security/credential-filter-failure.test.ts @@ -58,6 +58,11 @@ describe("credential filter no-follow boundary", () => { writeFileSync(jsonPath, jsonSource); writeFileSync(yamlPath, yamlSource); writeFileSync(envPath, envSource); + + expect(sanitizeEnvFile(envPath)).toBe(true); + expect(readFileSync(envPath, "utf-8")).toBe("API_KEY=[STRIPPED_BY_MIGRATION]\n"); + writeFileSync(envPath, envSource); + fsControl.noFollowUnavailable = true; expect(sanitizeConfigFile(jsonPath)).toBe(false); From 883474458d1afd540a42c011c652395da768d343 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 21:23:02 -0500 Subject: [PATCH 66/71] fix(e2e): reject candidate qualification rows --- ...ative-runtime-qualification-case-helpers.ts | 2 +- ...-runtime-qualification-case-helpers.test.ts | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts index c50d928f784..a476ed87a64 100644 --- a/test/e2e/live/native-runtime-qualification-case-helpers.ts +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -213,7 +213,7 @@ export function parseNativeRuntimeQualificationRow( typeof source.baseSha !== "string" || !SHA.test(source.baseSha) || source.candidateSha === source.baseSha || - (source.workflowSha !== source.baseSha && source.workflowSha !== source.candidateSha) || + source.workflowSha !== source.baseSha || !/^[1-9][0-9]{0,19}$/u.test(String(source.producerRunId)) || source.producerRunAttempt !== 1 ) { diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts index 5ef5c5df4e0..e94edbe8efa 100644 --- a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -144,15 +144,15 @@ describe("native runtime qualification case boundaries", () => { ); }); - it("accepts an exact administrator-authorized candidate workflow row", () => { - const candidateSource = { ...SOURCE, workflowSha: SOURCE.candidateSha }; - const candidateRow = buildNativeRuntimeQualificationProducerPlan({ - source: candidateSource, - installerSha256: "d".repeat(64), - arm64GpuRunner: "native-arm64-gpu", - } satisfies NativeRuntimeQualificationProducerPlanInput).include[0]!; - - expect(parseNativeRuntimeQualificationRow(JSON.stringify(candidateRow))).toEqual(candidateRow); + it("rejects candidate workflow authority in a forged row", () => { + const candidateRow = JSON.parse(JSON.stringify(row())) as { + source: { candidateSha: string; workflowSha: string }; + }; + candidateRow.source.workflowSha = candidateRow.source.candidateSha; + + expect(() => parseNativeRuntimeQualificationRow(JSON.stringify(candidateRow))).toThrow( + "Native runtime qualification source identity is invalid", + ); }); it("accepts only the run-owned rootless Podman executable path for the current uid", () => { From 1fb09c49f9718f87570d55d90f05581d7375746c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 16 Aug 2026 21:33:26 -0500 Subject: [PATCH 67/71] docs(e2e): clarify manual dispatch inputs --- .github/workflows/e2e.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 93f5263ca6c..842e57bb1d4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -64,12 +64,12 @@ on: default: false type: boolean checkout_sha: - description: Optional lowercase latest PR commit SHA for manual exact-revision E2E. + description: Optional PR commit SHA for manual E2E. required: false default: "" type: string checkout_repository: - description: Optional PR source repository for manual exact-revision E2E. + description: Optional PR source repository for manual E2E. required: false default: "" type: string @@ -84,7 +84,7 @@ on: default: "" type: string workflow_sha: - description: Optional trusted main workflow SHA for manual exact-revision E2E. + description: Optional trusted main workflow SHA for manual E2E. required: false default: "" type: string From d674820ab5d3586f9bc84f4c897d0810eb4e1ee7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 03:17:41 -0700 Subject: [PATCH 68/71] docs(e2e): correct qualification evidence guidance --- .github/workflows/e2e.yaml | 4 +-- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 2 ++ test/e2e/README.md | 34 ++++++++++++++------ test/e2e/docs/README.md | 6 ++-- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 842e57bb1d4..9e8aa10b95a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -64,7 +64,7 @@ on: default: false type: boolean checkout_sha: - description: Optional PR commit SHA for manual E2E. + description: Optional lowercase 40-character latest PR commit SHA for manual E2E. required: false default: "" type: string @@ -1246,7 +1246,7 @@ jobs: run: | set -euo pipefail [[ -x /usr/bin/apt-get ]] || { - echo "::error::Protected runner cannot install Podman from its signed OS repository" >&2 + echo "::error::Protected runner cannot install rootless Podman prerequisites from its signed OS repository" >&2 exit 1 } sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 32c0f687604..052fdfd9f17 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -180,6 +180,8 @@ It skips the generic Docker availability probe and all generic Docker container, It does not remove the user's Podman installation, storage, networks, or unrelated containers. It does not disable a user-managed socket. On every successful portable cleanup, the final retirement operation removes the exact portable lifecycle receipts, their matching `sandboxes.json` rows, and `~/.config/nemoclaw/portable/containers.conf`, regardless of `--destroy-user-data`. +After NemoClaw removes `containers.conf`, it removes `~/.config/nemoclaw/portable/` and `~/.config/nemoclaw/` only when each directory is empty. +NemoClaw preserves either directory when it contains an unrelated entry. Other preserved user data follows the normal `--destroy-user-data` behavior. After NemoClaw releases the locks, later uninstall-plan cleanup never recursively revisits the canonical receipt, sandbox registry, or `~/.config/nemoclaw/portable/containers.conf` paths. This preserves any new lifecycle generation published after lock release. diff --git a/test/e2e/README.md b/test/e2e/README.md index 387f924eb9a..ec12288bbed 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1148,30 +1148,44 @@ After a failure, inspect the workflow artifacts and remove resources that target For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary, cohort-owned NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. Before starting NIM or vLLM, the live fixture rejects a pre-existing cohort container name. It records the full container ID, requested image, immutable image ID, cohort owner, and provider label, then removes only that exact container after revalidating every field. Missing, ambiguous, name-reused, drifted, or indeterminate cleanup evidence fails the test, as does any retained exact ID or name. A fail-closed refusal can leave the secret-bearing NIM container alive until runner teardown; inspect the redacted artifacts and remove only the verified container. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. -For `native-runtime-qualification-producer`, dispatch the workflow from trusted `main` for a same-repository open PR and the first workflow attempt. The executing workflow commit and `workflow_sha` input must equal the exact PR-recorded base commit. The actor must have repository `maintain` or `admin` permission. If `github.triggering_actor` differs from the actor, it must also have one of those permissions. - -The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. -The host-side preparation step receives the long-lived `NVIDIA_API_KEY` repository secret in its environment. -It creates runner-local registry authentication and pulls pinned GPU images. -It then deletes the registry authentication file and unsets the variable before the separate candidate installer or live-test process starts. -Cleanup removes runner-local registry authentication but does not revoke the key. +Before you dispatch `native-runtime-qualification-producer`, review the `NVIDIA_API_KEY` boundary below. +The host-side preparation step receives the long-lived repository secret and uses it to create runner-local registry authentication and pull pinned GPU images. +The step deletes the registry authentication file and unsets the variable before candidate execution. +The workflow does not revoke the API key. The key remains valid in the issuing NVIDIA service until it expires or that service revokes it. If exposure occurs or cleanup cannot be confirmed, revoke the key in the issuing NVIDIA service. Alternatively, rotate the key and invalidate the old value. Verify that the old value is invalid. +After you accept this credential boundary, dispatch `native-runtime-qualification-producer` from trusted `main` for a same-repository open PR. +Use the first workflow attempt. +The executing workflow commit and `workflow_sha` input must equal the exact PR-recorded base commit. +The actor must have repository `maintain` or `admin` permission. +If `github.triggering_actor` differs from the actor, it must also have one of those permissions. + +The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The unprivileged installer and live-test processes run with `env -i` under a temporary account. They receive no GitHub, inference provider, API, or messaging credential. Docker is unavailable to these processes. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. -Each case uploads installer and execution receipts. -The aggregate job rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. +Each successful case uploads the validated installer, runtime, operation, and optional NVIDIA CDI receipts. +The workflow does not upload the candidate `execution.json` or `case-evidence.json` staging files. +A failed case uploads no case-evidence artifact. +The aggregate job runs only after all 24 cases succeed. +It rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. This qualification does not register or select Podman in production and does not establish public Podman support. -Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. It uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. Dispatch a new run after recovery. +Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. +Cleanup terminates processes owned by the candidate account and removes that account. +If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. +Recover or replace the runner before dispatching a new run. +Do not rerun the same workflow attempt; the producer rejects attempts after the first. +Dispatch a new run after recovery. +If a case fails, use the GitHub Actions job log. +Inspect a case artifact only when its upload step completed. For a manual PR run, provide the current PR number, lowercase 40-character candidate commit SHA, PR source repository, lowercase 40-character base commit SHA, exact trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. For a native runtime producer run, the executing workflow SHA and `workflow_sha` input must both equal the PR-recorded base SHA. Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index e7787d047d7..de37dfdbc6f 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -311,10 +311,8 @@ test/e2e/ this default selection. If the DGX Spark flag is `true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. - Accepted nonempty `jobs` values are `inference-routing`, - `managed-image-protected-runtime`, and - `native-runtime-qualification-producer`. The `jetson-nvmap-gpu` target is also accepted when - `allow_jetson_dispatch` is `true`. + Accepted nonempty `jobs` values are `inference-routing`, `managed-image-protected-runtime`, and `native-runtime-qualification-producer`. + The `jetson-nvmap-gpu` target is also accepted when `allow_jetson_dispatch` is `true`. Refer to [NemoClaw E2E CI](../README.md). - [Jetson dispatch controller](jetson-dispatch.md) defines the NemoClaw-owned From 0ed8b98ce3a65c1c992b08da565338b4930a0a30 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 03:23:01 -0700 Subject: [PATCH 69/71] docs(e2e): identify qualification inputs --- test/e2e/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index ec12288bbed..577f83ae967 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1167,8 +1167,9 @@ The trusted workflow binds the candidate commit, base commit, workflow commit, r The unprivileged installer and live-test processes run with `env -i` under a temporary account. They receive no GitHub, inference provider, API, or messaging credential. Docker is unavailable to these processes. -Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. -The ARM64 GPU cases also require `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. +For each self-hosted qualification runner, set the GitHub Actions repository variable `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL` to `enabled`. +Set the repository variable `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL` to the reviewed ARM64 GPU runner label. +The workflow provides no ARM64 GPU fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Each successful case uploads the validated installer, runtime, operation, and optional NVIDIA CDI receipts. The workflow does not upload the candidate `execution.json` or `case-evidence.json` staging files. @@ -1187,7 +1188,16 @@ Dispatch a new run after recovery. If a case fails, use the GitHub Actions job log. Inspect a case artifact only when its upload step completed. -For a manual PR run, provide the current PR number, lowercase 40-character candidate commit SHA, PR source repository, lowercase 40-character base commit SHA, exact trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. For a native runtime producer run, the executing workflow SHA and `workflow_sha` input must both equal the PR-recorded base SHA. +For a manual PR run, provide these inputs: + +- The current PR number. +- The lowercase 40-character SHA of the latest PR commit. +- The PR source repository. +- The lowercase 40-character PR base SHA. +- The exact SHA of the trusted workflow commit on `main`. +- A review reason containing 10 to 500 printable characters. + +For a native runtime producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. From 5a13fecf50329fa59d99552d07c6c92d5b98c5e0 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 03:27:07 -0700 Subject: [PATCH 70/71] docs(e2e): separate qualification selectors --- test/e2e/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 577f83ae967..f25e0a96168 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1197,8 +1197,7 @@ For a manual PR run, provide these inputs: - The exact SHA of the trusted workflow commit on `main`. - A review reason containing 10 to 500 printable characters. -For a native runtime producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. -Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. +For the default PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. @@ -1206,8 +1205,9 @@ To select the protected managed-image runtime qualification, set `jobs=managed-i Leave `targets` empty. Keep `include_staging_brev_launchable=false`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -To select native-runtime qualification evidence production, set `jobs=native-runtime-qualification-producer`. +To select native runtime qualification evidence production, set `jobs=native-runtime-qualification-producer`. Leave `targets` empty and keep `include_staging_brev_launchable=false`. +For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. A trusted `main` workflow pre-checkout step requires current `maintain` or `admin` permission. The workflow validates the exact open PR and selected mode before candidate code runs. A second validation after checkout rejects a changed candidate commit, base commit, or PR source repository before preparation. From 54a3c3b548d86035b752e499b227719f3eb961c2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 03:31:21 -0700 Subject: [PATCH 71/71] docs(e2e): name credential-bearing preparation --- .github/workflows/e2e.yaml | 4 ++-- test/e2e/README.md | 2 +- test/e2e/docs/README.md | 13 ++++++++----- ...-runtime-qualification-producer-workflow.test.ts | 8 ++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9e8aa10b95a..ce0ed3dd5dd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1704,7 +1704,7 @@ jobs: printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT" - - name: Materialize exact credential-free GPU resources + - name: Prepare GPU resources with the NVIDIA API key id: gpu_resources env: ACCOUNT: ${{ steps.boundary.outputs.account }} @@ -1730,7 +1730,7 @@ jobs: exit 0 fi [[ -n "$NVIDIA_API_KEY" ]] || { - echo "::error::Protected GPU resource preparation requires the existing NVIDIA registry credential" >&2 + echo "::error::Native runtime GPU preparation requires the NVIDIA_API_KEY repository secret" >&2 exit 1 } uid="$(id -u "$ACCOUNT")" diff --git a/test/e2e/README.md b/test/e2e/README.md index f25e0a96168..b80f1d8e93e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1167,7 +1167,7 @@ The trusted workflow binds the candidate commit, base commit, workflow commit, r The unprivileged installer and live-test processes run with `env -i` under a temporary account. They receive no GitHub, inference provider, API, or messaging credential. Docker is unavailable to these processes. -For each self-hosted qualification runner, set the GitHub Actions repository variable `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL` to `enabled`. +Before any self-hosted qualification job runs, set the GitHub Actions repository variable `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL` to `enabled`. Set the repository variable `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL` to the reviewed ARM64 GPU runner label. The workflow provides no ARM64 GPU fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index de37dfdbc6f..a8d5de979dc 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -311,7 +311,11 @@ test/e2e/ this default selection. If the DGX Spark flag is `true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. - Accepted nonempty `jobs` values are `inference-routing`, `managed-image-protected-runtime`, and `native-runtime-qualification-producer`. + Accepted nonempty `jobs` values are: + + - `inference-routing` + - `managed-image-protected-runtime` + - `native-runtime-qualification-producer` The `jetson-nvmap-gpu` target is also accepted when `allow_jetson_dispatch` is `true`. Refer to [NemoClaw E2E CI](../README.md). @@ -320,10 +324,9 @@ test/e2e/ evidence for `jetson-nvmap-gpu`. The service behind that contract is operator-owned infrastructure. -- `.github/workflows/e2e.yaml` runs selected or all supported - live E2E targets and uploads an explicit artifact allowlist with - JSON summaries plus action, log, and shell command-evidence directories under - 14-day retention. +- `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist. + The shared E2E uploader retains per-target JSON summaries and command-evidence directories for 14 days. + The native runtime aggregate upload retains `native-runtime-qualification-` for 30 days. Final OpenShell gateway-auth artifacts pass a fail-closed safety scan after cleanup. The scanner copies safe files into a private staging directory, scans that copy again, and adds a marker bound to the current Actions run ID diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index d7ba8e3b292..572cb2744e1 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -98,7 +98,7 @@ describe("native runtime qualification producer workflow", () => { it("keeps secret-bearing GPU preparation downstream of the trusted-main plan", () => { const producer = job("native-runtime-qualification-producer"); - const gpuResources = step(producer, "Materialize exact credential-free GPU resources"); + const gpuResources = step(producer, "Prepare GPU resources with the NVIDIA API key"); expect(producer.needs).toContain("native-runtime-qualification-producer-plan"); expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); @@ -228,7 +228,7 @@ describe("native runtime qualification producer workflow", () => { producer, "Install locked candidate test dependencies without scripts", ); - const gpuResources = step(producer, "Materialize exact credential-free GPU resources"); + const gpuResources = step(producer, "Prepare GPU resources with the NVIDIA API key"); const installer = step(producer, "Run the authenticated installer qualification"); const execute = step(producer, "Execute the candidate qualification case without credentials"); const validate = step(producer, "Validate receipts and emit bounded evidence"); @@ -237,7 +237,7 @@ describe("native runtime qualification producer workflow", () => { const credentialFreeSource = JSON.stringify({ ...producer, steps: producer.steps?.filter( - (entry) => entry.name !== "Materialize exact credential-free GPU resources", + (entry) => entry.name !== "Prepare GPU resources with the NVIDIA API key", ), }); const boundaryRun = boundary.run ?? ""; @@ -263,7 +263,7 @@ describe("native runtime qualification producer workflow", () => { /NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u, ); expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); - expect(gpuResources.run).toContain("existing NVIDIA registry credential"); + expect(gpuResources.run).toContain("NVIDIA_API_KEY repository secret"); expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); expect(gpuResources.run).not.toContain("logout --all"); expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"');