diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3bc0b805054..ce0ed3dd5dd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -64,12 +64,12 @@ on: default: false type: boolean checkout_sha: - description: Optional lowercase PR head SHA for manual exact-revision E2E. + description: Optional lowercase 40-character latest PR commit SHA for manual E2E. required: false default: "" type: string checkout_repository: - description: Optional PR head repository for manual exact-revision E2E. + description: Optional PR source repository for manual E2E. required: false default: "" type: string @@ -84,7 +84,7 @@ on: default: "" type: string workflow_sha: - description: Optional trusted main workflow SHA for manual exact-revision E2E. + description: Optional trusted main workflow SHA for manual E2E. required: false default: "" type: string @@ -327,7 +327,6 @@ jobs: { echo "::error::review_reason must contain 10 to 500 printable characters" >&2; exit 1; } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; } - require_maintainer "$ACTOR" if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then require_maintainer "$TRIGGERING_ACTOR" @@ -339,8 +338,8 @@ jobs: --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } - [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR head repository" >&2; exit 1; } - [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the PR head SHA" >&2; exit 1; } + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } - name: Authorize release qualification waiver @@ -844,6 +843,7 @@ jobs: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts test/e2e/registry/native-runtime-qualification.ts tools/e2e/native-runtime-qualification-producer-plan.mts sparse-checkout-cone-mode: false @@ -957,9 +957,210 @@ jobs: WORKFLOW_SHA: ${{ github.workflow_sha }} run: node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-plan.mts --ci-output + native-runtime-qualification-podman-toolchain: + name: Build pinned native Podman toolchain / ${{ matrix.architecture }} + needs: [generate-matrix, native-runtime-qualification-producer-plan] + if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runner: ubuntu-24.04 + - architecture: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Check out the pinned Podman source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: podman-container-tools/podman + ref: cade97a52ebdf9dbf9e81de8009015776837a074 # v6.1.0 + path: .podman-source + fetch-depth: 1 + persist-credentials: false + + - name: Check out the pinned Netavark source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: containers/netavark + ref: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab # v2.1.0 + path: .netavark-source + fetch-depth: 1 + persist-credentials: false + + - name: Check out the pinned Aardvark DNS source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: containers/aardvark-dns + ref: cd7417681229219059939bdd9f0b3bd9ac9abb08 # v2.1.0 + path: .aardvark-source + fetch-depth: 1 + persist-credentials: false + + - name: Set up pinned Go for the Podman build + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: 1.25.9 + cache: false + + - name: Set up pinned Rust for the network helper builds + uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + toolchain: 1.88.0 + cache: false + rustflags: "" + + - name: Install build dependencies from the signed runner OS repository + shell: bash + run: | + set -euo pipefail + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ + --yes --no-install-recommends \ + curl gcc git libapparmor-dev libbtrfs-dev libc6-dev \ + libdevmapper-dev libglib2.0-dev \ + libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ + libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler + + - name: Build and package the pinned native toolchain + env: + AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08 + EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} + NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab + PASTA_SOURCE_ARCHIVE_SHA256: 54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958 + PASTA_SOURCE_SHA: f8df3f1b228fe19a74a269334fdfe6cc7d0605ce + PASTA_VERSION: 2026_07_28.f8df3f1 + PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074 + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain + shell: bash + run: | + set -euo pipefail + [[ "$(dpkg --print-architecture)" == "$EXPECTED_ARCHITECTURE" ]] + [[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]] + [[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]] + [[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]] + [[ ! -e .passt-source && ! -L .passt-source ]] + pasta_source_archive="${RUNNER_TEMP}/passt-${PASTA_SOURCE_SHA}.tar.gz" + [[ ! -e "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] + /usr/bin/curl \ + --fail --location --proto '=https' --proto-redir '=https' \ + --retry 3 --show-error --silent --tlsv1.2 \ + --output "$pasta_source_archive" \ + "https://passt.top/passt/snapshot/passt-${PASTA_SOURCE_SHA}.tar.gz" + [[ -f "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] + [[ "$(sha256sum "$pasta_source_archive" | cut -d' ' -f1)" == "$PASTA_SOURCE_ARCHIVE_SHA256" ]] + mkdir .passt-source + tar \ + --extract --gzip --file="$pasta_source_archive" \ + --directory=.passt-source --strip-components=1 \ + --no-same-owner --no-same-permissions + [[ -f .passt-source/Makefile && ! -L .passt-source/Makefile ]] + [[ -f .passt-source/passt.c && ! -L .passt-source/passt.c ]] + [[ ! -e .passt-source/passt && ! -L .passt-source/passt ]] + for source in .podman-source .netavark-source .aardvark-source; do + [[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]] + done + [[ "$(go version)" == go\ version\ go1.25.9\ * ]] + [[ "$(rustc --version)" == rustc\ 1.88.0\ * ]] + + SOURCE_DATE_EPOCH=1786554266 \ + BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \ + EXTRA_BUILDTAGS=containers_image_openpgp \ + make --directory=.podman-source --jobs=2 \ + podman rootlessport PREFIX=/usr/local + SOURCE_DATE_EPOCH=1785940686 CI=1 \ + make --directory=.netavark-source --jobs=2 build + SOURCE_DATE_EPOCH=1785940850 CI=1 \ + make --directory=.aardvark-source --jobs=2 build + SOURCE_DATE_EPOCH=1785255008 \ + make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt + + podman_dependencies="$(ldd .podman-source/bin/podman)" + printf '%s\n' "$podman_dependencies" + if grep -F "not found" <<<"$podman_dependencies"; then + echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2 + exit 1 + fi + if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then + echo "::error::Pinned Podman build must not require an optional host ABI" >&2 + exit 1 + fi + + install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" + [[ -f .passt-source/passt && ! -L .passt-source/passt ]] + install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta" + install -D -m 0755 .podman-source/bin/rootlessport \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" + install -D -m 0755 .netavark-source/bin/netavark \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" + install -D -m 0755 .aardvark-source/bin/aardvark-dns \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" + install -D -m 0644 \ + .podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \ + "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" + + [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]] + pasta_version_output="$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)" + [[ "${pasta_version_output%%$'\n'*}" == "pasta $PASTA_VERSION" ]] || { + echo "::error::Pinned qualification pasta version is invalid" >&2 + exit 1 + } + [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]] + [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]] + jq -n \ + --arg architecture "$EXPECTED_ARCHITECTURE" \ + --arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \ + --arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \ + --arg pastaSourceArchiveSha256 "$PASTA_SOURCE_ARCHIVE_SHA256" \ + --arg pastaSourceSha "$PASTA_SOURCE_SHA" \ + --arg pastaVersion "$PASTA_VERSION" \ + --arg podmanSourceSha "$PODMAN_SOURCE_SHA" ' + { + schemaVersion: 1, + kind: "nemoclaw-native-podman-toolchain-v1", + architecture: $architecture, + podmanVersion: "6.1.0", + podmanSourceSha: $podmanSourceSha, + netavarkVersion: "2.1.0", + netavarkSourceSha: $netavarkSourceSha, + aardvarkDnsVersion: "2.1.0", + aardvarkDnsSourceSha: $aardvarkDnsSourceSha, + pastaVersion: $pastaVersion, + pastaSourceArchiveSha256: $pastaSourceArchiveSha256, + pastaSourceSha: $pastaSourceSha, + goVersion: "1.25.9", + rustVersion: "1.88.0" + } + ' >"$TOOLCHAIN_DIRECTORY/manifest.json" + ( + cd "$TOOLCHAIN_DIRECTORY" + sha256sum \ + bin/pasta \ + bin/podman \ + libexec/podman/aardvark-dns \ + libexec/podman/netavark \ + libexec/podman/rootlessport \ + manifest.json \ + share/containers/containers.conf >SHA256SUMS + ) + + - name: Upload the pinned native Podman toolchain + if: success() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: native-runtime-podman-toolchain-${{ matrix.architecture }} + path: ${{ runner.temp }}/native-runtime-podman-toolchain/ + native-runtime-qualification-producer: name: ${{ matrix.jobName }} - needs: [generate-matrix, native-runtime-qualification-producer-plan] + needs: + - generate-matrix + - native-runtime-qualification-podman-toolchain + - native-runtime-qualification-producer-plan if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} strategy: fail-fast: false @@ -988,10 +1189,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.source.workflowSha }} - path: .trusted-qualification + path: .qualification-workflow persist-credentials: false sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts scripts/checks/run-native-runtime-installer-qualification.sh + test/e2e/registry/native-runtime-qualification.ts + tools/e2e/native-runtime-qualification-producer-plan.mts tools/e2e/native-runtime-qualification-producer-evidence.mts sparse-checkout-cone-mode: false @@ -1009,25 +1213,153 @@ jobs: with: node-version: 22.19.0 - - name: Install locked candidate test dependencies without scripts - working-directory: .candidate-runtime - run: npm ci --ignore-scripts + - name: Require a reviewed Ubuntu runtime host + shell: bash + run: | + set -euo pipefail + [[ -r /etc/os-release ]] || { + echo "::error::Protected runner does not expose an OS release identity" >&2 + exit 1 + } + # shellcheck disable=SC1091 + source /etc/os-release + [[ "${ID:-}" == "ubuntu" ]] || { + echo "::error::Protected runner must use a reviewed Ubuntu image" >&2 + exit 1 + } + [[ "${VERSION_ID:-}" == "24.04" || "${VERSION_ID:-}" == "26.04" ]] || { + echo "::error::Protected runner Ubuntu release is not reviewed for Podman qualification" >&2 + exit 1 + } + + - name: Download the pinned native Podman toolchain + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-runtime-podman-toolchain-${{ matrix.case.architecture }} + path: ${{ runner.temp }}/native-runtime-podman-toolchain + + - name: Install the pinned native Podman toolchain and rootless prerequisites + env: + EXPECTED_ARCHITECTURE: ${{ matrix.case.architecture }} + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain + shell: bash + run: | + set -euo pipefail + [[ -x /usr/bin/apt-get ]] || { + echo "::error::Protected runner cannot install rootless Podman prerequisites from its signed OS repository" >&2 + exit 1 + } + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ + --yes --no-install-recommends \ + acl apparmor btrfs-progs conmon \ + golang-github-containers-common iptables nftables runc slirp4netns uidmap + + [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] + [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] + mapfile -t actual_files < <( + cd "$TOOLCHAIN_DIRECTORY" + find . -type f -print | LC_ALL=C sort + ) + expected_files=( + ./SHA256SUMS + ./bin/pasta + ./bin/podman + ./libexec/podman/aardvark-dns + ./libexec/podman/netavark + ./libexec/podman/rootlessport + ./manifest.json + ./share/containers/containers.conf + ) + [[ "${actual_files[*]}" == "${expected_files[*]}" ]] || { + echo "::error::Downloaded native Podman toolchain contains unexpected files" >&2 + exit 1 + } + ( + cd "$TOOLCHAIN_DIRECTORY" + sha256sum --check --strict SHA256SUMS + ) + jq -e \ + --arg architecture "$EXPECTED_ARCHITECTURE" ' + type == "object" and + keys == [ + "aardvarkDnsSourceSha", + "aardvarkDnsVersion", + "architecture", + "goVersion", + "kind", + "netavarkSourceSha", + "netavarkVersion", + "pastaSourceArchiveSha256", + "pastaSourceSha", + "pastaVersion", + "podmanSourceSha", + "podmanVersion", + "rustVersion", + "schemaVersion" + ] and + .schemaVersion == 1 and + .kind == "nemoclaw-native-podman-toolchain-v1" and + .architecture == $architecture and + .podmanVersion == "6.1.0" and + .podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and + .netavarkVersion == "2.1.0" and + .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and + .aardvarkDnsVersion == "2.1.0" and + .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and + .pastaVersion == "2026_07_28.f8df3f1" and + .pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and + .pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and + .goVersion == "1.25.9" and + .rustVersion == "1.88.0" + ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null + for target in \ + /usr/local/bin/podman \ + /usr/local/libexec/podman/aardvark-dns \ + /usr/local/libexec/podman/netavark \ + /usr/local/libexec/podman/rootlessport \ + /usr/share/containers/containers.conf; do + [[ ! -L "$target" ]] || { + echo "::error::Native Podman toolchain target must not be a symlink: $target" >&2 + exit 1 + } + done + sudo install --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/bin/podman" /usr/local/bin/podman + for helper in aardvark-dns netavark rootlessport; do + sudo install -D --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \ + "/usr/local/libexec/podman/$helper" + done + sudo install --owner=root --group=root --mode=0644 \ + "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \ + /usr/share/containers/containers.conf + [[ "$(command -v podman)" == "/usr/local/bin/podman" ]] + conmon_version="$(conmon --version | awk 'NR == 1 { print $NF }')" + runc_version="$(runc --version | awk 'NR == 1 { print $NF }')" + dpkg --compare-versions "$conmon_version" ge 2.1.7 + dpkg --compare-versions "$runc_version" ge 1.1.11 + [[ "$(/usr/local/libexec/podman/netavark --version)" == "netavark 2.1.0" ]] + [[ "$(/usr/local/libexec/podman/aardvark-dns --version)" == "aardvark-dns 2.1.0" ]] + version="$(podman --version)" + [[ "$version" == "podman version 6.1.0" ]] - name: Prepare the credential-free execution account and disable Docker id: boundary env: CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail - for command in git jq node pgrep podman sha256sum systemctl; do + for command in apparmor_parser awk cat curl getent git grep groupdel id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 } done - [[ "$(podman --version)" =~ ^podman\ version\ 5[.] ]] || { - echo "::error::Protected runner must provide Podman 5" >&2 + [[ "$(podman --version)" == "podman version 6.1.0" ]] || { + echo "::error::Protected runner must provide Podman 6.1.0" >&2 exit 1 } sudo systemctl stop docker.service docker.socket 2>/dev/null || true @@ -1039,88 +1371,730 @@ jobs: ! pgrep -x dockerd >/dev/null [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] account="nemoclawq" - printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" - sudo useradd --create-home --shell /usr/sbin/nologin "$account" + ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Qualification account, group, or subordinate-ID authorization already exists" >&2 + exit 1 + fi + [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { + echo "::error::Qualification account ownership marker already exists" >&2 + exit 1 + } + account_created_without_marker=0 + rollback_unmarked_account() { + local result="$?" + trap - EXIT + if ((result != 0 && account_created_without_marker == 1)); then + sudo userdel --remove "$account" 2>/dev/null || true + sudo groupdel "$account" 2>/dev/null || true + sudo rm -f -- "$ownership_marker" + if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Partially created qualification account could not be rolled back" >&2 + exit 1 + fi + fi + exit "$result" + } + trap rollback_unmarked_account EXIT + sudo useradd --create-home --shell /usr/sbin/nologin --user-group "$account" + account_created_without_marker=1 uid="$(id -u "$account")" + gid="$(id -g "$account")" home="$(getent passwd "$account" | cut -d: -f6)" + group_entry="$(getent group "$account")" + [[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { + echo "::error::Qualification account identity is missing or invalid" >&2 + exit 1 + } + [[ "$group_entry" == "${account}:x:${gid}:" ]] || { + echo "::error::Qualification private group identity is missing or invalid" >&2 + exit 1 + } + printf '%s:%s:%s\n' "$account" "$uid" "$gid" | sudo tee "$ownership_marker" >/dev/null + sudo chown root:root "$ownership_marker" + sudo chmod 0400 "$ownership_marker" + [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { + echo "::error::Qualification account ownership marker is invalid" >&2 + exit 1 + } + account_created_without_marker=0 + trap - EXIT + ensure_subordinate_range() { + local file="$1" + local option="$2" + local range_start=100000 + local range_end + local conflict_end + [[ -f "$file" && ! -L "$file" ]] || { + echo "::error::Rootless Podman subordinate-ID file is missing or invalid: $file" >&2 + exit 1 + } + if awk -F: -v account="$account" ' + $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } + END { exit found ? 0 : 1 } + ' "$file"; then + return + fi + while :; do + ((range_start <= 4294901760)) || { + echo "::error::Protected runner has no free subordinate-ID range for rootless Podman" >&2 + exit 1 + } + range_end=$((range_start + 65535)) + conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" ' + $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ { + current_end = $2 + $3 - 1 + if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end + } + END { if (maximum != "") print maximum } + ' "$file")" + [[ -n "$conflict_end" ]] || break + range_start=$((conflict_end + 1)) + done + range_end=$((range_start + 65535)) + sudo usermod "$option" "${range_start}-${range_end}" "$account" + awk -F: -v account="$account" ' + $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } + END { exit found ? 0 : 1 } + ' "$file" || { + echo "::error::Protected runner did not provision rootless Podman subordinate IDs" >&2 + exit 1 + } + } + ensure_subordinate_range /etc/subuid --add-subuids + ensure_subordinate_range /etc/subgid --add-subgids + printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" + printf 'account_created=true\n' >>"$GITHUB_OUTPUT" + printf 'uid=%s\n' "$uid" >>"$GITHUB_OUTPUT" + printf 'gid=%s\n' "$gid" >>"$GITHUB_OUTPUT" runtime_dir="/run/user/${uid}" - sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir" - sudo chown -R "$uid:$uid" "$CANDIDATE_DIRECTORY" + runtime_directory_unit="user-runtime-dir@${uid}.service" + user_manager_unit="user@${uid}.service" + user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" + user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" + trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" + [[ ! -e "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" ]] || { + echo "::error::Qualification systemd user-manager drop-in directory already exists" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0755 "$user_manager_dropin_directory" + printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"\n' "$trusted_user_unit_path" | + sudo tee "$user_manager_dropin" >/dev/null + sudo chown root:root "$user_manager_dropin" + sudo chmod 0444 "$user_manager_dropin" + [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { + echo "::error::Qualification systemd user-manager drop-in is invalid" >&2 + exit 1 + } + sudo systemctl daemon-reload + verify_user_manager_unit_path() { + local environment + environment="$(sudo -u "$1" env -i \ + HOME="$2" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$3" \ + /usr/bin/systemctl --user show-environment)" + tr ' ' '\n' <<<"$environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$trusted_user_unit_path" >/dev/null || { + echo "::error::Qualification systemd user manager did not inherit the trusted unit path" >&2 + return 1 + } + } + verify_user_bus() { + local execution_account="$1" + local expected_uid="$2" + local bus="$3" + local context="$4" + sudo /usr/bin/test -S "$bus" && + sudo /usr/bin/test ! -L "$bus" && + [[ "$(sudo stat -c '%u' -- "$bus")" == "$expected_uid" ]] && + sudo -u "$execution_account" /usr/bin/test -S "$bus" || { + echo "::error::Qualification systemd user bus $context" >&2 + return 1 + } + } + sudo systemctl start "$user_manager_unit" + systemctl is-active --quiet "$runtime_directory_unit" || { + echo "::error::Qualification systemd runtime-directory unit is not active" >&2 + exit 1 + } + [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${gid}:700" ]] || { + echo "::error::Qualification runtime directory is missing or invalid" >&2 + exit 1 + } + systemctl is-active --quiet "$user_manager_unit" || { + echo "::error::Qualification systemd user manager is not active" >&2 + exit 1 + } + sudo -u "$account" env -i \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$runtime_dir" \ + /usr/bin/systemctl --user start dbus.socket + sudo -u "$account" env -i \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$runtime_dir" \ + /usr/bin/systemctl --user is-active --quiet dbus.socket || { + echo "::error::Qualification systemd user bus socket unit is not active" >&2 + exit 1 + } + verify_user_bus "$account" "$uid" "$runtime_dir/bus" \ + "is invalid or inaccessible to the execution account" + verify_user_manager_unit_path "$account" "$home" "$runtime_dir" + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + storage_config="${storage_config_directory}/storage.conf" + containers_config="${storage_config_directory}/containers.conf" + podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + apparmor_profile="${storage_config_directory}/podman.apparmor" + apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + pasta_executable="${helper_directory}/pasta" + pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" + pasta_apparmor_profile_name="nemoclaw-native-pasta-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration directory already exists" >&2 + exit 1 + } + [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { + echo "::error::Qualification helper directory already exists" >&2 + exit 1 + } + sudo install -d -o root -g root -m 0755 "$storage_config_directory" + [[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || { + echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2 + exit 1 + } + [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { + echo "::error::Run-owned qualification Podman executable already exists" >&2 + exit 1 + } + sudo install --owner=root --group=root --mode=0555 \ + "$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable" + [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification Podman executable is invalid" >&2 + exit 1 + } + [[ "$(sha256sum "$podman_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/podman" | cut -d' ' -f1)" ]] || { + echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2 + exit 1 + } + [[ -f "$TOOLCHAIN_DIRECTORY/bin/pasta" && ! -L "$TOOLCHAIN_DIRECTORY/bin/pasta" ]] || { + echo "::error::Pinned qualification pasta executable source is missing or invalid" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0555 "$helper_directory" + sudo install --owner=root --group=root --mode=0555 \ + "$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable" + [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification helper directory is invalid" >&2 + exit 1 + } + [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { + echo "::error::Run-owned qualification pasta executable is invalid" >&2 + exit 1 + } + [[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/pasta" | cut -d' ' -f1)" ]] || { + echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2 + exit 1 + } + printf '%s\n' \ + '[storage]' \ + 'driver = "overlay"' \ + "runroot = \"${home}/.local/share/containers/runroot\"" \ + "graphroot = \"${home}/.local/share/containers/storage\"" \ + "rootless_storage_path = \"${home}/.local/share/containers/storage\"" | sudo tee "$storage_config" >/dev/null + sudo chown root:root "$storage_config" + sudo chmod 0444 "$storage_config" + [[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || { + echo "::error::Qualification storage configuration is not root-owned and read-only" >&2 + exit 1 + } + printf '%s\n' \ + '[network]' \ + 'firewall_driver = "nftables"' | sudo tee "$containers_config" >/dev/null + sudo chown root:root "$containers_config" + sudo chmod 0444 "$containers_config" + [[ -f "$containers_config" && ! -L "$containers_config" && "$(stat -c '%u:%g:%a' "$containers_config")" == "0:0:444" ]] || { + echo "::error::Qualification containers configuration is not root-owned and read-only" >&2 + exit 1 + } + if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then + printf '%s\n' \ + '# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \ + '' \ + 'abi ,' \ + 'include ' \ + '' \ + "profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined) {" \ + ' userns,' \ + '}' | sudo tee "$apparmor_profile" >/dev/null + sudo chown root:root "$apparmor_profile" + sudo chmod 0444 "$apparmor_profile" + [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification AppArmor profile is not root-owned and read-only" >&2 + exit 1 + } + sudo apparmor_parser -r "$apparmor_profile" + printf '%s\n' \ + '# This ephemeral profile is limited to the immutable run-owned pasta helper.' \ + '' \ + 'abi ,' \ + 'include ' \ + '' \ + "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined) {" \ + ' userns,' \ + '}' | sudo tee "$pasta_apparmor_profile" >/dev/null + sudo chown root:root "$pasta_apparmor_profile" + sudo chmod 0444 "$pasta_apparmor_profile" + [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification pasta AppArmor profile is not root-owned and read-only" >&2 + exit 1 + } + sudo apparmor_parser -r "$pasta_apparmor_profile" + fi + ancestor="$(dirname "$CANDIDATE_DIRECTORY")" + while [[ "$ancestor" != "/home" ]]; do + [[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || { + echo "::error::Candidate checkout ancestor is outside the reviewed runner workspace" >&2 + exit 1 + } + sudo setfacl --modify "u:${account}:--x" "$ancestor" + ancestor="$(dirname "$ancestor")" + done + sudo chown -R "$uid:$gid" "$CANDIDATE_DIRECTORY" + node_directory="$(dirname "$(command -v node)")" + [[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || { + echo "::error::Pinned Node toolchain path is invalid" >&2 + exit 1 + } guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard" - install -d -m 0700 "$guard_dir" + install -d -m 0755 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" - chmod 0500 "$guard_dir/docker" + chmod 0555 "$guard_dir/docker" + podman_info="$(sudo -u "$account" env -i \ + CONTAINERS_CONF="$containers_config" \ + CONTAINERS_STORAGE_CONF="$storage_config" \ + HOME="$home" \ + LANG=C.UTF-8 \ + PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin" \ + XDG_RUNTIME_DIR="$runtime_dir" \ + "$podman_executable" info --format json)" || { + echo "::error::Credential-free rootless Podman readiness failed" >&2 + exit 1 + } + jq -e ' + .host.security.rootless == true and + .store.graphDriverName == "overlay" and + (((.store.graphOptions // {})["overlay.mount_program"].Executable? // "") == "") + ' <<<"$podman_info" >/dev/null || { + echo "::error::Qualification requires native rootless overlay storage" >&2 + exit 1 + } printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" + printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT" + printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" + printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" + printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT" + printf 'runtime_directory_unit=%s\n' "$runtime_directory_unit" >>"$GITHUB_OUTPUT" + printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" + printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT" + + - name: Prepare GPU resources with the NVIDIA API key + id: gpu_resources + env: + ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} + ACCELERATION: ${{ matrix.case.acceleration }} + ARCHITECTURE: ${{ matrix.case.architecture }} + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} + GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} + INFERENCE: ${{ matrix.case.inference }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} + QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + shell: bash + run: | + set -euo pipefail + umask 077 + if [[ "$ACCELERATION" != "nvidia-gpu" ]]; then + printf 'runner_contract=\n' >>"$GITHUB_OUTPUT" + exit 0 + fi + [[ -n "$NVIDIA_API_KEY" ]] || { + echo "::error::Native runtime GPU preparation requires the NVIDIA_API_KEY repository secret" >&2 + exit 1 + } + uid="$(id -u "$ACCOUNT")" + gid="$(id -g "$ACCOUNT")" + [[ "$uid" == "$ACCOUNT_UID" && "$gid" == "$ACCOUNT_GID" ]] || { + echo "::error::Qualification account identity changed before GPU resource preparation" >&2 + exit 1 + } + storage_config_directory="$(dirname "$STORAGE_CONFIG")" + runner_contract="${storage_config_directory}/runner-contract.json" + registry_auth_directory="${storage_config_directory}/registry-auth" + registry_auth_file="${registry_auth_directory}/auth.json" + resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + model_directory="${resource_directory}/model" + model_revision="7ae557604adf67be50417f59c2c2f167def9a775" + model_name="Qwen/Qwen2.5-0.5B-Instruct" + case "$ARCHITECTURE" in + amd64) + probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:9855f4c8500addf185360474184b9efdaf4384284779aa9173dcd70164a4ae6f" + nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:a0fdbecdf51792dadc48d284fde3199a58d5a2067007ad5b80319975fe81ce93" + vllm_image="nvcr.io/nvidia/vllm@sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773" + ;; + arm64) + probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:a54fdceac3bc2a8d177f07db942defc2f7237e18d07fca2ff00718ba5aee4940" + nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:8342257b9744e9bc23a02e0f45badad6b88474727473965b4dc83e8fee45956a" + vllm_image="nvcr.io/nvidia/vllm@sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4" + ;; + *) + echo "::error::GPU resource architecture is unsupported" >&2 + exit 1 + ;; + esac + [[ ! -e "$runner_contract" && ! -L "$runner_contract" ]] || { + echo "::error::Run-owned GPU runner contract already exists" >&2 + exit 1 + } + [[ ! -e "$registry_auth_directory" && ! -L "$registry_auth_directory" ]] || { + echo "::error::Run-owned registry authentication directory already exists" >&2 + exit 1 + } + sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$registry_auth_directory" + cleanup_registry_auth() { + local result="$?" + trap - EXIT + if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 + return 1 + } + sudo unlink "$registry_auth_file" + fi + sudo rmdir "$registry_auth_directory" || { + echo "::error::Run-owned registry authentication directory was not empty" >&2 + return 1 + } + return "$result" + } + trap cleanup_registry_auth EXIT + printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + REGISTRY_AUTH_FILE="$registry_auth_file" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%h' -- "$registry_auth_file")" == "${uid}:${gid}:1" ]] || { + echo "::error::Run-owned registry authentication file is invalid" >&2 + exit 1 + } + sudo chmod 0600 -- "$registry_auth_file" + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + echo "::error::Run-owned registry authentication file permissions are invalid" >&2 + exit 1 + } + image_to_pull="$probe_image" + if [[ "$INFERENCE" == "nim" ]]; then + image_to_pull="$nim_image" + elif [[ "$INFERENCE" == "vllm" ]]; then + image_to_pull="$vllm_image" + fi + images=("$probe_image") + [[ "$image_to_pull" == "$probe_image" ]] || images+=("$image_to_pull") + for image in "${images[@]}"; do + sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + REGISTRY_AUTH_FILE="$registry_auth_file" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + "$PODMAN_EXECUTABLE" pull "$image" + done + cleanup_registry_auth + trap - EXIT + unset NVIDIA_API_KEY + + if [[ "$INFERENCE" == "nim" || "$INFERENCE" == "vllm" ]]; then + [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { + echo "::error::Run-owned GPU model resource already exists" >&2 + exit 1 + } + sudo install -d --owner=root --group=root --mode=0711 "$resource_directory" + sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$model_directory" + download_model_file() { + local file="$1" + local size="$2" + local algorithm="$3" + local digest="$4" + local target="${model_directory}/${file}" + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + /usr/bin/curl \ + --fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \ + --output "$target" \ + "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" + sudo test -f "$target" && + sudo test ! -L "$target" && + [[ "$(sudo stat -c '%u:%g:%h:%s' -- "$target")" == "${uid}:${gid}:1:${size}" ]] || { + echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 + exit 1 + } + sudo chmod 0600 -- "$target" + [[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { + echo "::error::Downloaded GPU model file permissions are invalid: $file" >&2 + exit 1 + } + if [[ "$algorithm" == "sha256" ]]; then + [[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]] + else + [[ "$(sudo git hash-object --no-filters -- "$target")" == "$digest" ]] + fi || { + echo "::error::Downloaded GPU model file digest is invalid: $file" >&2 + exit 1 + } + } + download_model_file config.json 659 sha1 0dbb161213629a23f0fc00ef286e6b1e366d180f + download_model_file generation_config.json 242 sha1 dfc11073787daf1b0f9c0f1499487ab5f4c93738 + download_model_file merges.txt 1671839 sha1 20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0 + download_model_file model.safetensors 988097824 sha256 fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe + download_model_file tokenizer.json 7031645 sha1 443909a61d429dff23010e5bddd28ff530edda00 + download_model_file tokenizer_config.json 7305 sha1 07bfe0640cb5a0037f9322287fbfc682806cf672 + download_model_file vocab.json 2776833 sha1 4783fe10ac3adce15ac8f358ef5462739852c569 + for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do + sudo chown root:root "${model_directory}/${file}" + sudo chmod 0444 "${model_directory}/${file}" + done + sudo chown root:root "$model_directory" + sudo chmod 0555 "$model_directory" + sudo chmod 0555 "$resource_directory" + fi + + jq -n \ + --arg architecture "$ARCHITECTURE" \ + --arg gpuProbeImageRef "$probe_image" \ + --arg model "$model_name" \ + --arg modelPath "$model_directory" \ + --arg modelRevision "$model_revision" \ + --arg nimImageRef "$nim_image" \ + --arg vllmImageRef "$vllm_image" ' + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture: $architecture, + gpuProbeImageRef: $gpuProbeImageRef, + nim: { + imageRef: $nimImageRef, + model: $model, + modelPath: $modelPath, + modelRevision: $modelRevision + }, + vllm: { + imageRef: $vllmImageRef, + model: $model, + modelPath: $modelPath, + modelRevision: $modelRevision + } + } + ' | sudo tee "$runner_contract" >/dev/null + sudo chown root:root "$runner_contract" + sudo chmod 0444 "$runner_contract" + [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { + echo "::error::Run-owned GPU runner contract is invalid" >&2 + exit 1 + } + printf 'runner_contract=%s\n' "$runner_contract" >>"$GITHUB_OUTPUT" + + - name: Install locked candidate test dependencies without scripts + env: + ACCOUNT: ${{ steps.boundary.outputs.account }} + CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} + QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + shell: bash + run: | + set -euo pipefail + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + /bin/bash --noprofile --norc -c ' + set -euo pipefail + cd "$1" + for file in package.json package-lock.json; do + [[ -f "$file" && ! -L "$file" && -O "$file" ]] || { + echo "Candidate dependency manifest is missing or invalid: $file" >&2 + exit 1 + } + done + exec npm --prefix "$1" ci --ignore-scripts + ' bash "$CANDIDATE_DIRECTORY" - name: Run the authenticated installer qualification env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} ARCHITECTURE: ${{ matrix.case.architecture }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CANDIDATE_SHA: ${{ matrix.source.candidateSha }} + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer INSTALLER_SHA256: ${{ matrix.installerSha256 }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + TRUSTED_USER_UNIT_PATH: /usr/lib/systemd/user:/lib/systemd/user + USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail install -d -m 0700 "$INSTALLER_RECEIPT_PARENT" - sudo chown "$ACCOUNT:$ACCOUNT" "$INSTALLER_RECEIPT_PARENT" + sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$INSTALLER_RECEIPT_PARENT" sudo -u "$ACCOUNT" env -i \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ - PATH="$GUARD_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ - bash .trusted-qualification/scripts/checks/run-native-runtime-installer-qualification.sh \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + bash .qualification-workflow/scripts/checks/run-native-runtime-installer-qualification.sh \ --candidate-checkout "$CANDIDATE_DIRECTORY" \ --candidate-sha "$CANDIDATE_SHA" \ --installer-sha256 "$INSTALLER_SHA256" \ --architecture "$ARCHITECTURE" \ --artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts" - sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true - [[ -d "$INSTALLER_RECEIPT_PARENT/receipts" && ! -L "$INSTALLER_RECEIPT_PARENT/receipts" ]] || { + sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" + ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle remained active after installer isolation" >&2 + exit 1 + } + sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true + sudo systemctl start "$USER_MANAGER_UNIT" + systemctl is-active --quiet "$USER_MANAGER_UNIT" && systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2 + exit 1 + } + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user start dbus.socket + sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user is-active --quiet dbus.socket || { + echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 + exit 1 + } + sudo /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" && + sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus" && + [[ "$(sudo stat -c '%u' -- "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] && + sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || { + echo "::error::Qualification systemd user bus is invalid or inaccessible after installer isolation" >&2 + exit 1 + } + manager_environment="$(sudo -u "$ACCOUNT" env -i \ + HOME="$QUALIFICATION_HOME" \ + LANG=C.UTF-8 \ + PATH=/usr/bin:/bin \ + XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ + /usr/bin/systemctl --user show-environment)" + tr ' ' '\n' <<<"$manager_environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$TRUSTED_USER_UNIT_PATH" >/dev/null || { + echo "::error::Qualification systemd user manager lost the trusted unit path after installer isolation" >&2 + exit 1 + } + sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { echo "::error::Installer receipt directory is missing or invalid" >&2 exit 1 } - sudo chown -R -h root:root "$INSTALLER_RECEIPT_PARENT/receipts" + sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts" || { + echo "::error::Installer receipt directory must not be a symlink" >&2 + exit 1 + } - name: Execute the candidate qualification case without credentials env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} + ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime + CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} + HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} + PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} + RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} + RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} + STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} + USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail - live_test="$CANDIDATE_DIRECTORY/test/e2e/live/native-runtime-qualification-case.test.ts" - [[ -f "$live_test" && ! -L "$live_test" ]] || { + live_test="test/e2e/live/native-runtime-qualification-case.test.ts" + [[ -f "$CANDIDATE_DIRECTORY/$live_test" && ! -L "$CANDIDATE_DIRECTORY/$live_test" ]] || { echo "::error::Candidate commit does not provide the native runtime qualification case executor" >&2 exit 1 } receipt_directory="${RUNNER_TEMP}/native-runtime-case" install -d -m 0700 "$receipt_directory" - sudo chown "$ACCOUNT:$ACCOUNT" "$receipt_directory" + sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$receipt_directory" + cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ + CONTAINERS_CONF="$CONTAINERS_CONFIG" \ + CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ E2E_DEFAULT_ENABLED=0 \ E2E_JOB=1 \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \ + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \ + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT="$RUNNER_CONTRACT" \ NEMOCLAW_RUN_LIVE_E2E=1 \ - PATH="$GUARD_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ + PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \ --config "$CANDIDATE_DIRECTORY/vitest.config.ts" \ --project e2e-live \ "$live_test" - sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true - sudo chown -R root:root "$receipt_directory" + sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" + ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { + echo "::error::Qualification systemd user lifecycle remained active after candidate execution" >&2 + exit 1 + } + sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true - name: Verify Docker stayed unavailable shell: bash @@ -1136,38 +2110,380 @@ jobs: EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts + NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} shell: bash run: | set -euo pipefail - sudo chown -R "$(id -u):$(id -g)" \ - "$INSTALLER_RECEIPT_DIRECTORY" \ - "$(dirname "$EXECUTION_RECEIPT_PATH")" - node --experimental-strip-types --no-warnings \ - .trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts + sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ + "$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \ + .qualification-workflow/tools/e2e/native-runtime-qualification-producer-evidence.mts + sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources if: always() env: ACCOUNT: ${{ steps.boundary.outputs.account }} + ACCOUNT_CREATED: ${{ steps.boundary.outputs.account_created }} shell: bash run: | set -euo pipefail - account="${ACCOUNT:-nemoclawq}" - if id "$account" >/dev/null 2>&1; then - sudo pkill -KILL -u "$(id -u "$account")" 2>/dev/null || true - sudo userdel --remove "$account" - fi - if id "$account" >/dev/null 2>&1; then - echo "::error::Qualification account still exists after cleanup" >&2 + reported_account="${ACCOUNT:-}" + reported_created="${ACCOUNT_CREATED:-}" + ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + account="" + uid="" + gid="" + if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then + [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { + echo "::error::Qualification account ownership marker cleanup target is invalid" >&2 + exit 1 + } + ownership="$(sudo cat "$ownership_marker")" + [[ "$ownership" =~ ^nemoclawq:([0-9]+):([0-9]+)$ ]] || { + echo "::error::Qualification account ownership marker content is invalid" >&2 + exit 1 + } + account="nemoclawq" + uid="${BASH_REMATCH[1]}" + gid="${BASH_REMATCH[2]}" + [[ -z "$reported_account" || "$reported_account" == "$account" ]] || { + echo "::error::Qualification account output does not match its ownership marker" >&2 + exit 1 + } + if getent passwd "$uid" >/dev/null && ! getent passwd "$account" >/dev/null; then + echo "::error::Qualification account UID belongs to a different host account" >&2 + exit 1 + fi + if getent passwd "$account" >/dev/null; then + [[ "$(id -u "$account")" == "$uid" && "$(id -g "$account")" == "$gid" ]] || { + echo "::error::Qualification account identity changed before cleanup" >&2 + exit 1 + } + fi + if getent group "$account" >/dev/null; then + [[ "$(getent group "$account")" == "${account}:x:${gid}:" ]] || { + echo "::error::Qualification private group identity changed before cleanup" >&2 + exit 1 + } + elif getent passwd "$account" >/dev/null; then + echo "::error::Qualification private group disappeared before cleanup" >&2 + exit 1 + fi + runtime_dir="/run/user/${uid}" + runtime_directory_unit="user-runtime-dir@${uid}.service" + user_manager_unit="user@${uid}.service" + user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" + user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" + trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + apparmor_profile="${storage_config_directory}/podman.apparmor" + pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" + helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + pasta_executable="${helper_directory}/pasta" + registry_auth_directory="${storage_config_directory}/registry-auth" + registry_auth_file="${registry_auth_directory}/auth.json" + runner_contract="${storage_config_directory}/runner-contract.json" + resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + model_directory="${resource_directory}/model" + sudo systemctl stop "$user_manager_unit" "$runtime_directory_unit" 2>/dev/null || true + ! systemctl is-active --quiet "$user_manager_unit" && ! systemctl is-active --quiet "$runtime_directory_unit" || { + echo "::error::Qualification systemd user lifecycle remained active during cleanup" >&2 + exit 1 + } + if getent passwd "$account" >/dev/null; then + sudo pkill -KILL -u "$uid" 2>/dev/null || true + fi + if [[ -e "$user_manager_dropin_directory" || -L "$user_manager_dropin_directory" ]]; then + [[ -d "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" && "$(stat -c '%u:%g:%a' "$user_manager_dropin_directory")" == "0:0:755" ]] || { + echo "::error::Qualification systemd user-manager drop-in directory cleanup target is invalid" >&2 + exit 1 + } + [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { + echo "::error::Qualification systemd user-manager drop-in cleanup target is invalid" >&2 + exit 1 + } + expected_user_manager_dropin="$(printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"' "$trusted_user_unit_path")" + [[ "$(cat "$user_manager_dropin")" == "$expected_user_manager_dropin" ]] || { + echo "::error::Qualification systemd user-manager drop-in content changed before cleanup" >&2 + exit 1 + } + sudo unlink "$user_manager_dropin" + sudo rmdir "$user_manager_dropin_directory" + sudo systemctl daemon-reload + fi + if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then + [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration cleanup target is invalid" >&2 + exit 1 + } + if [[ -e "$apparmor_profile" || -L "$apparmor_profile" ]]; then + [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification AppArmor profile cleanup target is invalid" >&2 + exit 1 + } + sudo apparmor_parser -R "$apparmor_profile" + sudo rm -f -- "$apparmor_profile" + fi + if [[ -e "$pasta_apparmor_profile" || -L "$pasta_apparmor_profile" ]]; then + [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { + echo "::error::Qualification pasta AppArmor profile cleanup target is invalid" >&2 + exit 1 + } + sudo apparmor_parser -R "$pasta_apparmor_profile" + sudo rm -f -- "$pasta_apparmor_profile" + fi + if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then + [[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${gid}:700" ]] || { + echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2 + exit 1 + } + if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then + sudo test -f "$registry_auth_file" && + sudo test ! -L "$registry_auth_file" && + [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { + echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$registry_auth_file" + fi + sudo rmdir "$registry_auth_directory" + fi + if [[ -e "$runner_contract" || -L "$runner_contract" ]]; then + [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { + echo "::error::Qualification GPU runner contract cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$runner_contract" + fi + if [[ -e "$storage_config_directory/containers.conf" || -L "$storage_config_directory/containers.conf" ]]; then + [[ -f "$storage_config_directory/containers.conf" && ! -L "$storage_config_directory/containers.conf" && "$(stat -c '%u:%g:%a' "$storage_config_directory/containers.conf")" == "0:0:444" ]] || { + echo "::error::Qualification containers configuration cleanup target is invalid" >&2 + exit 1 + } + sudo unlink "$storage_config_directory/containers.conf" + fi + sudo rm -f -- "$storage_config_directory/storage.conf" + sudo rmdir "$storage_config_directory" + fi + if [[ -e "$resource_directory" || -L "$resource_directory" ]]; then + [[ -d "$resource_directory" && ! -L "$resource_directory" && ("$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:711" || "$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:555") ]] || { + echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2 + exit 1 + } + if sudo test -e "$model_directory" || sudo test -L "$model_directory"; then + model_mode="$(sudo stat -c '%u:%g:%a' -- "$model_directory")" + sudo test -d "$model_directory" && + sudo test ! -L "$model_directory" && + [[ "$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555" ]] || { + echo "::error::Qualification GPU model directory cleanup target is invalid" >&2 + exit 1 + } + for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do + target="${model_directory}/${file}" + if sudo test -e "$target" || sudo test -L "$target"; then + file_mode="$(sudo stat -c '%u:%g:%a:%h' -- "$target")" + sudo test -f "$target" && + sudo test ! -L "$target" && + [[ "$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1" ]] || { + echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2 + exit 1 + } + sudo unlink "$target" + fi + done + sudo rmdir "$model_directory" || { + echo "::error::Qualification GPU model directory contains unexpected entries" >&2 + exit 1 + } + fi + sudo rmdir "$resource_directory" || { + echo "::error::Qualification GPU resource directory contains unexpected entries" >&2 + exit 1 + } + fi + if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then + [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { + echo "::error::Qualification Podman executable cleanup target is invalid" >&2 + exit 1 + } + sudo rm -f -- "$podman_executable" + fi + if [[ -e "$helper_directory" || -L "$helper_directory" ]]; then + [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { + echo "::error::Qualification helper directory cleanup target is invalid" >&2 + exit 1 + } + [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { + echo "::error::Qualification pasta executable cleanup target is invalid" >&2 + exit 1 + } + sudo rm -f -- "$pasta_executable" + sudo rmdir "$helper_directory" + fi + if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then + echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2 + exit 1 + fi + if getent passwd "$account" >/dev/null; then + sudo userdel --remove "$account" + fi + if getent group "$account" >/dev/null; then + sudo groupdel "$account" + fi + if getent passwd "$account" >/dev/null; then + echo "::error::Qualification account still exists after cleanup" >&2 + exit 1 + fi + if getent group "$account" >/dev/null; then + echo "::error::Qualification private group still exists after cleanup" >&2 + exit 1 + fi + if grep -q "^${account}:" /etc/subuid /etc/subgid; then + echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2 + exit 1 + fi + [[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || { + echo "::error::Qualification runtime directory remains after cleanup" >&2 + exit 1 + } + storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" + [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { + echo "::error::Qualification storage configuration remains after cleanup" >&2 + exit 1 + } + [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { + echo "::error::Qualification Podman executable remains after cleanup" >&2 + exit 1 + } + [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { + echo "::error::Qualification helper directory remains after cleanup" >&2 + exit 1 + } + [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { + echo "::error::Qualification GPU resource directory remains after cleanup" >&2 + exit 1 + } + sudo rm -f -- "$ownership_marker" + elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then + echo "::error::Qualification account output exists without its ownership marker" >&2 exit 1 fi + [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { + echo "::error::Qualification account ownership marker remains after cleanup" >&2 + exit 1 + } - name: Upload the qualification case evidence - if: always() + if: success() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: ${{ matrix.artifactName }} - path: ${{ runner.temp }}/native-runtime-evidence/evidence.json + path: ${{ runner.temp }}/native-runtime-evidence/ + + native-runtime-qualification-producer-aggregate: + name: Aggregate native runtime qualification evidence + needs: + [ + generate-matrix, + native-runtime-qualification-producer-plan, + native-runtime-qualification-producer, + ] + if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + actions: read + contents: read + pull-requests: read + steps: + - name: Check out the qualification aggregator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.repository }} + ref: ${{ github.workflow_sha }} + path: .qualification-aggregate + persist-credentials: false + sparse-checkout: | + src/lib/onboard/runtime-provider/native-qualification-authority.ts + test/e2e/registry/native-runtime-qualification.ts + tools/e2e/native-runtime-qualification-producer-plan.mts + tools/e2e/native-runtime-qualification-producer-aggregate.mts + sparse-checkout-cone-mode: false + + - name: Download the exact case evidence cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-* + path: ${{ runner.temp }}/native-runtime-case-artifacts + merge-multiple: false + + - name: Resolve this aggregate job identity + id: aggregate-job + env: + GH_TOKEN: ${{ github.token }} + PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} + PRODUCER_RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + jobs="$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \ + -f per_page=100)" + total_count="$(jq -er '.total_count | select(type == "number" and . >= 1)' <<<"$jobs")" || { + echo "::error::Aggregate job lookup returned an invalid job count" >&2 + exit 1 + } + (( total_count <= 100 )) || { + echo "::error::Aggregate job lookup exceeds the bounded 100-job page" >&2 + exit 1 + } + job_id="$(jq -er \ + --arg name 'Aggregate native runtime qualification evidence' \ + --argjson runId "$PRODUCER_RUN_ID" \ + --argjson attempt "$PRODUCER_RUN_ATTEMPT" ' + [.jobs[] | select( + .name == $name and + .run_id == $runId and + .run_attempt == $attempt and + .status == "in_progress" + )] | + select(length == 1) | + .[0].id + ' <<<"$jobs")" || { + echo "::error::Could not resolve one in-progress aggregate job identity" >&2 + exit 1 + } + [[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || { + echo "::error::Aggregate job identity is invalid" >&2 + exit 1 + } + printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT" + + - name: Set up Node for qualification aggregation + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + + - name: Validate and aggregate all 24 case receipts + working-directory: .qualification-aggregate + env: + AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }} + CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts + EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate + QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/native-runtime-qualification-producer-aggregate.mts + + - name: Upload aggregate evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-runtime-qualification-${{ inputs.checkout_sha }} + path: ${{ runner.temp }}/native-runtime-aggregate/ + if-no-files-found: error + retention-days: 30 + compression-level: 9 retired-selector-compatibility: needs: generate-matrix @@ -4145,6 +5461,7 @@ jobs: openclaw-plugin-runtime-exdev-release, openclaw-plugin-runtime-exdev, native-runtime-qualification-producer, + native-runtime-qualification-producer-aggregate, ] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} permissions: diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 32c0f687604..052fdfd9f17 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -180,6 +180,8 @@ It skips the generic Docker availability probe and all generic Docker container, It does not remove the user's Podman installation, storage, networks, or unrelated containers. It does not disable a user-managed socket. On every successful portable cleanup, the final retirement operation removes the exact portable lifecycle receipts, their matching `sandboxes.json` rows, and `~/.config/nemoclaw/portable/containers.conf`, regardless of `--destroy-user-data`. +After NemoClaw removes `containers.conf`, it removes `~/.config/nemoclaw/portable/` and `~/.config/nemoclaw/` only when each directory is empty. +NemoClaw preserves either directory when it contains an unrelated entry. Other preserved user data follows the normal `--destroy-user-data` behavior. After NemoClaw releases the locks, later uninstall-plan cleanup never recursively revisits the canonical receipt, sandbox registry, or `~/.config/nemoclaw/portable/containers.conf` paths. This preserves any new lifecycle generation published after lock release. diff --git a/scripts/checks/run-native-runtime-installer-qualification.sh b/scripts/checks/run-native-runtime-installer-qualification.sh index 0dfd610e6e8..61db3065be6 100755 --- a/scripts/checks/run-native-runtime-installer-qualification.sh +++ b/scripts/checks/run-native-runtime-installer-qualification.sh @@ -98,6 +98,7 @@ verify_checkout() { *) fail "$label has an unexpected origin repository." ;; esac assert_checkout_has_no_git_credentials "$checkout" "$label" + printf '%s\n' "$revision" } verify_committed_file() { @@ -191,14 +192,51 @@ assert_docker_unavailable() { [[ ! -S "$socket_path" ]] \ || fail "A Docker socket exists during the ${phase} check." done < <(docker_socket_paths) + + printf '%s\n' \ + '{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}' } -run_native_runtime_installer_qualification() { - candidate_checkout="" - candidate_sha="" - expected_installer_sha256="" - expected_architecture="" - artifact_dir_input="" +run_native_runtime_installer_qualification() ( + local candidate_checkout="" + local candidate_sha="" + local expected_installer_sha256="" + local expected_architecture="" + local artifact_dir_input="" + local artifact_parent="" + local artifact_name="" + local artifact_dir="" + local runner_architecture="" + local candidate_installer="" + local candidate_setup_script="" + local qualification_root="" + local qualification_home="" + local qualification_tmp="" + local docker_guard_dir="" + local managed_payload_root="" + local verified_script_dir="" + local verified_installer="" + local verified_setup_script="" + local installed_checkout="" + local receipt_stage="" + local docker_guard="" + local docker_guard_sha256="" + local candidate_status=0 + local verified_candidate_revision="" + local installed_revision="" + local pre_execution_docker_posture="" + local post_execution_docker_posture="" + + cleanup() { + if [[ -n "$receipt_stage" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then + rm -rf -- "$receipt_stage" + fi + if [[ -n "$qualification_root" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then + rm -rf -- "$qualification_root" + fi + } + trap cleanup EXIT + while [[ "$#" -gt 0 ]]; do case "$1" in --candidate-checkout) @@ -272,7 +310,9 @@ run_native_runtime_installer_qualification() { candidate_installer="${candidate_checkout}/scripts/install.sh" candidate_setup_script="${candidate_checkout}/scripts/setup-jetson.sh" - verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout" + verified_candidate_revision="$( + verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout" + )" verify_installer \ "$candidate_checkout" \ "$candidate_sha" \ @@ -303,16 +343,6 @@ run_native_runtime_installer_qualification() { "$managed_payload_root" \ "$verified_script_dir" - cleanup() { - if [[ -n "${receipt_stage:-}" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then - rm -rf -- "$receipt_stage" - fi - if [[ -n "${qualification_root:-}" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then - rm -rf -- "$qualification_root" - fi - } - trap cleanup EXIT - cp -- "$candidate_installer" "$verified_installer" cp -- "$candidate_setup_script" "$verified_setup_script" chmod 500 "$verified_installer" "$verified_setup_script" @@ -336,9 +366,10 @@ run_native_runtime_installer_qualification() { PATH="${docker_guard_dir}:${PATH}" export PATH - assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256" + pre_execution_docker_posture="$( + assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256" + )" - candidate_status=0 # The child shell expands positional parameters inside this literal program. # shellcheck disable=SC2016 env -i \ @@ -366,11 +397,15 @@ run_native_runtime_installer_qualification() { install_nemoclaw_before_onboarding ' _ "$verified_installer" "$verified_script_dir" || candidate_status=$? - assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256" + post_execution_docker_posture="$( + assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256" + )" [[ "$candidate_status" -eq 0 ]] \ || fail "The candidate installer phase executor exited with status ${candidate_status}." - verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout" + installed_revision="$( + verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout" + )" verify_installer \ "$installed_checkout" \ "$candidate_sha" \ @@ -382,16 +417,16 @@ run_native_runtime_installer_qualification() { "$expected_installer_sha256" "$candidate_sha" "$runner_architecture" \ >"${receipt_stage}/invocation.json" printf '{"receiptVersion":1,"repository":"%s","revision":"%s","installerSha256":"%s"}\n' \ - "$CANONICAL_REPOSITORY" "$candidate_sha" "$expected_installer_sha256" \ + "$CANONICAL_REPOSITORY" "$verified_candidate_revision" "$expected_installer_sha256" \ >"${receipt_stage}/candidate-source.json" printf '{"receiptVersion":1,"repository":"%s","requestedRevision":"%s","installedRevision":"%s","installMode":"managed","installerSha256":"%s"}\n' \ - "$CANONICAL_REPOSITORY" "$candidate_sha" "$candidate_sha" "$expected_installer_sha256" \ + "$CANONICAL_REPOSITORY" "$candidate_sha" "$installed_revision" "$expected_installer_sha256" \ >"${receipt_stage}/installed-source.json" printf '{"receiptVersion":1,"requested":"%s","runner":"%s"}\n' \ "$expected_architecture" "$runner_architecture" \ >"${receipt_stage}/architecture.json" - printf '%s\n' \ - '{"receiptVersion":1,"preExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true},"postExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}}' \ + printf '{"receiptVersion":1,"preExecution":%s,"postExecution":%s}\n' \ + "$pre_execution_docker_posture" "$post_execution_docker_posture" \ >"${receipt_stage}/docker-absence.json" bounded_file "${receipt_stage}/installer.sh" "$MAX_INSTALLER_BYTES" @@ -409,7 +444,10 @@ run_native_runtime_installer_qualification() { receipt_stage="" printf 'Native runtime installer qualification receipts: %s\n' "$artifact_dir" -} + cleanup + trap - EXIT + unset -f cleanup +) if [[ "${BASH_SOURCE[0]:-}" == "$0" ]]; then run_native_runtime_installer_qualification "$@" diff --git a/src/lib/onboard/runtime-provider/native-qualification-authority.ts b/src/lib/onboard/runtime-provider/native-qualification-authority.ts index 4c811882afc..b6fe3d5b5fd 100644 --- a/src/lib/onboard/runtime-provider/native-qualification-authority.ts +++ b/src/lib/onboard/runtime-provider/native-qualification-authority.ts @@ -10,7 +10,7 @@ export const NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY = "NVIDIA/NemoClaw"; /** The trusted collector is separate and rejects evidence emitted by its own workflow. */ export const NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW = - ".github/workflows/native-runtime-qualification.yaml"; + ".github/workflows/e2e.yaml"; export interface NativeRuntimeQualificationProtectedRun { readonly repository: string; diff --git a/src/lib/onboard/runtime-provider/podman-preflight.test.ts b/src/lib/onboard/runtime-provider/podman-preflight.test.ts index 80d8b253068..c37a9280a39 100644 --- a/src/lib/onboard/runtime-provider/podman-preflight.test.ts +++ b/src/lib/onboard/runtime-provider/podman-preflight.test.ts @@ -19,6 +19,16 @@ const INFO = JSON.stringify({ arch: "amd64", os: "linux", cgroupVersion: "v2", + idMappings: { + uidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + gidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + }, networkBackend: "netavark", security: { rootless: true }, discoveredDevices: [ @@ -33,7 +43,6 @@ function engine( readonly info?: string; readonly serverVersion?: string; readonly version?: string; - readonly idMap?: string; } = {}, ): ContainerEngine { const capture = vi.fn((args: readonly string[]) => { @@ -55,10 +64,7 @@ function engine( }); const captureHost = vi.fn((args: readonly string[]) => ({ status: 0, - stdout: - args[0] === "--version" - ? (overrides.version ?? "podman version 5.6.2\n") - : (overrides.idMap ?? "0 1000 1\n1 100000 65536\n"), + stdout: args[0] === "--version" ? (overrides.version ?? "podman version 5.6.2\n") : "", stderr: "", })); return { @@ -98,14 +104,7 @@ describe("Podman host preflight", () => { expect(runtime.capture).toHaveBeenCalledWith(["info", "--format", "json"], 15_000); expect(runtime.capture).toHaveBeenCalledWith(["version", "--format", "json"], 10_000); expect(runtime.captureHost).toHaveBeenCalledWith(["--version"], 10_000); - expect(runtime.captureHost).toHaveBeenCalledWith( - ["unshare", "cat", "/proc/self/uid_map"], - 10_000, - ); - expect(runtime.captureHost).toHaveBeenCalledWith( - ["unshare", "cat", "/proc/self/gid_map"], - 10_000, - ); + expect(runtime.captureHost).toHaveBeenCalledTimes(1); }); it("keeps the CPU receipt server version canonical while preserving exact inference authority", () => { @@ -196,12 +195,41 @@ describe("Podman host preflight", () => { }); it("rejects missing subordinate user mappings", () => { + const info = JSON.stringify({ + ...JSON.parse(INFO), + host: { + ...JSON.parse(INFO).host, + idMappings: { + ...JSON.parse(INFO).host.idMappings, + uidmap: [{ container_id: 0, host_id: 1000, size: 1 }], + }, + }, + }); + expect(() => + qualifyPodmanHost(engine({ info }), { + platform: "linux", + architecture: "x64", + }), + ).toThrow("subordinate UID range for the API service user"); + }); + + it("rejects malformed API-service ID mappings", () => { + const info = JSON.stringify({ + ...JSON.parse(INFO), + host: { + ...JSON.parse(INFO).host, + idMappings: { + ...JSON.parse(INFO).host.idMappings, + gidmap: [{ container_id: 0, host_id: 1000, size: "65536" }], + }, + }, + }); expect(() => - qualifyPodmanHost(engine({ idMap: "0 1000 1\n" }), { + qualifyPodmanHost(engine({ info }), { platform: "linux", architecture: "x64", }), - ).toThrow("subordinate UID range"); + ).toThrow("Podman API returned malformed gidmap"); }); it("fails before commands for another engine scope or unsupported host platform", () => { diff --git a/src/lib/onboard/runtime-provider/podman-preflight.ts b/src/lib/onboard/runtime-provider/podman-preflight.ts index 97741cb5e1f..e393cf0d95a 100644 --- a/src/lib/onboard/runtime-provider/podman-preflight.ts +++ b/src/lib/onboard/runtime-provider/podman-preflight.ts @@ -177,25 +177,42 @@ function normalizeArchitecture(value: string): "amd64" | "arm64" | null { return null; } -function hasSubordinateIdMapping(output: string): boolean { - return output - .trim() - .split(/\r?\n/u) - .some((line) => { - const values = line.trim().split(/\s+/u).map(Number); - return values.length === 3 && values.every(Number.isFinite) && (values[2] ?? 0) > 1; - }); -} - -function requireSubordinateIdMappings(engine: ContainerEngine): void { - for (const mapping of ["uid_map", "gid_map"] as const) { - const result = requireSuccessful( - `${mapping} inspection`, - engine.captureHost(["unshare", "cat", `/proc/self/${mapping}`], 10_000), +function requireSubordinateIdMappings(host: unknown): void { + // Bind this check to the same rootless API service as the rest of the + // preflight. A local `podman unshare` can resolve different storage and user + // authority than an explicitly bound service endpoint. + const mappings = field(host, "idMappings", "IDMappings"); + for (const mapping of ["uidmap", "gidmap"] as const) { + const entries = field( + mappings, + mapping, + mapping === "uidmap" ? "UIDMap" : "GIDMap", ); - if (!hasSubordinateIdMapping(result.stdout)) { + if (!Array.isArray(entries) || entries.length === 0 || entries.length > 1_024) { + throw new PodmanHostPreflightError(`the Podman API returned malformed ${mapping}`); + } + let hasSubordinateRange = false; + for (const value of entries) { + const entry = record(value); + const containerId = field(entry, "container_id", "containerID", "ContainerID"); + const hostId = field(entry, "host_id", "hostID", "HostID"); + const size = field(entry, "size", "Size"); + if ( + !entry || + !Number.isSafeInteger(containerId) || + !Number.isSafeInteger(hostId) || + !Number.isSafeInteger(size) || + (containerId as number) < 0 || + (hostId as number) < 0 || + (size as number) <= 0 + ) { + throw new PodmanHostPreflightError(`the Podman API returned malformed ${mapping}`); + } + if ((size as number) > 1) hasSubordinateRange = true; + } + if (!hasSubordinateRange) { throw new PodmanHostPreflightError( - `rootless Podman requires a subordinate ${mapping === "uid_map" ? "UID" : "GID"} range for the current user`, + `rootless Podman requires a subordinate ${mapping === "uidmap" ? "UID" : "GID"} range for the API service user`, ); } } @@ -437,7 +454,7 @@ export function qualifyPodmanHost( `the Podman service architecture '${normalizedArchitecture}' does not match host '${expectedArchitecture}'`, ); } - requireSubordinateIdMappings(engine); + requireSubordinateIdMappings(host); return Object.freeze({ providerId: "podman", diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 2d99815ba69..9710ea2b099 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -128,6 +128,16 @@ function hostDoctorEngine(authorityId = AUTHORITY_ID): PodmanContainerEngine { arch: "amd64", os: "linux", cgroupVersion: "v2", + idMappings: { + uidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + gidmap: [ + { container_id: 0, host_id: 1000, size: 1 }, + { container_id: 1, host_id: 100000, size: 65536 }, + ], + }, networkBackend: "netavark", security: { rootless: true }, }, @@ -140,7 +150,7 @@ function hostDoctorEngine(authorityId = AUTHORITY_ID): PodmanContainerEngine { }), captureHost: vi.fn((args: readonly string[]) => ({ status: 0, - stdout: args[0] === "--version" ? "podman version 5.6.2\n" : "0 1000 1\n1 100000 65536\n", + stdout: args[0] === "--version" ? "podman version 5.6.2\n" : "", stderr: "", })), }; diff --git a/src/lib/security/credential-filter.test.ts b/src/lib/security/credential-filter.test.ts index a9cce4b052f..9dee73c0b2f 100644 --- a/src/lib/security/credential-filter.test.ts +++ b/src/lib/security/credential-filter.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "n import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { parse as parseYaml } from "yaml"; import { makeEmptyClaimsJwtFixture, @@ -458,7 +459,31 @@ describe("sanitizeConfigFile", () => { expect(readFileSync(configPath, "utf-8")).toContain("api_key:"); }); - it("fails closed for array-root Hermes YAML", () => { + it("sanitizes nested YAML arrays and rejects non-object documents", () => { + const sanitized = sanitizeYamlConfigContent( + [ + "items:", + " - safe", + " - api_key: sk-secret-value-long-enough", + " enabled: true", + " count: 2", + " optional: null", + "", + ].join("\n"), + ); + + expect(parseYaml(sanitized as string)).toEqual({ + items: [ + "safe", + { + api_key: "[STRIPPED_BY_MIGRATION]", + enabled: true, + count: 2, + optional: null, + }, + ], + }); + expect(sanitizeYamlConfigContent("42\n")).toBeNull(); expect(sanitizeYamlConfigContent("- first\n- second\n")).toBeNull(); }); @@ -567,10 +592,10 @@ describe("sanitizeEnvFile", () => { it("rewrites .env credentials in place", () => { const envPath = join(tmpDir, ".env"); - writeFileSync(envPath, "DB_PASS=secret\nLOG_LEVEL=info\n"); + writeFileSync(envPath, "DB_PASS=secret\nAPI_KEY=sk-secret-value\nLOG_LEVEL=info\n"); expect(sanitizeEnvFile(envPath)).toBe(true); expect(readFileSync(envPath, "utf-8")).toBe( - "DB_PASS=[STRIPPED_BY_MIGRATION]\nLOG_LEVEL=info\n", + "DB_PASS=[STRIPPED_BY_MIGRATION]\nAPI_KEY=[STRIPPED_BY_MIGRATION]\nLOG_LEVEL=info\n", ); }); diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts index 448d4338804..cc933aad846 100644 --- a/src/lib/state/portable-uninstall-retirement.test.ts +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -39,6 +39,7 @@ function fixture() { type Fixture = ReturnType; type TargetRole = "config" | "receipt" | "registry"; +const noMutation = (): void => undefined; function prepareFixture(test: Fixture) { return preparePortableRetirement(test.homeDir, [RECEIPT_BASENAME]); @@ -173,6 +174,8 @@ describe("portable uninstall retirement state", () => { expect( [test.config, test.receipt, test.registryFile].every((target) => !fs.existsSync(target)), ).toBe(true); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(false); expect(inspectPortableRetirementRecovery(test.homeDir)).toEqual({ artifacts: [], fixedState: "1000", @@ -182,6 +185,172 @@ describe("portable uninstall retirement state", () => { expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); }); + it.each([ + [ + "portable configuration", + (test: Fixture) => path.join(path.dirname(test.config), "kept.conf"), + true, + ], + [ + "NemoClaw configuration", + (test: Fixture) => path.join(path.dirname(path.dirname(test.config)), "kept.conf"), + false, + ], + ])( + "preserves unrelated %s content during retirement (#9189)", + (_label, markerPath, portableDirectoryRemains) => { + const test = fixture(); + const marker = markerPath(test); + fs.writeFileSync(marker, "operator-owned\n", { mode: 0o600 }); + + publishAndRetirePortableEvidence(prepareFixture(test)); + + expect(fs.existsSync(test.config)).toBe(false); + expect(fs.readFileSync(marker, "utf8")).toBe("operator-owned\n"); + expect(fs.existsSync(path.dirname(test.config))).toBe(portableDirectoryRemains); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(true); + }, + ); + + it("preserves a NemoClaw configuration directory with more than 1,024 entries (#9189)", () => { + const test = fixture(); + const prepared = prepareFixture(test); + const configDir = path.dirname(path.dirname(test.config)); + const readdir = fs.readdirSync.bind(fs); + vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => + String(target) === configDir + ? new Array(1_025).fill("operator-owned.conf") + : readdir(target, options as never)) as typeof fs.readdirSync); + + expect(() => publishAndRetirePortableEvidence(prepared)).not.toThrow(); + expect(fs.existsSync(test.config)).toBe(false); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(configDir)).toBe(true); + }); + + it("rejects a symlink that replaces the portable configuration directory (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const outside = path.join(test.homeDir, "outside"); + fs.mkdirSync(outside, { mode: 0o700 }); + const unlink = fs.unlinkSync.bind(fs); + const replacePortableDirectory = () => { + fs.rmdirSync(portableDir); + fs.symlinkSync(outside, portableDir, "dir"); + }; + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + unlink(target); + (String(target).includes(".containers.conf.portable-uninstall-") + ? replacePortableDirectory + : noMutation)(); + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); + expect(fs.lstatSync(portableDir).isSymbolicLink()).toBe(true); + expect(fs.statSync(outside).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects a symlink that replaces the NemoClaw configuration directory (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const configDir = path.dirname(portableDir); + const outside = path.join(test.homeDir, "outside"); + fs.mkdirSync(path.join(outside, "portable"), { mode: 0o700, recursive: true }); + const unlink = fs.unlinkSync.bind(fs); + const replaceConfigDirectory = () => { + fs.rmdirSync(portableDir); + fs.rmdirSync(configDir); + fs.symlinkSync(outside, configDir, "dir"); + }; + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + unlink(target); + (String(target).includes(".containers.conf.portable-uninstall-") + ? replaceConfigDirectory + : noMutation)(); + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(); + expect(fs.lstatSync(configDir).isSymbolicLink()).toBe(true); + expect(fs.statSync(path.join(outside, "portable")).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects group-writable portable configuration authority (#9189)", () => { + const test = fixture(); + fs.chmodSync(path.dirname(test.config), 0o770); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/Unsafe/); + expect(fs.existsSync(path.dirname(test.config))).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects portable configuration ownership drift (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const lstat = fs.lstatSync.bind(fs); + vi.spyOn(fs, "lstatSync").mockImplementation(((target, options) => { + const stat = lstat(target, options as never); + return String(target) === portableDir && typeof stat.uid === "bigint" + ? new Proxy(stat, { + get(current, property) { + const value = Reflect.get(current, property, current) as unknown; + return property === "uid" + ? current.uid + 1n + : typeof value === "function" + ? value.bind(current) + : value; + }, + }) + : stat; + }) as typeof fs.lstatSync); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/Unsafe/); + expect(fs.existsSync(portableDir)).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("preserves an entry inserted before empty-directory removal (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const marker = path.join(portableDir, "concurrent.conf"); + const rmdir = fs.rmdirSync.bind(fs); + let inserted = false; + const insertMarker = () => { + inserted = true; + fs.writeFileSync(marker, "concurrent\n", { mode: 0o600 }); + }; + vi.spyOn(fs, "rmdirSync").mockImplementation((target) => { + (!inserted && String(target) === portableDir ? insertMarker : noMutation)(); + return rmdir(target); + }); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).not.toThrow(); + expect(fs.readFileSync(marker, "utf8")).toBe("concurrent\n"); + expect(fs.existsSync(portableDir)).toBe(true); + }); + + it("rejects portable configuration directory replacement between identity checks (#9189)", () => { + const test = fixture(); + const portableDir = path.dirname(test.config); + const readdir = fs.readdirSync.bind(fs); + let replaced = false; + const replacePortableDirectory = () => { + replaced = true; + fs.rmdirSync(portableDir); + fs.mkdirSync(portableDir, { mode: 0o700 }); + }; + vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => { + const entries = readdir(target, options as never); + (!replaced && String(target) === portableDir ? replacePortableDirectory : noMutation)(); + return entries; + }) as typeof fs.readdirSync); + + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/changed/); + expect(fs.statSync(portableDir).isDirectory()).toBe(true); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + it("rejects non-UTF-8 retirement and authority JSON before cleanup (#9189)", () => { const malformedRecord = fixture(); fs.writeFileSync(retirementRecordPath(malformedRecord), Buffer.from([0xff]), { mode: 0o600 }); @@ -385,7 +554,13 @@ describe("portable uninstall retirement state", () => { }; retirement.publishAndRetirePortableEvidence(prepared); `; - const boundaries = { fsyncSync: 10, linkSync: 1, renameSync: 4, unlinkSync: 4 } as const; + const boundaries = { + fsyncSync: 12, + linkSync: 1, + renameSync: 4, + rmdirSync: 2, + unlinkSync: 4, + } as const; const cases = Object.entries(boundaries).flatMap(([operation, count]) => Array.from({ length: count }, (_value, index) => [operation, index + 1] as const), ); @@ -413,6 +588,8 @@ describe("portable uninstall retirement state", () => { const assertRecovered = () => { resumePortableEvidenceRetirement(test.homeDir); expect(targets.every((target) => !fs.existsSync(target))).toBe(true); + expect(fs.existsSync(path.dirname(test.config))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(test.config)))).toBe(false); }; const assertPrior = () => expect(targets.every(fs.existsSync)).toBe(true); (hasPortableRetirementRecord(test.homeDir) ? assertRecovered : assertPrior)(); diff --git a/src/lib/state/portable-uninstall-retirement.ts b/src/lib/state/portable-uninstall-retirement.ts index 11d46d4029c..d0d8122443a 100644 --- a/src/lib/state/portable-uninstall-retirement.ts +++ b/src/lib/state/portable-uninstall-retirement.ts @@ -180,7 +180,10 @@ const paths = (homeDir: string) => [Key in keyof typeof NAMES]: string; }; function fsyncDirectory(directory: string): void { - const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + const descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_DIRECTORY, + ); try { fs.fsyncSync(descriptor); } finally { @@ -593,6 +596,125 @@ function detachDelete( throw new Error(`Portable uninstall survivor changed: ${survivor.path}`); } } +interface OwnedDirectoryHandle { + readonly descriptor: number; + readonly identity: fs.BigIntStats; + readonly path: string; +} +function sameDirectoryIdentity(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.isDirectory() && + right.isDirectory() && + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.uid === right.uid + ); +} +function assertOwnedDirectory(handle: OwnedDirectoryHandle): void { + const descriptorStat = fs.fstatSync(handle.descriptor, { bigint: true }); + const namedStat = fs.lstatSync(handle.path, { bigint: true }); + if ( + namedStat.isSymbolicLink() || + !sameDirectoryIdentity(handle.identity, descriptorStat) || + !sameDirectoryIdentity(handle.identity, namedStat) + ) + throw new Error(`Portable uninstall directory changed: ${handle.path}`); +} +function openOwnedDirectory( + directory: string, + required: boolean, + requirePrivateMode = false, +): OwnedDirectoryHandle | null { + let descriptor: number | null = null; + try { + descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_DIRECTORY, + ); + const identity = fs.fstatSync(descriptor, { bigint: true }); + const named = fs.lstatSync(directory, { bigint: true }); + const uid = process.getuid?.(); + const permissions = identity.mode & 0o777n; + if ( + uid === undefined || + named.isSymbolicLink() || + !sameStat(identity, named) || + identity.uid !== BigInt(uid) || + identity.nlink < 1n || + (requirePrivateMode ? permissions !== 0o700n : (permissions & 0o022n) !== 0n) + ) + throw new Error(`Unsafe portable uninstall directory: ${directory}`); + return { descriptor, identity, path: directory }; + } catch (error) { + if (descriptor !== null) fs.closeSync(descriptor); + if (isErrnoException(error) && error.code === "ENOENT" && !required) return null; + throw error; + } +} +function ownedDirectoryIsEmpty(handle: OwnedDirectoryHandle): boolean { + assertOwnedDirectory(handle); + const isEmpty = fs.readdirSync(handle.path).length === 0; + assertOwnedDirectory(handle); + return isEmpty; +} +function removeOwnedEmptyDirectory( + handle: OwnedDirectoryHandle, + parent: OwnedDirectoryHandle, +): boolean { + if (!ownedDirectoryIsEmpty(handle)) return false; + assertOwnedDirectory(parent); + assertOwnedDirectory(handle); + try { + fs.rmdirSync(handle.path); + } catch (error) { + if (isErrnoException(error) && (error.code === "ENOTEMPTY" || error.code === "EEXIST")) { + assertOwnedDirectory(parent); + assertOwnedDirectory(handle); + return false; + } + if (!(isErrnoException(error) && error.code === "ENOENT")) throw error; + } + fs.fsyncSync(parent.descriptor); + assertOwnedDirectory(parent); + if (entryExists(handle.path)) + throw new Error(`Portable uninstall directory was replaced: ${handle.path}`); + return true; +} +function removeRetiredPortableConfigDirectories(homeDir: string): void { + const home = openOwnedDirectory(homeDir, true)!; + const configHomePath = path.join(homeDir, ".config"); + const configDir = path.join(homeDir, ".config", "nemoclaw"); + let configHome: OwnedDirectoryHandle | null = null; + let nemoclawConfig: OwnedDirectoryHandle | null = null; + let portableConfig: OwnedDirectoryHandle | null = null; + try { + configHome = openOwnedDirectory(configHomePath, true)!; + assertOwnedDirectory(home); + nemoclawConfig = openOwnedDirectory(configDir, false); + assertOwnedDirectory(configHome); + assertOwnedDirectory(home); + if (!nemoclawConfig) { + fs.fsyncSync(configHome.descriptor); + assertOwnedDirectory(configHome); + return; + } + portableConfig = openOwnedDirectory(path.join(configDir, "portable"), false, true); + assertOwnedDirectory(nemoclawConfig); + assertOwnedDirectory(configHome); + if (portableConfig && !removeOwnedEmptyDirectory(portableConfig, nemoclawConfig)) return; + assertOwnedDirectory(nemoclawConfig); + if (!ownedDirectoryIsEmpty(nemoclawConfig)) return; + removeOwnedEmptyDirectory(nemoclawConfig, configHome); + assertOwnedDirectory(configHome); + assertOwnedDirectory(home); + } finally { + if (portableConfig) fs.closeSync(portableConfig.descriptor); + if (nemoclawConfig) fs.closeSync(nemoclawConfig.descriptor); + if (configHome) fs.closeSync(configHome.descriptor); + fs.closeSync(home.descriptor); + } +} function recoverTemp(homeDir: string): ExactFile | null { const state = paths(homeDir); const pendingPaths = [state.T, state.TC].filter(entryExists); @@ -654,7 +776,10 @@ function retireTargets( const target = record.targets[index]!; const state = states[index]!; if (state === "retired") { - fsyncDirectory(path.dirname(canonical(homeDir, target))); + const parent = path.dirname(canonical(homeDir, target)); + if (target[0] === "config" && !entryExists(parent)) + fsyncDirectory(path.join(homeDir, ".config")); + else fsyncDirectory(parent); continue; } const source = canonical(homeDir, target); @@ -669,11 +794,15 @@ function retireTargets( detachDelete(replacement ? null : source, staged, exact, roleLimit(target[0]), undefined, true); } } +function finishPortableEvidenceRetirement(homeDir: string, record: RecordState): void { + retireTargets(homeDir, record); + removeRetiredPortableConfigDirectories(homeDir); +} export function publishAndRetirePortableEvidence(prepared: PreparedPortableRetirement): void { if (!allTargetsExact(prepared.homeDir, prepared.record)) throw new Error("Portable uninstall authority changed before publication"); publish(prepared); - retireTargets(prepared.homeDir, prepared.record); + finishPortableEvidenceRetirement(prepared.homeDir, prepared.record); } function load(homeDir: string): { file: ExactFile; record: RecordState } | null { @@ -727,7 +856,7 @@ export function resumePortableOnboardReplacementEvidence(homeDir: string): void export function resumePortableEvidenceRetirement(homeDir: string): void { const recorded = load(homeDir); if (!recorded) throw new Error("Portable uninstall retirement record is missing"); - retireTargets(homeDir, recorded.record); + finishPortableEvidenceRetirement(homeDir, recorded.record); } function durableFile(target: string): ExactFile { diff --git a/test/e2e/README.md b/test/e2e/README.md index c95da338c80..b80f1d8e93e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1116,7 +1116,7 @@ The PR selection does not forward an NVIDIA API key, `BRAVE_API_KEY`, or `GITHUB The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. -The trusted workflow definition remains on `main` and binds the candidate head to the current PR base SHA. +The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. It does not run GitHub's synthetic merge commit. Before candidate execution, the workflow uploads a `nemoclaw-e2e-dispatch-v2` receipt for the trusted manual run. OpenShell PR qualification uses that receipt to bind the candidate repository, candidate commit SHA, base SHA, workflow SHA, run, and selectors. @@ -1148,12 +1148,56 @@ After a failure, inspect the workflow artifacts and remove resources that target For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary, cohort-owned NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. Before starting NIM or vLLM, the live fixture rejects a pre-existing cohort container name. It records the full container ID, requested image, immutable image ID, cohort owner, and provider label, then removes only that exact container after revalidating every field. Missing, ambiguous, name-reused, drifted, or indeterminate cleanup evidence fails the test, as does any retained exact ID or name. A fail-closed refusal can leave the secret-bearing NIM container alive until runner teardown; inspect the redacted artifacts and remove only the verified container. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. -For `native-runtime-qualification-producer`, use a same-repository open PR and the first workflow attempt. The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, and plan from `main`. It passes no GitHub, model-provider, API, or messaging credentials to candidate code. Candidate execution uses `env -i` under a temporary unprivileged account on a reviewed ephemeral runner. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU case also requires `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. Until that executor and the required runner capacity exist, the producer fails closed instead of claiming qualification. - -Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. It uploads one evidence artifact for each planned case. Cleanup terminates processes owned by the candidate account and removes that account. If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. Recover or replace the runner before dispatching a new run. Do not rerun the same workflow attempt; the producer rejects attempts after the first. Dispatch a new run after recovery. - -For a manual PR run, provide the current PR number, lowercase 40-character candidate commit SHA, PR source repository, lowercase 40-character base commit SHA, trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. -Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. +Before you dispatch `native-runtime-qualification-producer`, review the `NVIDIA_API_KEY` boundary below. +The host-side preparation step receives the long-lived repository secret and uses it to create runner-local registry authentication and pull pinned GPU images. +The step deletes the registry authentication file and unsets the variable before candidate execution. +The workflow does not revoke the API key. +The key remains valid in the issuing NVIDIA service until it expires or that service revokes it. +If exposure occurs or cleanup cannot be confirmed, revoke the key in the issuing NVIDIA service. +Alternatively, rotate the key and invalidate the old value. +Verify that the old value is invalid. + +After you accept this credential boundary, dispatch `native-runtime-qualification-producer` from trusted `main` for a same-repository open PR. +Use the first workflow attempt. +The executing workflow commit and `workflow_sha` input must equal the exact PR-recorded base commit. +The actor must have repository `maintain` or `admin` permission. +If `github.triggering_actor` differs from the actor, it must also have one of those permissions. + +The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. +The unprivileged installer and live-test processes run with `env -i` under a temporary account. +They receive no GitHub, inference provider, API, or messaging credential. +Docker is unavailable to these processes. +Before any self-hosted qualification job runs, set the GitHub Actions repository variable `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL` to `enabled`. +Set the repository variable `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL` to the reviewed ARM64 GPU runner label. +The workflow provides no ARM64 GPU fallback runner. +The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. +Each successful case uploads the validated installer, runtime, operation, and optional NVIDIA CDI receipts. +The workflow does not upload the candidate `execution.json` or `case-evidence.json` staging files. +A failed case uploads no case-evidence artifact. +The aggregate job runs only after all 24 cases succeed. +It rejects an incomplete or mixed cohort before it emits the 24-case evidence artifact. +If the executor or required runner capacity is absent, the producer fails closed instead of claiming qualification. +This qualification does not register or select Podman in production and does not establish public Podman support. + +Before candidate execution, the producer stops Docker, masks its service and socket, removes Docker sockets, and rejects a usable `docker` command. +Cleanup terminates processes owned by the candidate account and removes that account. +If cleanup fails or the runner becomes unavailable, inspect the host and remove the ephemeral runner from service. +Recover or replace the runner before dispatching a new run. +Do not rerun the same workflow attempt; the producer rejects attempts after the first. +Dispatch a new run after recovery. +If a case fails, use the GitHub Actions job log. +Inspect a case artifact only when its upload step completed. + +For a manual PR run, provide these inputs: + +- The current PR number. +- The lowercase 40-character SHA of the latest PR commit. +- The PR source repository. +- The lowercase 40-character PR base SHA. +- The exact SHA of the trusted workflow commit on `main`. +- A review reason containing 10 to 500 printable characters. + +For the default PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. @@ -1161,14 +1205,15 @@ To select the protected managed-image runtime qualification, set `jobs=managed-i Leave `targets` empty. Keep `include_staging_brev_launchable=false`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -To select native-runtime qualification evidence production, set `jobs=native-runtime-qualification-producer`. +To select native runtime qualification evidence production, set `jobs=native-runtime-qualification-producer`. Leave `targets` empty and keep `include_staging_brev_launchable=false`. +For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -The trusted pre-checkout step requires current `maintain` or `admin` permission and validates the exact open PR and selected mode before candidate code runs. +A trusted `main` workflow pre-checkout step requires current `maintain` or `admin` permission. The workflow validates the exact open PR and selected mode before candidate code runs. A second validation after checkout rejects a changed candidate commit, base commit, or PR source repository before preparation. The Actions run is advisory for the pull request and is not a required merge context. -Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and trusted workflow SHA. +Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and executing workflow SHA. A changed PR source repository, candidate commit SHA, or base commit SHA invalidates the evidence and requires a new manual run. The platform-evidence workflow runs on configured pushes to `main` and supports manual dispatch for branch diagnosis. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 271aafb0350..a8d5de979dc 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -286,8 +286,19 @@ test/e2e/ A maintainer can also dispatch the trusted `main` workflow against the latest commit from an open internal or fork PR. The manual path validates the actor, PR number, PR source repository, candidate commit SHA, base commit SHA, - workflow SHA, review reason, and - allowed jobs, targets, and Launchable combination before candidate checkout. + workflow SHA, review reason, and allowed jobs, targets, and Launchable + combination before candidate checkout. + A trusted `main` native runtime producer run requires the executing workflow + commit and `workflow_sha` input to equal the exact PR-recorded base commit. + The producer accepts only a same-repository PR and the first workflow attempt. + The host-side preparation step receives the long-lived `NVIDIA_API_KEY` + repository secret in its environment. It creates runner-local registry + authentication and pulls pinned GPU images. It then deletes the registry + authentication file and unsets the variable before the separate candidate + installer or live-test process starts. Cleanup removes runner-local registry + authentication but does not revoke the key. The key remains valid in the + issuing NVIDIA service until it expires or that service revokes it. + For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`, every catalogue target in the @@ -300,9 +311,12 @@ test/e2e/ this default selection. If the DGX Spark flag is `true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. - Accepted nonempty `jobs` values are `inference-routing` and - `managed-image-protected-runtime`. The `jetson-nvmap-gpu` target is also - accepted when `allow_jetson_dispatch` is `true`. + Accepted nonempty `jobs` values are: + + - `inference-routing` + - `managed-image-protected-runtime` + - `native-runtime-qualification-producer` + The `jetson-nvmap-gpu` target is also accepted when `allow_jetson_dispatch` is `true`. Refer to [NemoClaw E2E CI](../README.md). - [Jetson dispatch controller](jetson-dispatch.md) defines the NemoClaw-owned @@ -310,10 +324,9 @@ test/e2e/ evidence for `jetson-nvmap-gpu`. The service behind that contract is operator-owned infrastructure. -- `.github/workflows/e2e.yaml` runs selected or all supported - live E2E targets and uploads an explicit artifact allowlist with - JSON summaries plus action, log, and shell command-evidence directories under - 14-day retention. +- `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist. + The shared E2E uploader retains per-target JSON summaries and command-evidence directories for 14 days. + The native runtime aggregate upload retains `native-runtime-qualification-` for 30 days. Final OpenShell gateway-auth artifacts pass a fail-closed safety scan after cleanup. The scanner copies safe files into a private staging directory, scans that copy again, and adds a marker bound to the current Actions run ID diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts new file mode 100644 index 00000000000..daedbc420c8 --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -0,0 +1,1406 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + capturePodmanSocketAuthority, + createPodmanContainerEngine, + type PodmanBoundContainerEngine, +} from "../../../src/lib/adapters/podman/index.ts"; +import type { RuntimeProviderLifecycleInput } from "../../../src/lib/onboard/runtime-provider/contract.ts"; +import { createPodmanRuntimeProviderBundle } from "../../../src/lib/onboard/runtime-provider/podman.ts"; +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE, + PODMAN_SANDBOX_NAMESPACE_LABEL, + PODMAN_SANDBOX_WORKSPACE, + PODMAN_SANDBOX_WORKSPACE_LABEL, +} from "../../../src/lib/onboard/runtime-provider/podman-lifecycle.ts"; +import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; +import type { + NativeRuntimeQualificationAcceleration, + NativeRuntimeQualificationAgent, + NativeRuntimeQualificationInference, + NativeRuntimeQualificationObligation, +} from "../registry/native-runtime-qualification.ts"; +import { nativeRuntimeQualificationOperationFile } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + assertCredentialFreeQualificationEnvironment, + assertNativeRuntimeQualificationModelResource, + digestFromImageReference, + nativeRuntimeQualificationAgentImage, + nativeRuntimeQualificationInferenceImage, + nativeRuntimeQualificationPodmanExecutable, + nativeRuntimeQualificationRunnerContractPath, + parseNativeRuntimeQualificationRow, + readNativeRuntimeQualificationRunnerContract, +} from "./native-runtime-qualification-case-helpers.ts"; + +const FULL_ID = /^[a-f0-9]{64}$/u; +const CONTROL = /[\u0000-\u001f\u007f-\u009f]/gu; +// nvidia-smi defines the immutable UUID as alphanumeric; retain the repository's +// bounded physical GPU identifier envelope while excluding MIG device names. +const PHYSICAL_GPU_UUID = /^GPU-[A-Za-z0-9][A-Za-z0-9-]{6,121}[A-Za-z0-9]$/u; +const COMMAND_TIMEOUT = 60_000; +const INFERENCE_TIMEOUT = 900_000; +const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const LIFECYCLE_SANDBOX_NAMES = Object.freeze({ + hermes: "q-hermes", + "langchain-deepagents-code": "q-deepagents", + openclaw: "q-openclaw", +} as const satisfies Record); +export const NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES = [ + "validate credential-free Docker-unavailable isolation", + "bind the rootless Podman engine", + "launch exact local inference", + "onboard the managed agent image", + "exercise sandbox lifecycle and state recovery", + "restart and reconcile inference", + "prove exact cleanup", + "emit bounded case evidence", +] as const; + +interface PodmanNetworkAuthority { + readonly id: string; + readonly name: string; + readonly gateway: string; +} + +interface PodmanQualificationService { + readonly child: ChildProcess; + readonly diagnostic: () => string; +} + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +interface GpuComputeProcess { + readonly gpuUuid: string; + readonly pid: number; + readonly processName: string; + readonly usedMemoryMiB: number; +} + +function bounded(value: string): string { + return value.replace(CONTROL, " ").replace(/\s+/gu, " ").trim().slice(-500); +} + +function command(command: string, args: readonly string[]): CommandResult { + const result = spawnSync(command, [...args], { + encoding: "utf8", + env: process.env, + timeout: 10_000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + }); + return { + status: result.status ?? (result.signal ? 128 : 127), + stdout: result.stdout ?? "", + stderr: result.stderr ?? result.error?.message ?? "", + }; +} + +function requireCommand(executable: string, args: readonly string[], label: string): string { + const result = command(executable, args); + if (result.status !== 0) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout)}`, + ); + } + return result.stdout.trim(); +} + +function capture( + engine: PodmanBoundContainerEngine, + args: readonly string[], + label: string, + timeout = COMMAND_TIMEOUT, +): string { + const result = engine.capture(args, timeout); + if (result.status !== 0 || result.error) { + throw new Error( + `${label} failed with exit ${String(result.status)}: ${bounded(result.stderr || result.stdout || result.error?.message || "unknown failure")}`, + ); + } + return result.stdout.trim(); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function exactDirectory(directory: string): void { + const metadata = fs.lstatSync(directory); + const uid = process.getuid?.() ?? -1; + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + metadata.uid !== uid || + (metadata.mode & 0o077) !== 0 + ) { + throw new Error("Qualification receipt directory must be private and current-user owned"); + } +} + +function removeQualificationSnapshot(snapshot: string | null): void { + if (snapshot !== null) fs.rmSync(snapshot, { force: true }); +} + +function writeJson(directory: string, file: string, value: unknown): void { + const target = path.join(directory, file); + const temporary = `${target}.tmp`; + const serialized = `${JSON.stringify(value, null, 2)}\n`; + fs.writeFileSync(temporary, serialized, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, target); +} + +function assertDockerUnavailable(): Record { + const guarded = command("docker", ["version"]); + if (guarded.status !== 97) { + throw new Error(`Docker PATH invocation guard returned ${String(guarded.status)}, expected 97`); + } + for (const executable of ["/usr/bin/docker", "/usr/local/bin/docker", "/snap/bin/docker"]) { + if (!fs.existsSync(executable)) continue; + const result = command(executable, ["version"]); + if (result.status === 0) + throw new Error(`Absolute Docker client remained usable: ${executable}`); + } + for (const socket of ["/var/run/docker.sock", "/run/docker.sock"]) { + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) throw new Error(`Docker socket remained available: ${socket}`); + } + for (const unit of ["docker.service", "docker.socket"]) { + if (command("systemctl", ["is-active", "--quiet", unit]).status === 0) { + throw new Error(`Docker unit remained active: ${unit}`); + } + } + const proc = fs.readdirSync("/proc").filter((entry) => /^[1-9][0-9]*$/u.test(entry)); + for (const pid of proc) { + try { + if (fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim() === "dockerd") { + throw new Error("Docker daemon process remained active"); + } + } catch (error) { + if (error instanceof Error && error.message === "Docker daemon process remained active") { + throw error; + } + } + } + return { + dockerCommandGuarded: true, + dockerServiceInactive: true, + dockerSocketUnitInactive: true, + dockerdProcessNameAbsent: true, + defaultSocketPathsAbsent: true, + }; +} + +async function waitForSocket(socket: string, service: PodmanQualificationService): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const { child } = service; + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error( + `Rootless Podman API service exited before its socket became ready (exit=${String(child.exitCode)}, signal=${String(child.signalCode)}): ${service.diagnostic() || "no bounded diagnostic"}`, + ); + } + const metadata = fs.lstatSync(socket, { throwIfNoEntry: false }); + if (metadata?.isSocket()) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("Rootless Podman API service did not create its socket"); +} + +function startPodmanQualificationService( + socket: string, + podmanExecutable: string, + progress: TestProgress, +): PodmanQualificationService { + let diagnostic = ""; + const child = spawnObservedChild( + podmanExecutable, + ["system", "service", "--time=0", `unix://${socket}`], + { + activityLabel: "command: rootless Podman qualification service", + progress, + spawn: { env: process.env, stdio: ["ignore", "pipe", "pipe"] }, + }, + ); + child.stderr?.on("data", (value: Buffer | string) => { + diagnostic = bounded(`${diagnostic} ${String(value)}`); + }); + return Object.freeze({ child, diagnostic: () => diagnostic }); +} + +async function stopService(child: ChildProcess | null, socket: string): Promise { + if (child && child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (child.exitCode === null && child.signalCode === null) { + if (!child.kill("SIGKILL")) { + throw new Error("Rootless Podman API service rejected SIGKILL"); + } + const killDeadline = Date.now() + 10_000; + while (Date.now() < killDeadline && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (child.exitCode === null && child.signalCode === null) { + throw new Error("Rootless Podman API service remained alive after SIGKILL"); + } + } + } + fs.rmSync(socket, { force: true }); +} + +function createProviderNetwork( + engine: PodmanBoundContainerEngine, + name: string, + caseId: string, +): PodmanNetworkAuthority { + let created = false; + try { + const createdIdentity = capture( + engine, + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${caseId}`, name], + "provider network creation", + ); + created = true; + if (createdIdentity !== name && !FULL_ID.test(createdIdentity)) { + throw new Error("Provider network creation returned an unexpected identity"); + } + type NetworkInspection = { + id?: unknown; + labels?: unknown; + name?: unknown; + subnets?: Array<{ gateway?: unknown }>; + }; + const inspect = (identity: string, label: string): NetworkInspection => { + const inspected = JSON.parse(capture(engine, ["network", "inspect", identity], label)) as + | NetworkInspection[] + | unknown; + if (!Array.isArray(inspected) || inspected.length !== 1) { + throw new Error("Provider network inspection lacks one exact identity"); + } + return inspected[0] as NetworkInspection; + }; + const entry = inspect(createdIdentity, "provider network creation inspection"); + const id = typeof entry.id === "string" ? entry.id : ""; + const gateway = entry?.subnets?.[0]?.gateway; + const labels = + typeof entry.labels === "object" && entry.labels !== null && !Array.isArray(entry.labels) + ? (entry.labels as Record) + : null; + if ( + !FULL_ID.test(id) || + (FULL_ID.test(createdIdentity) && createdIdentity !== id) || + entry.name !== name || + typeof gateway !== "string" || + labels?.[QUALIFICATION_LABEL] !== caseId + ) { + throw new Error("Provider network inspection lacks exact identity"); + } + const immutable = inspect(id, "provider network immutable-ID inspection"); + if ( + immutable.id !== id || + immutable.name !== name || + immutable.subnets?.[0]?.gateway !== gateway || + typeof immutable.labels !== "object" || + immutable.labels === null || + Array.isArray(immutable.labels) || + (immutable.labels as Record)[QUALIFICATION_LABEL] !== caseId + ) { + throw new Error("Provider network identity changed after immutable-ID resolution"); + } + return Object.freeze({ id, name, gateway }); + } catch (error) { + if (created) { + const removalOutcome = (() => { + try { + const removal = engine.capture(["network", "rm", "--force", name], COMMAND_TIMEOUT); + return `exit ${String(removal.status)}`; + } catch (cleanupError) { + return `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`; + } + })(); + const existence = (() => { + try { + const result = engine.capture(["network", "exists", name], COMMAND_TIMEOUT); + return { outcome: `exit ${String(result.status)}`, removalProven: result.status === 1 }; + } catch (cleanupError) { + return { + outcome: `threw ${bounded( + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + )}`, + removalProven: false, + }; + } + })(); + if (!existence.removalProven) { + const validationFailure = bounded(error instanceof Error ? error.message : String(error)); + throw new Error( + `${validationFailure}; provider network cleanup could not prove removal (remove ${removalOutcome}; exists ${existence.outcome})`, + ); + } + } + throw error; + } +} + +function pullPublicImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["pull", imageRef], `pull ${imageRef}`, INFERENCE_TIMEOUT); + capture(engine, ["image", "exists", imageRef], `inspect pulled image ${imageRef}`); +} + +function requirePreloadedImage(engine: PodmanBoundContainerEngine, imageRef: string): void { + capture(engine, ["image", "exists", imageRef], `inspect preloaded image ${imageRef}`); +} + +function parsePhysicalGpuDevices(output: string): readonly string[] { + const devices = output + .split(/\r?\n/u) + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); + if ( + devices.length === 0 || + new Set(devices).size !== devices.length || + devices.some((device) => !PHYSICAL_GPU_UUID.test(device)) + ) { + throw new Error( + `NVIDIA CDI runtime proof did not return exact physical GPU UUIDs: ${bounded(JSON.stringify(devices))}`, + ); + } + return Object.freeze(devices); +} + +function proveGpuDevices( + engine: PodmanBoundContainerEngine, + probeImageRef: string, +): readonly string[] { + return parsePhysicalGpuDevices( + capture( + engine, + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + "--entrypoint", + "nvidia-smi", + probeImageRef, + "--query-gpu=uuid", + "--format=csv,noheader", + ], + "NVIDIA CDI runtime proof", + ), + ); +} + +function proveGpuBackedInference( + engine: PodmanBoundContainerEngine, + containerId: string, + selectedDevices: readonly string[], +): readonly GpuComputeProcess[] { + const output = capture( + engine, + [ + "exec", + containerId, + "nvidia-smi", + "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", + "--format=csv,noheader,nounits", + ], + "GPU-backed inference process proof", + ); + const processes = output + .split(/\r?\n/u) + .map((line) => line.split(",").map((field) => field.trim())) + .filter((fields) => fields.length === 4) + .map(([gpuUuid, pid, processName, usedMemoryMiB]) => ({ + gpuUuid: gpuUuid ?? "", + pid: Number(pid), + processName: processName ?? "", + usedMemoryMiB: Number(usedMemoryMiB), + })) + .filter( + (entry) => + selectedDevices.includes(entry.gpuUuid) && + Number.isSafeInteger(entry.pid) && + entry.pid > 0 && + entry.processName.length > 0 && + !/[\u0000-\u001f\u007f-\u009f]/u.test(entry.processName) && + Number.isSafeInteger(entry.usedMemoryMiB) && + entry.usedMemoryMiB > 0, + ); + if (processes.length === 0) { + throw new Error("Inference turn did not leave an exact GPU compute process proof"); + } + return Object.freeze(processes.map((entry) => Object.freeze(entry))); +} + +function inferenceFailureDiagnostic( + engine: PodmanBoundContainerEngine, + containerId: string, +): string { + let result: ReturnType; + try { + result = engine.capture( + ["inspect", "--format", "{{json .State}}", containerId], + COMMAND_TIMEOUT, + ); + } catch { + return "state=unavailable; inspect=threw"; + } + if (result.status !== 0) { + return `state=unavailable; inspectExit=${String(result.status)}`; + } + try { + const parsed = JSON.parse(result.stdout) as Record; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return "state=unparseable"; + } + const knownStatuses = new Set([ + "configured", + "created", + "exited", + "initialized", + "paused", + "removing", + "running", + "stopped", + "stopping", + "unknown", + ]); + return `state=${JSON.stringify({ + status: + typeof parsed.Status === "string" && knownStatuses.has(parsed.Status) + ? parsed.Status + : "unknown", + exitCode: Number.isSafeInteger(parsed.ExitCode) ? parsed.ExitCode : null, + oomKilled: parsed.OOMKilled === true, + running: parsed.Running === true, + })}`; + } catch { + return "state=unparseable"; + } +} + +type OwnedResourceKind = "container" | "network" | "volume"; + +type OwnedResourceGroup = { + readonly engine: PodmanBoundContainerEngine; + readonly identities: readonly string[]; + readonly kind: OwnedResourceKind; +}; + +function collectOwnedResourceCleanupFailures(groups: readonly OwnedResourceGroup[]): Error[] { + const failures: Error[] = []; + for (const { engine, identities, kind } of groups) { + for (const identity of identities) { + const outcomes: string[] = []; + const removeArgs = + kind === "container" ? ["rm", "--force", identity] : [kind, "rm", "--force", identity]; + try { + const removal = engine.capture(removeArgs, COMMAND_TIMEOUT); + if (removal.status !== 0) outcomes.push(`remove exit ${String(removal.status)}`); + } catch { + outcomes.push("remove threw"); + } + try { + const existence = engine.capture([kind, "exists", identity], COMMAND_TIMEOUT); + if (existence.status !== 1) outcomes.push(`exists exit ${String(existence.status)}`); + } catch { + outcomes.push("exists threw"); + } + if (outcomes.length > 0) { + failures.push( + new Error( + `Native runtime qualification ${kind} cleanup failed for ${identity} (${outcomes.join("; ")})`, + ), + ); + } + } + } + return failures; +} + +function collectQualificationResourceCleanupFailures(input: { + readonly inferenceContainers: readonly string[]; + readonly inferenceEngine: PodmanBoundContainerEngine | null; + readonly lifecycleContainers: readonly string[]; + readonly lifecycleEngine: PodmanBoundContainerEngine | null; + readonly networks: readonly string[]; + readonly volumes: readonly string[]; +}): Error[] { + return collectOwnedResourceCleanupFailures([ + ...(input.lifecycleEngine + ? [ + { + engine: input.lifecycleEngine, + identities: input.lifecycleContainers, + kind: "container" as const, + }, + { + engine: input.lifecycleEngine, + identities: input.volumes, + kind: "volume" as const, + }, + ] + : []), + ...(input.inferenceEngine + ? [ + { + engine: input.inferenceEngine, + identities: input.inferenceContainers, + kind: "container" as const, + }, + { + engine: input.inferenceEngine, + identities: input.networks, + kind: "network" as const, + }, + ] + : []), + ]); +} + +function vllmServeArguments(model: string, port: number): readonly string[] { + return [ + "vllm", + "serve", + "/models", + "--served-model-name", + model, + "--host", + "0.0.0.0", + "--port", + String(port), + "--max-model-len", + "2048", + ]; +} + +function inferenceContainerPlan(input: { + readonly acceleration: NativeRuntimeQualificationAcceleration; + readonly caseId: string; + readonly imageRef: string; + readonly inference: NativeRuntimeQualificationInference; + readonly model: string; + readonly modelPath?: string; + readonly name: string; + readonly network: string; + readonly port: number; +}): { readonly arguments: readonly string[]; readonly endpoint: string } { + if ((input.inference === "nim" || input.inference === "vllm") && !input.modelPath) { + throw new Error("Native runtime qualification GPU inference requires a model path"); + } + return { + arguments: [ + "run", + "--detach", + "--pull=never", + "--name", + input.name, + "--network", + input.network, + "--label", + `${QUALIFICATION_LABEL}=${input.caseId}`, + ...(input.acceleration === "nvidia-gpu" ? ["--device", "nvidia.com/gpu=all"] : []), + ...(input.inference === "nim" + ? [ + "--shm-size", + "16g", + "--env", + "NIM_MODEL_PATH=/models", + "--env", + `NIM_SERVED_MODEL_NAME=${input.model}`, + "--volume", + `${input.modelPath}:/models:ro`, + ] + : []), + ...(input.inference === "vllm" + ? ["--shm-size", "16g", "--volume", `${input.modelPath}:/models:ro`] + : []), + input.imageRef, + ...(input.inference === "vllm" ? vllmServeArguments(input.model, input.port) : []), + ], + endpoint: `http://${input.name}:${String(input.port)}`, + }; +} + +function createAgentContainer(input: { + readonly engine: PodmanBoundContainerEngine; + readonly imageRef: string; + readonly name: string; + readonly network: string; + readonly qualificationId: string; + readonly sandboxId: string; + readonly sandboxName: string; + readonly volume: string; +}): string { + const id = capture( + input.engine, + [ + "run", + "--detach", + "--pull=never", + "--name", + input.name, + "--network", + input.network, + "--label", + `${PODMAN_MANAGED_LABEL}=true`, + "--label", + `${PODMAN_SANDBOX_NAME_LABEL}=${input.sandboxName}`, + "--label", + `${PODMAN_SANDBOX_ID_LABEL}=${input.sandboxId}`, + "--label", + `${PODMAN_SANDBOX_NAMESPACE_LABEL}=${PODMAN_SANDBOX_NAMESPACE}`, + "--label", + `${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, + "--label", + `${QUALIFICATION_LABEL}=${input.qualificationId}`, + "--volume", + `${input.volume}:/qualification`, + "--entrypoint", + "/bin/sh", + input.imageRef, + "-c", + "trap 'exit 0' TERM INT; while :; do sleep 3600 & wait $!; done", + ], + "agent container creation", + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(id)) throw new Error("Agent container did not return a full immutable ID"); + return id; +} + +async function agentTurn( + engine: PodmanBoundContainerEngine, + containerId: string, + endpoint: string, + model: string, + inference: NativeRuntimeQualificationInference, +): Promise { + const body = JSON.stringify({ + model, + ...(inference === "ollama" ? { reasoning_effort: "none" } : {}), + messages: [ + { + role: "user", + content: + inference === "ollama" + ? "/no_think\nReply with the single word qualified." + : "Reply with the single word qualified.", + }, + ], + max_tokens: 128, + stream: false, + }); + const args = [ + "exec", + containerId, + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--connect-timeout", + "5", + "--max-time", + "60", + "--header", + "Content-Type: application/json", + "--data-binary", + body, + `${endpoint}/v1/chat/completions`, + ]; + const deadline = Date.now() + 600_000; + let output = ""; + let lastFailure = "inference request was not attempted"; + while (Date.now() < deadline) { + const result = engine.capture(args, 90_000); + if (result.status === 0 && !result.error) { + output = result.stdout.trim(); + break; + } + lastFailure = bounded(result.stderr || result.stdout || result.error?.message || "failed"); + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + if (!output) throw new Error(`Agent inference turn did not become ready: ${lastFailure}`); + const response = JSON.parse(output) as { + model?: unknown; + choices?: Array<{ + finish_reason?: unknown; + message?: { content?: unknown; reasoning?: unknown; tool_calls?: unknown }; + }>; + }; + const first = response.choices?.[0]; + const completeMessage = + typeof first?.message?.content === "string" || + typeof first?.message?.reasoning === "string" || + Array.isArray(first?.message?.tool_calls); + if ( + response.model !== model || + typeof first?.finish_reason !== "string" || + first.finish_reason === "length" || + !completeMessage + ) { + throw new Error( + `Agent turn did not return a complete exact-model inference response (modelMatch=${String(response.model === model)}; finishReason=${typeof first?.finish_reason === "string" ? bounded(first.finish_reason) : typeof first?.finish_reason}; contentType=${typeof first?.message?.content}; reasoningType=${typeof first?.message?.reasoning}; toolCalls=${String(Array.isArray(first?.message?.tool_calls))})`, + ); + } + return sha256(output); +} + +function lifecycleInput(agent: string, sandboxName: string): RuntimeProviderLifecycleInput { + const sandbox: SandboxEntry = { + agent, + name: sandboxName, + openshellDriver: "podman", + }; + return { + environment: process.env, + log: () => undefined, + sandbox, + sandboxName, + }; +} + +function lifecycleSandboxName(agent: NativeRuntimeQualificationAgent): string { + return LIFECYCLE_SANDBOX_NAMES[agent]; +} + +function assertNoQualificationResidue(engine: PodmanBoundContainerEngine, caseId: string): void { + for (const [resource, args] of [ + ["container", ["ps", "--all", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["volume", ["volume", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ["network", ["network", "ls", "--quiet", "--filter", `label=${QUALIFICATION_LABEL}=${caseId}`]], + ] as const) { + if (capture(engine, args, `qualification ${resource} residue inspection`) !== "") { + throw new Error(`Qualification cleanup left an owned ${resource}`); + } + } +} + +export async function executeNativeRuntimeQualificationCase(progress: TestProgress): Promise { + progress.phase("validate credential-free Docker-unavailable isolation"); + assertCredentialFreeQualificationEnvironment(process.env); + expect(process.platform).toBe("linux"); + const uid = process.getuid?.() ?? 0; + expect(uid, "Native runtime qualification must execute as an unprivileged UID").toBeGreaterThan( + 0, + ); + const row = parseNativeRuntimeQualificationRow( + process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW ?? "", + ); + const expectedArchitecture = process.arch === "x64" ? "amd64" : process.arch; + expect(expectedArchitecture).toBe(row.case.architecture); + const receiptPath = process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT ?? ""; + expect(path.basename(receiptPath)).toBe("execution.json"); + const receiptDirectory = path.dirname(receiptPath); + exactDirectory(receiptDirectory); + + const dockerBefore = assertDockerUnavailable(); + const runtimeDirectory = process.env.XDG_RUNTIME_DIR ?? ""; + expect(runtimeDirectory).toBe(`/run/user/${String(uid)}`); + const podmanExecutable = nativeRuntimeQualificationPodmanExecutable(process.env, uid); + const socket = path.join(runtimeDirectory, "podman", "podman.sock"); + fs.mkdirSync(path.dirname(socket), { recursive: true, mode: 0o700 }); + + let service: PodmanQualificationService | null = startPodmanQualificationService( + socket, + podmanExecutable, + progress, + ); + let hostEngine: PodmanBoundContainerEngine | null = null; + let inferenceEngine: PodmanBoundContainerEngine | null = null; + let lifecycleEngine: PodmanBoundContainerEngine | null = null; + const ownedInferenceContainers = new Set(); + const ownedLifecycleContainers = new Set(); + const ownedVolumes = new Set(); + const ownedNetworks = new Set(); + let inferenceContainerId = ""; + let gpuDevices: readonly string[] = []; + let gpuComputeProcesses: readonly GpuComputeProcess[] = []; + let completed = false; + let qualificationFailure: unknown; + const cleanupFailures: unknown[] = []; + let snapshot: string | null = null; + const operationDetails = new Map>(); + + try { + progress.phase("bind the rootless Podman engine"); + await waitForSocket(socket, service); + const socketAuthority = capturePodmanSocketAuthority(socket); + hostEngine = createPodmanContainerEngine({ + executable: podmanExecutable, + operation: "host-doctor", + socketAuthority, + }); + inferenceEngine = createPodmanContainerEngine({ + executable: podmanExecutable, + operation: "host-local-inference", + socketAuthority, + }); + lifecycleEngine = createPodmanContainerEngine({ + executable: podmanExecutable, + operation: "sandbox-lifecycle", + socketAuthority, + }); + const bundle = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: hostEngine, + sandboxLifecycle: lifecycleEngine, + }, + }); + expect(bundle.identity.id).toBe("podman"); + expect(bundle.workload.profile.support).toBeNull(); + const hostInspection = bundle.preflightDoctor.inspectHost(); + if (hostInspection.status !== "ok") { + throw new Error(`Podman host qualification failed: ${bounded(hostInspection.detail)}`); + } + const caseSuffix = sha256(row.id).slice(0, 12); + const networkName = `nemoclaw-q-${caseSuffix}`; + const network = createProviderNetwork(inferenceEngine, networkName, row.id); + ownedNetworks.add(network.id); + const runnerContractFile = + row.case.acceleration === "nvidia-gpu" + ? nativeRuntimeQualificationRunnerContractPath(process.env, uid) + : undefined; + const runnerContract = runnerContractFile + ? readNativeRuntimeQualificationRunnerContract(row.case.architecture, runnerContractFile) + : undefined; + const agentImage = nativeRuntimeQualificationAgentImage(row.case.architecture, row.case.agent); + const inference = nativeRuntimeQualificationInferenceImage({ + architecture: row.case.architecture, + acceleration: row.case.acceleration, + inference: row.case.inference, + ...(runnerContract ? { runnerContract } : {}), + }); + pullPublicImage(inferenceEngine, agentImage); + if (row.case.inference === "ollama") pullPublicImage(inferenceEngine, inference.imageRef); + else { + if (!inference.modelPath || !runnerContractFile || !path.isAbsolute(inference.modelPath)) { + throw new Error("Native runtime qualification GPU model path must be absolute"); + } + requirePreloadedImage(inferenceEngine, inference.imageRef); + assertNativeRuntimeQualificationModelResource(inference.modelPath, uid, runnerContractFile); + } + if (runnerContract) { + requirePreloadedImage(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const inferenceName = `nemoclaw-inference-${caseSuffix}`; + progress.phase("launch exact local inference"); + const inferencePort = row.case.inference === "ollama" ? 11434 : 8000; + const inferencePlan = inferenceContainerPlan({ + acceleration: row.case.acceleration, + caseId: row.id, + imageRef: inference.imageRef, + inference: row.case.inference, + model: inference.model, + ...(inference.modelPath ? { modelPath: inference.modelPath } : {}), + name: inferenceName, + network: network.name, + port: inferencePort, + }); + inferenceContainerId = capture( + inferenceEngine, + inferencePlan.arguments, + `${row.case.inference} container start`, + INFERENCE_TIMEOUT, + ); + if (!FULL_ID.test(inferenceContainerId)) { + throw new Error("Inference container did not return a full immutable ID"); + } + ownedInferenceContainers.add(inferenceContainerId); + if (row.case.inference === "ollama") { + capture( + inferenceEngine, + ["exec", inferenceContainerId, "ollama", "pull", inference.model], + "Ollama model acquisition", + INFERENCE_TIMEOUT, + ); + } + if (row.case.acceleration === "nvidia-gpu") { + if (!runnerContract) throw new Error("GPU runner contract is unavailable"); + gpuDevices = proveGpuDevices(inferenceEngine, runnerContract.gpuProbeImageRef); + } + + const sandboxId = caseSuffix; + progress.phase("onboard the managed agent image"); + const sandboxName = lifecycleSandboxName(row.case.agent); + const agentName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; + const volumeName = `nemoclaw-q-state-${caseSuffix}`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "agent volume creation", + ); + ownedVolumes.add(volumeName); + let agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedLifecycleContainers.add(agentId); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' qualified >/qualification/state"], + "agent state initialization", + ); + operationDetails.set("agent.onboard", { + containerId: agentId, + agent: row.case.agent, + imageDigest: digestFromImageReference(agentImage), + }); + + const turnSha256 = await agentTurn( + lifecycleEngine, + agentId, + inferencePlan.endpoint, + inference.model, + row.case.inference, + ); + if (row.case.acceleration === "nvidia-gpu") { + gpuComputeProcesses = proveGpuBackedInference( + inferenceEngine, + inferenceContainerId, + gpuDevices, + ); + } + operationDetails.set("agent.turn", { + protocol: "openai-chat-completions", + model: inference.model, + responseSha256: turnSha256, + route: "provider-network-dns", + }); + + if (!bundle.lifecycle.supported) throw new Error("Podman lifecycle surface is unavailable"); + progress.phase("exercise sandbox lifecycle and state recovery"); + const lifecycle = bundle.lifecycle; + const input = lifecycleInput(row.case.agent, sandboxName); + let beforeStopCalled = false; + const firstStop = lifecycle.stop(input, { + beforeStop: () => { + beforeStopCalled = true; + }, + }); + if (firstStop.exitCode !== 0) { + throw new Error(`Initial sandbox stop failed: ${bounded(firstStop.message ?? "unknown")}`); + } + expect(firstStop.state).toBe("stopped"); + expect(beforeStopCalled).toBe(true); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + operationDetails.set("sandbox.stop-start", { + containerId: agentId, + executionPath: "runtime-provider-bundle", + stoppedAndStarted: true, + }); + + snapshot = path.join(os.tmpdir(), `nemoclaw-q-${caseSuffix}.tar`); + const snapshotStop = lifecycle.stop(input, { beforeStop: () => undefined }); + if (snapshotStop.exitCode !== 0) { + throw new Error( + `Snapshot sandbox stop failed: ${bounded(snapshotStop.message ?? "unknown")}`, + ); + } + capture( + lifecycleEngine, + ["volume", "export", "--output", snapshot, volumeName], + "sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + const snapshotBytes = fs.readFileSync(snapshot); + const snapshotSha256 = sha256(snapshotBytes); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + capture( + lifecycleEngine, + ["exec", agentId, "/bin/sh", "-c", "printf '%s\\n' drifted >/qualification/state"], + "sandbox state mutation", + ); + capture(lifecycleEngine, ["rm", "--force", agentId], "remove sandbox before rebuild"); + ownedLifecycleContainers.delete(agentId); + capture(lifecycleEngine, ["volume", "rm", volumeName], "remove sandbox volume"); + ownedVolumes.delete(volumeName); + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, volumeName], + "recreate sandbox volume", + ); + ownedVolumes.add(volumeName); + capture( + lifecycleEngine, + ["volume", "import", volumeName, snapshot], + "restore sandbox volume snapshot", + INFERENCE_TIMEOUT, + ); + agentId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: agentName, + network: network.name, + qualificationId: row.id, + sandboxId, + sandboxName, + volume: volumeName, + }); + ownedLifecycleContainers.add(agentId); + expect( + capture(lifecycleEngine, ["exec", agentId, "cat", "/qualification/state"], "restored state"), + ).toBe("qualified"); + operationDetails.set("sandbox.snapshot-restore", { + snapshotSha256, + restoredStateSha256: sha256("qualified\n"), + }); + operationDetails.set("sandbox.rebuild", { + priorContainerReplaced: true, + rebuiltContainerId: agentId, + preservedState: true, + }); + + const focusedResults: Record = Object.create(null); + if (row.focusedOperations.length > 0) { + const cloneVolume = `${volumeName}-clone`; + const cloneName = `${agentName}-clone`; + capture( + lifecycleEngine, + ["volume", "create", "--label", `${QUALIFICATION_LABEL}=${row.id}`, cloneVolume], + "clone volume creation", + ); + ownedVolumes.add(cloneVolume); + capture( + lifecycleEngine, + ["volume", "import", cloneVolume, snapshot], + "clone volume restore", + INFERENCE_TIMEOUT, + ); + const cloneId = createAgentContainer({ + engine: lifecycleEngine, + imageRef: agentImage, + name: cloneName, + network: network.name, + qualificationId: row.id, + sandboxId: `${sandboxId}c`, + sandboxName: `${sandboxName}-clone`, + volume: cloneVolume, + }); + ownedLifecycleContainers.add(cloneId); + expect( + capture(lifecycleEngine, ["exec", cloneId, "cat", "/qualification/state"], "clone state"), + ).toBe("qualified"); + const duplicate = lifecycleEngine.capture([ + "run", + "--detach", + "--pull=never", + "--name", + agentName, + "--entrypoint", + "/bin/sh", + agentImage, + "-c", + "exit 0", + ]); + if (duplicate.status === 0) throw new Error("Podman allowed unsafe managed-name reuse"); + capture(lifecycleEngine, ["kill", "--signal", "KILL", agentId], "sandbox crash injection"); + expect(lifecycle.start(input)).toEqual({ exitCode: 0 }); + expect( + capture( + lifecycleEngine, + ["exec", agentId, "cat", "/qualification/state"], + "recovered state", + ), + ).toBe("qualified"); + focusedResults.clone = { cloneContainerId: cloneId, restored: true }; + focusedResults.backup = { + sha256: snapshotSha256, + bytes: snapshotBytes.length, + }; + focusedResults["crash-recovery"] = { + signal: "SIGKILL", + recovered: true, + }; + focusedResults.rollback = { restoredSnapshotSha256: snapshotSha256 }; + focusedResults["name-reuse"] = { rejected: true }; + } + + if (!inferenceContainerId) throw new Error("Inference runtime identity is missing"); + progress.phase("restart and reconcile inference"); + capture( + inferenceEngine, + ["restart", inferenceContainerId], + `${row.case.inference} runtime restart`, + INFERENCE_TIMEOUT, + ); + const reconciledTurnSha256 = await agentTurn( + lifecycleEngine, + agentId, + inferencePlan.endpoint, + inference.model, + row.case.inference, + ); + const reconciledGpuComputeProcesses = + row.case.acceleration === "nvidia-gpu" + ? proveGpuBackedInference(inferenceEngine, inferenceContainerId, gpuDevices) + : []; + operationDetails.set("runtime.restart-reconcile", { + service: row.case.inference, + runtimeIdentity: inferenceContainerId, + responseSha256: reconciledTurnSha256, + gpuComputeProcesses: reconciledGpuComputeProcesses, + revalidated: true, + }); + + operationDetails.set("installer.install", { + authority: "trusted-installer-step", + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + }); + const rootfulSelectionDenied = row.rootModes.includes("rootful") + ? command("podman", ["--root", "/var/lib/containers/storage", "info"]).status !== 0 + : true; + if (!rootfulSelectionDenied) { + throw new Error( + "Unprivileged qualification unexpectedly obtained a rootful Podman storage authority", + ); + } + operationDetails.set("runtime.docker-unavailable", { + beforeCandidate: dockerBefore, + rootfulSelectionDenied, + executedRootMode: "rootless", + }); + + progress.phase("prove exact cleanup"); + capture(lifecycleEngine, ["rm", "--force", agentId], "agent cleanup"); + ownedLifecycleContainers.delete(agentId); + for (const containerId of [...ownedLifecycleContainers]) { + capture(lifecycleEngine, ["rm", "--force", containerId], "focused container cleanup"); + ownedLifecycleContainers.delete(containerId); + } + for (const volume of [...ownedVolumes]) { + capture(lifecycleEngine, ["volume", "rm", volume], "qualification volume cleanup"); + ownedVolumes.delete(volume); + } + capture(inferenceEngine, ["rm", "--force", inferenceContainerId], "inference runtime cleanup"); + ownedInferenceContainers.delete(inferenceContainerId); + capture(inferenceEngine, ["network", "rm", network.id], "provider network cleanup"); + ownedNetworks.delete(network.id); + removeQualificationSnapshot(snapshot); + snapshot = null; + assertNoQualificationResidue(lifecycleEngine, row.id); + operationDetails.set("cleanup.exact", { + containersRemaining: 0, + volumesRemaining: 0, + networksRemaining: 0, + }); + + if (row.focusedOperations.length > 0) { + Object.assign(focusedResults, { + restart: operationDetails.get("runtime.restart-reconcile"), + rebuild: operationDetails.get("sandbox.rebuild"), + "snapshot-restore": operationDetails.get("sandbox.snapshot-restore"), + installer: operationDetails.get("installer.install"), + cleanup: operationDetails.get("cleanup.exact"), + }); + const missing = row.focusedOperations.filter( + (operation) => !Object.hasOwn(focusedResults, operation), + ); + if (missing.length > 0) { + throw new Error(`Focused qualification operations are incomplete: ${missing.join(", ")}`); + } + } + + const dockerAfter = assertDockerUnavailable(); + progress.phase("emit bounded case evidence"); + const podmanVersion = requireCommand("podman", ["--version"], "Podman version"); + const managedImages = [ + { role: "agent", digest: digestFromImageReference(agentImage) }, + { + role: "inference", + digest: digestFromImageReference(inference.imageRef), + }, + ...(runnerContract + ? [ + { + role: "gpu-probe", + digest: digestFromImageReference(runnerContract.gpuProbeImageRef), + }, + ] + : []), + ]; + writeJson(receiptDirectory, "runtime-result.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { + providerId: "podman", + executionPath: "runtime-provider-bundle", + rootMode: "rootless", + podmanVersion, + inferenceService: row.case.inference, + modelRevision: inference.modelRevision ?? null, + focusedOperations: focusedResults, + dockerBefore, + dockerAfter, + }, + }); + for (const obligation of row.case.obligations) { + const details = operationDetails.get(obligation); + if (!details) throw new Error(`Qualification operation '${obligation}' was not executed`); + writeJson(receiptDirectory, nativeRuntimeQualificationOperationFile(obligation), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: obligation, + result: "passed", + details, + }); + } + if (row.case.acceleration === "nvidia-gpu") { + writeJson(receiptDirectory, "nvidia-cdi.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { + requested: "nvidia.com/gpu=all", + selectedDevices: gpuDevices, + inferenceRuntimeId: inferenceContainerId, + inferenceComputeProcesses: gpuComputeProcesses, + }, + }); + } + writeJson(receiptDirectory, "case-evidence.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: podmanVersion.replace(/^podman version\s+/u, ""), + managedImages, + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { + nvidiaCdi: { + device: "nvidia.com/gpu=all", + file: "nvidia-cdi.json", + }, + } + : {}), + }); + writeJson(receiptDirectory, "execution.json", { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-execution-v1", + caseId: row.id, + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + agent: row.case.agent, + inference: row.case.inference, + rootModes: row.rootModes, + obligations: row.case.obligations, + focusedOperations: row.focusedOperations, + evidenceKinds: row.case.evidenceKinds, + dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, + credentialBoundary: { + githubCredentialsAbsent: true, + modelCredentialsAbsent: true, + isolatedUid: true, + }, + result: "passed", + }); + completed = true; + } catch (error) { + qualificationFailure = error; + } finally { + try { + removeQualificationSnapshot(snapshot); + } catch (error) { + cleanupFailures.push(error); + } + if (!completed) { + if (inferenceEngine && FULL_ID.test(inferenceContainerId)) { + console.error( + `Native runtime qualification inference failure diagnostic: ${inferenceFailureDiagnostic(inferenceEngine, inferenceContainerId)}`, + ); + } + cleanupFailures.push( + ...collectQualificationResourceCleanupFailures({ + inferenceContainers: [...ownedInferenceContainers], + inferenceEngine, + lifecycleContainers: [...ownedLifecycleContainers], + lifecycleEngine, + networks: [...ownedNetworks], + volumes: [...ownedVolumes], + }), + ); + } + try { + await stopService(service?.child ?? null, socket); + } catch (error) { + cleanupFailures.push(error); + } + service = null; + } + if (cleanupFailures.length > 0 && qualificationFailure === undefined) { + throw new AggregateError(cleanupFailures, "Native runtime qualification cleanup failed"); + } + if (cleanupFailures.length > 0) { + throw new AggregateError( + [qualificationFailure, ...cleanupFailures], + "Native runtime qualification failed and cleanup could not be proven", + ); + } + if (qualificationFailure !== undefined) { + throw qualificationFailure; + } +} + +export const nativeRuntimeQualificationCaseInternals = Object.freeze({ + collectOwnedResourceCleanupFailures, + collectQualificationResourceCleanupFailures, + createProviderNetwork, + inferenceContainerPlan, + inferenceFailureDiagnostic, + lifecycleSandboxName, + parsePhysicalGpuDevices, + proveGpuDevices, + removeQualificationSnapshot, + vllmServeArguments, +}); diff --git a/test/e2e/live/native-runtime-qualification-case-helpers.ts b/test/e2e/live/native-runtime-qualification-case-helpers.ts new file mode 100644 index 00000000000..a476ed87a64 --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case-helpers.ts @@ -0,0 +1,558 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { + constants, + closeSync, + fstatSync, + openSync, + readFileSync, + readdirSync, + readSync, +} from "node:fs"; +import path from "node:path"; + +import { + NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, + NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS, + type NativeRuntimeQualificationProducerPlanRow, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, + type NativeRuntimeQualificationAcceleration, + type NativeRuntimeQualificationAgent, + type NativeRuntimeQualificationArchitecture, + type NativeRuntimeQualificationInference, +} from "../registry/native-runtime-qualification.ts"; + +export const NATIVE_RUNTIME_QUALIFICATION_MODEL_REVISION = + "7ae557604adf67be50417f59c2c2f167def9a775"; + +const SHA = /^[a-f0-9]{40}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const OCI_DIGEST = /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; +const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,511}$/u; +const ABSOLUTE_MODEL = + /^\/var\/tmp\/nemoclaw-native-runtime-resources-[1-9][0-9]*-[1-9][0-9]*-[1-9][0-9]*\/model$/u; +const MODEL_FILES = Object.freeze([ + Object.freeze({ + path: "config.json", + size: 659, + algorithm: "sha1" as const, + digest: "0dbb161213629a23f0fc00ef286e6b1e366d180f", + }), + Object.freeze({ + path: "generation_config.json", + size: 242, + algorithm: "sha1" as const, + digest: "dfc11073787daf1b0f9c0f1499487ab5f4c93738", + }), + Object.freeze({ + path: "merges.txt", + size: 1_671_839, + algorithm: "sha1" as const, + digest: "20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0", + }), + Object.freeze({ + path: "model.safetensors", + size: 988_097_824, + algorithm: "sha256" as const, + digest: "fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe", + }), + Object.freeze({ + path: "tokenizer.json", + size: 7_031_645, + algorithm: "sha1" as const, + digest: "443909a61d429dff23010e5bddd28ff530edda00", + }), + Object.freeze({ + path: "tokenizer_config.json", + size: 7_305, + algorithm: "sha1" as const, + digest: "07bfe0640cb5a0037f9322287fbfc682806cf672", + }), + Object.freeze({ + path: "vocab.json", + size: 2_776_833, + algorithm: "sha1" as const, + digest: "4783fe10ac3adce15ac8f358ef5462739852c569", + }), +]); +const SENSITIVE_ENVIRONMENT = + /^(?:GH_TOKEN|GITHUB_TOKEN|NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|NGC_API_KEY|NIM_NGC_API_KEY|HF_TOKEN|HUGGING_FACE_HUB_TOKEN|SSH_AUTH_SOCK|DOCKER_CERT_PATH|DOCKER_CONFIG|DOCKER_CONTEXT|DOCKER_HOST|DOCKER_TLS_VERIFY|CONTAINER_HOST|AWS_.+|AZURE_.+|GOOGLE_.+|.*(?:_API_KEY|_ACCESS_TOKEN|_AUTH_TOKEN|_PASSWORD|_PRIVATE_KEY|_SECRET|_SECRET_KEY))$/u; + +const AGENT_IMAGES = Object.freeze({ + amd64: Object.freeze({ + openclaw: + "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:2bca5955feb48f9b9170e51bbd5114c8ec481714b95a804d213957d4f5c3d069", + hermes: + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:28b9578ab9676ef046de37fa6feb9b7b61824b87d77fd08978758bd01c03cb54", + "langchain-deepagents-code": + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:f7ad7ddc95cea260cff02d26b873903805806ccfef5d27436cbec4eba3455eff", + }), + arm64: Object.freeze({ + openclaw: + "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:2f5bd4025b7cb61502d48f1fa02dd282d6d1156e7818c56e62ee675dc418c207", + hermes: + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:119076205d8ac366a1e0309a4c6a3822d616151d0c657b53df1e308ba690d46b", + "langchain-deepagents-code": + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:3ba92564fb17de4082745b9f32608188e28504cf421a8af7c5ef18e13680028e", + }), +}) satisfies Readonly< + Record< + NativeRuntimeQualificationArchitecture, + Readonly> + > +>; + +const OLLAMA_IMAGES = Object.freeze({ + amd64: + "docker.io/ollama/ollama@sha256:268c47cdc4718ded54babcd842579a7295ad79fd8d5c2ea64d7ba2e76872de6b", + arm64: + "docker.io/ollama/ollama@sha256:bcf5adbfacc0e13a975f981810959d05b6ee95632da0f27e5343bc868ad2c82d", +}) satisfies Readonly>; + +export interface NativeRuntimeQualificationRunnerContract { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-runner-v1"; + readonly architecture: NativeRuntimeQualificationArchitecture; + readonly gpuProbeImageRef: string; + readonly nim: { + readonly imageRef: string; + readonly model: string; + readonly modelPath: string; + readonly modelRevision: string; + }; + readonly vllm: { + readonly imageRef: string; + readonly model: string; + readonly modelPath: string; + readonly modelRevision: string; + }; +} + +type UnknownRecord = Record; + +function record(value: unknown, label: string): UnknownRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as UnknownRecord; +} + +function exactKeys(value: UnknownRecord, expected: readonly string[], label: string): void { + if (Object.keys(value).sort().join("\n") !== [...expected].sort().join("\n")) { + throw new Error(`${label} fields are invalid`); + } +} + +function exactJson(actual: unknown, expected: unknown, label: string): void { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label} does not match the trusted qualification definition`); + } +} + +export function parseNativeRuntimeQualificationRow( + serialized: string, +): NativeRuntimeQualificationProducerPlanRow { + let parsed: unknown; + try { + parsed = JSON.parse(serialized) as unknown; + } catch { + throw new Error("Native runtime qualification row is not valid JSON"); + } + const row = record(parsed, "Native runtime qualification row"); + exactKeys( + row, + [ + "id", + "jobName", + "artifactName", + "runner", + "installerSha256", + "source", + "case", + "rootModes", + "focusedOperations", + ], + "Native runtime qualification row", + ); + const qualificationCase = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.cases.find( + (entry) => entry.id === row.id, + ); + if (!qualificationCase) throw new Error("Native runtime qualification case is not canonical"); + exactJson(row.case, qualificationCase, "Native runtime qualification case"); + const source = record(row.source, "Native runtime qualification source"); + exactKeys( + source, + [ + "repository", + "producerWorkflow", + "pullRequestNumber", + "candidateRepository", + "candidateSha", + "baseRef", + "baseSha", + "workflowSha", + "producerRunId", + "producerRunAttempt", + "dispatchArtifact", + ], + "Native runtime qualification source", + ); + if ( + source.repository !== "NVIDIA/NemoClaw" || + source.producerWorkflow !== ".github/workflows/e2e.yaml" || + source.candidateRepository !== "NVIDIA/NemoClaw" || + source.baseRef !== "main" || + !Number.isSafeInteger(source.pullRequestNumber) || + Number(source.pullRequestNumber) < 1 || + typeof source.candidateSha !== "string" || + !SHA.test(source.candidateSha) || + typeof source.baseSha !== "string" || + !SHA.test(source.baseSha) || + source.candidateSha === source.baseSha || + source.workflowSha !== source.baseSha || + !/^[1-9][0-9]{0,19}$/u.test(String(source.producerRunId)) || + source.producerRunAttempt !== 1 + ) { + throw new Error("Native runtime qualification source identity is invalid"); + } + const artifact = record(source.dispatchArtifact, "Native runtime dispatch artifact"); + exactKeys(artifact, ["id", "name", "digest", "sizeInBytes"], "Native runtime dispatch artifact"); + if ( + !/^[1-9][0-9]{0,19}$/u.test(String(artifact.id)) || + artifact.name !== `e2e-dispatch-${String(source.producerRunId)}-1` || + typeof artifact.digest !== "string" || + !/^sha256:[a-f0-9]{64}$/u.test(artifact.digest) || + !Number.isSafeInteger(artifact.sizeInBytes) || + Number(artifact.sizeInBytes) < 1 || + Number(artifact.sizeInBytes) > 1_048_576 + ) { + throw new Error("Native runtime dispatch artifact identity is invalid"); + } + const focused = row.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE; + exactJson(row.rootModes, focused ? ["rootless", "rootful"] : ["rootless"], "Root modes"); + exactJson( + row.focusedOperations, + focused ? NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS : [], + "Focused operations", + ); + if ( + row.jobName !== `Native runtime qualification / ${String(row.id)}` || + row.artifactName !== + `native-runtime-qualification-evidence-${String(source.candidateSha)}-${String(row.id)}` || + typeof row.runner !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(row.runner) || + typeof row.installerSha256 !== "string" || + !SHA256.test(row.installerSha256) + ) { + throw new Error("Native runtime qualification plan metadata is invalid"); + } + return parsed as NativeRuntimeQualificationProducerPlanRow; +} + +export function assertCredentialFreeQualificationEnvironment(environment: NodeJS.ProcessEnv): void { + const present = Object.keys(environment).filter((name) => SENSITIVE_ENVIRONMENT.test(name)); + if (present.length > 0) { + throw new Error( + `Native runtime qualification environment contains forbidden credential names: ${present.sort().join(", ")}`, + ); + } +} + +export function nativeRuntimeQualificationPodmanExecutable( + environment: NodeJS.ProcessEnv, + uid: number, +): string { + const executable = environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE ?? ""; + const expected = new RegExp( + `^/nemoclaw-native-runtime-podman-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}$`, + "u", + ); + if (!Number.isSafeInteger(uid) || uid <= 0 || !expected.test(executable)) { + throw new Error("Native runtime qualification Podman executable path is invalid"); + } + return executable; +} + +function exactRunnerRuntime( + value: unknown, + label: "NIM" | "vLLM", +): { + readonly imageRef: string; + readonly model: string; + readonly modelPath: string; + readonly modelRevision: string; +} { + const runtime = record(value, `${label} runner contract`); + exactKeys( + runtime, + ["imageRef", "model", "modelPath", "modelRevision"], + `${label} runner contract`, + ); + if ( + typeof runtime.imageRef !== "string" || + !OCI_DIGEST.test(runtime.imageRef) || + typeof runtime.model !== "string" || + !MODEL.test(runtime.model) || + typeof runtime.modelPath !== "string" || + !ABSOLUTE_MODEL.test(runtime.modelPath) || + runtime.modelRevision !== NATIVE_RUNTIME_QUALIFICATION_MODEL_REVISION + ) { + throw new Error(`${label} runner contract is invalid`); + } + return Object.freeze({ + imageRef: runtime.imageRef, + model: runtime.model, + modelPath: runtime.modelPath, + modelRevision: runtime.modelRevision, + }); +} + +export function parseNativeRuntimeQualificationRunnerContract( + value: unknown, + architecture: NativeRuntimeQualificationArchitecture, +): NativeRuntimeQualificationRunnerContract { + const contract = record(value, "Native runtime qualification runner contract"); + exactKeys( + contract, + ["schemaVersion", "kind", "architecture", "gpuProbeImageRef", "nim", "vllm"], + "Native runtime qualification runner contract", + ); + if ( + contract.schemaVersion !== 1 || + contract.kind !== "nemoclaw-native-runtime-qualification-runner-v1" || + contract.architecture !== architecture || + typeof contract.gpuProbeImageRef !== "string" || + !OCI_DIGEST.test(contract.gpuProbeImageRef) + ) { + throw new Error("Native runtime qualification runner contract identity is invalid"); + } + const nim = exactRunnerRuntime(contract.nim, "NIM"); + const vllm = exactRunnerRuntime(contract.vllm, "vLLM"); + return Object.freeze({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture, + gpuProbeImageRef: contract.gpuProbeImageRef, + nim: Object.freeze({ + imageRef: nim.imageRef, + model: nim.model, + modelPath: nim.modelPath, + modelRevision: nim.modelRevision, + }), + vllm: Object.freeze({ + imageRef: vllm.imageRef, + model: vllm.model, + modelPath: vllm.modelPath, + modelRevision: vllm.modelRevision, + }), + }); +} + +export function readNativeRuntimeQualificationRunnerContract( + architecture: NativeRuntimeQualificationArchitecture, + file: string, +): NativeRuntimeQualificationRunnerContract { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== 0n || + before.gid !== 0n || + (before.mode & 0o777n) !== 0o444n || + before.size < 1n || + before.size > 65_536n + ) { + throw new Error("runner contract must be a bounded root-owned regular file"); + } + const bytes = readFileSync(descriptor); + const after = fstatSync(descriptor, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mode !== after.mode || + before.uid !== after.uid || + before.gid !== after.gid || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) { + throw new Error("runner contract changed during its stable read"); + } + return parseNativeRuntimeQualificationRunnerContract( + JSON.parse(bytes.toString("utf8")) as unknown, + architecture, + ); + } catch (error) { + throw new Error( + `Native runtime qualification runner contract is invalid: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +export function nativeRuntimeQualificationRunnerContractPath( + environment: NodeJS.ProcessEnv, + uid: number, +): string { + const file = environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT ?? ""; + const expected = new RegExp( + `^/run/nemoclaw-native-runtime-[1-9][0-9]*-[1-9][0-9]*-${String(uid)}/runner-contract\\.json$`, + "u", + ); + if (!Number.isSafeInteger(uid) || uid <= 0 || !expected.test(file)) { + throw new Error("Native runtime qualification runner contract path is invalid"); + } + return file; +} + +function stableModelFileDigest(file: string, expected: (typeof MODEL_FILES)[number]): string { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== 0n || + before.gid !== 0n || + (before.mode & 0o777n) !== 0o444n || + before.size !== BigInt(expected.size) + ) { + throw new Error(`model file metadata is invalid: ${expected.path}`); + } + const digest = createHash(expected.algorithm); + if (expected.algorithm === "sha1") digest.update(`blob ${String(expected.size)}\0`); + const buffer = Buffer.allocUnsafe(1024 * 1024); + for (;;) { + const count = readSync(descriptor, buffer, 0, buffer.length, null); + if (count === 0) break; + digest.update(buffer.subarray(0, count)); + } + const after = fstatSync(descriptor, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mode !== after.mode || + before.uid !== after.uid || + before.gid !== after.gid || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) { + throw new Error(`model file changed during its stable read: ${expected.path}`); + } + return digest.digest("hex"); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function assertStableRootOwnedDirectory(directory: string): void { + let descriptor: number | undefined; + try { + descriptor = openSync( + directory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + const metadata = fstatSync(descriptor); + if ( + !metadata.isDirectory() || + metadata.uid !== 0 || + metadata.gid !== 0 || + (metadata.mode & 0o777) !== 0o555 + ) { + throw new Error(`Runner model resource is not root-owned and read-only: ${directory}`); + } + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +export function assertNativeRuntimeQualificationModelResource( + directory: string, + uid: number, + contractFile: string, +): void { + const contractMatch = contractFile.match( + /^\/run\/nemoclaw-native-runtime-([1-9][0-9]*)-([1-9][0-9]*)-([1-9][0-9]*)\/runner-contract\.json$/u, + ); + const modelMatch = directory.match( + /^\/var\/tmp\/nemoclaw-native-runtime-resources-([1-9][0-9]*)-([1-9][0-9]*)-([1-9][0-9]*)\/model$/u, + ); + if ( + !contractMatch || + !modelMatch || + contractMatch.slice(1).join(":") !== modelMatch.slice(1).join(":") || + contractMatch[3] !== String(uid) + ) { + throw new Error("Runner model resource does not match the run-owned contract identity"); + } + assertStableRootOwnedDirectory(path.dirname(directory)); + assertStableRootOwnedDirectory(directory); + const actual = readdirSync(directory).sort(); + const expectedFiles = MODEL_FILES.map((entry) => entry.path).sort(); + if (actual.join("\n") !== expectedFiles.join("\n")) { + throw new Error("Runner model resource file set is invalid"); + } + for (const expected of MODEL_FILES) { + if (stableModelFileDigest(path.join(directory, expected.path), expected) !== expected.digest) { + throw new Error(`Runner model resource digest is invalid: ${expected.path}`); + } + } +} + +export function nativeRuntimeQualificationAgentImage( + architecture: NativeRuntimeQualificationArchitecture, + agent: NativeRuntimeQualificationAgent, +): string { + return AGENT_IMAGES[architecture][agent]; +} + +export function nativeRuntimeQualificationInferenceImage(input: { + readonly architecture: NativeRuntimeQualificationArchitecture; + readonly acceleration: NativeRuntimeQualificationAcceleration; + readonly inference: NativeRuntimeQualificationInference; + readonly runnerContract?: NativeRuntimeQualificationRunnerContract; +}): { + readonly imageRef: string; + readonly model: string; + readonly modelPath?: string; + readonly modelRevision?: string; +} { + if (input.inference === "ollama") { + return Object.freeze({ + imageRef: OLLAMA_IMAGES[input.architecture], + model: "qwen3:0.6b", + }); + } + if (input.acceleration !== "nvidia-gpu" || !input.runnerContract) { + throw new Error(`${input.inference} qualification requires the reviewed GPU runner contract`); + } + if (input.inference === "nim") { + return Object.freeze({ + imageRef: input.runnerContract.nim.imageRef, + model: input.runnerContract.nim.model, + modelPath: input.runnerContract.nim.modelPath, + modelRevision: input.runnerContract.nim.modelRevision, + }); + } + return Object.freeze({ + imageRef: input.runnerContract.vllm.imageRef, + model: input.runnerContract.vllm.model, + modelPath: input.runnerContract.vllm.modelPath, + modelRevision: input.runnerContract.vllm.modelRevision, + }); +} + +export function digestFromImageReference(reference: string): string { + if (!OCI_DIGEST.test(reference)) throw new Error("Managed image reference is not immutable"); + return reference.slice(reference.lastIndexOf("@") + 1); +} diff --git a/test/e2e/live/native-runtime-qualification-case.test.ts b/test/e2e/live/native-runtime-qualification-case.test.ts new file mode 100644 index 00000000000..677832c6508 --- /dev/null +++ b/test/e2e/live/native-runtime-qualification-case.test.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from "../fixtures/e2e-test.ts"; +import { + executeNativeRuntimeQualificationCase, + NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES, +} from "./native-runtime-qualification-case-executor.ts"; + +const ENABLED = + process.env.NEMOCLAW_RUN_LIVE_E2E === "1" && + typeof process.env.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW === "string"; + +test.skipIf(!ENABLED)( + "executes one exact credential-free native runtime qualification case", + { meta: { e2ePhases: NATIVE_RUNTIME_QUALIFICATION_E2E_PHASES }, timeout: 1_800_000 }, + async ({ progress }) => executeNativeRuntimeQualificationCase(progress), +); diff --git a/test/e2e/live/podman-portable-uninstall.test.ts b/test/e2e/live/podman-portable-uninstall.test.ts index 62dd1b654c0..bb9a1f4de3f 100644 --- a/test/e2e/live/podman-portable-uninstall.test.ts +++ b/test/e2e/live/podman-portable-uninstall.test.ts @@ -334,8 +334,8 @@ test( .filter((entry) => entry.isFile() || entry.isSymbolicLink()) .map((entry) => path.join(entry.parentPath, entry.name)); expect(residualFiles).toEqual([retirementRecord]); - expect(fs.readdirSync(path.dirname(expectedContainersConf))).toEqual([]); - expect(fs.statSync(path.dirname(expectedContainersConf)).mode & 0o777).toBe(0o700); + expect(fs.existsSync(path.dirname(expectedContainersConf))).toBe(false); + expect(fs.existsSync(path.dirname(path.dirname(expectedContainersConf)))).toBe(false); const managerEnvironment = await runCommand( shellProbe, "systemctl", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index e4c95b40b44..6e0e5dbfdf4 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -81,6 +81,15 @@ "test/e2e/support/podman-cpu-proof-workflow.test.ts" ] }, + { + "live": "test/e2e/live/native-runtime-qualification-case.test.ts", + "fast": [ + "test/e2e/support/native-runtime-qualification-case-helpers.test.ts", + "test/e2e/support/native-runtime-qualification-producer-workflow.test.ts", + "test/e2e/support/native-runtime-qualification.test.ts", + "test/install-native-runtime-qualification.test.ts" + ] + }, { "live": "test/e2e/live/podman-portable-uninstall.test.ts", "fast": [ diff --git a/test/e2e/registry/native-runtime-qualification.ts b/test/e2e/registry/native-runtime-qualification.ts index a5d2c3b6e8c..22db5a3ff43 100644 --- a/test/e2e/registry/native-runtime-qualification.ts +++ b/test/e2e/registry/native-runtime-qualification.ts @@ -6,18 +6,18 @@ import { createHash } from "node:crypto"; export { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; import type { NativeRuntimeQualificationAuthority, NativeRuntimeQualificationExpectedSource, NativeRuntimeQualificationProtectedRun, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; export type { NativeRuntimeQualificationAuthority, NativeRuntimeQualificationExpectedSource, NativeRuntimeQualificationProtectedRun, -} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority"; +} from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; export const NATIVE_RUNTIME_QUALIFICATION_AGENTS = [ "openclaw", @@ -436,7 +436,9 @@ function validatedArtifactReceipt( } const contents = readReceipt(artifactPath); if (contents === null) { - throw new Error(`${label} receipt '${artifactPath}' is missing from the authenticated artifact`); + throw new Error( + `${label} receipt '${artifactPath}' is missing from the authenticated artifact`, + ); } const actualSha256 = createHash("sha256").update(contents).digest("hex"); if (actualSha256 !== artifact.sha256) { diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 2d43b6e76c2..33f69ec1f18 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -350,15 +350,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; writeFileSync(output, ""); const result = spawnSync( "bash", - [ - "--noprofile", - "--norc", - "-e", - "-o", - "pipefail", - "-c", - credentialAuthorization.run!, - ], + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", credentialAuthorization.run!], { encoding: "utf8", env: { @@ -376,7 +368,9 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(`allowed=${expectedAllowed ? "true" : "false"}\n`); + expect(readFileSync(output, "utf8")).toBe( + `allowed=${expectedAllowed ? "true" : "false"}\n`, + ); } finally { rmSync(directory, { force: true, recursive: true }); } @@ -482,7 +476,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; INCLUDE_LAUNCHABLE: "false", JOBS: jobs, PR_NUMBER: "42", - REVIEW_REASON: "Reviewed PR head revision", + REVIEW_REASON: "Reviewed latest PR commit", RUN_ATTEMPT: "1", TARGETS: targets, TRIGGERING_ACTOR: "maintainer", @@ -498,6 +492,20 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }, ); + it("rejects candidate-workflow qualification when a downstream job receives a repository secret", () => { + const workflow = readE2eOperationsWorkflow(); + const plan = workflow.jobs["native-runtime-qualification-producer-plan"]; + const producer = workflow.jobs["native-runtime-qualification-producer"]; + + expect(JSON.stringify(producer)).toContain("${{ secrets.NVIDIA_API_KEY }}"); + plan.if = + "${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && (github.ref == 'refs/heads/main' || github.workflow_sha == inputs.checkout_sha) && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }}"; + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "Native runtime qualification producer plan must execute only from trusted main", + ); + }); + it("uses central maintainer authorization for protected managed-image qualification", () => { const workflow = readE2eOperationsWorkflow(); const guards = [ diff --git a/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts new file mode 100644 index 00000000000..70d84f0014f --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-account-lifecycle.test.ts @@ -0,0 +1,599 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as net from "node:net"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { readYaml, type Workflow } from "../../helpers/e2e-workflow-contract"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +function workflowScripts(): { boundary: string; cleanup: string } { + const workflow = readYaml(".github/workflows/e2e.yaml") as Workflow; + const steps = workflow.jobs["native-runtime-qualification-producer"]?.steps ?? []; + const run = (name: string): string => { + const source = steps.find((entry) => entry.name === name)?.run; + expect(source, `Missing workflow step ${name}`).toBeTruthy(); + return source!; + }; + return { + boundary: run("Prepare the credential-free execution account and disable Docker"), + cleanup: run("Remove qualification resources"), + }; +} + +function extractFunction(source: string, name: string): string { + const match = source.match(new RegExp(`${name}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); + expect(match, `Missing shell function ${name}`).toBeTruthy(); + return `${name}() {${match![1]}\n}`; +} + +const fixtureRewrites = { + testExecutable: ["/usr/bin/test", "/bin/test"], + systemctlExecutable: ["/usr/bin/systemctl", "systemctl"], + commandPath: ["PATH=/usr/bin:/bin", 'PATH="$FIXTURE_BIN:/usr/bin:/bin"'], + subuid: ["/etc/subuid", "${FIXTURE_ROOT}/etc/subuid"], + subgid: ["/etc/subgid", "${FIXTURE_ROOT}/etc/subgid"], + ownershipMarker: [ + 'ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + 'ownership_marker="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + ], + homeIdentity: ['"$home" == "/home/${account}"', '"$home" == "$FIXTURE_HOME"'], + runtimeDirectory: [ + 'runtime_dir="/run/user/${uid}"', + 'runtime_dir="${FIXTURE_ROOT}/run/user/${uid}"', + ], + userManagerDropinDirectory: [ + 'user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"', + 'user_manager_dropin_directory="${FIXTURE_ROOT}/run/systemd/system/${user_manager_unit}.d"', + ], + storageConfigDirectory: [ + 'storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'storage_config_directory="${FIXTURE_ROOT}/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + podmanExecutable: [ + 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'podman_executable="${FIXTURE_ROOT}/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + helperDirectory: [ + 'helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'helper_directory="${FIXTURE_ROOT}/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + resourceDirectory: [ + 'resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + 'resource_directory="${FIXTURE_ROOT}/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ], + userRuntimePath: ['"/run/user/${uid}"', '"${FIXTURE_ROOT}/run/user/${uid}"'], +} as const; + +type FixtureRewrite = keyof typeof fixtureRewrites; +type FixtureRewriteProfile = "bus" | "cleanup" | "provision" | "subgid" | "subuid" | "unitPath"; +type FixtureRewriteContract = { + readonly required: readonly FixtureRewrite[]; + readonly optional: readonly FixtureRewrite[]; +}; + +const fixtureRewriteProfiles: Record = { + bus: { required: ["testExecutable"], optional: [] }, + unitPath: { required: ["systemctlExecutable", "commandPath"], optional: [] }, + provision: { + required: ["subuid", "subgid", "ownershipMarker", "homeIdentity"], + optional: [], + }, + subuid: { required: ["subuid"], optional: [] }, + subgid: { required: ["subgid"], optional: [] }, + cleanup: { + required: [ + "subuid", + "subgid", + "ownershipMarker", + "runtimeDirectory", + "userManagerDropinDirectory", + "storageConfigDirectory", + "podmanExecutable", + "helperDirectory", + "resourceDirectory", + "userRuntimePath", + ], + optional: [], + }, +}; + +function rewriteRequiredFixtureTarget(source: string, target: FixtureRewrite): string { + const [from, to] = fixtureRewrites[target]; + expect(source, `Missing mandatory fixture rewrite '${target}'`).toContain(from); + return source.replaceAll(from, to); +} + +function fixtureSource(source: string, profile: FixtureRewriteProfile): string { + let rewritten = source; + for (const target of fixtureRewriteProfiles[profile].required) { + rewritten = rewriteRequiredFixtureTarget(rewritten, target); + } + for (const target of fixtureRewriteProfiles[profile].optional) { + const [from, to] = fixtureRewrites[target]; + rewritten = rewritten.replaceAll(from, to); + } + expect(rewritten, "Fixture source retains an unredirected destructive host path").not.toMatch( + /(?:\/usr\/bin\/(?:systemctl|test)|PATH=\/usr\/bin:\/bin|"\$home" == "\/home\/\$\{account\}"|(?:^|[\s"'=])\/(?:etc\/sub(?:uid|gid)|run\/(?:nemoclaw-native-runtime|systemd\/system|user\/)|(?:nemoclaw-native-runtime-(?:podman|helpers)|var\/tmp\/nemoclaw-native-runtime-resources)-))/mu, + ); + return rewritten; +} + +function writeExecutable(file: string, source: string): void { + fs.writeFileSync(file, `#!/usr/bin/env bash\nset -euo pipefail\n${source}\n`, { mode: 0o700 }); +} + +function createFixture(): { + root: string; + bin: string; + calls: string; + passwd: string; + group: string; + subuid: string; + subgid: string; + home: string; + marker: string; +} { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-account-fixture-")); + roots.push(root); + const bin = path.join(root, "bin"); + const etc = path.join(root, "etc"); + const run = path.join(root, "run"); + const home = path.join(root, "home", "nemoclawq"); + const calls = path.join(root, "calls.log"); + const passwd = path.join(etc, "passwd"); + const group = path.join(etc, "group"); + const subuid = path.join(etc, "subuid"); + const subgid = path.join(etc, "subgid"); + fs.mkdirSync(bin, { recursive: true }); + fs.mkdirSync(etc, { recursive: true }); + fs.mkdirSync(run, { recursive: true }); + for (const file of [calls, passwd, group, subuid, subgid]) fs.writeFileSync(file, ""); + + writeExecutable( + path.join(bin, "sudo"), + `printf 'sudo:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == -u ]]; then shift 2; fi +exec "$@"`, + ); + writeExecutable( + path.join(bin, "getent"), + `case "$1" in + passwd) file="$FIXTURE_ROOT/etc/passwd" ;; + group) file="$FIXTURE_ROOT/etc/group" ;; + *) exit 2 ;; +esac +awk -F: -v key="$2" '$1 == key || $3 == key { print; found = 1 } END { exit found ? 0 : 2 }' "$file"`, + ); + writeExecutable( + path.join(bin, "id"), + `[[ "$1" == -u || "$1" == -g ]] +[[ "\${FAIL_ID:-0}" != 1 ]] || exit 26 +field=3 +[[ "$1" == -u ]] || field=4 +awk -F: -v account="$2" -v field="$field" '$1 == account { print $field; found = 1 } END { exit found ? 0 : 1 }' "$FIXTURE_ROOT/etc/passwd"`, + ); + writeExecutable( + path.join(bin, "useradd"), + `printf 'useradd:%s\\n' "$*" >>"$FIXTURE_CALLS" +[[ "\${FAIL_USERADD:-0}" != 1 ]] || exit 23 +account="\${!#}" +mkdir -p "$FIXTURE_HOME" +printf '%s:x:1002:1007::%s:/usr/sbin/nologin\\n' "$account" "$FIXTURE_HOME" >>"$FIXTURE_ROOT/etc/passwd" +printf '%s:x:1007:\\n' "$account" >>"$FIXTURE_ROOT/etc/group"`, + ); + writeExecutable( + path.join(bin, "usermod"), + `printf 'usermod:%s\\n' "$*" >>"$FIXTURE_CALLS" +case "$1" in + --add-subuids) file="$FIXTURE_ROOT/etc/subuid" ;; + --add-subgids) file="$FIXTURE_ROOT/etc/subgid" ;; + *) exit 24 ;; +esac +start="\${2%-*}" +end="\${2#*-}" +printf '%s:%s:%s\\n' "$3" "$start" "$((end - start + 1))" >>"$file"`, + ); + writeExecutable(path.join(bin, "chown"), ":"); + writeExecutable(path.join(bin, "chmod"), `exec /bin/chmod "$@"`); + writeExecutable( + path.join(bin, "unlink"), + `/bin/chmod u+w "$(/usr/bin/dirname "$1")"\nexec /bin/unlink "$1"`, + ); + writeExecutable( + path.join(bin, "rmdir"), + `/bin/chmod u+w "$(/usr/bin/dirname "$1")"\nexec /bin/rmdir "$1"`, + ); + writeExecutable( + path.join(bin, "rm"), + `target="\${!#}"\n/bin/chmod u+w "$(/usr/bin/dirname "$target")"\nexec /bin/rm "$@"`, + ); + writeExecutable( + path.join(bin, "stat"), + `target="\${!#}" +case "$target" in + *bus) printf '%s\\n' "\${BUS_UID:-1002}" ;; + *user@*.service.d) printf '0:0:755\\n' ;; + *50-nemoclaw-native-runtime.conf) printf '0:0:444:1\\n' ;; + *native-runtime-owner-*) printf '0:0:400\\n' ;; + *native-runtime-podman-*) printf '0:0:555\\n' ;; + *native-runtime-helpers-*) printf '0:0:555\\n' ;; + *native-runtime-resources-*/model/*) + [[ "$2" == '%u:%g:%a:%h' ]] && printf '0:0:444:1\\n' || printf '0:0:444\\n' + ;; + *native-runtime-resources-*) printf '0:0:555\\n' ;; + *podman.apparmor|*pasta.apparmor|*runner-contract.json|*containers.conf|*storage.conf) printf '0:0:444\\n' ;; + *) exit 25 ;; +esac`, + ); + writeExecutable(path.join(bin, "pkill"), `printf 'pkill:%s\\n' "$*" >>"$FIXTURE_CALLS"`); + writeExecutable( + path.join(bin, "systemctl"), + `printf 'systemctl:%s\\n' "$*" >>"$FIXTURE_CALLS" +if [[ "$1" == --user ]]; then shift; fi +if [[ "$1" == stop ]]; then + shift + for unit in "$@"; do + if [[ "$unit" =~ ^user-runtime-dir@([0-9]+)\\.service$ ]]; then + /bin/rm -rf -- "$FIXTURE_ROOT/run/user/\${BASH_REMATCH[1]}" + fi + done +elif [[ "$1" == is-active ]]; then + exit 3 +elif [[ "$1" == show || "$1" == show-environment ]]; then + printf '%s\\n' "\${SYSTEMD_ENVIRONMENT:-}" +fi`, + ); + writeExecutable( + path.join(bin, "apparmor_parser"), + `printf 'apparmor:%s\\n' "$*" >>"$FIXTURE_CALLS"`, + ); + writeExecutable( + path.join(bin, "userdel"), + `printf 'userdel:%s\\n' "$*" >>"$FIXTURE_CALLS" +account="\${!#}" +for file in "$FIXTURE_ROOT/etc/passwd" "$FIXTURE_ROOT/etc/subuid" "$FIXTURE_ROOT/etc/subgid"; do + awk -F: -v account="$account" '$1 != account' "$file" >"$file.next" + mv "$file.next" "$file" +done +rmdir "$FIXTURE_HOME" 2>/dev/null || true`, + ); + writeExecutable( + path.join(bin, "groupdel"), + `printf 'groupdel:%s\\n' "$*" >>"$FIXTURE_CALLS" +awk -F: -v account="$1" '$1 != account' "$FIXTURE_ROOT/etc/group" >"$FIXTURE_ROOT/etc/group.next" +mv "$FIXTURE_ROOT/etc/group.next" "$FIXTURE_ROOT/etc/group"`, + ); + + return { + root, + bin, + calls, + passwd, + group, + subuid, + subgid, + home, + marker: path.join(run, "nemoclaw-native-runtime-owner-42-1"), + }; +} + +function runFixture( + fixture: ReturnType, + source: string, + profile: FixtureRewriteProfile, + extraEnv: Record = {}, +) { + return spawnSync("bash", ["-c", fixtureSource(source, profile)], { + encoding: "utf8", + timeout: 15_000, + env: { + ...process.env, + ACCOUNT: "nemoclawq", + ACCOUNT_CREATED: "true", + FIXTURE_CALLS: fixture.calls, + FIXTURE_BIN: fixture.bin, + FIXTURE_HOME: fixture.home, + FIXTURE_ROOT: fixture.root, + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "42", + PATH: `${fixture.bin}:${process.env.PATH ?? ""}`, + ...extraEnv, + }, + }); +} + +function provisionBlock(): string { + const source = workflowScripts().boundary; + const start = source.indexOf('account="nemoclawq"'); + const end = source.indexOf("ensure_subordinate_range()", start); + expect(start, "Missing qualification account provision start").toBeGreaterThanOrEqual(0); + expect(end, "Missing qualification account provision end").toBeGreaterThan(start); + return `set -euo pipefail\n${source.slice(start, end)}`; +} + +function expectPreExistingAccountStateRejected(): void { + for (const state of ["passwd", "group", "subuid", "subgid"] as const) { + const fixture = createFixture(); + const file = fixture[state]; + fs.appendFileSync( + file, + state === "passwd" + ? `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n` + : state === "group" + ? "nemoclawq:x:1007:\n" + : "nemoclawq:200000:65536\n", + ); + const result = runFixture(fixture, provisionBlock(), "provision"); + expect(result.status, `${state}: ${result.stderr}`).not.toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("useradd:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + } +} + +function writeModelFixtureFiles(model: string): void { + for (const file of [ + "config.json", + "generation_config.json", + "merges.txt", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + ]) { + fs.writeFileSync(path.join(model, file), "fixture", { mode: 0o444 }); + } +} + +describe("native runtime qualification account lifecycle", () => { + it("fails closed when a mandatory fixture rewrite no longer matches", () => { + expect(() => fixtureSource("set -euo pipefail", "bus")).toThrow( + "Missing mandatory fixture rewrite 'testExecutable'", + ); + }); + + it("rejects a symlinked or wrong-owner user bus while accepting the exact account socket", async () => { + const fixture = createFixture(); + const socketRoot = fs.mkdtempSync("/tmp/nrq-bus-"); + roots.push(socketRoot); + const socket = path.join(socketRoot, "bus"); + const socketLink = path.join(socketRoot, "bus-link"); + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socket, resolve); + }); + fs.symlinkSync(socket, socketLink); + const verifyUserBus = extractFunction(workflowScripts().boundary, "verify_user_bus"); + const source = (candidate: string) => + `set -euo pipefail\n${verifyUserBus}\nverify_user_bus nemoclawq 1002 ${JSON.stringify(candidate)} fixture`; + + try { + const exact = runFixture(fixture, source(socket), "bus"); + expect(exact.status, exact.stderr).toBe(0); + + const symlink = runFixture(fixture, source(socketLink), "bus"); + expect(symlink.status).not.toBe(0); + expect(symlink.stderr).toContain("Qualification systemd user bus fixture"); + + const wrongOwner = runFixture(fixture, source(socket), "bus", { BUS_UID: "1003" }); + expect(wrongOwner.status).not.toBe(0); + expect(wrongOwner.stderr).toContain("Qualification systemd user bus fixture"); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + + it("rejects a user manager whose unit path includes candidate-writable authority", () => { + const fixture = createFixture(); + const verifyUnitPath = extractFunction( + workflowScripts().boundary, + "verify_user_manager_unit_path", + ).replace("env -i", "env"); + const source = `set -euo pipefail +trusted_user_unit_path=/usr/lib/systemd/user:/lib/systemd/user +${verifyUnitPath} +verify_user_manager_unit_path nemoclawq "$FIXTURE_HOME" "$FIXTURE_ROOT/run/user/1002"`; + + const trusted = runFixture(fixture, source, "unitPath", { + SYSTEMD_ENVIRONMENT: "SYSTEMD_UNIT_PATH=/usr/lib/systemd/user:/lib/systemd/user", + }); + expect(trusted.status, trusted.stderr).toBe(0); + + const candidateWritable = runFixture(fixture, source, "unitPath", { + SYSTEMD_ENVIRONMENT: + "SYSTEMD_UNIT_PATH=/home/nemoclawq/.config/systemd/user:/usr/lib/systemd/user:/lib/systemd/user", + }); + expect(candidateWritable.status).not.toBe(0); + expect(candidateWritable.stderr).toContain("did not inherit the trusted unit path"); + }); + + it("rejects pre-existing accounts and stale subordinate-ID authorization before mutation", () => { + expectPreExistingAccountStateRejected(); + }); + + it("does not publish ownership when account creation fails", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock(), "provision", { FAIL_USERADD: "1" }); + expect(result.status).toBe(23); + expect(fs.existsSync(fixture.marker)).toBe(false); + expect(fs.readFileSync(fixture.passwd, "utf8")).toBe(""); + }); + + it("records the exact private group when its numeric GID differs from the UID", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock(), "provision"); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(fixture.marker, "utf8")).toBe("nemoclawq:1002:1007\n"); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain( + "useradd:--create-home --shell /usr/sbin/nologin --user-group nemoclawq", + ); + }); + + it("rolls back an account when identity validation fails before marker publication", () => { + const fixture = createFixture(); + const result = runFixture(fixture, provisionBlock(), "provision", { FAIL_ID: "1" }); + expect(result.status).not.toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); + expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.group, "utf8")).not.toContain("nemoclawq:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + }); + + it("keeps an existing valid range and advances past every overlapping range", () => { + const rangeFunction = extractFunction(workflowScripts().boundary, "ensure_subordinate_range"); + + const valid = createFixture(); + fs.writeFileSync(valid.subuid, "nemoclawq:200000:65536\n"); + const validResult = runFixture( + valid, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + "subuid", + ); + expect(validResult.status, validResult.stderr).toBe(0); + expect(fs.readFileSync(valid.calls, "utf8")).not.toContain("usermod:"); + + const overlapping = createFixture(); + fs.writeFileSync(overlapping.subuid, "runner-a:100000:65536\nrunner-b:165536:65536\n"); + const overlappingResult = runFixture( + overlapping, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subuid --add-subuids`, + "subuid", + ); + expect(overlappingResult.status, overlappingResult.stderr).toBe(0); + expect(fs.readFileSync(overlapping.subuid, "utf8")).toContain("nemoclawq:231072:65536"); + }); + + it("fails closed when no complete subordinate-ID range remains", () => { + const fixture = createFixture(); + fs.writeFileSync(fixture.subgid, "runner:100000:4294867296\n"); + const rangeFunction = extractFunction(workflowScripts().boundary, "ensure_subordinate_range"); + const result = runFixture( + fixture, + `set -euo pipefail\naccount=nemoclawq\n${rangeFunction}\nensure_subordinate_range /etc/subgid --add-subgids`, + "subgid", + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("no free subordinate-ID range"); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toContain("usermod:"); + }); + + it("removes partial account setup and verifies subordinate-ID revocation", () => { + const fixture = createFixture(); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1007:\n"); + fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); + fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(fixture.calls, "utf8")).toContain("userdel:--remove nemoclawq"); + expect(fs.readFileSync(fixture.passwd, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.group, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.subuid, "utf8")).not.toContain("nemoclawq:"); + expect(fs.readFileSync(fixture.subgid, "utf8")).not.toContain("nemoclawq:"); + expect(fs.existsSync(fixture.marker)).toBe(false); + }); + + it("removes the run-owned runtime, helper copies, GPU resources, and AppArmor profiles", () => { + const fixture = createFixture(); + const runtime = path.join(fixture.root, "run", "user", "1002", "libpod", "tmp"); + const storage = path.join(fixture.root, "run", "nemoclaw-native-runtime-42-1-1002"); + const podman = path.join(fixture.root, "nemoclaw-native-runtime-podman-42-1-1002"); + const helpers = path.join(fixture.root, "nemoclaw-native-runtime-helpers-42-1-1002"); + const userManagerDropinDirectory = path.join( + fixture.root, + "run", + "systemd", + "system", + "user@1002.service.d", + ); + const resources = path.join( + fixture.root, + "var", + "tmp", + "nemoclaw-native-runtime-resources-42-1-1002", + ); + const model = path.join(resources, "model"); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.mkdirSync(runtime, { recursive: true }); + fs.mkdirSync(storage, { recursive: true }); + fs.mkdirSync(userManagerDropinDirectory, { recursive: true, mode: 0o755 }); + fs.writeFileSync( + path.join(userManagerDropinDirectory, "50-nemoclaw-native-runtime.conf"), + '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=/usr/lib/systemd/user:/lib/systemd/user"\n', + { mode: 0o444 }, + ); + fs.writeFileSync(path.join(runtime, "alive"), "fixture"); + fs.writeFileSync(path.join(storage, "storage.conf"), "fixture"); + fs.writeFileSync(path.join(storage, "containers.conf"), "fixture"); + fs.writeFileSync(path.join(storage, "podman.apparmor"), "fixture"); + fs.writeFileSync(path.join(storage, "pasta.apparmor"), "fixture"); + fs.writeFileSync(path.join(storage, "runner-contract.json"), "fixture"); + fs.writeFileSync(podman, "fixture", { mode: 0o555 }); + fs.mkdirSync(helpers, { recursive: true, mode: 0o755 }); + fs.writeFileSync(path.join(helpers, "pasta"), "fixture", { mode: 0o555 }); + fs.chmodSync(helpers, 0o555); + fs.mkdirSync(model, { recursive: true, mode: 0o755 }); + writeModelFixtureFiles(model); + fs.chmodSync(model, 0o555); + fs.chmodSync(resources, 0o555); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1007:\n"); + fs.writeFileSync(fixture.subuid, "nemoclawq:200000:65536\n"); + fs.writeFileSync(fixture.subgid, "nemoclawq:300000:65536\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); + expect(result.status, result.stderr).toBe(0); + const calls = fs.readFileSync(fixture.calls, "utf8"); + expect(calls).toContain("systemctl:stop user@1002.service user-runtime-dir@1002.service"); + expect(calls).toContain("groupdel:nemoclawq"); + expect(calls).toContain("apparmor:-R"); + expect(fs.existsSync(path.join(fixture.root, "run", "user", "1002"))).toBe(false); + expect(fs.existsSync(storage)).toBe(false); + expect(fs.existsSync(podman)).toBe(false); + expect(fs.existsSync(helpers)).toBe(false); + expect(fs.existsSync(resources)).toBe(false); + expect(fs.existsSync(userManagerDropinDirectory)).toBe(false); + }, 15_000); + + it("does not run destructive cleanup when the run-owned marker is absent", () => { + const fixture = createFixture(); + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup", { + ACCOUNT_CREATED: "", + }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("output exists without its ownership marker"); + const calls = fs.readFileSync(fixture.calls, "utf8"); + expect(calls).not.toMatch(/pkill:|systemctl:|userdel:|groupdel:|apparmor:/u); + }); + + it("fails closed before destructive cleanup when the private group identity changes", () => { + const fixture = createFixture(); + fs.mkdirSync(fixture.home, { recursive: true }); + fs.writeFileSync(fixture.passwd, `nemoclawq:x:1002:1007::${fixture.home}:/usr/sbin/nologin\n`); + fs.writeFileSync(fixture.group, "nemoclawq:x:1008:\n"); + fs.writeFileSync(fixture.marker, "nemoclawq:1002:1007\n", { mode: 0o400 }); + + const result = runFixture(fixture, workflowScripts().cleanup, "cleanup"); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("private group identity changed before cleanup"); + expect(fs.readFileSync(fixture.calls, "utf8")).not.toMatch( + /pkill:|systemctl:|userdel:|groupdel:|apparmor:/u, + ); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-case-executor.test.ts b/test/e2e/support/native-runtime-qualification-case-executor.test.ts new file mode 100644 index 00000000000..4a41e0076f7 --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-case-executor.test.ts @@ -0,0 +1,407 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import type { PodmanBoundContainerEngine } from "../../../src/lib/adapters/podman/index.ts"; +import { isValidName } from "../../../src/lib/name-validation.ts"; +import { nativeRuntimeQualificationCaseInternals } from "../live/native-runtime-qualification-case-executor.ts"; +import { NATIVE_RUNTIME_QUALIFICATION_AGENTS } from "../registry/native-runtime-qualification.ts"; + +const NETWORK_ID = "a".repeat(64); +const NETWORK_NAME = "nemoclaw-q-0123456789ab"; +const CASE_ID = "podman-openclaw-linux-amd64-cpu-ollama"; +const QUALIFICATION_LABEL = "ai.nvidia.nemoclaw.qualification"; +const GPU_PROBE_IMAGE = `nvcr.io/nvidia/k8s/cuda-sample@sha256:${"d".repeat(64)}`; + +function inspection(overrides: Record = {}): string { + return JSON.stringify([ + { + id: NETWORK_ID, + labels: { [QUALIFICATION_LABEL]: CASE_ID }, + name: NETWORK_NAME, + subnets: [{ gateway: "10.89.0.1" }], + ...overrides, + }, + ]); +} + +type EngineOutput = + | string + | (() => never) + | { readonly status: number; readonly stdout?: string; readonly stderr?: string }; + +function engine(outputs: readonly EngineOutput[]): { + readonly capture: ReturnType; + readonly value: PodmanBoundContainerEngine; +} { + let index = 0; + const capture = vi.fn((args: readonly string[]) => { + const configured = outputs[index++]; + const output = typeof configured === "function" ? configured() : configured; + return typeof output === "object" + ? { status: output.status, stdout: output.stdout ?? "", stderr: output.stderr ?? "" } + : { + status: output === undefined && args[0] === "network" && args[1] === "exists" ? 1 : 0, + stdout: output ?? "", + stderr: "", + }; + }); + return { + capture, + value: { + operation: "host-local-inference", + engineId: "podman", + displayName: "Podman", + authorityId: `podman-sha256:${"b".repeat(64)}`, + endpointAuthorityId: `podman-sha256:${"c".repeat(64)}`, + capture, + captureHost: vi.fn(), + assertAuthority: vi.fn(), + } as unknown as PodmanBoundContainerEngine, + }; +} + +describe("native runtime GPU evidence", () => { + it("overrides the probe image entrypoint with the exact nvidia-smi UUID query", () => { + const gpuUuid = "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6"; + const runtime = engine([gpuUuid]); + + expect( + nativeRuntimeQualificationCaseInternals.proveGpuDevices(runtime.value, GPU_PROBE_IMAGE), + ).toEqual([gpuUuid]); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + [ + "run", + "--rm", + "--pull=never", + "--device", + "nvidia.com/gpu=all", + "--entrypoint", + "nvidia-smi", + GPU_PROBE_IMAGE, + "--query-gpu=uuid", + "--format=csv,noheader", + ], + ]); + }); + + it("accepts bounded NVIDIA physical GPU identities and sorts them exactly", () => { + expect( + nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices( + [ + "GPU-z9Y8x7W6-v5U4-t3S2-r1Q0-p9O8n7M6l5K4", + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + ].join("\n"), + ), + ).toEqual([ + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + "GPU-z9Y8x7W6-v5U4-t3S2-r1Q0-p9O8n7M6l5K4", + ]); + }); + + it.each([ + ["empty output", ""], + [ + "duplicate identities", + "GPU-8932f937-d72c-4106-c12f-20bd9faed9f6\nGPU-8932f937-d72c-4106-c12f-20bd9faed9f6", + ], + ["MIG identities", "MIG-8932f937-d72c-4106-c12f-20bd9faed9f6"], + ["leading hyphens", "GPU--932f937"], + ["trailing hyphens", "GPU-8932f937-"], + ["control characters", "GPU-8932f937\u0000"], + ])("rejects %s as physical GPU proof", (_label, output) => { + expect(() => nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices(output)).toThrow( + "NVIDIA CDI runtime proof did not return exact physical GPU UUIDs", + ); + }); + + it("reports the bounded rejected rows for protected-run diagnosis", () => { + expect(() => + nativeRuntimeQualificationCaseInternals.parsePhysicalGpuDevices("unexpected-row"), + ).toThrow( + 'NVIDIA CDI runtime proof did not return exact physical GPU UUIDs: ["unexpected-row"]', + ); + }); +}); + +describe("native runtime failure diagnosis", () => { + it("emits only allowlisted inference state without logs or child output", () => { + const containerId = "e".repeat(64); + const runtime = engine([ + JSON.stringify({ + Status: "exited", + ExitCode: 1, + OOMKilled: false, + Running: false, + Error: "Authorization: Bearer credential-like-value", + Request: "private request content", + }), + ]); + + const diagnostic = nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( + runtime.value, + containerId, + ); + + expect(diagnostic).toBe( + 'state={"status":"exited","exitCode":1,"oomKilled":false,"running":false}', + ); + expect(diagnostic).not.toContain("credential-like-value"); + expect(diagnostic).not.toContain("private request content"); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["inspect", "--format", "{{json .State}}", containerId], + ]); + }); + + it("does not echo an unexpected state string", () => { + const diagnostic = nativeRuntimeQualificationCaseInternals.inferenceFailureDiagnostic( + engine([JSON.stringify({ Status: "credential-like-value" })]).value, + "e".repeat(64), + ); + + expect(diagnostic).toBe( + 'state={"status":"unknown","exitCode":null,"oomKilled":false,"running":false}', + ); + expect(diagnostic).not.toContain("credential-like-value"); + }); +}); + +describe("native runtime failed-case cleanup", () => { + it.each([ + ["container", ["rm", "--force", "container-id"], ["container", "exists", "container-id"]], + ["volume", ["volume", "rm", "--force", "volume-id"], ["volume", "exists", "volume-id"]], + ["network", ["network", "rm", "--force", "network-id"], ["network", "exists", "network-id"]], + ] as const)("reports an unproven %s removal without child output", (kind, remove, exists) => { + const runtime = engine([ + { status: 1, stderr: "Authorization: Bearer cleanup-secret" }, + { status: 0 }, + ]); + + const failures = nativeRuntimeQualificationCaseInternals.collectOwnedResourceCleanupFailures([ + { engine: runtime.value, identities: [`${kind}-id`], kind }, + ]); + + expect(failures.map((failure) => failure.message)).toEqual([ + `Native runtime qualification ${kind} cleanup failed for ${kind}-id (remove exit 1; exists exit 0)`, + ]); + expect(failures[0]?.message).not.toContain("cleanup-secret"); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([remove, exists]); + }); + + it("routes each resource to its owning engine and continues after one removal throws", () => { + const lifecycle = engine([ + () => { + throw new Error("container removal failed"); + }, + { status: 0 }, + { status: 0 }, + { status: 1 }, + ]); + const inference = engine([{ status: 0 }, { status: 1 }, { status: 0 }, { status: 1 }]); + + const failures = + nativeRuntimeQualificationCaseInternals.collectQualificationResourceCleanupFailures({ + inferenceContainers: ["inference-id"], + inferenceEngine: inference.value, + lifecycleContainers: ["container-id"], + lifecycleEngine: lifecycle.value, + networks: ["network-id"], + volumes: ["volume-id"], + }); + + expect(failures.map((failure) => failure.message)).toEqual([ + "Native runtime qualification container cleanup failed for container-id (remove threw; exists exit 0)", + ]); + expect(lifecycle.capture.mock.calls.map(([args]) => args)).toEqual([ + ["rm", "--force", "container-id"], + ["container", "exists", "container-id"], + ["volume", "rm", "--force", "volume-id"], + ["volume", "exists", "volume-id"], + ]); + expect(inference.capture.mock.calls.map(([args]) => args)).toEqual([ + ["rm", "--force", "inference-id"], + ["container", "exists", "inference-id"], + ["network", "rm", "--force", "network-id"], + ["network", "exists", "network-id"], + ]); + }); +}); + +describe("native runtime vLLM launch", () => { + it("provides the pinned NGC image entrypoint with an exact serve command", () => { + expect( + nativeRuntimeQualificationCaseInternals.vllmServeArguments( + "Qwen/Qwen2.5-0.5B-Instruct", + 8000, + ), + ).toEqual([ + "vllm", + "serve", + "/models", + "--served-model-name", + "Qwen/Qwen2.5-0.5B-Instruct", + "--host", + "0.0.0.0", + "--port", + "8000", + "--max-model-len", + "2048", + ]); + }); +}); + +describe("native runtime provider-network authority", () => { + it("uses distinct canonical sandbox names for every qualified agent", () => { + const names = NATIVE_RUNTIME_QUALIFICATION_AGENTS.map((agent) => + nativeRuntimeQualificationCaseInternals.lifecycleSandboxName(agent), + ); + + expect(names).toEqual(["q-openclaw", "q-hermes", "q-deepagents"]); + expect(names.every((name) => isValidName(name))).toBe(true); + expect(new Set(names).size).toBe(names.length); + }); + + it("resolves Podman 6.1 name output to one immutable labeled network ID", () => { + const runtime = engine([NETWORK_NAME, inspection(), inspection()]); + + expect( + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toEqual({ id: NETWORK_ID, name: NETWORK_NAME, gateway: "10.89.0.1" }); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "inspect", NETWORK_NAME], + ["network", "inspect", NETWORK_ID], + ]); + }); + + it("also binds an implementation that returns the immutable network ID", () => { + const runtime = engine([NETWORK_ID, inspection(), inspection()]); + + expect( + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ).id, + ).toBe(NETWORK_ID); + }); + + it("rejects creation output outside the requested name or immutable-ID forms", () => { + const runtime = engine(["unexpected-network"]); + + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network creation returned an unexpected identity"); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "rm", "--force", NETWORK_NAME], + ["network", "exists", NETWORK_NAME], + ]); + }); + + it("rejects label or immutable re-inspection drift", () => { + const missingLabel = engine([NETWORK_NAME, inspection({ labels: {} }), inspection()]); + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + missingLabel.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network inspection lacks exact identity"); + + const changedGateway = engine([ + NETWORK_NAME, + inspection(), + inspection({ subnets: [{ gateway: "10.90.0.1" }] }), + ]); + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + changedGateway.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow("Provider network identity changed after immutable-ID resolution"); + expect(changedGateway.capture).toHaveBeenNthCalledWith( + 4, + ["network", "rm", "--force", NETWORK_NAME], + 60_000, + ); + expect(changedGateway.capture).toHaveBeenLastCalledWith( + ["network", "exists", NETWORK_NAME], + 60_000, + ); + }); + + it("reports validation and cleanup together when network removal cannot be proven", () => { + const runtime = engine([ + "unexpected-network", + { status: 1, stderr: "remove failed" }, + { status: 0 }, + ]); + + expect(() => + nativeRuntimeQualificationCaseInternals.createProviderNetwork( + runtime.value, + NETWORK_NAME, + CASE_ID, + ), + ).toThrow( + "Provider network creation returned an unexpected identity; provider network cleanup could not prove removal (remove exit 1; exists exit 0)", + ); + expect(runtime.capture.mock.calls.map(([args]) => args)).toEqual([ + ["network", "create", "--label", `${QUALIFICATION_LABEL}=${CASE_ID}`, NETWORK_NAME], + ["network", "rm", "--force", NETWORK_NAME], + ["network", "exists", NETWORK_NAME], + ]); + }); + + it("removes an exported snapshot without replacing the case failure", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-snapshot-cleanup-")); + const snapshot = path.join(root, "nemoclaw-q-fixture.tar"); + const original = new Error("failure after export"); + fs.writeFileSync(snapshot, "snapshot"); + + const failAfterExport = () => { + try { + throw original; + } finally { + nativeRuntimeQualificationCaseInternals.removeQualificationSnapshot(snapshot); + } + }; + + expect(failAfterExport).toThrow(original); + expect(fs.existsSync(snapshot)).toBe(false); + fs.rmSync(root, { force: true, recursive: true }); + }); + + it("routes inference inside the provider network without a host port publication", () => { + const plan = nativeRuntimeQualificationCaseInternals.inferenceContainerPlan({ + acceleration: "cpu", + caseId: CASE_ID, + imageRef: `docker.io/ollama/ollama@sha256:${"e".repeat(64)}`, + inference: "ollama", + model: "qwen2.5:0.5b", + name: "nemoclaw-inference-fixture", + network: NETWORK_NAME, + port: 11434, + }); + + expect(plan.endpoint).toBe("http://nemoclaw-inference-fixture:11434"); + expect(plan.arguments).toContain("--network"); + expect(plan.arguments).toContain(NETWORK_NAME); + expect(plan.arguments).not.toContain("--publish"); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-case-helpers.test.ts b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts new file mode 100644 index 00000000000..e94edbe8efa --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-case-helpers.test.ts @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildNativeRuntimeQualificationProducerPlan, + type NativeRuntimeQualificationProducerPlanInput, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; +import { + assertCredentialFreeQualificationEnvironment, + digestFromImageReference, + nativeRuntimeQualificationAgentImage, + nativeRuntimeQualificationInferenceImage, + nativeRuntimeQualificationPodmanExecutable, + nativeRuntimeQualificationRunnerContractPath, + parseNativeRuntimeQualificationRow, + parseNativeRuntimeQualificationRunnerContract, +} from "../live/native-runtime-qualification-case-helpers.ts"; + +const SOURCE = { + repository: "NVIDIA/NemoClaw", + producerWorkflow: ".github/workflows/e2e.yaml", + pullRequestNumber: 9144, + candidateRepository: "NVIDIA/NemoClaw", + candidateSha: "a".repeat(40), + baseRef: "main", + baseSha: "b".repeat(40), + workflowSha: "b".repeat(40), + producerRunId: "123456", + producerRunAttempt: 1, + dispatchArtifact: { + id: "987654", + name: "e2e-dispatch-123456-1", + digest: `sha256:${"c".repeat(64)}`, + sizeInBytes: 4096, + }, +} as const; + +function row() { + const plan = buildNativeRuntimeQualificationProducerPlan({ + source: SOURCE, + installerSha256: "d".repeat(64), + arm64GpuRunner: "native-arm64-gpu", + } satisfies NativeRuntimeQualificationProducerPlanInput); + return plan.include.find((entry) => entry.id === "podman-hermes-linux-amd64-cpu-ollama")!; +} + +function runnerContract() { + return { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runner-v1", + architecture: "amd64", + gpuProbeImageRef: `nvcr.io/nvidia/cuda@sha256:${"3".repeat(64)}`, + nim: { + imageRef: `nvcr.io/nim/nvidia/model@sha256:${"1".repeat(64)}`, + model: "nvidia/model", + modelPath: "/var/tmp/nemoclaw-native-runtime-resources-123456-1-1002/model", + modelRevision: "7ae557604adf67be50417f59c2c2f167def9a775", + }, + vllm: { + imageRef: `docker.io/vllm/vllm-openai@sha256:${"2".repeat(64)}`, + model: "qualification", + modelPath: "/var/tmp/nemoclaw-native-runtime-resources-123456-1-1002/model", + modelRevision: "7ae557604adf67be50417f59c2c2f167def9a775", + }, + } as const; +} + +function expectInvalidPodmanExecutablesRejected(): void { + for (const executable of [ + "/usr/local/bin/podman", + "/nemoclaw-native-runtime-podman-123456-1-0", + "/nemoclaw-native-runtime-podman-123456-1-1003", + "/nemoclaw-native-runtime-podman-123456-1-1002/../podman", + ]) { + expect(() => + nativeRuntimeQualificationPodmanExecutable( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: executable }, + 1002, + ), + ).toThrow("Podman executable path is invalid"); + } +} + +function expectCredentialEnvironmentNamesRejected(): void { + for (const name of [ + "GITHUB_TOKEN", + "NGC_API_KEY", + "HF_TOKEN", + "AWS_SECRET_ACCESS_KEY", + "SSH_AUTH_SOCK", + "DOCKER_CONFIG", + "DOCKER_HOST", + "CUSTOM_API_KEY", + ]) { + expect(() => assertCredentialFreeQualificationEnvironment({ [name]: "forbidden" })).toThrow( + name, + ); + } +} + +function expectInvalidRunnerContractPathsRejected(): void { + for (const file of [ + "/etc/nemoclaw/native-runtime-qualification-v1.json", + "/run/nemoclaw-native-runtime-123456-1-1003/runner-contract.json", + "/run/nemoclaw-native-runtime-123456-1-1002/../runner-contract.json", + ]) { + expect(() => + nativeRuntimeQualificationRunnerContractPath( + { NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: file }, + 1002, + ), + ).toThrow("runner contract path is invalid"); + } +} + +function expectPublicCaseImagesPinned(): void { + for (const architecture of ["amd64", "arm64"] as const) { + for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + expect(nativeRuntimeQualificationAgentImage(architecture, agent)).toMatch( + /@sha256:[a-f0-9]{64}$/u, + ); + } + const ollama = nativeRuntimeQualificationInferenceImage({ + architecture, + acceleration: "cpu", + inference: "ollama", + }); + expect(ollama).toMatchObject({ model: "qwen3:0.6b" }); + expect(digestFromImageReference(ollama.imageRef)).toMatch(/^sha256:[a-f0-9]{64}$/u); + } +} + +describe("native runtime qualification case boundaries", () => { + it("accepts only an exact canonical trusted-plan row", () => { + const expected = row(); + expect(parseNativeRuntimeQualificationRow(JSON.stringify(expected))).toEqual(expected); + + const forged = JSON.parse(JSON.stringify(expected)) as Record; + forged.rootModes = ["rootless", "rootful"]; + expect(() => parseNativeRuntimeQualificationRow(JSON.stringify(forged))).toThrow( + "Root modes does not match", + ); + }); + + it("rejects candidate workflow authority in a forged row", () => { + const candidateRow = JSON.parse(JSON.stringify(row())) as { + source: { candidateSha: string; workflowSha: string }; + }; + candidateRow.source.workflowSha = candidateRow.source.candidateSha; + + expect(() => parseNativeRuntimeQualificationRow(JSON.stringify(candidateRow))).toThrow( + "Native runtime qualification source identity is invalid", + ); + }); + + it("accepts only the run-owned rootless Podman executable path for the current uid", () => { + const environment = { + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE: + "/nemoclaw-native-runtime-podman-123456-1-1002", + }; + expect(nativeRuntimeQualificationPodmanExecutable(environment, 1002)).toBe( + environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE, + ); + expectInvalidPodmanExecutablesRejected(); + }); + + it("rejects credential and alternate runtime authority environment names", () => { + expect(() => + assertCredentialFreeQualificationEnvironment({ + HOME: "/tmp/home", + PATH: "/usr/bin", + }), + ).not.toThrow(); + expectCredentialEnvironmentNamesRejected(); + }); + + it("accepts only typed immutable GPU runner resources", () => { + const parsed = parseNativeRuntimeQualificationRunnerContract(runnerContract(), "amd64"); + expect(parsed.nim.imageRef).toContain("@sha256:"); + expect(parsed.vllm.modelPath).toMatch(/^\/var\/tmp\/nemoclaw-native-runtime-resources-/u); + + expect(() => + parseNativeRuntimeQualificationRunnerContract( + { + ...runnerContract(), + nim: { ...runnerContract().nim, command: ["bash", "-c", "id"] }, + }, + "amd64", + ), + ).toThrow("NIM runner contract fields are invalid"); + expect(() => + parseNativeRuntimeQualificationRunnerContract( + { + ...runnerContract(), + vllm: { ...runnerContract().vllm, modelPath: "/tmp/model" }, + }, + "amd64", + ), + ).toThrow("vLLM runner contract is invalid"); + }); + + it("accepts only the current uid's run-owned GPU contract path", () => { + const environment = { + NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT: + "/run/nemoclaw-native-runtime-123456-1-1002/runner-contract.json", + }; + expect(nativeRuntimeQualificationRunnerContractPath(environment, 1002)).toBe( + environment.NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT, + ); + expectInvalidRunnerContractPathsRejected(); + }); + + it("pins every public case image to architecture-specific immutable digests", () => { + expectPublicCaseImagesPinned(); + }); + + it("requires the root-owned typed contract for NIM and vLLM", () => { + expect(() => + nativeRuntimeQualificationInferenceImage({ + architecture: "amd64", + acceleration: "nvidia-gpu", + inference: "nim", + }), + ).toThrow("reviewed GPU runner contract"); + const contract = parseNativeRuntimeQualificationRunnerContract(runnerContract(), "amd64"); + expect( + nativeRuntimeQualificationInferenceImage({ + architecture: "amd64", + acceleration: "nvidia-gpu", + inference: "vllm", + runnerContract: contract, + }), + ).toMatchObject({ model: "qualification" }); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-collector.test.ts b/test/e2e/support/native-runtime-qualification-collector.test.ts index 7ac90810263..61b73abd412 100644 --- a/test/e2e/support/native-runtime-qualification-collector.test.ts +++ b/test/e2e/support/native-runtime-qualification-collector.test.ts @@ -22,7 +22,7 @@ import type { NativeRuntimeQualificationEvidenceEnvelope } from "../registry/nat const REPOSITORY = "NVIDIA/NemoClaw"; const ACTOR = "maintainer"; -const WORKFLOW = ".github/workflows/native-runtime-qualification.yaml"; +const WORKFLOW = ".github/workflows/e2e.yaml"; const JOB_NAME = "Aggregate native runtime qualification evidence"; const ARTIFACT_NAME = "native-runtime-qualification-9143"; @@ -137,7 +137,7 @@ function githubFixture( [`repos/${REPOSITORY}/pulls/9143`, pull], [`repos/${REPOSITORY}/commits/main`, { sha: NATIVE_QUALIFICATION_BASE_SHA }], [ - `repos/${REPOSITORY}/actions/workflows/native-runtime-qualification.yaml`, + `repos/${REPOSITORY}/actions/workflows/${WORKFLOW.split("/").at(-1)!}`, { id: 101, path: WORKFLOW, state: "active" }, ], [`repos/${REPOSITORY}/actions/runs/7001`, run], diff --git a/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts new file mode 100644 index 00000000000..e16649ac1bf --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-producer-aggregate.test.ts @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW } from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; +import { + aggregateNativeRuntimeQualificationProducerEvidence, + NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE, +} from "../../../tools/e2e/native-runtime-qualification-producer-aggregate.mts"; +import { writeNativeRuntimeQualificationProducerEvidence } from "../../../tools/e2e/native-runtime-qualification-producer-evidence.mts"; +import { + buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, + type NativeRuntimeQualificationProducerPlanRow, +} from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; + +const roots: string[] = []; +const INSTALLER = "#!/usr/bin/env bash\nexit 0\n"; +const INSTALLER_SHA256 = createHash("sha256").update(INSTALLER).digest("hex"); +const AGGREGATE_TEST_OPTIONS = { timeout: 15_000 } as const; + +function writeJson(file: string, value: unknown): void { + fs.writeFileSync(file, `${JSON.stringify(value)}\n`); +} + +function installerReceipts( + directory: string, + row: NativeRuntimeQualificationProducerPlanRow, +): void { + fs.mkdirSync(directory); + fs.writeFileSync(path.join(directory, "installer.sh"), INSTALLER); + writeJson(path.join(directory, "invocation.json"), { + receiptVersion: 1, + script: "scripts/install.sh", + scriptSha256: INSTALLER_SHA256, + candidateSha: row.source.candidateSha, + architecture: row.case.architecture, + }); + writeJson(path.join(directory, "candidate-source.json"), { + receiptVersion: 1, + repository: "https://github.com/NVIDIA/NemoClaw.git", + revision: row.source.candidateSha, + installerSha256: INSTALLER_SHA256, + }); + writeJson(path.join(directory, "installed-source.json"), { + receiptVersion: 1, + repository: "https://github.com/NVIDIA/NemoClaw.git", + requestedRevision: row.source.candidateSha, + installedRevision: row.source.candidateSha, + installMode: "managed", + installerSha256: INSTALLER_SHA256, + }); + writeJson(path.join(directory, "architecture.json"), { + receiptVersion: 1, + requested: row.case.architecture, + runner: row.case.architecture, + }); + const posture = { + dockerCommandGuarded: true, + dockerEnvironmentVariablesUnset: true, + dockerServiceInactive: true, + dockerSocketUnitInactive: true, + dockerdProcessNameAbsent: true, + defaultSocketPathsAbsent: true, + }; + writeJson(path.join(directory, "docker-absence.json"), { + receiptVersion: 1, + preExecution: posture, + postExecution: posture, + }); +} + +function candidateReceipts( + directory: string, + row: NativeRuntimeQualificationProducerPlanRow, +): string { + fs.mkdirSync(directory); + const executionPath = path.join(directory, "execution.json"); + writeJson(executionPath, { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-execution-v1", + caseId: row.id, + candidateSha: row.source.candidateSha, + installerSha256: row.installerSha256, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + agent: row.case.agent, + inference: row.case.inference, + rootModes: row.rootModes, + obligations: row.case.obligations, + focusedOperations: row.focusedOperations, + evidenceKinds: row.case.evidenceKinds, + dockerUnavailable: { beforeCandidate: true, afterCandidate: true }, + credentialBoundary: { + githubCredentialsAbsent: true, + modelCredentialsAbsent: true, + isolatedUid: true, + }, + result: "passed", + }); + writeJson(path.join(directory, "runtime-result.json"), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { engineAuthority: `podman-sha256:${"9".repeat(64)}` }, + }); + for (const id of row.case.obligations) { + writeJson(path.join(directory, nativeRuntimeQualificationOperationFile(id)), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: id, + result: "passed", + details: { proof: id }, + }); + } + const cdiReceipts = + row.case.acceleration === "nvidia-gpu" + ? [ + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }, + ] + : []; + for (const receipt of cdiReceipts) { + writeJson(path.join(directory, "nvidia-cdi.json"), receipt); + } + writeJson(path.join(directory, "case-evidence.json"), { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: "5.6.2", + managedImages: [ + { role: "agent", digest: `sha256:${"1".repeat(64)}` }, + { role: "inference", digest: `sha256:${"2".repeat(64)}` }, + ], + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { nvidiaCdi: { device: "nvidia.com/gpu=all", file: "nvidia-cdi.json" } } + : {}), + }); + return executionPath; +} + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-aggregate-")); + roots.push(root); + const plan = buildNativeRuntimeQualificationProducerPlan({ + source: { + repository: "NVIDIA/NemoClaw", + producerWorkflow: NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, + pullRequestNumber: 9144, + candidateRepository: "NVIDIA/NemoClaw", + candidateSha: "a".repeat(40), + baseRef: "main", + baseSha: "b".repeat(40), + workflowSha: "b".repeat(40), + producerRunId: "7001", + producerRunAttempt: 1, + dispatchArtifact: { + id: "42", + name: "e2e-dispatch-7001-1", + digest: `sha256:${"c".repeat(64)}`, + sizeInBytes: 4096, + }, + }, + installerSha256: INSTALLER_SHA256, + arm64GpuRunner: "reviewed-arm64-gpu", + }); + const artifactRoot = path.join(root, "case-artifacts"); + fs.mkdirSync(artifactRoot); + for (const row of plan.include) { + const rowRoot = path.join(root, "rows", row.id); + fs.mkdirSync(rowRoot, { recursive: true }); + const installer = path.join(rowRoot, "installer"); + const candidate = path.join(rowRoot, "candidate"); + installerReceipts(installer, row); + const execution = candidateReceipts(candidate, row); + writeNativeRuntimeQualificationProducerEvidence( + row, + installer, + execution, + path.join(artifactRoot, row.artifactName), + ); + } + const evidenceDirectory = path.join(root, "aggregate"); + return { artifactRoot, evidenceDirectory, plan, root }; +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +describe("native runtime qualification producer aggregate", () => { + it( + "binds the exact 24-case cohort to one protected aggregate job", + AGGREGATE_TEST_OPTIONS, + () => { + const value = fixture(); + + const envelope = aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }); + + expect(envelope.cases).toHaveLength(24); + expect(new Set(envelope.cases.map((entry) => entry.caseId)).size).toBe(24); + expect( + envelope.cases.every( + (entry) => + entry.protectedRun.runId === 7001 && + entry.protectedRun.attempt === 1 && + entry.protectedRun.jobId === 811, + ), + ).toBe(true); + expect( + fs.existsSync( + path.join(value.evidenceDirectory, NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE), + ), + ).toBe(true); + }, + ); + + it("rejects replacing an existing aggregate output", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const sentinel = path.join(value.evidenceDirectory, "sentinel.txt"); + fs.mkdirSync(value.evidenceDirectory); + fs.writeFileSync(sentinel, "preserve me"); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("output must not already exist"); + expect(fs.readFileSync(sentinel, "utf8")).toBe("preserve me"); + }); + + it("rejects an omitted case artifact", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + fs.renameSync( + path.join(value.artifactRoot, value.plan.include[0]!.artifactName), + path.join(value.root, "omitted"), + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("cohort is incomplete or mixed"); + }); + + it("rejects a symlink substituted for a candidate receipt", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const row = value.plan.include[0]!; + const artifact = path.join(value.artifactRoot, row.artifactName); + const receipt = path.join(artifact, "receipts", row.id, "runtime", "runtime-result.json"); + const target = path.join(value.root, "substituted.json"); + fs.renameSync(receipt, target); + fs.symlinkSync(target, receipt); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("cannot contain symlinks"); + }); + + it("rejects a trusted plan with a mixed source cohort", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + const first = value.plan.include[0]!; + const mixedPlan = { + include: [ + { ...first, source: { ...first.source, candidateSha: "d".repeat(40) } }, + ...value.plan.include.slice(1), + ], + }; + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: mixedPlan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("source cohort"); + }); + + it("rejects an unexpected file in a case artifact", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + fs.writeFileSync( + path.join(value.artifactRoot, value.plan.include[0]!.artifactName, "candidate.log"), + "unexpected", + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("invalid files"); + }); + + it( + "rejects a receipt whose bytes no longer match its trusted fragment", + AGGREGATE_TEST_OPTIONS, + () => { + const value = fixture(); + const row = value.plan.include[0]!; + fs.appendFileSync( + path.join( + value.artifactRoot, + row.artifactName, + "receipts", + row.id, + "runtime", + "runtime-result.json", + ), + " ", + ); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 811, + }), + ).toThrow("does not match its SHA-256 digest"); + }, + ); + + it("rejects an invalid aggregate job identity", AGGREGATE_TEST_OPTIONS, () => { + const value = fixture(); + + expect(() => + aggregateNativeRuntimeQualificationProducerEvidence({ + plan: value.plan, + caseArtifactRoot: value.artifactRoot, + evidenceDirectory: value.evidenceDirectory, + aggregateJobId: 0, + }), + ).toThrow("aggregate job id is invalid"); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts index 62babcb43aa..b69391d3100 100644 --- a/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-evidence.test.ts @@ -8,9 +8,11 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW } from "../../../src/lib/onboard/runtime-provider/native-qualification-authority.ts"; import { writeNativeRuntimeQualificationProducerEvidence } from "../../../tools/e2e/native-runtime-qualification-producer-evidence.mts"; import { buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, } from "../../../tools/e2e/native-runtime-qualification-producer-plan.mts"; @@ -18,13 +20,13 @@ const roots: string[] = []; const INSTALLER = "#!/usr/bin/env bash\nexit 0\n"; const INSTALLER_SHA256 = createHash("sha256").update(INSTALLER).digest("hex"); -function fixture() { +function fixture(options: { readonly gpu?: boolean } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "native-runtime-producer-evidence-")); roots.push(root); - const row = buildNativeRuntimeQualificationProducerPlan({ + const plan = buildNativeRuntimeQualificationProducerPlan({ source: { repository: "NVIDIA/NemoClaw", - producerWorkflow: ".github/workflows/e2e.yaml", + producerWorkflow: NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, pullRequestNumber: 8064, candidateRepository: "NVIDIA/NemoClaw", candidateSha: "a".repeat(40), @@ -42,11 +44,18 @@ function fixture() { }, installerSha256: INSTALLER_SHA256, arm64GpuRunner: "reviewed-native-arm64-gpu-runner", - }).include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; + }); + const row = options.gpu + ? plan.include.find( + (entry) => entry.case.architecture === "arm64" && entry.case.acceleration === "nvidia-gpu", + )! + : plan.include.find((entry) => entry.id === NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE)!; const installerDirectory = path.join(root, "installer"); - const executionPath = path.join(root, "execution.json"); + const executionDirectory = path.join(root, "candidate"); + const executionPath = path.join(executionDirectory, "execution.json"); const evidenceDirectory = path.join(root, "evidence"); fs.mkdirSync(installerDirectory); + fs.mkdirSync(executionDirectory); fs.writeFileSync(path.join(installerDirectory, "installer.sh"), INSTALLER); fs.writeFileSync( path.join(installerDirectory, "invocation.json"), @@ -80,7 +89,11 @@ function fixture() { ); fs.writeFileSync( path.join(installerDirectory, "architecture.json"), - JSON.stringify({ receiptVersion: 1, requested: "amd64", runner: "amd64" }), + JSON.stringify({ + receiptVersion: 1, + requested: row.case.architecture, + runner: row.case.architecture, + }), ); const dockerPosture = { dockerCommandGuarded: true, @@ -121,6 +134,68 @@ function fixture() { result: "passed", }; fs.writeFileSync(executionPath, JSON.stringify(execution)); + fs.writeFileSync( + path.join(executionDirectory, "runtime-result.json"), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + caseId: row.id, + result: "passed", + details: { endpointAuthority: `podman-sha256:${"f".repeat(64)}` }, + }), + ); + for (const id of row.case.obligations) { + fs.writeFileSync( + path.join(executionDirectory, nativeRuntimeQualificationOperationFile(id)), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + caseId: row.id, + operationId: id, + result: "passed", + details: { proof: id }, + }), + ); + } + const cdiReceipts = + row.case.acceleration === "nvidia-gpu" + ? [ + { + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + caseId: row.id, + result: "passed", + details: { device: "nvidia.com/gpu=all" }, + }, + ] + : []; + for (const receipt of cdiReceipts) { + fs.writeFileSync(path.join(executionDirectory, "nvidia-cdi.json"), JSON.stringify(receipt)); + } + fs.writeFileSync( + path.join(executionDirectory, "case-evidence.json"), + JSON.stringify({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-details-v1", + caseId: row.id, + runtime: { + engineName: "Podman", + engineVersion: "5.6.2", + managedImages: [ + { role: "agent", digest: `sha256:${"1".repeat(64)}` }, + { role: "inference", digest: `sha256:${"2".repeat(64)}` }, + ], + resultFile: "runtime-result.json", + }, + operations: row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })), + ...(row.case.acceleration === "nvidia-gpu" + ? { nvidiaCdi: { device: "nvidia.com/gpu=all", file: "nvidia-cdi.json" } } + : {}), + }), + ); return { evidenceDirectory, execution, executionPath, installerDirectory, root, row }; } @@ -139,21 +214,81 @@ describe("native runtime qualification producer evidence", () => { value.evidenceDirectory, ); - expect(fs.readdirSync(value.evidenceDirectory)).toEqual(["evidence.json"]); + expect(fs.readdirSync(value.evidenceDirectory).sort()).toEqual([ + "case-fragment.json", + "receipts", + ]); expect( - JSON.parse(fs.readFileSync(path.join(value.evidenceDirectory, "evidence.json"), "utf8")), - ).toEqual({ + JSON.parse(fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8")), + ).toMatchObject({ schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-case-evidence-v1", + kind: "nemoclaw-native-runtime-qualification-case-fragment-v1", qualificationId: "podman-protected-host-local-inference", providerId: "podman", source: value.row.source, case: value.row.case, - result: "passed", + installer: { + architecture: value.row.case.architecture, + dockerAvailability: "unavailable", + exitCode: 0, + providerId: "podman", + }, + runtime: { + agent: "openclaw", + engineName: "Podman", + engineVersion: "5.6.2", + providerId: "podman", + }, }); - expect(fs.statSync(path.join(value.evidenceDirectory, "evidence.json")).mode & 0o777).toBe( + expect(fs.statSync(path.join(value.evidenceDirectory, "case-fragment.json")).mode & 0o777).toBe( 0o600, ); + expect( + fs.existsSync( + path.join(value.evidenceDirectory, "receipts", value.row.id, "installer", "installer.sh"), + ), + ).toBe(true); + const fragment = JSON.parse( + fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8"), + ) as { installer: { script: { path: string; sha256: string } } }; + const copied = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.installer.script.path), + ); + expect(createHash("sha256").update(copied).digest("hex")).toBe( + fragment.installer.script.sha256, + ); + }); + + it("emits the NVIDIA CDI receipt for a GPU case", () => { + const value = fixture({ gpu: true }); + + writeNativeRuntimeQualificationProducerEvidence( + value.row, + value.installerDirectory, + value.executionPath, + value.evidenceDirectory, + ); + + const fragment = JSON.parse( + fs.readFileSync(path.join(value.evidenceDirectory, "case-fragment.json"), "utf8"), + ) as { + installer: { architecture: string; script: { path: string; sha256: string } }; + nvidiaCdi: { artifact: { path: string; sha256: string } }; + }; + expect(fragment.installer.architecture).toBe("arm64"); + const copiedInstaller = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.installer.script.path), + ); + expect(createHash("sha256").update(copiedInstaller).digest("hex")).toBe( + fragment.installer.script.sha256, + ); + const copiedCdi = fs.readFileSync( + path.join(value.evidenceDirectory, fragment.nvidiaCdi.artifact.path), + ); + expect(fragment.nvidiaCdi.artifact.path).toContain("/runtime/nvidia-cdi.json"); + expect(createHash("sha256").update(copiedCdi).digest("hex")).toBe( + fragment.nvidiaCdi.artifact.sha256, + ); }); it.each([ @@ -247,6 +382,23 @@ describe("native runtime qualification producer evidence", () => { value.executionPath, value.evidenceDirectory, ), - ).toThrow("receipt is missing or invalid"); + ).toThrow("receipt file is invalid"); + }); + + it("rejects an unexpected candidate-controlled receipt file", () => { + const value = fixture(); + fs.writeFileSync( + path.join(path.dirname(value.executionPath), "candidate.log"), + "candidate output", + ); + + expect(() => + writeNativeRuntimeQualificationProducerEvidence( + value.row, + value.installerDirectory, + value.executionPath, + value.evidenceDirectory, + ), + ).toThrow("receipt files are invalid"); }); }); diff --git a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts index 7075a8c7424..75544024954 100644 --- a/test/e2e/support/native-runtime-qualification-producer-plan.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-plan.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { buildNativeRuntimeQualificationProducerPlan, + nativeRuntimeQualificationOperationFile, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_CASE, NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS, type NativeRuntimeQualificationProducerPlanInput, @@ -51,6 +52,9 @@ describe("native runtime qualification producer plan", () => { expect(entry.source.candidateSha).toBe(CANDIDATE_SHA); expect(entry.source.baseSha).toBe(entry.source.workflowSha); expect(entry.case.id).toBe(entry.id); + expect( + new Set(entry.case.obligations.map(nativeRuntimeQualificationOperationFile)).size, + ).toBe(entry.case.obligations.length); expect(Object.isFrozen(entry)).toBe(true); } expect( @@ -62,7 +66,7 @@ describe("native runtime qualification producer plan", () => { plan.include.find( (entry) => entry.case.architecture === "arm64" && entry.case.acceleration === "cpu", )?.runner, - ).toBe("ubuntu-24.04-arm"); + ).toBe("ubuntu-26.04-arm"); expect( plan.include.find( (entry) => entry.case.architecture === "amd64" && entry.case.acceleration === "nvidia-gpu", @@ -94,10 +98,22 @@ describe("native runtime qualification producer plan", () => { ).toBe(true); }); + it("rejects a candidate workflow SHA as qualification authority", () => { + const baseInput = input(); + const candidateWorkflow = { + ...baseInput, + source: { ...baseInput.source, workflowSha: CANDIDATE_SHA }, + } satisfies NativeRuntimeQualificationProducerPlanInput; + + expect(() => buildNativeRuntimeQualificationProducerPlan(candidateWorkflow)).toThrow( + "Native runtime qualification producer source is invalid", + ); + }); + it.each([ ["fork candidate", { source: { ...input().source, candidateRepository: "fork/NemoClaw" } }], ["candidate commit", { source: { ...input().source, candidateSha: "A".repeat(40) } }], - ["base authority", { source: { ...input().source, workflowSha: "e".repeat(40) } }], + ["unbound workflow", { source: { ...input().source, workflowSha: "e".repeat(40) } }], ["run attempt", { source: { ...input().source, producerRunAttempt: 2 } }], ["installer digest", { installerSha256: "short" }], ["ARM64 GPU runner", { arm64GpuRunner: "" }], diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index d6cbd25fb79..572cb2744e1 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -26,6 +26,20 @@ function step(owner: WorkflowJob, name: string): WorkflowStep { return value!; } +function expectRequiredPodmanPackages(run: string): void { + for (const requiredPackage of [ + "acl", + "apparmor", + "conmon", + "golang-github-containers-common", + "runc", + "slirp4netns", + "uidmap", + ]) { + expect(run).toContain(requiredPackage); + } +} + describe("native runtime qualification producer workflow", () => { it("keeps candidate execution out of the authenticated controller", () => { const generate = job("generate-matrix"); @@ -54,8 +68,9 @@ describe("native runtime qualification producer workflow", () => { const authenticate = step(plan, "Authenticate the candidate and dispatch artifact"); const compile = step(plan, "Compile the trusted qualification producer matrix"); - expect(plan.if).toContain("github.ref == 'refs/heads/main'"); - expect(plan.if).toContain("inputs.jobs == 'native-runtime-qualification-producer'"); + expect(plan.if).toBe( + "${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }}", + ); expect(plan.permissions).toEqual({ actions: "read", contents: "read", @@ -63,6 +78,8 @@ describe("native runtime qualification producer workflow", () => { }); expect(authenticate.run).toContain('"$CANDIDATE_REPOSITORY" == "NVIDIA/NemoClaw"'); expect(authenticate.run).toContain('"$BASE_SHA" == "$WORKFLOW_SHA"'); + expect(authenticate.run).toContain('"$CANDIDATE_SHA" != "$WORKFLOW_SHA"'); + expect(authenticate.run).not.toContain('"$WORKFLOW_SHA" == "$CANDIDATE_SHA"'); expect(authenticate.run).toContain(".head.sha == $candidateSha"); expect(authenticate.run).toContain(".base.sha == $baseSha"); expect(authenticate.run).toContain(".total_count == 1"); @@ -73,52 +90,499 @@ describe("native runtime qualification producer workflow", () => { ); expect(compile.run).toContain("native-runtime-qualification-producer-plan.mts --ci-output"); expect(JSON.stringify(plan)).not.toContain("linux-arm64-gpu-dgx-spark-gb10-protected-1"); + const producerCheckout = step(plan, "Check out the trusted qualification producer"); + expect(producerCheckout.with?.["sparse-checkout"]).toContain( + "src/lib/onboard/runtime-provider/native-qualification-authority.ts", + ); + }); + + it("keeps secret-bearing GPU preparation downstream of the trusted-main plan", () => { + const producer = job("native-runtime-qualification-producer"); + const gpuResources = step(producer, "Prepare GPU resources with the NVIDIA API key"); + + expect(producer.needs).toContain("native-runtime-qualification-producer-plan"); + expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); + }); + + it("builds one pinned Podman 6 toolchain for each qualified architecture", () => { + const toolchain = job("native-runtime-qualification-podman-toolchain"); + const podmanSource = step(toolchain, "Check out the pinned Podman source"); + const netavarkSource = step(toolchain, "Check out the pinned Netavark source"); + const aardvarkSource = step(toolchain, "Check out the pinned Aardvark DNS source"); + const setupGo = step(toolchain, "Set up pinned Go for the Podman build"); + const setupRust = step(toolchain, "Set up pinned Rust for the network helper builds"); + const buildDependencies = step( + toolchain, + "Install build dependencies from the signed runner OS repository", + ); + const build = step(toolchain, "Build and package the pinned native toolchain"); + const upload = step(toolchain, "Upload the pinned native Podman toolchain"); + + expect(toolchain.name).toBe( + "Build pinned native Podman toolchain / ${{ matrix.architecture }}", + ); + expect(toolchain.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-producer-plan", + ]); + expect(toolchain["runs-on"]).toBe("${{ matrix.runner }}"); + expect(toolchain.permissions).toEqual({ contents: "read" }); + expect(toolchain.strategy).toMatchObject({ + "fail-fast": false, + matrix: { + include: [ + { architecture: "amd64", runner: "ubuntu-24.04" }, + { architecture: "arm64", runner: "ubuntu-24.04-arm" }, + ], + }, + }); + expect(podmanSource.with).toMatchObject({ + repository: "podman-container-tools/podman", + ref: "cade97a52ebdf9dbf9e81de8009015776837a074", + path: ".podman-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(netavarkSource.with).toMatchObject({ + repository: "containers/netavark", + ref: "8e91ad1d947ed325327b638f0cb906bea1f7d0ab", + path: ".netavark-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(aardvarkSource.with).toMatchObject({ + repository: "containers/aardvark-dns", + ref: "cd7417681229219059939bdd9f0b3bd9ac9abb08", + path: ".aardvark-source", + "fetch-depth": 1, + "persist-credentials": false, + }); + expect(setupGo.uses).toBe("actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00"); + expect(setupGo.with).toEqual({ "go-version": "1.25.9", cache: false }); + expect(setupRust.uses).toBe( + "actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659", + ); + expect(setupRust.with).toEqual({ toolchain: "1.88.0", cache: false, rustflags: "" }); + expect(buildDependencies.run).not.toContain("libsubid-dev"); + expect(buildDependencies.run).not.toContain("libgpgme-dev"); + expect(buildDependencies.run).not.toContain("libassuan-dev"); + expect(buildDependencies.run).not.toContain("libgpg-error-dev"); + expect(buildDependencies.run).toContain("curl"); + expect(build.run).toContain("podman rootlessport PREFIX=/usr/local"); + expect(build.run).toContain("EXTRA_BUILDTAGS=containers_image_openpgp"); + expect(build.run).not.toContain("quadlet"); + expect(build.run).toContain("make --directory=.netavark-source --jobs=2 build"); + expect(build.run).toContain("make --directory=.aardvark-source --jobs=2 build"); + expect(build.run).toContain("https://passt.top/passt"); + expect(build.run).toContain("/usr/bin/curl"); + expect(build.env?.PASTA_SOURCE_ARCHIVE_SHA256).toBe( + "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958", + ); + expect(build.env?.PASTA_SOURCE_SHA).toBe("f8df3f1b228fe19a74a269334fdfe6cc7d0605ce"); + expect(build.env?.PASTA_VERSION).toBe("2026_07_28.f8df3f1"); + expect(build.run).toContain('sha256sum "$pasta_source_archive"'); + expect(build.run).toContain("--no-same-owner --no-same-permissions"); + expect(build.run).toContain("[[ ! -e .passt-source/passt && ! -L .passt-source/passt ]]"); + expect(build.run).not.toMatch(/\bgit\s+fetch\b/u); + expect(build.run).toContain( + 'make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt', + ); + expect(build.run).toContain( + 'install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta"', + ); + expect(build.run).toContain("pasta_version_output="); + expect(build.run).toContain("pasta_version_output%%$'\\n'*"); + expect(build.run).toContain('"pasta $PASTA_VERSION"'); + expect(build.run).toContain("Pinned qualification pasta version is invalid"); + expect(build.run).toMatch(/sha256sum[\s\S]+bin\/pasta[\s\S]+manifest\.json/u); + expect(build.run).toContain("sha256sum"); + expect(build.run).toContain("Pinned Podman build has an unresolved runtime dependency"); + expect(build.run).toContain("Pinned Podman build must not require an optional host ABI"); + expect(build.run).toContain('grep -E "libgpgme|libsubid"'); + expect(build.run).toMatch(/sha256sum[\s\S]+manifest\.json/u); + expect(build.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); + expect(upload.with).toMatchObject({ + name: "native-runtime-podman-toolchain-${{ matrix.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain/", + }); + expect(upload.if).toBe("success()"); + expect(upload.uses).toBe( + "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57", + ); }); - it("runs each candidate case in an isolated account and emits one trusted artifact", () => { + it("runs each candidate case in an isolated account and emits one bounded evidence artifact", () => { const producer = job("native-runtime-qualification-producer"); + const harness = step(producer, "Check out the trusted qualification harness"); + const podmanHost = step(producer, "Require a reviewed Ubuntu runtime host"); + const podmanDownload = step(producer, "Download the pinned native Podman toolchain"); + const podman = step( + producer, + "Install the pinned native Podman toolchain and rootless prerequisites", + ); const boundary = step( producer, "Prepare the credential-free execution account and disable Docker", ); + const dependencies = step( + producer, + "Install locked candidate test dependencies without scripts", + ); + const gpuResources = step(producer, "Prepare GPU resources with the NVIDIA API key"); const installer = step(producer, "Run the authenticated installer qualification"); const execute = step(producer, "Execute the candidate qualification case without credentials"); const validate = step(producer, "Validate receipts and emit bounded evidence"); const upload = step(producer, "Upload the qualification case evidence"); const cleanup = step(producer, "Remove qualification resources"); - const source = JSON.stringify(producer); + const credentialFreeSource = JSON.stringify({ + ...producer, + steps: producer.steps?.filter( + (entry) => entry.name !== "Prepare GPU resources with the NVIDIA API key", + ), + }); const boundaryRun = boundary.run ?? ""; - const installerRun = installer.run ?? ""; + const executeRun = execute.run ?? ""; + const gpuResourcesRun = gpuResources.run ?? ""; expect(producer.name).toBe("${{ matrix.jobName }}"); + expect(producer.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-podman-toolchain", + "native-runtime-qualification-producer-plan", + ]); expect(producer["runs-on"]).toBe("${{ matrix.runner }}"); expect(producer.permissions).toEqual({ contents: "read" }); expect(producer.strategy).toMatchObject({ "fail-fast": false }); - expect(source).not.toMatch(/NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u); + expect(harness.with?.["sparse-checkout"]).toContain( + "tools/e2e/native-runtime-qualification-producer-plan.mts", + ); + expect(harness.with?.["sparse-checkout"]).toContain( + "test/e2e/registry/native-runtime-qualification.ts", + ); + expect(credentialFreeSource).not.toMatch( + /NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|DOCKERHUB_TOKEN/u, + ); + expect(gpuResources.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); + expect(gpuResources.run).toContain("NVIDIA_API_KEY repository secret"); + expect(gpuResources.run).toContain("login nvcr.io --username '$oauthtoken' --password-stdin"); + expect(gpuResources.run).not.toContain("logout --all"); + expect(gpuResources.run).toContain('sudo unlink "$registry_auth_file"'); + expect(gpuResources.run).toContain("$(sudo stat -c '%u:%g:%h' -- \"$registry_auth_file\")"); + expect(gpuResources.run).toContain('sudo chmod 0600 -- "$registry_auth_file"'); + expect(gpuResources.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(gpuResources.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(gpuResources.run).toContain("${uid}:${gid}:600:1"); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$registry_auth_file"')).toBeGreaterThan( + gpuResourcesRun.indexOf("login nvcr.io --username '$oauthtoken' --password-stdin"), + ); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$registry_auth_file"')).toBeLessThan( + gpuResourcesRun.indexOf('"$PODMAN_EXECUTABLE" pull "$image"'), + ); + expect(gpuResources.run).toContain("unset NVIDIA_API_KEY"); + expect(gpuResources.run).toContain("7ae557604adf67be50417f59c2c2f167def9a775"); + expect(gpuResources.run).toContain("sudo stat -c '%u:%g:%a:%h:%s' -- \"$target\""); + expect(gpuResources.run).toContain("sudo stat -c '%u:%g:%h:%s' -- \"$target\""); + expect(gpuResources.run).toContain('sudo chmod 0600 -- "$target"'); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$target"')).toBeGreaterThan( + gpuResourcesRun.indexOf("sudo stat -c '%u:%g:%h:%s'"), + ); + expect(gpuResourcesRun.indexOf('sudo chmod 0600 -- "$target"')).toBeLessThan( + gpuResourcesRun.indexOf("sudo stat -c '%u:%g:%a:%h:%s'"), + ); + expect(gpuResources.run).toContain('sudo git hash-object --no-filters -- "$target"'); + expect(gpuResources.run).toContain('sudo sha256sum -- "$target"'); + expect(gpuResources.run).toContain("model-free-nim@sha256:"); + expect(gpuResources.run).toContain("nvcr.io/nvidia/vllm@sha256:"); + expect(gpuResources.run).toContain("runner-contract.json"); + expect(gpuResources.run).toContain( + 'install -d --owner=root --group=root --mode=0711 "$resource_directory"', + ); + expect(gpuResources.run).toContain('chmod 0555 "$resource_directory"'); + expect(podmanHost.run).toContain('[[ "${ID:-}" == "ubuntu" ]]'); + expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "24.04"'); + expect(podmanHost.run).toContain('"${VERSION_ID:-}" == "26.04"'); + expect(podmanHost.run).toContain("Ubuntu release is not reviewed"); + expect(podmanDownload.with).toMatchObject({ + name: "native-runtime-podman-toolchain-${{ matrix.case.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain", + }); + expect(podman.run).toContain("/usr/bin/apt-get install"); + expectRequiredPodmanPackages(podman.run ?? ""); + expect(podman.run).not.toMatch(/\s+passt(?:\s|$)/u); + expect(podman.run).not.toContain("fuse-overlayfs"); + expect(podman.run).toContain("find -P"); + expect(podman.run).toContain("sha256sum --check --strict SHA256SUMS"); + expect(podman.run).toContain("./bin/pasta"); + expect(podman.run).toContain('"nemoclaw-native-podman-toolchain-v1"'); + expect(podman.run).toContain("Downloaded native Podman toolchain contains unexpected files"); + expect(podman.run).toContain("Native Podman toolchain target must not be a symlink"); + expect(podman.run).toContain('dpkg --compare-versions "$conmon_version" ge 2.1.7'); + expect(podman.run).toContain('dpkg --compare-versions "$runc_version" ge 1.1.11'); + expect(podman.run).toContain('"netavark 2.1.0"'); + expect(podman.run).toContain('"aardvark-dns 2.1.0"'); + expect(podman.run).toContain('"2026_07_28.f8df3f1"'); + expect(podman.run).toContain( + '"54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958"', + ); + expect(podman.run).toContain('"f8df3f1b228fe19a74a269334fdfe6cc7d0605ce"'); + expect(podman.run).toContain('[[ "$version" == "podman version 6.1.0" ]]'); + expect(podman.run).not.toContain("CANDIDATE_DIRECTORY"); expect(boundary.run).toContain("mask --runtime docker.service docker.socket"); - expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin"); - expect(boundaryRun.indexOf("printf 'account=%s")).toBeLessThan( + expect(boundary.run).toContain("useradd --create-home --shell /usr/sbin/nologin --user-group"); + expect(boundary.run).toContain('getent passwd "$account"'); + expect(boundary.run).toContain('getent group "$account"'); + expect(boundary.run).toContain('grep -q "^${account}:" /etc/subuid /etc/subgid'); + expect(boundary.run).toContain( + "Qualification account, group, or subordinate-ID authorization already exists", + ); + expect(boundary.run).toContain('ownership_marker="/run/nemoclaw-native-runtime-owner-'); + expect(boundary.run).toContain("0:0:400"); + expect(boundary.run).toContain("Qualification account ownership marker is invalid"); + expect(boundary.run).toContain("rollback_unmarked_account"); + expect(boundary.run).toContain( + "Partially created qualification account could not be rolled back", + ); + expect(boundary.run).toContain("ensure_subordinate_range /etc/subuid --add-subuids"); + expect(boundary.run).toContain("ensure_subordinate_range /etc/subgid --add-subgids"); + expect(boundary.run).toContain("has no free subordinate-ID range for rootless Podman"); + expect(boundary.run).toContain('runtime_directory_unit="user-runtime-dir@${uid}.service"'); + expect(boundary.run).toContain("Qualification runtime directory is missing or invalid"); + expect(boundary.run).toContain('user_manager_unit="user@${uid}.service"'); + expect(boundary.run).toContain('systemctl start "$user_manager_unit"'); + expect(boundary.run).toContain("/usr/bin/systemctl --user start dbus.socket"); + expect(boundary.run).toContain("/usr/bin/systemctl --user is-active --quiet dbus.socket"); + expect(boundary.run).toContain("Qualification systemd user bus socket unit is not active"); + expect(boundary.run).toContain('sudo -u "$execution_account" /usr/bin/test -S "$bus"'); + expect(boundary.run).toContain('sudo /usr/bin/test ! -L "$bus"'); + expect(boundary.run).toContain("sudo stat -c '%u' -- \"$bus\""); + expect(boundary.run).toContain("Qualification systemd user bus $context"); + expect(boundary.run).toContain( + 'trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"', + ); + expect(boundary.run).toContain('Environment="SYSTEMD_UNIT_PATH=%s"'); + expect(boundary.run).toContain("/usr/bin/systemctl --user show-environment"); + expect(boundary.run).toContain('sudo -u "$account" env -i'); + expect(boundary.run).toContain('CONTAINERS_CONF="$containers_config"'); + expect(boundary.run).toContain('CONTAINERS_STORAGE_CONF="$storage_config"'); + expect(boundary.run).toContain('firewall_driver = "nftables"'); + expect(boundary.run).toContain( + "Qualification containers configuration is not root-owned and read-only", + ); + expect(boundary.run).toContain("rootless_storage_path"); + expect(boundary.run).toContain("${home}/.local/share/containers/storage"); + expect(boundary.run).not.toContain('mount_program = "/usr/bin/fuse-overlayfs"'); + expect(boundary.run).toContain('.store.graphDriverName == "overlay"'); + expect(boundary.run).toContain('["overlay.mount_program"].Executable?'); + expect(boundary.run).toContain("Qualification requires native rootless overlay storage"); + expect(boundary.run).toContain("0:0:444"); + expect(boundary.run).toContain("/sys/module/apparmor/parameters/enabled"); + expect(boundary.run).toContain( + "profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined)", + ); + expect(boundary.run).toContain( + "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined)", + ); + expect(boundary.run).toContain("Run-owned qualification pasta executable digest changed"); + expect(boundary.run).toContain('"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"'); + expect(boundary.run).not.toContain("/usr/bin/pasta"); + expect(boundary.run).toContain( + 'PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin"', + ); + expect(boundary.env?.TOOLCHAIN_DIRECTORY).toBe( + "${{ runner.temp }}/native-runtime-podman-toolchain", + ); + expect(boundary.run).toContain( + 'podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"', + ); + expect(boundary.run).toContain("sudo install --owner=root --group=root --mode=0555"); + expect(boundary.run).toContain("0:0:555"); + expect(boundary.run).toContain('"$podman_executable" info --format json'); + expect(boundary.run).toContain("userns,"); + expect(boundary.run).toContain('apparmor_parser -r "$apparmor_profile"'); + expect(boundary.run).not.toContain("apparmor_restrict_unprivileged_userns="); + expect(boundary.run).toContain("Credential-free rootless Podman readiness failed"); + expect(boundary.run).toContain('install -d -m 0755 "$guard_dir"'); + expect(boundary.run).toContain('chmod 0555 "$guard_dir/docker"'); + expect(boundary.run).toContain('setfacl --modify "u:${account}:--x"'); + expect(boundary.run).not.toContain("chmod o+x"); + expect(boundaryRun.indexOf("useradd --create-home")).toBeLessThan( + boundaryRun.indexOf("printf 'account=%s"), + ); + expect(boundaryRun.indexOf("Qualification account ownership marker is invalid")).toBeLessThan( + boundaryRun.indexOf("printf 'account=%s"), + ); + expect(boundaryRun.indexOf("printf 'account=%s")).toBeGreaterThan( boundaryRun.indexOf("useradd --create-home"), ); + expect(dependencies.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(dependencies.run).toContain('cd "$1"'); + expect(dependencies.run).toContain("package.json package-lock.json"); + expect(dependencies.run).toContain('! -L "$file" && -O "$file"'); + expect(dependencies.run).toContain("npm --prefix"); + expect(dependencies.run).toContain("ci --ignore-scripts"); expect(installer.run).toContain('sudo -u "$ACCOUNT" env -i'); - expect(installer.run).toContain("run-native-runtime-installer-qualification.sh"); - expect(installerRun.indexOf("pkill -KILL -u")).toBeLessThan( - installerRun.indexOf("chown -R -h root:root"), + expect(installer.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(installer.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(installer.env?.RUNTIME_DIRECTORY_UNIT).toBe( + "${{ steps.boundary.outputs.runtime_directory_unit }}", + ); + expect(installer.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); + expect(installer.run).toContain("/usr/bin/systemctl --user start dbus.socket"); + expect(installer.run).toContain("Qualification systemd user bus socket unit did not restart"); + expect(installer.run).toContain('sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus"'); + expect(installer.run).toContain('sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus"'); + expect(installer.run).toContain("sudo stat -c '%u' -- \"$RUNTIME_DIRECTORY/bus\""); + expect(installer.run).toContain( + "Qualification systemd user bus is invalid or inaccessible after installer isolation", + ); + expect(installer.env?.TRUSTED_USER_UNIT_PATH).toBe("/usr/lib/systemd/user:/lib/systemd/user"); + expect(installer.run).toContain("/usr/bin/systemctl --user show-environment"); + expect(installer.env?.CONTAINERS_CONFIG).toBe( + "${{ steps.boundary.outputs.containers_config }}", ); - expect(installer.run).toContain('[[ -d "$INSTALLER_RECEIPT_PARENT/receipts" && ! -L'); + expect(installer.run).toContain('CONTAINERS_CONF="$CONTAINERS_CONFIG"'); + expect(installer.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); + expect(installer.run).toContain('XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY"'); + expect(installer.run).toContain("run-native-runtime-installer-qualification.sh"); + expect(installer.run).not.toContain("chown -R"); + expect(installer.run).toContain('sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts"'); + expect(installer.run).toContain('sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts"'); expect(execute.run).toContain('sudo -u "$ACCOUNT" env -i'); + expect(execute.env?.ACCOUNT_GID).toBe("${{ steps.boundary.outputs.gid }}"); + expect(execute.env?.ACCOUNT_UID).toBe("${{ steps.boundary.outputs.uid }}"); + expect(execute.env?.RUNTIME_DIRECTORY_UNIT).toBe( + "${{ steps.boundary.outputs.runtime_directory_unit }}", + ); + expect(execute.run).toContain('sudo chown "$ACCOUNT_UID:$ACCOUNT_GID"'); + expect(execute.run).toContain( + 'live_test="test/e2e/live/native-runtime-qualification-case.test.ts"', + ); + expect(execute.run).toContain('cd "$CANDIDATE_DIRECTORY"'); + expect(executeRun.indexOf('cd "$CANDIDATE_DIRECTORY"')).toBeLessThan( + executeRun.indexOf('sudo -u "$ACCOUNT" env -i'), + ); expect(execute.run).toContain("native-runtime-qualification-case.test.ts"); expect(execute.run).not.toContain("GITHUB_TOKEN"); expect(execute.run).not.toContain("GH_TOKEN"); + expect(execute.run).not.toContain("chown -R"); + expect(execute.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(execute.env?.CONTAINERS_CONFIG).toBe("${{ steps.boundary.outputs.containers_config }}"); + expect(execute.env?.PODMAN_EXECUTABLE).toBe("${{ steps.boundary.outputs.podman_executable }}"); + expect(execute.env?.STORAGE_CONFIG).toBe("${{ steps.boundary.outputs.storage_config }}"); + expect(execute.env?.RUNNER_CONTRACT).toBe("${{ steps.gpu_resources.outputs.runner_contract }}"); + expect(execute.run).toContain('CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG"'); + expect(execute.run).toContain('CONTAINERS_CONF="$CONTAINERS_CONFIG"'); + expect(execute.run).toContain( + 'NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE"', + ); + expect(execute.run).toContain( + 'PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin"', + ); + expect(validate.env?.NODE_DIRECTORY).toBe("${{ steps.boundary.outputs.node_dir }}"); + expect(validate.run).not.toContain('chown -R -h "$(id -u):$(id -g)"'); + expect(validate.run).toContain("sudo --preserve-env="); + expect(validate.run).toContain('"$NODE_DIRECTORY/node"'); expect(validate.run).toContain("native-runtime-qualification-producer-evidence.mts"); expect(upload.with).toMatchObject({ name: "${{ matrix.artifactName }}", - path: "${{ runner.temp }}/native-runtime-evidence/evidence.json", + path: "${{ runner.temp }}/native-runtime-evidence/", }); expect(cleanup.if).toBe("always()"); - expect(cleanup.run).toContain('account="${ACCOUNT:-nemoclawq}"'); + expect(cleanup.env?.ACCOUNT_CREATED).toBe("${{ steps.boundary.outputs.account_created }}"); + expect(cleanup.run).toContain('reported_account="${ACCOUNT:-}"'); + expect(cleanup.run).not.toContain("ACCOUNT:-nemoclawq"); + expect(cleanup.run).toContain( + "Qualification account ownership marker cleanup target is invalid", + ); + expect(cleanup.run).toContain('ownership="$(sudo cat "$ownership_marker")"'); expect(cleanup.run).toContain("pkill -KILL -u"); + expect(cleanup.run).not.toContain("rm -rf"); + expect(cleanup.run).not.toContain("find "); + expect(cleanup.run).toContain('systemctl stop "$user_manager_unit" "$runtime_directory_unit"'); + expect(cleanup.run).toContain('sudo unlink "$user_manager_dropin"'); + expect(cleanup.run).toContain('sudo rmdir "$user_manager_dropin_directory"'); + expect(cleanup.run).toContain("sudo systemctl daemon-reload"); + expect(cleanup.run).toContain( + "Qualification systemd user lifecycle remained active during cleanup", + ); + expect(cleanup.run).toContain('apparmor_parser -R "$apparmor_profile"'); + expect(cleanup.run).toContain('apparmor_parser -R "$pasta_apparmor_profile"'); + expect(cleanup.run).toContain('sudo rm -f -- "$apparmor_profile"'); + expect(cleanup.run).toContain('sudo unlink "$storage_config_directory/containers.conf"'); + expect(cleanup.run).toContain('sudo rm -f -- "$storage_config_directory/storage.conf"'); + expect(cleanup.run).toContain('sudo rm -f -- "$podman_executable"'); + expect(cleanup.run).toContain('sudo test -e "$model_directory"'); + expect(cleanup.run).toContain("sudo stat -c '%u:%g:%a' -- \"$model_directory\""); + expect(cleanup.run).toContain('sudo test -f "$target"'); + expect(cleanup.run).toContain('sudo test ! -L "$target"'); + expect(cleanup.run).toContain("Qualification Podman executable remains after cleanup"); + expect(cleanup.run).toContain("Qualification GPU resource directory remains after cleanup"); + expect(cleanup.run).toContain( + "Qualification runtime directory remains after its systemd cleanup", + ); + expect(cleanup.run).toContain("Qualification storage configuration remains after cleanup"); expect(cleanup.run).toContain("userdel --remove"); expect(cleanup.run).toContain("Qualification account still exists after cleanup"); + expect(cleanup.run).toContain("Qualification private group still exists after cleanup"); + expect(cleanup.run).toContain( + "Qualification subordinate-ID authorization remains after cleanup", + ); + expect(cleanup.run).toContain( + "Qualification account output exists without its ownership marker", + ); + expect(cleanup.run).toContain('sudo rm -f -- "$ownership_marker"'); + const accountOwnershipSource = [ + boundary.run, + gpuResources.run, + installer.run, + execute.run, + cleanup.run, + ].join("\n"); + expect(accountOwnershipSource).not.toContain("${uid}:${uid}"); + expect(accountOwnershipSource).not.toContain("$uid:$uid"); + expect(accountOwnershipSource).not.toContain("$ACCOUNT:$ACCOUNT"); + }); + + it("aggregates the exact successful 24-case cohort in a separate workflow job", () => { + const aggregate = job("native-runtime-qualification-producer-aggregate"); + const download = step(aggregate, "Download the exact case evidence cohort"); + const identity = step(aggregate, "Resolve this aggregate job identity"); + const setupNode = step(aggregate, "Set up Node for qualification aggregation"); + const collect = step(aggregate, "Validate and aggregate all 24 case receipts"); + const upload = step(aggregate, "Upload aggregate evidence"); + const aggregateCheckout = step(aggregate, "Check out the qualification aggregator"); + + expect(aggregate.name).toBe("Aggregate native runtime qualification evidence"); + expect(aggregate.needs).toEqual([ + "generate-matrix", + "native-runtime-qualification-producer-plan", + "native-runtime-qualification-producer", + ]); + expect(aggregate.if).toContain( + "needs.native-runtime-qualification-producer.result == 'success'", + ); + expect(aggregate.permissions).toEqual({ + actions: "read", + contents: "read", + "pull-requests": "read", + }); + expect(aggregateCheckout.with?.repository).toBe("${{ github.repository }}"); + expect(download.with).toMatchObject({ + pattern: "native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*", + "merge-multiple": false, + }); + expect(identity.run).toContain('.status == "in_progress"'); + expect(identity.run).toContain("select(length == 1)"); + expect(identity.run).toContain("Aggregate job lookup exceeds the bounded 100-job page"); + expect(setupNode.with?.["node-version"]).toBe("22.19.0"); + expect(collect.run).toContain("native-runtime-qualification-producer-aggregate.mts"); + expect(collect.env?.QUALIFICATION_PLAN).toBe( + "${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}", + ); + expect(upload.with).toMatchObject({ + name: "native-runtime-qualification-${{ inputs.checkout_sha }}", + path: "${{ runner.temp }}/native-runtime-aggregate/", + "if-no-files-found": "error", + }); }); }); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 0d7b8db2710..1f8c128b35e 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -95,7 +95,6 @@ describe("E2E artifact uploads", () => { expect(policyErrors).toContain( "upload-e2e-artifacts must preserve artifact defaults, hidden-file policy, missing-file behavior, and retention", ); - }); it("uploads artifacts even when an earlier step fails", () => { @@ -151,6 +150,19 @@ describe("E2E artifact uploads", () => { ); }); + it("allows only the exact 30-day native runtime aggregate upload", () => { + const workflow = mutableWorkflow(); + const upload = workflow.jobs["native-runtime-qualification-producer-aggregate"].steps?.find( + (step) => step.name === "Upload aggregate evidence", + ); + expect(upload).toBeDefined(); + upload!.with!["retention-days"] = 14; + + expect(validateUploadE2eArtifactsInvocations(workflow)).toContain( + "native-runtime-qualification-producer-aggregate must not invoke actions/upload-artifact directly", + ); + }); + it.each([ ["name", "another-cache-artifact"], ["path", "another-cache-path/"], diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 51863863d93..a7795771ccf 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -391,6 +391,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map; + +function record(value: unknown, label: string): UnknownRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as UnknownRecord; +} + +function exactKeys(value: UnknownRecord, keys: readonly string[], label: string): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label} fields are invalid`); + } +} + +function readBoundedBytes(file: string, maximum: number): Buffer { + let descriptor: number | undefined; + try { + descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const status = fstatSync(descriptor); + if (!status.isFile() || status.size < 1 || status.size > maximum) { + throw new Error(`Native runtime qualification aggregate input is invalid: ${file}`); + } + return readFileSync(descriptor); + } catch (error) { + if (error instanceof Error && error.message.startsWith("Native runtime qualification")) { + throw error; + } + throw new Error(`Native runtime qualification aggregate input is invalid: ${file}`); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function readJson(file: string, maximum = MAX_FRAGMENT_BYTES): unknown { + try { + return JSON.parse(readBoundedBytes(file, maximum).toString("utf8")) as unknown; + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Native runtime qualification aggregate input is not JSON: ${file}`); + } + throw error; + } +} + +function positiveInteger(value: string, label: string): number { + if (!POSITIVE_INTEGER.test(value)) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } + return parsed; +} + +function exactJson(actual: unknown, expected: unknown, label: string): void { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`Native runtime qualification ${label} does not match the trusted plan`); + } +} + +function assertDirectory(directory: string, label: string): void { + const status = lstatSync(directory, { throwIfNoEntry: false }); + if (!status?.isDirectory() || status.isSymbolicLink()) { + throw new Error(`Native runtime qualification ${label} is invalid`); + } +} + +function walkRegularFiles(root: string): readonly string[] { + const files: string[] = []; + const visit = (directory: string, relative: string): void => { + assertDirectory(directory, "aggregate artifact directory"); + for (const name of readdirSync(directory).sort()) { + const child = path.join(directory, name); + const childRelative = relative ? `${relative}/${name}` : name; + const status = lstatSync(child); + if (status.isSymbolicLink()) { + throw new Error( + `Native runtime qualification aggregate input cannot contain symlinks: ${childRelative}`, + ); + } + if (status.isDirectory()) visit(child, childRelative); + else if (status.isFile() && status.size >= 1 && status.size <= MAX_RECEIPT_BYTES) + files.push(childRelative); + else + throw new Error( + `Native runtime qualification aggregate input is invalid: ${childRelative}`, + ); + } + }; + visit(root, ""); + return Object.freeze(files); +} + +function expectedReceiptFiles( + row: NativeRuntimeQualificationProducerPlan["include"][number], +): readonly string[] { + const caseId = row.id; + const operations = row.case.obligations.map( + (id) => `receipts/${caseId}/operations/${nativeRuntimeQualificationOperationFile(id)}`, + ); + return Object.freeze([ + "case-fragment.json", + `receipts/${caseId}/installer/architecture.json`, + `receipts/${caseId}/installer/candidate-source.json`, + `receipts/${caseId}/installer/docker-absence.json`, + `receipts/${caseId}/installer/installed-source.json`, + `receipts/${caseId}/installer/installer.sh`, + `receipts/${caseId}/installer/invocation.json`, + `receipts/${caseId}/runtime/runtime-result.json`, + ...operations, + ...(row.case.acceleration === "nvidia-gpu" + ? [`receipts/${caseId}/runtime/nvidia-cdi.json`] + : []), + ]); +} + +function validateFragment( + value: unknown, + row: NativeRuntimeQualificationProducerPlan["include"][number], +): NativeRuntimeQualificationCaseFragment { + const fragment = record(value, `Native runtime qualification fragment '${row.id}'`); + exactKeys( + fragment, + [ + "schemaVersion", + "kind", + "qualificationId", + "providerId", + "source", + "case", + "installer", + "runtime", + "operations", + ...(row.case.acceleration === "nvidia-gpu" ? ["nvidiaCdi"] : []), + ], + `Native runtime qualification fragment '${row.id}'`, + ); + if ( + fragment.schemaVersion !== 1 || + fragment.kind !== "nemoclaw-native-runtime-qualification-case-fragment-v1" || + fragment.qualificationId !== "podman-protected-host-local-inference" || + fragment.providerId !== "podman" + ) { + throw new Error(`Native runtime qualification fragment '${row.id}' identity is invalid`); + } + exactJson(fragment.source, row.source, `fragment '${row.id}' source`); + exactJson(fragment.case, row.case, `fragment '${row.id}' case`); + return fragment as unknown as NativeRuntimeQualificationCaseFragment; +} + +function expectedSource( + plan: NativeRuntimeQualificationProducerPlan, + aggregateJobId: number, +): NativeRuntimeQualificationExpectedSource { + const first = plan.include[0]; + if (!first) throw new Error("Native runtime qualification plan is empty"); + return Object.freeze({ + repository: first.source.repository, + workflow: first.source.producerWorkflow, + pullRequestNumber: first.source.pullRequestNumber, + candidateRepository: first.source.candidateRepository, + headSha: first.source.candidateSha, + baseRef: "main" as const, + baseSha: first.source.baseSha, + runId: positiveInteger(first.source.producerRunId, "producer run id"), + attempt: first.source.producerRunAttempt, + jobId: aggregateJobId, + // The aggregate job cannot know the GitHub artifact identity before upload. + // The collector replaces these placeholders with the independently resolved identity. + artifact: Object.freeze({ + id: 1, + name: "native-runtime-qualification-pre-upload", + digest: `sha256:${"0".repeat(64)}`, + }), + }); +} + +function caseEvidence( + fragment: NativeRuntimeQualificationCaseFragment, + source: NativeRuntimeQualificationExpectedSource, +): NativeRuntimeQualificationEvidenceEnvelope["cases"][number] { + return Object.freeze({ + schemaVersion: 1, + caseId: fragment.case.id, + protectedRun: Object.freeze({ + repository: source.repository, + workflow: source.workflow, + pullRequestNumber: source.pullRequestNumber, + candidateRepository: source.candidateRepository, + headSha: source.headSha, + baseRef: source.baseRef, + baseSha: source.baseSha, + runId: source.runId, + attempt: source.attempt, + jobId: source.jobId, + }), + installer: + fragment.installer as NativeRuntimeQualificationEvidenceEnvelope["cases"][number]["installer"], + runtime: + fragment.runtime as NativeRuntimeQualificationEvidenceEnvelope["cases"][number]["runtime"], + operations: fragment.operations, + ...(fragment.nvidiaCdi ? { nvidiaCdi: fragment.nvidiaCdi } : {}), + }); +} + +export function aggregateNativeRuntimeQualificationProducerEvidence(input: { + readonly plan: NativeRuntimeQualificationProducerPlan; + readonly caseArtifactRoot: string; + readonly evidenceDirectory: string; + readonly aggregateJobId: number; +}): NativeRuntimeQualificationEvidenceEnvelope { + const { plan } = input; + const expectedCaseCount = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.cases.length; + if ( + plan.include.length !== expectedCaseCount || + new Set(plan.include.map((row) => row.id)).size !== expectedCaseCount + ) { + throw new Error("Native runtime qualification aggregate requires the exact 24-case plan"); + } + const first = plan.include[0]!; + for (const row of plan.include) { + exactJson(row.source, first.source, `plan row '${row.id}' source cohort`); + } + if (!Number.isSafeInteger(input.aggregateJobId) || input.aggregateJobId < 1) { + throw new Error("Native runtime qualification aggregate job id is invalid"); + } + assertDirectory(input.caseArtifactRoot, "aggregate artifact root"); + const expectedDirectories = plan.include.map((row) => row.artifactName).sort(); + const actualDirectories = readdirSync(input.caseArtifactRoot).sort(); + if (JSON.stringify(actualDirectories) !== JSON.stringify(expectedDirectories)) { + throw new Error( + "Native runtime qualification aggregate artifact cohort is incomplete or mixed", + ); + } + if (lstatSync(input.evidenceDirectory, { throwIfNoEntry: false })) { + throw new Error("Native runtime qualification aggregate output must not already exist"); + } + const outputParent = path.dirname(input.evidenceDirectory); + assertDirectory(outputParent, "aggregate output parent"); + mkdirSync(input.evidenceDirectory, { mode: 0o700 }); + + const source = expectedSource(plan, input.aggregateJobId); + const cases: NativeRuntimeQualificationEvidenceEnvelope["cases"][number][] = []; + let totalBytes = 0; + const copiedPaths = new Set(); + for (const row of plan.include) { + const artifactDirectory = path.join(input.caseArtifactRoot, row.artifactName); + assertDirectory(artifactDirectory, `case artifact '${row.artifactName}'`); + const actualFiles = walkRegularFiles(artifactDirectory); + const expectedFiles = [...expectedReceiptFiles(row)].sort(); + if (JSON.stringify([...actualFiles].sort()) !== JSON.stringify(expectedFiles)) { + throw new Error(`Native runtime qualification case artifact '${row.id}' has invalid files`); + } + const fragment = validateFragment( + readJson(path.join(artifactDirectory, "case-fragment.json")), + row, + ); + for (const relativePath of actualFiles.filter((file) => file !== "case-fragment.json")) { + if (copiedPaths.has(relativePath)) { + throw new Error(`Native runtime qualification aggregate repeats receipt '${relativePath}'`); + } + copiedPaths.add(relativePath); + const bytes = readBoundedBytes(path.join(artifactDirectory, relativePath), MAX_RECEIPT_BYTES); + totalBytes += bytes.length; + if (totalBytes > MAX_TOTAL_BYTES) { + throw new Error("Native runtime qualification aggregate receipts exceed their byte limit"); + } + const target = path.join(input.evidenceDirectory, relativePath); + mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + writeFileSync(target, bytes, { flag: "wx", mode: 0o600 }); + } + cases.push(caseEvidence(fragment, source)); + } + const envelope: NativeRuntimeQualificationEvidenceEnvelope = Object.freeze({ + schemaVersion: 1, + qualificationId: "podman-protected-host-local-inference", + providerId: "podman", + cases: Object.freeze(cases), + }); + const definition = PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION; + consumeNativeRuntimeQualificationEvidence(definition, envelope, source, (receiptPath) => { + try { + return readBoundedBytes(path.join(input.evidenceDirectory, receiptPath), MAX_RECEIPT_BYTES); + } catch { + return null; + } + }); + writeFileSync( + path.join(input.evidenceDirectory, NATIVE_RUNTIME_QUALIFICATION_AGGREGATE_EVIDENCE_FILE), + `${JSON.stringify(envelope)}\n`, + { flag: "wx", mode: 0o600 }, + ); + return envelope; +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Native runtime qualification environment '${name}' is missing`); + return value; +} + +if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? "")) { + try { + if (process.argv.length !== 2) { + throw new Error("Usage: native-runtime-qualification-producer-aggregate.mts"); + } + const plan = JSON.parse( + requiredEnvironment("QUALIFICATION_PLAN"), + ) as NativeRuntimeQualificationProducerPlan; + aggregateNativeRuntimeQualificationProducerEvidence({ + plan, + caseArtifactRoot: requiredEnvironment("CASE_ARTIFACT_ROOT"), + evidenceDirectory: requiredEnvironment("EVIDENCE_DIRECTORY"), + aggregateJobId: positiveInteger(requiredEnvironment("AGGREGATE_JOB_ID"), "aggregate job id"), + }); + } catch (error) { + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/tools/e2e/native-runtime-qualification-producer-evidence.mts b/tools/e2e/native-runtime-qualification-producer-evidence.mts index a17f36095f2..85b2f12adf1 100644 --- a/tools/e2e/native-runtime-qualification-producer-evidence.mts +++ b/tools/e2e/native-runtime-qualification-producer-evidence.mts @@ -15,9 +15,19 @@ import { } from "node:fs"; import path from "node:path"; -import type { NativeRuntimeQualificationProducerPlanRow } from "./native-runtime-qualification-producer-plan.mts"; +import type { + NativeRuntimeQualificationObligation, + NativeRuntimeQualificationArtifactReceipt, +} from "../../test/e2e/registry/native-runtime-qualification.ts"; +import { + nativeRuntimeQualificationOperationFile, + NATIVE_RUNTIME_QUALIFICATION_ID, + NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID, + type NativeRuntimeQualificationProducerPlanRow, +} from "./native-runtime-qualification-producer-plan.mts"; const MAX_RECEIPT_BYTES = 65_536; +const MAX_INSTALLER_BYTES = 524_288; const MAX_RECEIPT_DIRECTORY_BYTES = 1_048_576; const EXPECTED_INSTALLER_FILES = [ "architecture.json", @@ -27,6 +37,14 @@ const EXPECTED_INSTALLER_FILES = [ "installer.sh", "invocation.json", ] as const; +const DETAIL_FILE = "case-evidence.json"; +const EXECUTION_FILE = "execution.json"; +const RUNTIME_FILE = "runtime-result.json"; +const CDI_FILE = "nvidia-cdi.json"; +const IMAGE_DIGEST = /^sha256:[a-f0-9]{64}$/u; +const SAFE_ENGINE = /^[A-Za-z0-9][A-Za-z0-9 ._/-]{0,127}$/u; +const FORBIDDEN_RECEIPT_TEXT = + /(?:github_pat_|gh[pousr]_[A-Za-z0-9]{20}|nvapi-[A-Za-z0-9_-]{12}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----)/u; interface CaseExecutionReceipt { readonly schemaVersion: 1; @@ -54,14 +72,79 @@ interface CaseExecutionReceipt { readonly result: "passed"; } -function record(value: unknown, label: string): Record { +interface CandidateCaseDetails { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-case-details-v1"; + readonly caseId: string; + readonly runtime: { + readonly engineName: string; + readonly engineVersion: string; + readonly managedImages: readonly { + readonly role: string; + readonly digest: string; + }[]; + readonly resultFile: typeof RUNTIME_FILE; + }; + readonly operations: readonly { + readonly id: NativeRuntimeQualificationObligation; + readonly file: string; + }[]; + readonly nvidiaCdi?: { + readonly device: "nvidia.com/gpu=all"; + readonly file: typeof CDI_FILE; + }; +} + +export interface NativeRuntimeQualificationCaseFragment { + readonly schemaVersion: 1; + readonly kind: "nemoclaw-native-runtime-qualification-case-fragment-v1"; + readonly qualificationId: string; + readonly providerId: string; + readonly source: NativeRuntimeQualificationProducerPlanRow["source"]; + readonly case: NativeRuntimeQualificationProducerPlanRow["case"]; + readonly installer: { + readonly providerId: string; + readonly architecture: string; + readonly dockerAvailability: "unavailable"; + readonly exitCode: 0; + readonly invocation: NativeRuntimeQualificationArtifactReceipt; + readonly script: NativeRuntimeQualificationArtifactReceipt; + }; + readonly runtime: { + readonly providerId: string; + readonly agent: string; + readonly inference: string; + readonly architecture: string; + readonly acceleration: string; + readonly rootMode: "rootless"; + readonly engineName: string; + readonly engineVersion: string; + readonly managedImages: readonly { + readonly role: string; + readonly digest: string; + }[]; + readonly result: NativeRuntimeQualificationArtifactReceipt; + }; + readonly operations: readonly { + readonly id: NativeRuntimeQualificationObligation; + readonly artifact: NativeRuntimeQualificationArtifactReceipt; + }[]; + readonly nvidiaCdi?: { + readonly device: "nvidia.com/gpu=all"; + readonly artifact: NativeRuntimeQualificationArtifactReceipt; + }; +} + +type UnknownRecord = Record; + +function record(value: unknown, label: string): UnknownRecord { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`${label} must be an object`); } - return value as Record; + return value as UnknownRecord; } -function exactKeys(value: Record, keys: readonly string[], label: string): void { +function exactKeys(value: UnknownRecord, keys: readonly string[], label: string): void { const actual = Object.keys(value).sort(); const expected = [...keys].sort(); if (JSON.stringify(actual) !== JSON.stringify(expected)) { @@ -79,7 +162,7 @@ function exactStrings(actual: unknown, expected: readonly string[], label: strin } } -function readBoundedFile(file: string, maximum = MAX_RECEIPT_BYTES): string { +function readBoundedBytes(file: string, maximum = MAX_RECEIPT_BYTES): Buffer { let descriptor: number | undefined; try { descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); @@ -87,7 +170,11 @@ function readBoundedFile(file: string, maximum = MAX_RECEIPT_BYTES): string { if (!status.isFile() || status.size < 1 || status.size > maximum) { throw new Error(`Native runtime qualification receipt is missing or invalid: ${file}`); } - return readFileSync(descriptor, "utf8"); + const bytes = readFileSync(descriptor); + if (FORBIDDEN_RECEIPT_TEXT.test(bytes.toString("utf8"))) { + throw new Error(`Native runtime qualification receipt contains credential material: ${file}`); + } + return bytes; } catch (error) { if (error instanceof Error && error.message.startsWith("Native runtime qualification")) { throw error; @@ -98,9 +185,9 @@ function readBoundedFile(file: string, maximum = MAX_RECEIPT_BYTES): string { } } -function readJson(file: string): unknown { +function parseJsonBytes(bytes: Buffer, file: string): unknown { try { - return JSON.parse(readBoundedFile(file)) as unknown; + return JSON.parse(bytes.toString("utf8")) as unknown; } catch (error) { if (error instanceof SyntaxError) { throw new Error(`Native runtime qualification receipt is not valid JSON: ${file}`); @@ -135,10 +222,19 @@ function validateDirectory(directory: string, expectedFiles: readonly string[]): function validateInstallerReceipts( row: NativeRuntimeQualificationProducerPlanRow, directory: string, -) { +): Readonly> { validateDirectory(directory, EXPECTED_INSTALLER_FILES); + const receipts = Object.fromEntries( + EXPECTED_INSTALLER_FILES.map((file) => [ + file, + readBoundedBytes( + path.join(directory, file), + file === "installer.sh" ? MAX_INSTALLER_BYTES : MAX_RECEIPT_BYTES, + ), + ]), + ) as Record<(typeof EXPECTED_INSTALLER_FILES)[number], Buffer>; const invocation = record( - readJson(path.join(directory, "invocation.json")), + parseJsonBytes(receipts["invocation.json"], "invocation.json"), "Installer invocation", ); exactKeys( @@ -156,7 +252,7 @@ function validateInstallerReceipts( throw new Error("Native runtime qualification installer invocation is invalid"); } const architecture = record( - readJson(path.join(directory, "architecture.json")), + parseJsonBytes(receipts["architecture.json"], "architecture.json"), "Installer architecture", ); exactKeys(architecture, ["receiptVersion", "requested", "runner"], "Installer architecture"); @@ -168,11 +264,11 @@ function validateInstallerReceipts( throw new Error("Native runtime qualification installer architecture is invalid"); } const candidate = record( - readJson(path.join(directory, "candidate-source.json")), + parseJsonBytes(receipts["candidate-source.json"], "candidate-source.json"), "Installer candidate source", ); const installed = record( - readJson(path.join(directory, "installed-source.json")), + parseJsonBytes(receipts["installed-source.json"], "installed-source.json"), "Installed source", ); exactKeys( @@ -208,7 +304,7 @@ function validateInstallerReceipts( throw new Error("Native runtime qualification installer source identity is invalid"); } const docker = record( - readJson(path.join(directory, "docker-absence.json")), + parseJsonBytes(receipts["docker-absence.json"], "docker-absence.json"), "Installer Docker absence", ); exactKeys( @@ -234,13 +330,14 @@ function validateInstallerReceipts( if (docker.receiptVersion !== 1) { throw new Error("Native runtime qualification installer Docker absence is invalid"); } - const installer = readBoundedFile(path.join(directory, "installer.sh"), 524_288); + const installer = receipts["installer.sh"]; if ( - !installer.startsWith("#!/") || + !installer.toString("utf8").startsWith("#!/") || createHash("sha256").update(installer).digest("hex") !== row.installerSha256 ) { throw new Error("Native runtime qualification installer receipt is invalid"); } + return Object.freeze(receipts); } function validateCaseExecution( @@ -316,14 +413,196 @@ function validateCaseExecution( return receipt as unknown as CaseExecutionReceipt; } +function expectedCaseFiles(row: NativeRuntimeQualificationProducerPlanRow): string[] { + return [ + DETAIL_FILE, + EXECUTION_FILE, + RUNTIME_FILE, + ...row.case.obligations.map(nativeRuntimeQualificationOperationFile), + ...(row.case.acceleration === "nvidia-gpu" ? [CDI_FILE] : []), + ]; +} + +function validateEvidencePayload( + bytes: Buffer, + file: string, + expected: { + readonly caseId: string; + readonly kind: string; + readonly operationId?: string; + }, +): void { + const value = record(parseJsonBytes(bytes, file), `Candidate evidence '${path.basename(file)}'`); + exactKeys( + value, + [ + "schemaVersion", + "kind", + "caseId", + ...(expected.operationId ? ["operationId"] : []), + "result", + "details", + ], + `Candidate evidence '${path.basename(file)}'`, + ); + record(value.details, `Candidate evidence '${path.basename(file)}' details`); + if ( + value.schemaVersion !== 1 || + value.kind !== expected.kind || + value.caseId !== expected.caseId || + value.result !== "passed" || + (expected.operationId !== undefined && value.operationId !== expected.operationId) + ) { + throw new Error(`Native runtime qualification candidate evidence is invalid: ${file}`); + } +} + +function validateCandidateDetails( + row: NativeRuntimeQualificationProducerPlanRow, + directory: string, +): { + readonly details: CandidateCaseDetails; + readonly receipts: Readonly>; +} { + const expectedFiles = expectedCaseFiles(row); + validateDirectory(directory, expectedFiles); + const receipts = Object.fromEntries( + expectedFiles.map((file) => [file, readBoundedBytes(path.join(directory, file))]), + ) as Record; + const details = record( + parseJsonBytes(receipts[DETAIL_FILE]!, DETAIL_FILE), + "Candidate case details", + ); + exactKeys( + details, + [ + "schemaVersion", + "kind", + "caseId", + "runtime", + "operations", + ...(row.case.acceleration === "nvidia-gpu" ? ["nvidiaCdi"] : []), + ], + "Candidate case details", + ); + const runtime = record(details.runtime, "Candidate runtime details"); + exactKeys( + runtime, + ["engineName", "engineVersion", "managedImages", "resultFile"], + "Candidate runtime details", + ); + if ( + details.schemaVersion !== 1 || + details.kind !== "nemoclaw-native-runtime-qualification-case-details-v1" || + details.caseId !== row.id || + typeof runtime.engineName !== "string" || + !SAFE_ENGINE.test(runtime.engineName) || + typeof runtime.engineVersion !== "string" || + !SAFE_ENGINE.test(runtime.engineVersion) || + runtime.resultFile !== RUNTIME_FILE || + !Array.isArray(runtime.managedImages) || + runtime.managedImages.length < 2 || + runtime.managedImages.length > 8 + ) { + throw new Error("Native runtime qualification candidate runtime details are invalid"); + } + const roles = new Set(); + for (const entry of runtime.managedImages) { + const image = record(entry, "Candidate managed image"); + exactKeys(image, ["role", "digest"], "Candidate managed image"); + if ( + typeof image.role !== "string" || + !/^[a-z][a-z0-9-]{0,62}$/u.test(image.role) || + roles.has(image.role) || + typeof image.digest !== "string" || + !IMAGE_DIGEST.test(image.digest) + ) { + throw new Error("Native runtime qualification candidate managed image is invalid"); + } + roles.add(image.role); + } + if ( + !Array.isArray(details.operations) || + details.operations.length !== row.case.obligations.length + ) { + throw new Error("Native runtime qualification candidate operations are incomplete"); + } + const expectedOperations = row.case.obligations.map((id) => ({ + id, + file: nativeRuntimeQualificationOperationFile(id), + })); + for (const [index, entry] of details.operations.entries()) { + const operation = record(entry, "Candidate operation detail"); + exactKeys(operation, ["id", "file"], "Candidate operation detail"); + if ( + operation.id !== expectedOperations[index]?.id || + operation.file !== expectedOperations[index]?.file + ) { + throw new Error( + "Native runtime qualification candidate operations do not match the trusted plan", + ); + } + const file = String(operation.file); + validateEvidencePayload(receipts[file]!, file, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-operation-v1", + operationId: String(operation.id), + }); + } + validateEvidencePayload(receipts[RUNTIME_FILE]!, RUNTIME_FILE, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-runtime-v1", + }); + if (row.case.acceleration === "nvidia-gpu") { + const cdi = record(details.nvidiaCdi, "Candidate NVIDIA CDI details"); + exactKeys(cdi, ["device", "file"], "Candidate NVIDIA CDI details"); + if (cdi.device !== "nvidia.com/gpu=all" || cdi.file !== CDI_FILE) { + throw new Error("Native runtime qualification candidate NVIDIA CDI details are invalid"); + } + validateEvidencePayload(receipts[CDI_FILE]!, CDI_FILE, { + caseId: row.id, + kind: "nemoclaw-native-runtime-qualification-nvidia-cdi-v1", + }); + } + return Object.freeze({ + details: details as unknown as CandidateCaseDetails, + receipts: Object.freeze(receipts), + }); +} + +function receiptPath(caseId: string, category: string, file: string): string { + return `receipts/${caseId}/${category}/${file}`; +} + +function copyReceipt( + bytes: Buffer, + outputRoot: string, + relativePath: string, +): NativeRuntimeQualificationArtifactReceipt { + const target = path.join(outputRoot, relativePath); + const parent = path.dirname(target); + mkdirSync(parent, { mode: 0o700, recursive: true }); + writeFileSync(target, bytes, { mode: 0o600, flag: "wx" }); + return Object.freeze({ + path: relativePath, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); +} + export function writeNativeRuntimeQualificationProducerEvidence( row: NativeRuntimeQualificationProducerPlanRow, installerReceiptDirectory: string, executionReceiptPath: string, evidenceDirectory: string, ): void { - validateInstallerReceipts(row, installerReceiptDirectory); - validateCaseExecution(row, readJson(executionReceiptPath)); + const installerBytes = validateInstallerReceipts(row, installerReceiptDirectory); + const executionDirectory = path.dirname(executionReceiptPath); + if (path.basename(executionReceiptPath) !== EXECUTION_FILE) { + throw new Error("Native runtime qualification execution receipt path is invalid"); + } + const candidate = validateCandidateDetails(row, executionDirectory); + validateCaseExecution(row, parseJsonBytes(candidate.receipts[EXECUTION_FILE]!, EXECUTION_FILE)); + const { details } = candidate; if (lstatSync(evidenceDirectory, { throwIfNoEntry: false })) { throw new Error("Native runtime qualification evidence directory must not already exist"); } @@ -337,18 +616,84 @@ export function writeNativeRuntimeQualificationProducerEvidence( throw new Error("Native runtime qualification evidence parent is invalid"); } mkdirSync(evidenceDirectory, { mode: 0o700 }); + + const installerReceipts = Object.fromEntries( + EXPECTED_INSTALLER_FILES.map((file) => [ + file, + copyReceipt(installerBytes[file], evidenceDirectory, receiptPath(row.id, "installer", file)), + ]), + ) as Record<(typeof EXPECTED_INSTALLER_FILES)[number], NativeRuntimeQualificationArtifactReceipt>; + const runtimeReceipt = copyReceipt( + candidate.receipts[RUNTIME_FILE]!, + evidenceDirectory, + receiptPath(row.id, "runtime", RUNTIME_FILE), + ); + const operations = row.case.obligations.map((id) => { + const file = nativeRuntimeQualificationOperationFile(id); + return Object.freeze({ + id, + artifact: copyReceipt( + candidate.receipts[file]!, + evidenceDirectory, + receiptPath(row.id, "operations", file), + ), + }); + }); + const cdiReceipt = + row.case.acceleration === "nvidia-gpu" + ? copyReceipt( + candidate.receipts[CDI_FILE]!, + evidenceDirectory, + receiptPath(row.id, "runtime", CDI_FILE), + ) + : undefined; + const providerId = NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID; + const fragment: NativeRuntimeQualificationCaseFragment = Object.freeze({ + schemaVersion: 1, + kind: "nemoclaw-native-runtime-qualification-case-fragment-v1", + qualificationId: NATIVE_RUNTIME_QUALIFICATION_ID, + providerId, + source: row.source, + case: row.case, + installer: Object.freeze({ + providerId, + architecture: row.case.architecture, + dockerAvailability: "unavailable", + exitCode: 0, + invocation: installerReceipts["invocation.json"], + script: installerReceipts["installer.sh"], + }), + runtime: Object.freeze({ + providerId, + agent: row.case.agent, + inference: row.case.inference, + architecture: row.case.architecture, + acceleration: row.case.acceleration, + rootMode: "rootless", + engineName: details.runtime.engineName, + engineVersion: details.runtime.engineVersion, + managedImages: Object.freeze( + details.runtime.managedImages.map((entry) => Object.freeze({ ...entry })), + ), + result: runtimeReceipt, + }), + operations: Object.freeze(operations), + ...(cdiReceipt + ? { + nvidiaCdi: Object.freeze({ + device: "nvidia.com/gpu=all" as const, + artifact: cdiReceipt, + }), + } + : {}), + }); writeFileSync( - path.join(evidenceDirectory, "evidence.json"), - `${JSON.stringify({ - schemaVersion: 1, - kind: "nemoclaw-native-runtime-qualification-case-evidence-v1", - qualificationId: `${row.case.id.slice(0, row.case.id.indexOf("-"))}-protected-host-local-inference`, - providerId: row.case.id.slice(0, row.case.id.indexOf("-")), - source: row.source, - case: row.case, - result: "passed", - })}\n`, - { mode: 0o600 }, + path.join(evidenceDirectory, "case-fragment.json"), + `${JSON.stringify(fragment)}\n`, + { + mode: 0o600, + flag: "wx", + }, ); } diff --git a/tools/e2e/native-runtime-qualification-producer-plan.mts b/tools/e2e/native-runtime-qualification-producer-plan.mts index 26b282e4414..679ac338120 100644 --- a/tools/e2e/native-runtime-qualification-producer-plan.mts +++ b/tools/e2e/native-runtime-qualification-producer-plan.mts @@ -31,6 +31,15 @@ export const NATIVE_RUNTIME_QUALIFICATION_FOCUSED_OPERATIONS = [ "cleanup", ] as const; +export const NATIVE_RUNTIME_QUALIFICATION_ID = + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.id; +export const NATIVE_RUNTIME_QUALIFICATION_PROVIDER_ID = + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION.providerId; + +export function nativeRuntimeQualificationOperationFile(id: string): string { + return `operation-${id.replaceAll(".", "-")}.json`; +} + export interface NativeRuntimeQualificationDispatchArtifact { readonly id: string; readonly name: string; @@ -105,7 +114,7 @@ function validateSource( !COMMIT_SHA.test(value.baseSha) || !COMMIT_SHA.test(value.workflowSha) || value.candidateSha === value.baseSha || - value.baseSha !== value.workflowSha || + value.workflowSha !== value.baseSha || !RUN_ID.test(value.producerRunId) || value.producerRunAttempt !== 1 ) { @@ -120,7 +129,7 @@ function validateSource( function runnerForCase(entry: NativeRuntimeQualificationCase, arm64GpuRunner: string): string { if (entry.architecture === "amd64" && entry.acceleration === "cpu") return "ubuntu-26.04"; if (entry.architecture === "arm64" && entry.acceleration === "cpu") { - return "ubuntu-24.04-arm"; + return "ubuntu-26.04-arm"; } if (entry.architecture === "amd64") return "linux-amd64-gpu-rtxpro6000-latest-1"; if (!RUNNER_LABEL.test(arm64GpuRunner)) { @@ -132,6 +141,12 @@ function runnerForCase(entry: NativeRuntimeQualificationCase, arm64GpuRunner: st } function immutableCase(value: NativeRuntimeQualificationCase): NativeRuntimeQualificationCase { + const operationFiles = value.obligations.map(nativeRuntimeQualificationOperationFile); + if (new Set(operationFiles).size !== operationFiles.length) { + throw new Error( + `Native runtime qualification case '${value.id}' has colliding operation files`, + ); + } return Object.freeze({ ...value, capabilities: Object.freeze([...value.capabilities]), diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index ff57174011e..f89dbf28db3 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -16,6 +16,7 @@ const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); const DEFAULT_ADVISOR_PATH = join(REPO_ROOT, ".github", "workflows", "pr-review-advisor.yaml"); const META_JOBS = new Set([ + "native-runtime-qualification-podman-toolchain", "native-runtime-qualification-producer-plan", "release-qualification", "relevant-e2e", @@ -57,6 +58,7 @@ const GH_API_WRITE_METHOD = /\bgh\s+api\b[\s\S]{0,160}?(?:(?:--method|-X)\s+(?:POST|PUT|PATCH|DELETE)\b|graphql\b[\s\S]{0,160}?\bmutation\b)/iu; const NATIVE_RUNTIME_QUALIFICATION_READ_JOBS = new Set([ "native-runtime-qualification-producer-plan", + "native-runtime-qualification-producer-aggregate", ]); const GENERIC_ISSUE_REST_MUTATION = /github\.request\s*\(\s*["'`](?:POST|PATCH|PUT|DELETE)\s+\/repos\/[^/\s]+\/[^/\s]+\/issues(?:\/|\b)/u; @@ -354,6 +356,23 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push(`Manual PR authentication must retain ${fragment}`); } + const qualificationPlanName = "native-runtime-qualification-producer-plan"; + const qualificationPlan = workflow.jobs[qualificationPlanName] ?? {}; + const trustedMainPlanCondition = + "${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }}"; + if (qualificationPlan.if !== trustedMainPlanCondition) { + errors.push("Native runtime qualification producer plan must execute only from trusted main"); + } + for (const jobName of [ + "native-runtime-qualification-podman-toolchain", + "native-runtime-qualification-producer", + "native-runtime-qualification-producer-aggregate", + ]) { + if (!needs(workflow.jobs[jobName] ?? {}).includes(qualificationPlanName)) { + errors.push(`${jobName} must depend on the trusted-main qualification producer plan`); + } + } + const validation = validationIndex >= 0 ? steps[validationIndex] : {}; if ( validation.if !== @@ -378,8 +397,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const credentialAuthorization = credentialAuthorizationIndex >= 0 ? steps[credentialAuthorizationIndex] : {}; if ( - matrixJob.outputs?.e2e_credentials_allowed !== - "${{ steps.e2e_credentials.outputs.allowed }}" || + matrixJob.outputs?.e2e_credentials_allowed !== "${{ steps.e2e_credentials.outputs.allowed }}" || credentialAuthorization.id !== "e2e_credentials" || credentialAuthorization.if !== "${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}" || @@ -396,12 +414,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow WORKFLOW_REPOSITORY: "${{ github.repository }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", }; - if ( - !isDeepStrictEqual( - credentialAuthorization.env, - expectedCredentialAuthorizationEnvironment, - ) - ) { + if (!isDeepStrictEqual(credentialAuthorization.env, expectedCredentialAuthorizationEnvironment)) { errors.push( "Manual PR credential authorization must bind the workflow and checkout identities", ); @@ -479,17 +492,42 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.repository === "${{ github.repository }}" && step.with?.ref === "${{ inputs.workflow_sha || github.workflow_sha }}" && step.with?.path === ".trusted-openshell-dev-artifact"; - const trustedNativeRuntimeCheckout = - ((jobName === "native-runtime-qualification-producer-plan" && + const nativeRuntimeQualificationCheckout = + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Podman source" && + step.with?.repository === "podman-container-tools/podman" && + step.with?.ref === "cade97a52ebdf9dbf9e81de8009015776837a074" && + step.with?.path === ".podman-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Netavark source" && + step.with?.repository === "containers/netavark" && + step.with?.ref === "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" && + step.with?.path === ".netavark-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || + (jobName === "native-runtime-qualification-podman-toolchain" && + step.name === "Check out the pinned Aardvark DNS source" && + step.with?.repository === "containers/aardvark-dns" && + step.with?.ref === "cd7417681229219059939bdd9f0b3bd9ac9abb08" && + step.with?.path === ".aardvark-source" && + step.with?.["fetch-depth"] === 1 && + step.with?.["persist-credentials"] === false) || + (jobName === "native-runtime-qualification-producer-plan" && step.name === "Check out the trusted qualification producer" && step.with?.ref === "${{ github.workflow_sha }}") || - (jobName === "native-runtime-qualification-producer" && - step.name === "Check out the trusted qualification harness" && - step.with?.ref === "${{ matrix.source.workflowSha }}") || - (jobName === "native-runtime-qualification-producer" && - step.name === "Check out the candidate commit" && - step.with?.repository === "${{ matrix.source.candidateRepository }}" && - step.with?.ref === "${{ matrix.source.candidateSha }}")); + (jobName === "native-runtime-qualification-producer" && + step.name === "Check out the trusted qualification harness" && + step.with?.ref === "${{ matrix.source.workflowSha }}") || + (jobName === "native-runtime-qualification-producer" && + step.name === "Check out the candidate commit" && + step.with?.repository === "${{ matrix.source.candidateRepository }}" && + step.with?.ref === "${{ matrix.source.candidateSha }}") || + (jobName === "native-runtime-qualification-producer-aggregate" && + step.name === "Check out the qualification aggregator" && + step.with?.repository === "${{ github.repository }}" && + step.with?.ref === "${{ github.workflow_sha }}"); const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || @@ -501,7 +539,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow trustedLlamaCppPlanCheckout || trustedLlamaCppQualificationCheckout || trustedJetsonControllerCheckout || - trustedNativeRuntimeCheckout || + nativeRuntimeQualificationCheckout || trustedOpenShellDevToolingCheckout; if ( step.uses?.startsWith("actions/checkout@") && diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 23d25e6bc05..db30bb8a04c 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -52,6 +52,17 @@ const RELEASE_QUALIFICATION_WAIVER_UPLOAD_CONTRACT: WorkflowStep = { "retention-days": 30, }, }; +const NATIVE_RUNTIME_AGGREGATE_UPLOAD_CONTRACT: WorkflowStep = { + name: "Upload aggregate evidence", + uses: UPLOAD_ARTIFACT_ACTION, + with: { + name: "native-runtime-qualification-${{ inputs.checkout_sha }}", + path: "${{ runner.temp }}/native-runtime-aggregate/", + "if-no-files-found": "error", + "retention-days": 30, + "compression-level": 9, + }, +}; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const RETIRED_SELECTOR_COMPATIBILITY_JOB = "retired-selector-compatibility"; @@ -108,6 +119,13 @@ function isExactReleaseQualificationWaiverUpload(jobName: string, step: Workflow ); } +function isExactNativeRuntimeAggregateUpload(jobName: string, step: WorkflowStep): boolean { + return ( + jobName === "native-runtime-qualification-producer-aggregate" && + isDeepStrictEqual(step, NATIVE_RUNTIME_AGGREGATE_UPLOAD_CONTRACT) + ); +} + const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "generate-matrix", @@ -177,11 +195,18 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/managed-image-protected-runtime/", }, ], + [ + "native-runtime-qualification-podman-toolchain", + { + name: "native-runtime-podman-toolchain-${{ matrix.architecture }}", + path: "${{ runner.temp }}/native-runtime-podman-toolchain/", + }, + ], [ "native-runtime-qualification-producer", { name: "${{ matrix.artifactName }}", - path: "${{ runner.temp }}/native-runtime-evidence/evidence.json", + path: "${{ runner.temp }}/native-runtime-evidence/", }, ], [ @@ -237,6 +262,8 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["generate-matrix", "${{ github.event_name == 'workflow_dispatch' }}"], + ["native-runtime-qualification-podman-toolchain", "success()"], + ["native-runtime-qualification-producer", "success()"], ["staging-brev-launchable", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], @@ -371,6 +398,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): jobName === "generate-matrix" || jobName === "jetson-nvmap-gpu" || jobName === "live" || + jobName === "native-runtime-qualification-podman-toolchain" || jobName === "openshell-dev-artifact" || jobName === RETIRED_SELECTOR_COMPATIBILITY_JOB || env.E2E_JOB === "1" || @@ -436,7 +464,8 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): uses.startsWith(UPLOAD_ARTIFACT_ACTION_PREFIX) && !isExactCommitCliArtifactUpload && !isExactManagedImageBuildCacheUpload(jobName, step) && - !isExactReleaseQualificationWaiverUpload(jobName, step) + !isExactReleaseQualificationWaiverUpload(jobName, step) && + !isExactNativeRuntimeAggregateUpload(jobName, step) ) { errors.push(`${jobName} must not invoke actions/upload-artifact directly`); }