diff --git a/.github/actions/_lib/compute-deps-hash/action.yml b/.github/actions/_lib/compute-deps-hash/action.yml new file mode 100644 index 000000000000..7df3607c4e54 --- /dev/null +++ b/.github/actions/_lib/compute-deps-hash/action.yml @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Compute deps hash' +description: > + Compute the deps-cache hash for the Isaac Lab Docker build. Shared by the + docker-build (local-store check) and ecr-build-push-pull (registry check) + actions so a local hit and a registry hit always agree on the same + `deps-` tag. Hashes the install-relevant files plus the resolved base + image digest. + +inputs: + dockerfile-path: + description: 'Path to Dockerfile' + required: true + isaacsim-base-image: + description: 'IsaacSim base image' + required: true + isaacsim-version: + description: 'IsaacSim version' + required: true + +outputs: + hash: + description: '16-char deps-cache hash' + value: ${{ steps.compute.outputs.hash }} + +runs: + using: composite + steps: + - id: compute + shell: bash + env: + DOCKERFILE_PATH: ${{ inputs.dockerfile-path }} + ISAACSIM_BASE_IMAGE: ${{ inputs.isaacsim-base-image }} + ISAACSIM_VERSION: ${{ inputs.isaacsim-version }} + run: | + set -euo pipefail + + # Exact files/dirs whose full content is hashed. The Dockerfile is first. + deps_files=( + "${DOCKERFILE_PATH}" + isaaclab.sh + environment.yml + source/isaaclab/isaaclab/cli + ) + deps_manifest_pattern='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$' + + # Resolve the actual base image digest so a new push of a mutable tag + # (e.g. latest-develop) invalidates the deps cache automatically. + base_image_digest=$(docker buildx imagetools inspect \ + "${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}" \ + --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true) + if [ -n "${base_image_digest}" ]; then + base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}:${base_image_digest}" + else + echo "🟠 Could not resolve base image digest, falling back to tag string" + base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}" + fi + + mapfile -t manifest_files < <(git ls-files | grep -E "${deps_manifest_pattern}" || true) + file_hash=$(git ls-files -s "${deps_files[@]}" "${manifest_files[@]}" 2>/dev/null \ + | sha256sum | cut -c1-16) + deps_hash=$(printf '%s %s' "${file_hash}" "${base_image_uniq_id}" | sha256sum | cut -c1-16) + + echo "🔵 Deps hash: ${deps_hash}" + echo "hash=${deps_hash}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/_lib/setup-docker-config/action.yml b/.github/actions/_lib/setup-docker-config/action.yml new file mode 100644 index 000000000000..09effa56eaa3 --- /dev/null +++ b/.github/actions/_lib/setup-docker-config/action.yml @@ -0,0 +1,43 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Setup docker config' +description: > + Point DOCKER_CONFIG at a temp config with the credential helper disabled and + log into nvcr.io. Shared by the docker-build and ecr-build-push-pull actions. + Idempotent: re-invoking it in the same job is a no-op, so callers (e.g. + ecr-build-push-pull delegating to docker-build) don't need to coordinate. + Reads NGC_API_KEY from the environment (optional; warns when missing). + +runs: + using: composite + steps: + - shell: bash + run: | + # The runner's credential helper backend is broken ("not implemented") + # and causes docker login calls to fail unless we point DOCKER_CONFIG at + # a temp config with credsStore disabled. The value is written to + # $GITHUB_ENV so subsequent steps in the job inherit it; a second + # invocation sees it already set and short-circuits. + if [ -n "${DOCKER_CONFIG:-}" ] && [ -f "${DOCKER_CONFIG}/config.json" ]; then + echo "🟢 Docker config already set up at ${DOCKER_CONFIG}, skipping" + exit 0 + fi + + DOCKER_CONFIG_DIR=$(mktemp -d) + if [ -f "${HOME}/.docker/config.json" ]; then + python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))" + else + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + fi + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV" + + if [ -n "${NGC_API_KEY:-}" ]; then + echo "🔵 Logging into nvcr.io..." + docker login -u '$oauthtoken' -p "${NGC_API_KEY}" nvcr.io + else + echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)" + fi diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 7f88241cfb8c..cbf69590c815 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -24,61 +24,221 @@ inputs: description: 'Build context path' default: '.' required: false + platform: + description: 'Target platform for `docker buildx build --platform`.' + default: 'linux/amd64' + required: false + cache-from: + description: > + Optional value for `docker buildx build --cache-from`. Typically a + `type=registry,ref=` for cross-host layer cache. Leave empty for + pure local-only builds. + default: '' + required: false + cache-to: + description: > + Optional value for `docker buildx build --cache-to`. Pairs with + `cache-from` for registry-backed layer cache writes. + default: '' + required: false + deps-hash: + description: > + Pre-computed deps-hash to use for the local deps-tag check. When empty, + this action computes the hash itself via the `_lib/compute-deps-hash` + action. Set by callers (e.g. `ecr-build-push-pull`) that already compute + the hash for a registry-side check, to avoid recomputing here. + default: '' + required: false + evict-stale-cache: + description: > + When 'true', evict `isaac-lab*:deps-*` tags older than 14 days at the + end of the build to bound disk growth on long-lived self-hosted + runners. Default 'false' — no implicit cleanup. + default: 'false' + required: false runs: using: composite steps: - - name: NGC Login - shell: sh + + ##### 1: Setup docker config + login to nvcr.io (optional) ##### + + - name: Setup docker config and login to nvcr.io + uses: ./.github/actions/_lib/setup-docker-config + + ##### 2: Host disk snapshot (pre) ##### + + - name: Host disk snapshot (pre) + shell: bash + run: | + set +e + docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker") + deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l) + commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l) + { + echo "## Disk snapshot (pre)" + echo '```' + echo "Filesystem:" + df -h / "${docker_root}" 2>/dev/null | sort -u + echo + echo "docker system df:" + docker system df + echo + echo "Tag counts:" + echo " isaac-lab* (commit + deps tags): ${commit_count}" + echo " isaac-lab*:deps-* (deps cache) : ${deps_count}" + echo + echo "Deps tags (newest first):" + docker images --filter 'reference=isaac-lab*:deps-*' \ + --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \ + | head -20 + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + ##### 3: Local exact-tag short-circuit ##### + + - name: Check image locally + id: local + shell: bash + run: | + if docker image inspect "${{ inputs.image-tag }}" >/dev/null 2>&1; then + echo "🟢 Image already in local docker store: ${{ inputs.image-tag }}" + echo "hit=true" >> "$GITHUB_OUTPUT" + else + echo "🔵 Image not present locally, will check deps-cache / build" + fi + + ##### 4: Local deps-tag short-circuit ##### + + - name: Compute deps hash + id: deps-hash + if: steps.local.outputs.hit != 'true' && inputs.deps-hash == '' + uses: ./.github/actions/_lib/compute-deps-hash + with: + dockerfile-path: ${{ inputs.dockerfile-path }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + + - name: Check deps-tag locally + id: local-deps + if: steps.local.outputs.hit != 'true' + shell: bash run: | - # Only attempt NGC login if API key is available - if [ -n "${{ env.NGC_API_KEY }}" ]; then - echo "Logging into NGC registry..." - docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io - echo "✅ Successfully logged into NGC registry" + DEPS_HASH="${{ inputs.deps-hash || steps.deps-hash.outputs.hash }}" + LOCAL_DEPS_TAG="$(echo "${{ inputs.image-tag }}" | cut -d: -f1):deps-${DEPS_HASH}" + + echo "🔵 Local deps tag: ${LOCAL_DEPS_TAG}" + echo "LOCAL_DEPS_TAG=${LOCAL_DEPS_TAG}" >> "$GITHUB_ENV" + + if docker image inspect "${LOCAL_DEPS_TAG}" >/dev/null 2>&1; then + echo "🟢 Local deps-cache HIT! Retagging as ${{ inputs.image-tag }}" + docker tag "${LOCAL_DEPS_TAG}" "${{ inputs.image-tag }}" + echo "hit=true" >> "$GITHUB_OUTPUT" + else + echo "🟠 Local deps-cache MISS (will build then tag for future hits)" + fi + + ##### 5: Full build ##### + + - name: Build image + id: build + if: > + steps.local.outputs.hit != 'true' && + steps.local-deps.outputs.hit != 'true' + shell: bash + run: | + BUILD_ARGS=( + --progress=plain + --platform "${{ inputs.platform }}" + -f "${{ inputs.dockerfile-path }}" + --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" + --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" + --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim" + --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab" + --build-arg "DOCKER_USER_HOME_ARG=/root" + -t "${{ inputs.image-tag }}" + ) + if [ -n "${{ inputs.cache-from }}" ]; then + BUILD_ARGS+=( --cache-from "${{ inputs.cache-from }}" ) + fi + if [ -n "${{ inputs.cache-to }}" ]; then + BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" ) + fi + + BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" + docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ + || docker buildx use "${BUILDER_NAME}" + trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + + echo "🔵 Building ${{ inputs.image-tag }}..." + docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" + echo "was-built=true" >> "$GITHUB_OUTPUT" + + ##### 6: Tag built image with local deps-tag ##### + + # Runs only when a real build happened (not on cache hits). Populates the + # deps-tag so the next build with identical deps short-circuits at step 4. + + - name: Tag built image with local deps-tag + if: steps.build.outputs.was-built == 'true' + shell: bash + run: | + if [ -n "${LOCAL_DEPS_TAG:-}" ]; then + docker tag "${{ inputs.image-tag }}" "${LOCAL_DEPS_TAG}" + echo "🟢 Tagged local deps-cache: ${LOCAL_DEPS_TAG}" else - echo "⚠️ NGC_API_KEY not available - skipping NGC login" - echo "This is normal for PRs from forks or when secrets are not configured" + echo "🟠 LOCAL_DEPS_TAG not set, skipping local deps-cache tag" fi - - name: Build Docker Image - shell: sh + ##### 7: Evict stale local deps-cache tags (>14d) — opt-in ##### + + - name: Evict stale local deps-cache tags (>14d) + if: always() && inputs.evict-stale-cache == 'true' + shell: bash run: | - # Function to build Docker image - build_docker_image() { - local image_tag="$1" - local isaacsim_base_image="$2" - local isaacsim_version="$3" - local dockerfile_path="$4" - local context_path="$5" - - # Skip build if image already exists locally (e.g. built by a prior job on the same runner) - if docker image inspect "$image_tag" > /dev/null 2>&1; then - echo "Image $image_tag already exists locally, skipping build." - return 0 + set +e + TTL_DAYS=14 + cutoff=$(date -u -d "${TTL_DAYS} days ago" +%s) + evicted=0 + while IFS='|' read -r created tag; do + [ -z "$tag" ] && continue + created_epoch=$(date -d "$created" +%s 2>/dev/null) || continue + if [ "$created_epoch" -lt "$cutoff" ]; then + days_old=$(( (cutoff - created_epoch) / 86400 + TTL_DAYS )) + echo "🟠 Evicting deps tag (~${days_old}d old): ${tag}" + docker rmi -f "$tag" >/dev/null 2>&1 || true + evicted=$(( evicted + 1 )) fi + done < <(docker images --filter 'reference=isaac-lab*:deps-*' \ + --format '{{.CreatedAt}}|{{.Repository}}:{{.Tag}}' 2>/dev/null) + echo "🔵 Evicted ${evicted} deps tag(s) older than ${TTL_DAYS}d" - echo "Building Docker image: $image_tag" - echo "Using Dockerfile: $dockerfile_path" - echo "Build context: $context_path" - - # Build Docker image - docker buildx build --progress=plain --platform linux/amd64 \ - -t $image_tag \ - --build-arg ISAACSIM_BASE_IMAGE_ARG="$isaacsim_base_image" \ - --build-arg ISAACSIM_VERSION_ARG="$isaacsim_version" \ - --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \ - --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \ - --build-arg DOCKER_USER_HOME_ARG=/root \ - --cache-from type=gha \ - --cache-to type=gha,mode=max \ - -f $dockerfile_path \ - --load $context_path - - echo "✅ Docker image built successfully: $image_tag" - echo "Current local Docker images:" - docker images - } - - # Call the function with provided parameters - build_docker_image "${{ inputs.image-tag }}" "${{ inputs.isaacsim-base-image }}" "${{ inputs.isaacsim-version }}" "${{ inputs.dockerfile-path }}" "${{ inputs.context-path }}" + ##### 8: Host disk snapshot (post) ##### + + - name: Host disk snapshot (post) + if: always() + shell: bash + run: | + set +e + docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker") + deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l) + commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l) + { + echo "## Disk snapshot (post)" + echo '```' + echo "Filesystem:" + df -h / "${docker_root}" 2>/dev/null | sort -u + echo + echo "docker system df:" + docker system df + echo + echo "Tag counts:" + echo " isaac-lab* (commit + deps tags): ${commit_count}" + echo " isaac-lab*:deps-* (deps cache) : ${deps_count}" + echo + echo "Deps tags (newest first):" + docker images --filter 'reference=isaac-lab*:deps-*' \ + --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \ + | head -20 + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml index b661d4b9fd62..d6fe0cb27987 100644 --- a/.github/actions/ecr-build-push-pull/action.yml +++ b/.github/actions/ecr-build-push-pull/action.yml @@ -50,23 +50,7 @@ runs: # (including ECR login in step 3) inherit it automatically. - name: Setup docker config and login to nvcr.io - shell: bash - run: | - DOCKER_CONFIG_DIR=$(mktemp -d) - if [ -f "${HOME}/.docker/config.json" ]; then - python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))" - else - echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" - fi - echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV" - export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" - - if [ -n "${{ env.NGC_API_KEY }}" ]; then - echo "🔵 Logging into nvcr.io..." - docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io - else - echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)" - fi + uses: ./.github/actions/_lib/setup-docker-config ##### 2: Resolve ECR URL ##### @@ -191,40 +175,21 @@ runs: # Edit DEPS_FILES or DEPS_MANIFEST_PATTERN when install # inputs change (new packages, new manifests, etc.). + - name: Compute deps hash + id: deps-hash + if: steps.resolve-ecr.outputs.available == 'true' && steps.pull-exact.outputs.hit != 'true' + uses: ./.github/actions/_lib/compute-deps-hash + with: + dockerfile-path: ${{ inputs.dockerfile-path }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + - name: Check deps cache id: deps-cache if: steps.resolve-ecr.outputs.available == 'true' && steps.pull-exact.outputs.hit != 'true' shell: bash run: | - ##### Deps-hash configuration ##### - # Exact files/dirs whose full content is hashed. The Dockerfile is first. - DEPS_FILES=( - "${{ inputs.dockerfile-path }}" - isaaclab.sh - environment.yml - source/isaaclab/isaaclab/cli - ) - # Manifest files matched repo-wide via git ls-files. - DEPS_MANIFEST_PATTERN='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$' - - # Resolve the actual base image digest so a new push of a mutable tag - # (e.g. latest-develop) invalidates the deps cache automatically. - BASE_IMAGE_DIGEST=$(docker buildx imagetools inspect \ - "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \ - --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true) - if [ -n "${BASE_IMAGE_DIGEST}" ]; then - BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}:${BASE_IMAGE_DIGEST}" - else - echo "🟠 Could not resolve base image digest, falling back to tag string" - BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" - fi - - echo "🔵 Base image ID: ${BASE_IMAGE_UNIQ_ID}" - - MANIFEST_FILES=$(git ls-files | grep -E "${DEPS_MANIFEST_PATTERN}" || true) - FILE_HASH=$(git ls-files -s "${DEPS_FILES[@]}" ${MANIFEST_FILES} 2>/dev/null \ - | sha256sum | cut -c1-16) - DEPS_HASH=$(printf '%s %s' "${FILE_HASH}" "${BASE_IMAGE_UNIQ_ID}" | sha256sum | cut -c1-16) + DEPS_HASH="${{ steps.deps-hash.outputs.hash }}" DEPS_ECR_IMAGE="${ECR_URL}:deps-${DEPS_HASH}" echo "🔵 Deps hash: ${DEPS_HASH}" echo "🔵 Checking if deps image ${DEPS_ECR_IMAGE} exists in ECR..." @@ -245,41 +210,34 @@ runs: echo "PUSH_DEPS_IMAGE=true" >> "$GITHUB_ENV" fi - ##### 6: Full build ##### + ##### 6: Full build (delegated to docker-build) ##### # Runs when neither the exact image nor the deps cache was available. - # Uses ECR layer cache (--cache-from/--cache-to) when ECR is available. + # docker-build does the actual buildx invocation; we pass ECR layer-cache + # refs and the ECR-prefixed tag so the push steps below have something to + # push. - name: Full build if: steps.pull-exact.outputs.hit != 'true' && steps.deps-cache.outputs.deps-cache-hit != 'true' + uses: ./.github/actions/docker-build + with: + image-tag: ${{ inputs.image-tag }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + dockerfile-path: ${{ inputs.dockerfile-path }} + cache-from: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0}', env.CACHE_IMAGE) || '' }} + cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }} + deps-hash: ${{ steps.deps-hash.outputs.hash }} + + - name: Tag built image with ECR-prefixed name + if: > + steps.resolve-ecr.outputs.available == 'true' && + steps.pull-exact.outputs.hit != 'true' && + steps.deps-cache.outputs.deps-cache-hit != 'true' shell: bash run: | - BUILD_ARGS=( - --progress=plain - --platform linux/amd64 - -f "${{ inputs.dockerfile-path }}" - --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" - --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" - --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim" - --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab" - --build-arg "DOCKER_USER_HOME_ARG=/root" - -t "${{ inputs.image-tag }}" - ) - if [ -n "${ECR_URL:-}" ]; then - BUILD_ARGS+=( - --cache-from "type=registry,ref=${CACHE_IMAGE}" - --cache-to "type=registry,ref=${CACHE_IMAGE},mode=max" - -t "${ECR_IMAGE}" - ) - fi - - BUILDER_NAME="ci-builder-${{ github.run_id }}-${{ github.job }}" - docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ - || docker buildx use "${BUILDER_NAME}" - trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT - - echo "🔵 Building ${{ inputs.image-tag }}..." - docker buildx build --load "${BUILD_ARGS[@]}" . + docker tag "${{ inputs.image-tag }}" "${ECR_IMAGE}" + echo "🟢 Tagged ${ECR_IMAGE}" ##### 7: Push to ECR ##### diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ceea3a1c4e65..f416ef6d5e5f 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,6 +69,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + ci-marker: + description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)' + default: '' + required: false runs: using: composite @@ -93,6 +97,7 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" + local ci_marker="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -204,6 +209,11 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + if [ -n "$ci_marker" ]; then + docker_env_vars="$docker_env_vars -e CI_MARKER=$ci_marker" + echo "Setting CI_MARKER=$ci_marker" + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -392,7 +402,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.ci-marker }}" - name: Kill container on cancellation if: cancelled() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5d715d4e2945..fceda8c740f8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -238,6 +238,52 @@ jobs: dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo + # aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners. + # Build and test must share one runner because ECR is not wired for arm64 — + # the locally-built image cannot be handed off across machines. + arm-ci: + name: arm-ci + runs-on: [self-hosted, arm64] + needs: [changes, config] + if: needs.changes.outputs.run_docker_tests == 'true' + timeout-minutes: 60 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + - name: Build base image (linux/arm64) + uses: ./.github/actions/docker-build + with: + image-tag: ${{ env.CI_IMAGE_TAG }}-arm64 + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + platform: linux/arm64 + # The Spark runner is long-lived self-hosted with no ECR, so its local + # deps-cache tags accumulate; evict ones older than 14 days each run. + evict-stale-cache: "true" + + - name: Run arm_ci marker tests + uses: ./.github/actions/run-tests + with: + test-path: tools + result-file: arm-ci-report.xml + container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ env.CI_IMAGE_TAG }}-arm64 + ci-marker: arm_ci + volume-mount-source: ${{ github.workspace }} + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: arm-ci-reports + path: reports/ + retention-days: 7 + #endregion #region test jobs diff --git a/.github/workflows/daily-compatibility.yml b/.github/workflows/daily-compatibility.yml index 472b07300a5b..210abf47f7fa 100644 --- a/.github/workflows/daily-compatibility.yml +++ b/.github/workflows/daily-compatibility.yml @@ -3,6 +3,23 @@ # # SPDX-License-Identifier: BSD-3-Clause +# Nightly canary: runs the IsaacLab test suite against multiple pinned +# IsaacSim versions to catch backwards-compatibility regressions. +# +# Caveats for editors of this file or its action dependencies +# (`.github/actions/docker-build`, `run-tests`, `combine-results`): +# +# * No `pull_request:` trigger — changes are not validated automatically +# at PR time. Manually trigger against the PR branch before merge: +# gh workflow run "Backwards Compatibility Tests" --ref +# Otherwise breakage surfaces only on the next nightly cron after merge. +# +# * The build steps below pass `cache-from: type=gha` / `cache-to: +# type=gha,mode=max` to docker-build explicitly. Future migration to +# `./.github/actions/ecr-build-push-pull` would share build.yaml's +# cross-runner ECR layer cache, but that's deferred until daily-compat +# is ready to wire ECR auth. + name: Backwards Compatibility Tests on: @@ -101,6 +118,8 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Run IsaacLab Tasks Tests uses: ./.github/actions/run-tests @@ -159,6 +178,8 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Run General Tests uses: ./.github/actions/run-tests diff --git a/pyproject.toml b/pyproject.toml index a4a31197858c..64a45376d4c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,8 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty,segway" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", + "windows_ci: mark test to run on Windows platforms in CI", + "arm_ci: mark test to run on ARM platforms in CI (e.g. NVIDIA DGX Spark)", "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] diff --git a/source/isaaclab/changelog.d/jichuanh-arm-ci.rst b/source/isaaclab/changelog.d/jichuanh-arm-ci.rst new file mode 100644 index 000000000000..38548a771d35 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-arm-ci.rst @@ -0,0 +1,16 @@ +Fixed +^^^^^ + +* Added a defensive fallback in :class:`isaaclab.app.AppLauncher` so it derives + ``EXP_PATH`` from the installed ``isaacsim`` package when the env var is not + set. ``isaacsim.bootstrap_kernel`` normally sets ``EXP_PATH`` on first import, + but the early-return path in its bootstrap (triggered under some pip install + layouts on aarch64) skips the env-var setup. Previously this caused + ``KeyError: 'EXP_PATH'`` deep inside ``_resolve_experience_file``; now + AppLauncher resolves the path from ``isaacsim.__file__`` and stores it back + into the environment so subsequent code can rely on it. + +* Fixed :class:`~isaaclab.sim.converters.AssetConverterBase` hardcoding + ``/tmp`` for its default USD output directory. It now resolves the path under + ``tempfile.gettempdir()``, so it honors ``$TMPDIR`` on POSIX and works on + Windows (where the system temp dir is ``%TEMP%``). diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 511b46a3f816..9071af2eea5a 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1089,8 +1089,22 @@ def _resolve_experience_file(self, launcher_args: dict): launcher_args.get("deterministic", AppLauncher._APPLAUNCHER_CFG_INFO["deterministic"][1]) ) - # If nothing is provided resolve the experience file based on the headless flag - kit_app_exp_path = os.environ["EXP_PATH"] + # If nothing is provided resolve the experience file based on the headless flag. + # EXP_PATH is normally set by ``isaacsim.bootstrap_kernel()`` on first import. + # If it is not set (e.g. on aarch64 where the bootstrap early-return triggered + # under certain install layouts), derive it from the installed isaacsim package. + kit_app_exp_path = os.environ.get("EXP_PATH") + if not kit_app_exp_path: + try: + import isaacsim as _isaacsim_for_paths + except ImportError as e: + raise RuntimeError( + "EXP_PATH is not set and the 'isaacsim' package is not importable." + " Install Isaac Sim (`pip install isaacsim` or the binary distribution)" + " before launching AppLauncher." + ) from e + kit_app_exp_path = os.path.join(os.path.dirname(_isaacsim_for_paths.__file__), "apps") + os.environ["EXP_PATH"] = kit_app_exp_path isaaclab_app_exp_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), *[".."] * 4, "apps") # For Isaac Sim 4.5 compatibility, we use the 4.5 app files in a different folder # if launcher_args.get("use_isaacsim_45", False): @@ -1193,12 +1207,18 @@ def _create_app(self): sys.stdout = open(os.devnull, "w") # noqa: SIM115 # pytest may have left some things in sys.argv, this will check for some of those - # do a mark and sweep to remove any -m pytest and -m isaacsim_ci and -c **/pyproject.toml + # do a mark and sweep to remove any -m pytest, -m isaacsim_ci, -m windows_ci, -m arm_ci, + # and -c **/pyproject.toml indexes_to_remove = [] for idx, arg in enumerate(sys.argv[:-1]): if arg == "-m": value_for_dash_m = sys.argv[idx + 1] - if "pytest" in value_for_dash_m or "isaacsim_ci" in value_for_dash_m: + if ( + "pytest" in value_for_dash_m + or "isaacsim_ci" in value_for_dash_m + or "windows_ci" in value_for_dash_m + or "arm_ci" in value_for_dash_m + ): indexes_to_remove.append(idx) indexes_to_remove.append(idx + 1) if arg.startswith("--config-file=") and "pyproject.toml" in arg: diff --git a/source/isaaclab/isaaclab/sim/converters/asset_converter_base.py b/source/isaaclab/isaaclab/sim/converters/asset_converter_base.py index 11c200422391..703ef202e2a7 100644 --- a/source/isaaclab/isaaclab/sim/converters/asset_converter_base.py +++ b/source/isaaclab/isaaclab/sim/converters/asset_converter_base.py @@ -9,6 +9,7 @@ import os import pathlib import random +import tempfile from datetime import datetime from isaaclab.sim.converters.asset_converter_base_cfg import AssetConverterBaseCfg @@ -34,9 +35,10 @@ class AssetConverterBase(abc.ABC): can be set to True. When no output directory is defined, lazy conversion is deactivated and the generated USD file is - stored in folder ``/tmp/IsaacLab/usd_{date}_{time}_{random}``, where the parameters in braces are generated - at runtime. The random identifiers help avoid a race condition where two simultaneously triggered conversions - try to use the same directory for reading/writing the generated files. + stored in folder ``/IsaacLab/usd_{date}_{time}_{random}``, where ```` is the system + temporary directory (e.g. ``/tmp`` on POSIX, ``%TEMP%`` on Windows) and the parameters in braces are + generated at runtime. The random identifiers help avoid a race condition where two simultaneously + triggered conversions try to use the same directory for reading/writing the generated files. .. note:: Changes to the parameters :obj:`AssetConverterBaseCfg.asset_path`, :obj:`AssetConverterBaseCfg.usd_dir`, and @@ -64,9 +66,9 @@ def __init__(self, cfg: AssetConverterBaseCfg): # resolve USD directory name if cfg.usd_dir is None: - # a folder in "/tmp/IsaacLab" by the name: usd_{date}_{time}_{random} + # a folder in the system temp dir by the name: IsaacLab/usd_{date}_{time}_{random} time_tag = datetime.now().strftime("%Y%m%d_%H%M%S") - self._usd_dir = f"/tmp/IsaacLab/usd_{time_tag}_{random.randrange(10000)}" + self._usd_dir = os.path.join(tempfile.gettempdir(), "IsaacLab", f"usd_{time_tag}_{random.randrange(10000)}") else: self._usd_dir = cfg.usd_dir diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index bed0760271e7..c57611b08d34 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -16,6 +16,8 @@ import torch from flaky import flaky +pytestmark = pytest.mark.arm_ci + import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils from isaaclab import cloner diff --git a/source/isaaclab/test/deps/test_scipy.py b/source/isaaclab/test/deps/test_scipy.py index d697716aad7a..6b2d0dee6e81 100644 --- a/source/isaaclab/test/deps/test_scipy.py +++ b/source/isaaclab/test/deps/test_scipy.py @@ -13,6 +13,8 @@ import numpy as np import scipy.interpolate as interpolate +pytestmark = pytest.mark.arm_ci + @pytest.mark.isaacsim_ci def test_interpolation(): diff --git a/source/isaaclab/test/deps/test_torch.py b/source/isaaclab/test/deps/test_torch.py index 6a50110757de..9d1bf39da64f 100644 --- a/source/isaaclab/test/deps/test_torch.py +++ b/source/isaaclab/test/deps/test_torch.py @@ -7,6 +7,8 @@ import torch import torch.utils.benchmark as benchmark +pytestmark = pytest.mark.arm_ci + @pytest.mark.isaacsim_ci def test_array_slicing(): diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip new file mode 100644 index 000000000000..c312132ab201 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip @@ -0,0 +1,2 @@ +Test-only changes (arm_ci markers + aarch64 skip guards in rendering test +helpers); no user-facing changelog entry. diff --git a/source/isaaclab_tasks/test/core/test_cartpole_training_smoke.py b/source/isaaclab_tasks/test/core/test_cartpole_training_smoke.py new file mode 100644 index 000000000000..4b0ff1c054cc --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_cartpole_training_smoke.py @@ -0,0 +1,80 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Minimal end-to-end training smoke for cartpole. + +Two cases — state-only and perception (RGB tiled camera) — each spawn a +``scripts/reinforcement_learning//train.py`` for two PPO iterations +on a small env count. They validate the full pipeline (``./isaaclab.sh`` +wrapper, gym registration, env build, RL wrapper, optimizer step, checkpoint +write) without the cost of a real training run, so the orchestrator can +include them in every CI shape (Linux, ARM/Spark). + +The state case uses rsl_rl (matches Isaac-Cartpole-Direct's registered +``rsl_rl_cfg_entry_point``); the perception case uses rl_games against +Isaac-Cartpole-Camera-Direct's ``rl_games_cfg_entry_point``. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.arm_ci + +_REPO_ROOT = Path(__file__).resolve().parents[4] + + +def _run_train(train_script: str, task_name: str, extra_args: list[str] | None = None, timeout: int = 600) -> None: + """Spawn a trainer for two iterations and assert it exits cleanly.""" + cmd = [ + "./isaaclab.sh", + "-p", + train_script, + "--task", + task_name, + "--headless", + "--num_envs", + "16", + "--max_iterations", + "2", + "--seed", + "42", + ] + if extra_args: + cmd.extend(extra_args) + + result = subprocess.run( + cmd, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + assert result.returncode == 0, ( + f"Training command failed for {task_name}: {' '.join(cmd)}\n" + f"--- stdout (tail) ---\n{result.stdout[-4000:]}\n" + f"--- stderr (tail) ---\n{result.stderr[-4000:]}\n" + ) + + +def test_train_cartpole_state(): + """State-observation cartpole trains for two rsl_rl PPO iterations without errors.""" + _run_train("scripts/reinforcement_learning/rsl_rl/train.py", "Isaac-Cartpole-Direct") + + +def test_train_cartpole_perception(): + """RGB-camera cartpole trains for two rl_games PPO iterations without errors.""" + # The first camera-enabled run on a cold cache compiles shaders (~600 s) + # before training starts, so allow well beyond the state-case budget. + _run_train( + "scripts/reinforcement_learning/rl_games/train.py", + "Isaac-Cartpole-Camera-Direct", + extra_args=["--enable_cameras"], + timeout=1800, + ) diff --git a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py index 3930d0e2d998..56c269db1882 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py @@ -17,7 +17,7 @@ rendering_test_cartpole, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py index e5ba35a90e3c..9a56034ecea2 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py @@ -17,7 +17,7 @@ rendering_test_dexsuite_kuka, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py index 7652b9c92fa6..bd3f6ddebf8c 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py @@ -17,7 +17,7 @@ rendering_test_shadow_hand, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index 784d6b586d87..5190fe76c2a4 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -6,6 +6,7 @@ """Shared helpers for rendering correctness tests.""" import os +import platform from datetime import datetime from typing import Any @@ -420,6 +421,8 @@ def _require_ovlibs_install(request): print(f"ovrtx version: {ovrtx.__version__}") except ImportError as exc: + if platform.machine() == "aarch64": + pytest.skip("OVRTX has no aarch64 wheel; skipping renderer=ovrtx_renderer on this platform.") pytest.fail( "Kitless OVRTX rendering tests require the optional dependency ov[ovrtx]. " "Install with: ./isaaclab.sh -i 'ov[ovrtx]'\n" @@ -432,6 +435,8 @@ def _require_ovlibs_install(request): print(f"ovphysx version: {ovphysx.__version__}") except ImportError as exc: + if platform.machine() == "aarch64": + pytest.skip("OVPhysX has no aarch64 wheel; skipping physics_backend=ovphysx on this platform.") pytest.fail( "Kitless OVPhysX rendering tests require the optional dependency ov[ovphysx]. " "Install with: ./isaaclab.sh -i 'ov[ovphysx]'\n" diff --git a/tools/conftest.py b/tools/conftest.py index a45f8cd9183c..ead776ad3bbd 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -410,8 +410,9 @@ class _PassContext: test_file: Absolute path to the test file being driven. file_name: Basename of ``test_file`` (used for JUnit naming). workspace_root: Repository root; passed to pytest's ``--config-file``. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. + ci_marker: Optional pytest marker expression. When set, adds the + ``-m `` selector (e.g. ``isaacsim_ci``, ``windows_ci``, + ``arm_ci``); when falsy, no marker filter is applied. timeout: Per-pass hard timeout in seconds. startup_deadline: Per-pass startup-hang deadline in seconds. env: Environment passed to the pytest subprocess. @@ -420,7 +421,7 @@ class _PassContext: test_file: str file_name: str workspace_root: str - isaacsim_ci: bool + ci_marker: str | None timeout: int startup_deadline: int env: dict @@ -494,8 +495,8 @@ def _run_one_pass( f"--junitxml={report_file}", "--tb=short", ] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] + if ctx.ci_marker: + cmd += ["-m", ctx.ci_marker] if k_expr is not None: cmd += ["-k", k_expr] cmd.append(str(ctx.test_file)) @@ -729,7 +730,7 @@ def _run_one_pass( ) -def run_individual_tests(test_files, workspace_root, isaacsim_ci): +def run_individual_tests(test_files, workspace_root, ci_marker): """Run each test file separately, ensuring one finishes before starting the next.""" failed_tests = [] test_status = {} @@ -766,7 +767,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): test_file=test_file, file_name=file_name, workspace_root=workspace_root, - isaacsim_ci=isaacsim_ci, + ci_marker=ci_marker, timeout=timeout, startup_deadline=startup_deadline, env=env, @@ -892,6 +893,13 @@ def pytest_sessionstart(session): isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" + # CI_MARKER env var is a separate, parallel mechanism for cross-platform + # jobs (arm-ci, windows-ci, ...) to reuse this orchestrator with their own + # markers. Deliberately NOT aliased to ISAACSIM_CI_SHORT: the isaacsim_ci + # filter is owned by Isaac Sim's external CI pipeline; the CI_MARKER path + # leaves that contract untouched. + ci_marker = os.environ.get("CI_MARKER", "") + # Parse include files list (comma-separated paths) include_files = set() if include_files_str: @@ -939,6 +947,24 @@ def pytest_sessionstart(session): new_test_files.append(test_file) test_files = new_test_files + if ci_marker: + # Match both `@pytest.mark.` (per-function) and + # `pytestmark = pytest.mark.` / `pytestmark = [..., pytest.mark., ...]` + # (module-level) by looking for the common `pytest.mark.` substring. + marker_token = f"pytest.mark.{ci_marker}" + new_test_files = [] + for test_file in test_files: + try: + with open(test_file) as f: + if marker_token in f.read(): + new_test_files.append(test_file) + except OSError as exc: + raise RuntimeError( + f"ci_marker post-scan could not read {test_file}; refusing to" + f" silently drop a potentially marker-tagged file" + ) from exc + test_files = new_test_files + if not test_files: if quarantined_only: print("No quarantined tests configured — nothing to run.") @@ -955,8 +981,11 @@ def pytest_sessionstart(session): for test_file in test_files: print(f" - {test_file}") - # Run all tests individually - failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) + # Run all tests individually. CI_MARKER takes precedence when both env + # vars are set; falls back to "isaacsim_ci" when only ISAACSIM_CI_SHORT + # is set. The pytest -m flag only accepts one expression. + effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") + failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, effective_marker) print("failed tests:", failed_tests)