diff --git a/.github/workflows/_test_cadence.yml b/.github/workflows/_test_cadence.yml index d899e2ae585..963e2236700 100644 --- a/.github/workflows/_test_cadence.yml +++ b/.github/workflows/_test_cadence.yml @@ -1,17 +1,15 @@ name: Test Cadence permissions: - id-token: write contents: read on: workflow_call: inputs: docker-image: - description: 'Name of the docker image to use, without registry or tag suffix' - required: false + description: 'Fully qualified CI image (resolved by the caller)' + required: true type: string - default: ci-image:executorch-ubuntu-22.04-clang12 runner: description: 'Runner type' required: false @@ -29,20 +27,14 @@ on: default: 90 jobs: - docker-image: - name: Resolve CI docker image - uses: ./.github/workflows/_docker-image.yml - test-aot: - needs: docker-image uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: - id-token: write contents: read with: job-name: test-aot runner: ${{ inputs.runner }} - docker-image: ${{ needs.docker-image.outputs.docker-registry }}/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} + docker-image: ${{ inputs.docker-image }} submodules: recursive ref: ${{ inputs.ref }} timeout: ${{ inputs.timeout }} @@ -58,15 +50,13 @@ jobs: python -m pytest backends/cadence/aot/tests/ -v -n auto --reruns 2 --reruns-delay 1 test-ops: - needs: docker-image uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: - id-token: write contents: read with: job-name: test-ops runner: ${{ inputs.runner }} - docker-image: ${{ needs.docker-image.outputs.docker-registry }}/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} + docker-image: ${{ inputs.docker-image }} submodules: recursive ref: ${{ inputs.ref }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_xtensa_build.yml b/.github/workflows/_xtensa_build.yml index d8ccdd3ad7a..327cc7c9e25 100644 --- a/.github/workflows/_xtensa_build.yml +++ b/.github/workflows/_xtensa_build.yml @@ -20,25 +20,28 @@ on: required: false type: string default: "" + docker-image: + description: "Fully qualified CI image (resolved by the caller)" + required: true + type: string + allow-fork-checkout: + description: "Check out fork PR code on pull_request_target" + required: false + type: boolean + default: false jobs: - # The runner pod pulls the container before any step runs, so the image has to - # be a fully qualified reference resolved by a job this one depends on. - docker-image: - name: Resolve CI docker image - uses: ./.github/workflows/_docker-image.yml - build: name: ${{ inputs.backend }} - needs: docker-image - # Intel, not the x86iavx512 (r7a/AMD) default: the Cadence licence key that - # xt-clang computes for a core differs by CPU vendor, and only the Intel one - # (XT_XCC_TIE_ED4D1230) is in the bundled licence. On an AMD runner the - # vision core resolves to XT_XCC_TIE_ED4DB539 and the checkout fails with - # "No such feature exists". hifi4's older RI-2022.10 toolchain does not care. + # Keep this label at 18 characters or fewer. The pod hostname is the label plus + # 28 characters, and RJ-2025.5 xt-clang (vision) derives the wrong licence key + # once the hostname reaches 47: it asks for XT_XCC_TIE_ED4DB539 instead of the + # bundled ED4D1230 and fails with "No such feature exists". The CPU vendor is + # not the cause; mt-l-x86iavx512-8-64 (20 characters) fails on its hostname. + # hifi4's RI-2022.9 toolchain is unaffected. runs-on: mt-l-x86iamx-8-64 container: - image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} + image: ${{ inputs.docker-image }} environment: cadence permissions: id-token: write @@ -63,6 +66,8 @@ jobs: with: submodules: recursive ref: ${{ inputs.ref }} + persist-credentials: ${{ !inputs.allow-fork-checkout }} + allow-unsafe-pr-checkout: ${{ inputs.allow-fork-checkout }} - name: Assume Cadence artifacts role uses: aws-actions/configure-aws-credentials@v4 @@ -92,6 +97,7 @@ jobs: chmod -R a+rX cmake-out - name: Upload runner + if: ${{ !inputs.allow-fork-checkout }} uses: actions/upload-artifact@v4 with: name: cadence-xtensa-build-${{ inputs.backend }} diff --git a/.github/workflows/_xtensa_test.yml b/.github/workflows/_xtensa_test.yml index f415fce969b..e691826d80c 100644 --- a/.github/workflows/_xtensa_test.yml +++ b/.github/workflows/_xtensa_test.yml @@ -19,25 +19,28 @@ on: required: false type: string default: "" + docker-image: + description: "Fully qualified CI image (resolved by the caller)" + required: true + type: string + allow-fork-checkout: + description: "Check out fork PR code on pull_request_target" + required: false + type: boolean + default: false jobs: - # The runner pod pulls the container before any step runs, so the image has to - # be a fully qualified reference resolved by a job this one depends on. - docker-image: - name: Resolve CI docker image - uses: ./.github/workflows/_docker-image.yml - test: name: ${{ inputs.backend }} - needs: docker-image - # Intel, not the x86iavx512 (r7a/AMD) default: the Cadence licence key that - # xt-clang computes for a core differs by CPU vendor, and only the Intel one - # (XT_XCC_TIE_ED4D1230) is in the bundled licence. On an AMD runner the - # vision core resolves to XT_XCC_TIE_ED4DB539 and the checkout fails with - # "No such feature exists". hifi4's older RI-2022.10 toolchain does not care. + # Keep this label at 18 characters or fewer. The pod hostname is the label plus + # 28 characters, and RJ-2025.5 xt-clang (vision) derives the wrong licence key + # once the hostname reaches 47: it asks for XT_XCC_TIE_ED4DB539 instead of the + # bundled ED4D1230 and fails with "No such feature exists". The CPU vendor is + # not the cause; mt-l-x86iavx512-8-64 (20 characters) fails on its hostname. + # hifi4's RI-2022.9 toolchain is unaffected. runs-on: mt-l-x86iamx-8-64 container: - image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} + image: ${{ inputs.docker-image }} environment: cadence permissions: id-token: write @@ -62,6 +65,8 @@ jobs: with: submodules: recursive ref: ${{ inputs.ref }} + persist-credentials: ${{ !inputs.allow-fork-checkout }} + allow-unsafe-pr-checkout: ${{ inputs.allow-fork-checkout }} - name: Assume Cadence artifacts role uses: aws-actions/configure-aws-credentials@v4 diff --git a/.github/workflows/build-cadence-runner.yml b/.github/workflows/build-cadence-runner.yml index 6d1627c4279..723d7bd99d4 100644 --- a/.github/workflows/build-cadence-runner.yml +++ b/.github/workflows/build-cadence-runner.yml @@ -10,40 +10,75 @@ on: tags: - ciflow/nightly/* pull_request: + # Fork PRs, Meta exports from personal forks included. pull_request_target + # always runs this file from the default branch, so the paths filter below only + # affects fork PRs. pull_request_target: - types: [labeled] + types: [opened, synchronize, reopened] + paths: + - backends/cadence/** + - examples/cadence/** + - .ci/scripts/setup-xtensa-tools.sh + - .ci/scripts/build-cadence-xtensa.sh + - .ci/scripts/test-cadence-xtensa.sh workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }} cancel-in-progress: true +# Every job runs on the same paths. Same-repo PRs use pull_request, which reads +# the PR's own workflow AND code -- so CI changes, new test jobs, code, and tests +# are all validated pre-merge. Fork PRs can't get credentials (OIDC) there, so +# fork PRs whose author has contributed before (CONTRIBUTOR or above) run on +# pull_request_target instead. The CPU jobs hold no AWS credentials on any path +# (contents: read only), so linux_job_v3's role/arc never reaches fork code; only +# the Xtensa jobs assume a role, the Cadence artifacts one. The run condition is +# inlined per job (GitHub Actions has no YAML anchors and env is unavailable in +# job-level if), so keep the copies in sync. jobs: + # Resolves the image inline rather than through the shared _docker-image.yml, + # which checks out the PR head: actions/checkout refuses that for fork code on + # pull_request_target. There the image comes from the base branch (github.sha), + # which also keeps a fork from picking the image it runs in. Same-repo PRs read + # their own head, as _docker-image.yml does, so .ci/docker changes are tested. docker-image: name: Resolve CI docker image - uses: ./.github/workflows/_docker-image.yml + if: >- + github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + image: ${{ steps.hash.outputs.image }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - id: hash + run: | + set -eu + CI_DOCKER_HASH="$(git rev-parse HEAD:.ci/docker)" + echo "image=308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${CI_DOCKER_HASH}" >> "$GITHUB_OUTPUT" - # Same-repo PRs run on pull_request, which reads the PR's own workflow AND code - # -- so CI changes, new test jobs, code, and tests are all validated pre-merge. - # Fork PRs can't get credentials (OIDC) on pull_request, so Meta-exported forks - # (labeled CLA Signed + meta-exported) run on pull_request_target instead. The - # run condition is inlined per job (GitHub Actions has no YAML anchors and env - # is unavailable in job-level if), so keep the copies in sync. cpu-build: needs: docker-image if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && - contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: - id-token: write contents: read with: job-name: build runner: mt-l-x86iavx512-8-64 - docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} + docker-image: ${{ needs.docker-image.outputs.image }} submodules: recursive ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -61,17 +96,17 @@ jobs: cp cmake-out/backends/cadence/cadence_runner "${RUNNER_ARTIFACT_DIR}/" cpu-test: - needs: cpu-build + needs: [docker-image, cpu-build] if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && - contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) permissions: - id-token: write contents: read uses: ./.github/workflows/_test_cadence.yml with: + docker-image: ${{ needs.docker-image.outputs.image }} ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} # Cross-compile cadence_executor_runner for each Cadence Xtensa core, one job @@ -79,11 +114,12 @@ jobs: # lives in _xtensa_build.yml. fusion_g3 is omitted until the upstream fusion_g3 # <-> nnlib-FusionG3 API skew is fixed (its runner does not link). hifi-build: + needs: docker-image if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && - contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) permissions: id-token: write contents: read @@ -91,18 +127,20 @@ jobs: with: backend: hifi4 ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} + docker-image: ${{ needs.docker-image.outputs.image }} + allow-fork-checkout: ${{ github.event_name == 'pull_request_target' }} # Op-level gtest tests on the Xtensa ISS, mirroring cpu-build -> cpu-test. The # op tests are a self-contained cross-compile (the gtests need exceptions and # RTTI that the runner build disables), so this does not consume hifi-build's # artifact; needs: hifi-build only to fail fast when the backend cannot build. hifi-op-test: - needs: hifi-build + needs: [docker-image, hifi-build] if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && - contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) permissions: id-token: write contents: read @@ -110,13 +148,16 @@ jobs: with: backend: hifi4 ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} + docker-image: ${{ needs.docker-image.outputs.image }} + allow-fork-checkout: ${{ github.event_name == 'pull_request_target' }} vision-build: + needs: docker-image if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && - contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association)) permissions: id-token: write contents: read @@ -124,3 +165,5 @@ jobs: with: backend: vision ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} + docker-image: ${{ needs.docker-image.outputs.image }} + allow-fork-checkout: ${{ github.event_name == 'pull_request_target' }} diff --git a/.github/workflows/cadence-fork-pr.yml b/.github/workflows/cadence-fork-pr.yml deleted file mode 100644 index 512af3f799b..00000000000 --- a/.github/workflows/cadence-fork-pr.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Cadence Fork PR - -# Xtensa build and op tests for fork PRs that are not Meta exports (those run -# from build-cadence-runner.yml). The jobs need the licensed toolchain, so they -# run the fork's code with the Cadence artifacts role. That is limited to authors -# who have contributed to the repository before (CONTRIBUTOR or above); -# first-time contributors and unassociated authors get nothing. -# -# Self-contained on purpose: the shared reusable workflows check out the PR head -# on pull_request_target, which actions/checkout refuses for fork code, and they -# are used well beyond Cadence. -on: - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - backends/cadence/** - - examples/cadence/** - - .ci/scripts/setup-xtensa-tools.sh - - .ci/scripts/build-cadence-xtensa.sh - - .ci/scripts/test-cadence-xtensa.sh - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - # The image comes from the base branch rather than the fork, so the fork - # cannot pick which image its code runs in. - docker-image: - if: >- - github.event.pull_request.head.repo.full_name != github.repository && - !startsWith(github.event.pull_request.head.ref, 'export-D') && - !contains(github.event.pull_request.labels.*.name, 'meta-exported') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association) - runs-on: ubuntu-latest - permissions: - contents: read - outputs: - image: ${{ steps.hash.outputs.image }} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - id: hash - run: | - set -eu - CI_DOCKER_HASH="$(git rev-parse HEAD:.ci/docker)" - echo "image=308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${CI_DOCKER_HASH}" >> "$GITHUB_OUTPUT" - - xtensa: - name: ${{ matrix.name }} - needs: docker-image - # Same gate as docker-image, so the job that holds the role does not depend - # on needs-skip propagation alone. - if: >- - github.event.pull_request.head.repo.full_name != github.repository && - !startsWith(github.event.pull_request.head.ref, 'export-D') && - !contains(github.event.pull_request.labels.*.name, 'meta-exported') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.pull_request.author_association) - strategy: - fail-fast: false - matrix: - include: - - name: hifi4 - backend: hifi4 - script: .ci/scripts/build-cadence-xtensa.sh --no-run - - name: hifi4 op tests - backend: hifi4 - script: .ci/scripts/test-cadence-xtensa.sh - - name: vision - backend: vision - script: .ci/scripts/build-cadence-xtensa.sh --no-run - # Same runner as build-cadence-runner.yml: RJ-2025.5 xt-clang (vision) needs - # a pod hostname of 46 characters or fewer, i.e. a label of 18 or fewer. - runs-on: mt-l-x86iamx-8-64 - container: - image: ${{ needs.docker-image.outputs.image }} - environment: cadence - permissions: - id-token: write - contents: read - steps: - - name: Fix workspace permissions - shell: bash - run: sudo chmod -R 777 "${GITHUB_WORKSPACE}" 2>/dev/null || true - - - name: Check out the fork's head - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - persist-credentials: false - allow-unsafe-pr-checkout: true - - - name: Assume Cadence artifacts role - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.CADENCE_CI_AWS_ROLE }} - aws-region: ${{ vars.CADENCE_CI_AWS_REGION }} - - - name: Build and test - env: - BACKEND: ${{ matrix.backend }} - XTENSA_S3_BUCKET: ${{ vars.CADENCE_CI_S3_BUCKET }} - shell: bash - run: | - set -exo pipefail - eval "$(/opt/conda/bin/conda shell.bash hook)" - conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" - ./install_requirements.sh > /dev/null - pip install --quiet awscli - if [ "${BACKEND}" != "vision" ]; then - backends/cadence/install_requirements.sh - fi - source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" - ${{ matrix.script }}