IntegrationTest.yml

name: "IntegrationTest"

on:
  workflow_call:
    inputs:
      julia-version:
        description: "Julia version"
        default: "1"
        required: false
        type: string
      pkgs:
        description: "JSON-array string of package names (e.g. `'[\"BlockSparseArrays\", \"GradedArrays\"]'`) and/or git URLs (https/git@) for unregistered or private packages. Use `'[]'` when no downstream integration tests should run."
        required: true
        type: string
      localregistry:
        description: "Add local registries hosted on GitHub. Specified by providing the url (https/ssh) to the repositories as a newline (\n) seperated list. User is responsible for setting up the necessary SSH-Keys to access the repositories if necessary."
        default: ""
        required: false
        type: string
      extra-dev-paths:
        description: "Additional local package paths to develop alongside the repository root, specified as a newline-separated list."
        default: ""
        required: false
        type: string
      subdir:
        description: "Run the downstream tests against the package in this subdirectory."
        default: ""
        required: false
        type: string
      paths:
        description: "Newline-separated list of path prefixes / files defining positive scope for the substantive-PR check. A file triggers IntegrationTest iff it equals one of these paths or starts with `<path>/`. Empty means no positive-scope filter; the check falls back to `exclude-paths`. Mutually exclusive with `exclude-paths`."
        default: ""
        required: false
        type: string
      exclude-paths:
        description: "Newline-separated list of path prefixes / files defining negative scope for the substantive-PR check. A file does NOT trigger IntegrationTest if it matches one of these. Default ignores `.gitignore` and `.pre-commit-config.yaml`."
        default: |
          .gitignore
          .pre-commit-config.yaml
        required: false
        type: string
      run-on-draft:
        description: "When true, run integration tests even on draft PRs"
        default: false
        required: false
        type: boolean
    secrets:
      INTEGRATIONTEST_PAT:
        description: "GitHub PAT for cloning private repositories via HTTPS."
        required: false

jobs:
  integration-test:
    name: "${{ matrix.pkg }}"
    # Skip the matrix-driven leg entirely when `pkgs` is empty. Without this
    # guard, GitHub Actions reports an empty-matrix job as `failure`, which
    # propagates to the run's overall conclusion even though the aggregating
    # `gate` job correctly handles empty `pkgs` by exiting 0. Using
    # `fromJSON(inputs.pkgs)[0] != null` rather than a string comparison so
    # the check is robust against whitespace / block-scalar style.
    if: |
      fromJSON(inputs.pkgs)[0] != null &&
      (!github.event.pull_request.draft || inputs.run-on-draft)
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        pkg: ${{ fromJSON(inputs.pkgs) }}

    steps:
      - uses: ITensor/ITensorActions/.github/actions/classify-pr@main
        id: classify
        with:
          # `subdir` (the in-tree package under test) is auto-appended
          # to the positive scope so callers don't have to repeat it.
          paths: |
            ${{ inputs.subdir }}
            ${{ inputs.paths }}
          exclude-paths: ${{ inputs.exclude-paths }}

      - name: "Skip on out-of-scope PR"
        if: "${{ steps.classify.outputs.triggers != 'true' }}"
        run: |
          echo "PR has no changes in this caller's scope; integration test skipped."

      # Decide whether this matrix leg should run, and emit `skip=true|false`.
      # Skips when the entry is a URL that needs auth and the PR is from a
      # fork (no secrets are exposed to fork PRs under `pull_request:`).
      # Runs in every other case, including registered package names,
      # URL entries reachable anonymously, and URL entries on internal PRs /
      # push events / `pull_request_target` / `issue_comment` /
      # `workflow_dispatch` (where secrets are in scope).
      - name: "Decide whether to run"
        id: gate
        if: "${{ steps.classify.outputs.triggers == 'true' }}"
        env:
          PKG: ${{ matrix.pkg }}
          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
          BASE_REPO: ${{ github.repository }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          if [[ "$PKG" != https://* && "$PKG" != git@* ]]; then
            echo "Registered package '$PKG'; running test."
            echo "skip=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # URL-based dep: probe anonymously to detect whether auth is needed.
          # `credential.helper=` disables any helper that might silently
          # supply credentials; `timeout` bounds a slow network probe.
          if timeout 30 git -c credential.helper= ls-remote --exit-code "$PKG" HEAD &>/dev/null; then
            echo "URL '$PKG' is reachable anonymously; running test."
            echo "skip=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Anonymous probe failed: the URL needs auth (or is unreachable).
          # Skip only in the one context where the workflow has no secrets:
          # a `pull_request` event from a fork.
          if [ "$EVENT_NAME" = "pull_request" ] && [ -n "$HEAD_REPO" ] && [ "$HEAD_REPO" != "$BASE_REPO" ]; then
            echo "::notice::URL dependency '$PKG' requires authentication and this PR is from a fork. Skipping. Run \`/integrationtest $PKG\` on the PR to test it manually."
            echo "skip=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          echo "URL '$PKG' needs auth; running with secrets in scope (event '$EVENT_NAME')."
          echo "skip=false" >> "$GITHUB_OUTPUT"
      - name: "Record skipped dependency"
        id: skip_artifact
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip == 'true' }}"
        env:
          PKG: ${{ matrix.pkg }}
        run: |
          key=$(printf '%s' "$PKG" | shasum -a 256 | cut -d ' ' -f 1)
          path="integration-test-skip-${key}.txt"
          printf '%s\n' "$PKG" > "$path"
          echo "key=$key" >> "$GITHUB_OUTPUT"
          echo "path=$path" >> "$GITHUB_OUTPUT"
      - name: "Upload skipped-dependency marker"
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip == 'true' }}"
        uses: actions/upload-artifact@v7
        with:
          name: "integration-test-skip-${{ steps.skip_artifact.outputs.key }}"
          path: ${{ steps.skip_artifact.outputs.path }}
      - uses: actions/checkout@v7
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip != 'true' }}"
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
      - uses: julia-actions/setup-julia@v3
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip != 'true' }}"
        with:
          version: ${{ inputs.julia-version }}
          arch: x64
      - name: "Configure git authentication for private repository"
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip != 'true' }}"
        env:
          TOKEN: ${{ secrets.INTEGRATIONTEST_PAT }}
          PKG: ${{ matrix.pkg }}
        run: |
          if [[ "$PKG" == https://* ]] || [[ "$PKG" == git@* ]]; then
            if [ -n "$TOKEN" ]; then
              git config --global url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
            fi
          fi
      - name: "Run the downstream tests"
        if: "${{ steps.classify.outputs.triggers == 'true' && steps.gate.outputs.skip != 'true' }}"
        shell: julia --color=yes --project=downstream {0}
        env:
          PKG: ${{ matrix.pkg }}
          LOCALREGISTRY: ${{ inputs.localregistry }}
          EXTRA_DEV_PATHS: ${{ inputs.extra-dev-paths }}
          PKG_DIR: ${{ inputs.subdir || '.' }}
          JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
        run: |
          using Pkg
          Pkg.Registry.add("General")
          # If provided add local registries
          localregistry = get(ENV, "LOCALREGISTRY", "")
          if !isempty(localregistry)
            registry_urls = split(localregistry, "\n") .|> string
            for registry_url in registry_urls
              isempty(registry_url) && continue
              Pkg.Registry.add(Pkg.RegistrySpec(; url=registry_url))
            end
          end
          dev_paths = [get(ENV, "PKG_DIR", "."), split(get(ENV, "EXTRA_DEV_PATHS", ""), "\n")...]
          filter!(!isempty, dev_paths)
          dev_specs = [PackageSpec(; path) for path in dev_paths]
          pkg = get(ENV, "PKG", "")
          try
            if startswith(pkg, "https://") || startswith(pkg, "git@")
              # URL: unregistered or private package, skip version-pinning.
              pkg_name = replace(basename(pkg), r"\.jl$" => "")
              Pkg.add(PackageSpec(; url=pkg))  # resolver may fail with main deps
              Pkg.develop(dev_specs)  # resolver may fail with main deps
              Pkg.update()
              Pkg.test(PackageSpec(; name=pkg_name))  # resolver may fail with test time deps
            else
              Pkg.add(PackageSpec(; name=pkg))
              # Determine the version that gets installed in order to detect
              # if the package is forced to downgrade.
              eval(Expr(:using, :($(Expr(:., Symbol(pkg))))))
              version = pkgversion(eval(Symbol(pkg)))
              # force it to use this PR's version of the package
              Pkg.develop(dev_specs)  # resolver may fail with main deps
              Pkg.update()
              # Try forcing the installation of the package at the latest version
              # to see if there is a compatibility issue.
              Pkg.add(PackageSpec(; name=pkg, version=version))
              Pkg.test(PackageSpec(; name=pkg))  # resolver may fail with test time deps
            end
          catch err
            err isa Pkg.Resolve.ResolverError || rethrow()
            # If we can't resolve that means this is incompatible by SemVer and this is fine
            # It means we marked this as a breaking change, so we don't need to worry about
            # Mistakenly introducing a breaking change, as we have intentionally made one
            @info "Not compatible with this release. No problem." exception=err
            exit(0)  # Exit immediately, as a success
          end

  # Aggregate the matrix result into a single check named "IntegrationTest"
  # that branch protection can require. Behavior:
  #   - matrix succeeded            -> pass
  #   - `pkgs: '[]'`                -> pass (no downstream configured)
  #   - draft + run-on-draft=false  -> skip (matches the matrix-leg's draft
  #                                    skip; gate appears as Skipped on the PR
  #                                    rather than Failure for the expected
  #                                    no-signal case)
  #   - matrix all-skipped + fork   -> fail with /integrationtest hint
  #   - any failure                 -> fail
  # The fork-PR-all-skipped case is the trigger for the manual
  # `/integrationtest <pkg>` flow in IntegrationTestRequest.yml, which
  # posts a passing check with the same name to satisfy branch protection.
  gate:
    name: "IntegrationTest"
    needs: integration-test
    if: ${{ always() && (!github.event.pull_request.draft || inputs.run-on-draft) }}
    runs-on: ubuntu-latest
    permissions:
      actions: read
    steps:
      - name: "Download skipped-dependency markers"
        uses: actions/download-artifact@v8
        with:
          pattern: integration-test-skip-*
          path: skipped-integration-tests
          merge-multiple: true
        continue-on-error: true
      - name: "Verify integration tests"
        env:
          PKGS: ${{ inputs.pkgs }}
          RESULT: ${{ needs.integration-test.result }}
          EVENT_NAME: ${{ github.event_name }}
          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
          BASE_REPO: ${{ github.repository }}
        run: |
          echo "test.result=$RESULT pkgs=$PKGS event=$EVENT_NAME"
          pkg_count=$(printf '%s' "$PKGS" | jq 'length')
          if [ "$pkg_count" -eq 0 ]; then
            echo "No downstream integration tests configured; passing."
            exit 0
          fi
          skipped_count=0
          if [ -d skipped-integration-tests ]; then
            skipped_count=$(find skipped-integration-tests -type f | wc -l | tr -d ' ')
          fi
          echo "skipped dependency count: $skipped_count / $pkg_count"
          if [ "$RESULT" = "success" ]; then
            if [ "$skipped_count" -gt 0 ]; then
              if [ "$EVENT_NAME" = "pull_request" ] && [ -n "$HEAD_REPO" ] && [ "$HEAD_REPO" != "$BASE_REPO" ] && [ "$skipped_count" -eq "$pkg_count" ]; then
                echo "::error::All downstream integration tests skipped on this fork PR. A maintainer must run \`/integrationtest <pkg>\` for each private dep before merging."
                exit 1
              fi
              echo "::notice::$skipped_count downstream integration test(s) skipped because they require authentication in this event context."
            fi
            exit 0
          fi
          if [ "$RESULT" = "skipped" ]; then
            if [ "$pkg_count" -eq 0 ]; then
              echo "No downstream integration tests configured; passing."
              exit 0
            fi
            if [ "$EVENT_NAME" = "pull_request" ] && [ -n "$HEAD_REPO" ] && [ "$HEAD_REPO" != "$BASE_REPO" ]; then
              echo "::error::All downstream integration tests skipped on this fork PR. A maintainer must run \`/integrationtest <pkg>\` for each private dep before merging."
              exit 1
            fi
            echo "::error::Integration matrix unexpectedly all skipped (event=$EVENT_NAME)."
            exit 1
          fi
          echo "::error::Integration tests did not pass (result=$RESULT)."
          exit 1