Declare system LLVM as +python external instead of rebuilding it

.github/actions/setup-thapi-deps/action.yml

@@ -12,7 +12,7 @@ runs:
     - name: Install apt-get packages
       uses: awalsh128/cache-apt-pkgs-action@latest
       with:
-        packages: libnuma-dev
+        packages: libnuma-dev re2c protobuf-compiler libprotobuf-dev elfutils libelf-dev libdw-dev
     - name: Checkout Spack
       uses: actions/checkout@v6
       with:
@@ -30,9 +30,57 @@ runs:
         # Find external packages known to spack (avoid bzip2 and xz since
         # they cause build failures in elfutils, and libxml2 since externals
         # in system paths don't get added to PKG_CONFIG_PATH).
+        #
+        # llvm is declared by hand in a later step, because neither of these
+        # commands records the `+python` variant h2yaml needs. Any entry they
+        # leave behind would shadow it, so drop theirs. `compiler find` has no
+        # --exclude, hence the `rm`.
         . external/spack/share/spack/setup-env.sh
         spack compiler find
-        spack external find --all --exclude bzip2 --exclude xz --exclude libxml2
+        spack compiler rm -a llvm || true
+        spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2
+    - name: Pin the target microarchitecture
+      shell: bash
+      run: |
+        # Spack puts the microarchitecture in the dag hash, and the runner pool
+        # is a mix of cpu generations, so a package built on one runner does not
+        # match the same package on another and the build cache never hits.
+        #
+        # `targets:granularity:generic` is not enough: it still picks the *best*
+        # generic target the host supports, so an avx512 runner (icelake,
+        # sapphire rapids) concretizes to x86_64_v4 while a zen2/zen3 one gets
+        # x86_64_v3 -- the pool just splits two ways instead of five. Requiring
+        # an explicit target is what actually makes the hashes stable.
+        #
+        # x86_64_v3 (avx2) is the highest level every runner in the pool
+        # supports. If one ever does not, `host_compatible: true` makes the
+        # solve fail loudly rather than emit illegal instructions.
+        . external/spack/share/spack/setup-env.sh
+        spack config add "packages:all:require:target=x86_64_v3"
+    - name: Declare the apt re2c, protobuf and elfutils as externals
+      shell: bash
+      run: |
+        # The three most expensive packages in the dependency jobs (~17, ~17
+        # and ~6 min) that apt already ships. `spack external find` cannot see
+        # them -- no recipe declares `executables` or `libraries` -- so declare
+        # them by hand.
+        #
+        # This does pin PR CI to Noble's older protobuf and elfutils; the
+        # weekly cron `from_scratch` job installs with no externals and covers
+        # the current ones.
+        . external/spack/share/spack/setup-env.sh
+        for pkg_tool in re2c:re2c protobuf:protoc elfutils:eu-readelf; do
+          pkg=${pkg_tool%:*}
+          tool=${pkg_tool#*:}
+          python3 scripts/gen-external.py "${pkg}" --version-from "${tool}" \
+            > ${{ github.workspace }}/${pkg}-external.yaml
+          spack config add -f ${{ github.workspace }}/${pkg}-external.yaml
+          # The script only writes config, so check Spack took it: an
+          # external node has a `.external.path`, everything else null.
+          spack spec -j "${pkg}" | grep -v '^==>' \
+            | jq -e --arg p "${pkg}" '.spec.nodes[] | select(.name == $p) | .external.path' \
+            || { echo "${pkg} did not resolve to the external"; exit 1; }
+        done
     - name: Setup spack mirror
       shell: bash
       run: |
@@ -48,12 +96,20 @@ runs:
         # Source spack
         . external/spack/share/spack/setup-env.sh
 
-        # Find the llvm major version in the thapi@develop. We need to filter out
-        # the fetch lines generated by `spack spec` when it is run for the first time.
-        # These lines start with `==>`.
-        LLVM_VER=$( spack spec -j thapi@develop | grep -v '==>' | jq '.spec.nodes[] | select(.name == "llvm") | .version' )
-        # Extract major version (21) from "\"21.1.4\"".
-        LLVM_VER=$( echo $LLVM_VER | awk '{ split($0,a,"[\".]"); print a[2] }' )
+        # The llvm major version thapi@develop wants. `spack spec -j` writes its
+        # status lines ("==> Using cached archive: ...") to stdout alongside the
+        # json, so they have to be stripped before jq sees them.
+        #
+        # A spec can hold several llvm nodes (same version, different variants),
+        # so collapse them to the distinct majors and insist on exactly one --
+        # two would otherwise be concatenated into an unusable "22\n22".
+        LLVM_VER=$( spack spec -j thapi@develop | grep -v '^==>' \
+          | jq -r '[.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]]
+                   | unique | join(" ")' )
+        if [ "$( echo "${LLVM_VER}" | wc -w )" -ne 1 ]; then
+          echo "expected exactly one llvm major version, got: '${LLVM_VER}'"
+          exit 1
+        fi
 
         # Add llvm to apt repository.
         wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
@@ -65,24 +121,35 @@ runs:
         sudo apt-get install libllvm${LLVM_VER} llvm-${LLVM_VER} llvm-${LLVM_VER}-dev
         sudo apt-get install clang-${LLVM_VER} libclang-${LLVM_VER}-dev python3-clang-${LLVM_VER}
 
-        # Let spack find newly installed llvm/clang. Keep the same exclusions,
-        # otherwise this run re-adds them as externals.
+        # Pick up what apt pulled in, with the same exclusions as above.
         spack compiler find
-        spack external find --all --exclude bzip2 --exclude xz --exclude libxml2
+        spack compiler rm -a llvm || true
+        spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2
 
         LLVM_FULL_VER=`llvm-config-${LLVM_VER} --version`
         echo "LLVM_FULL_VER=${LLVM_FULL_VER}" >> ${GITHUB_ENV}
-    - name: Workaround for providing llvm+python in GitHub CI runners
+        # The next step needs the major version too.
+        echo "LLVM_VER=${LLVM_VER}" >> ${GITHUB_ENV}
+    - name: Declare the apt llvm as an external with +python
       shell: bash
       if: ${{ inputs.install-llvm == 'true' }}
       run: |
-        # Symlink python3-clang package to site-packages so h2yaml won't fail on
-        # `import clang`. This way, our llvm+python trick will work without an issue.
-        python_major=$(python -c "import sys; print(f'{sys.version_info.major}')")
-        python_minor=$(python -c "import sys; print(f'{sys.version_info.minor}')")
-        SITE_PACKAGES=/usr/lib/python${python_major}.${python_minor}/site-packages
-        sudo mkdir -p ${SITE_PACKAGES}
-        sudo ln -s /usr/lib/python${python_major}/dist-packages/clang ${SITE_PACKAGES}/clang
-
-        # Trick spack to think that llvm found in packages.yml has +python variant:
-        sed -i 's/llvm@[0-9][0-9]\.[0-9]\.[0-9]/&+python/' ~/.spack/packages.yaml
+        # `spack external find` never records +python (it picks variants from
+        # executables and never looks for clang's python bindings), so h2yaml's
+        # llvm@18:+clang+python would go unsatisfied and Spack would rebuild LLVM.
+        # The helper below declares the external properly; apt splits the bindings
+        # into /usr/lib/python3/dist-packages, which it finds and links in.
+        # Same script the README tells users to run.
+        . external/spack/share/spack/setup-env.sh
+        PYVER=$( spack spec -j h2yaml | grep -v '^==>' \
+          | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")]
+                   | unique | join(" ")' )
+        if [ "$( echo "${PYVER}" | wc -w )" -ne 1 ]; then
+          echo "expected exactly one python version, got: '${PYVER}'"
+          exit 1
+        fi
+        python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \
+          --python-version "${PYVER}" \
+          --overlay-dir ${{ github.workspace }} \
+          > ${{ github.workspace }}/llvm-external.yaml
+        spack config add -f ${{ github.workspace }}/llvm-external.yaml

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ on:
         required: false
         default: false
 env:
-  SPACK_OPT: "--concurrent-packages 2 --show-log-on-error --fail-fast"
+  SPACK_OPT: "--show-log-on-error --fail-fast"
   CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}"
   CACHE_DIR_LEGACY: "${{ github.workspace }}/thapi_cache_legacy_${{ github.run_id }}"
 jobs:
@@ -35,10 +35,23 @@ jobs:
     - name: Install dependencies
       run: |
         # Install the dependencies of the latest legacy version.
+        # --include-build-deps: with a mirror configured, the installer keeps
+        # build deps out of the graph and only pulls them in after a package
+        # misses the cache. `--only dependencies` never installs thapi itself,
+        # so that miss never happens and thapi's build-only deps (re2c,
+        # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed --
+        # leaving every downstream job to build them from source.
         . external/spack/share/spack/setup-env.sh
-        spack install --only dependencies $SPACK_OPT thapi@0.0.12
+        spack install --only dependencies --include-build-deps $SPACK_OPT thapi@0.0.12
+    - name: Index the build cache
+      run: |
+        # `--autopush` uploads packages but writes no index, and a mirror
+        # without one is skipped during concretization ("no index found"),
+        # leaving the downstream jobs to rebuild what is already here.
+        . external/spack/share/spack/setup-env.sh
+        spack buildcache update-index ${{ env.CACHE_DIR_LEGACY }}
     - name: Upload THAPI build cache
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: thapi_cache_legacy
         include-hidden-files: true
@@ -61,7 +74,7 @@ jobs:
       uses: mxschmitt/action-tmate@v3
       if: ${{ inputs.debug_enabled }}
     - name: Download THAPI build cache
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v7
       with:
         name: thapi_cache_legacy
         path: ${{ env.CACHE_DIR_LEGACY }}
@@ -98,9 +111,22 @@ jobs:
         # We have to use the exact llvm version from apt package as a direct dependency
         # as the versions in apt and spack doesn't agree to the patch version. The
         # latter will force spack to build llvm from scratch.
-        spack install --only dependencies $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER}
+        # --include-build-deps: with a mirror configured, the installer keeps
+        # build deps out of the graph and only pulls them in after a package
+        # misses the cache. `--only dependencies` never installs thapi itself,
+        # so that miss never happens and thapi's build-only deps (re2c,
+        # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed --
+        # leaving every downstream job to build them from source.
+        spack install --only dependencies --include-build-deps $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER}
+    - name: Index the build cache
+      run: |
+        # `--autopush` uploads packages but writes no index, and a mirror
+        # without one is skipped during concretization ("no index found"),
+        # leaving the downstream jobs to rebuild what is already here.
+        . external/spack/share/spack/setup-env.sh
+        spack buildcache update-index ${{ env.CACHE_DIR }}
     - name: Upload THAPI build cache
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: thapi_cache
         include-hidden-files: true
@@ -125,7 +151,7 @@ jobs:
       uses: mxschmitt/action-tmate@v3
       if: ${{ inputs.debug_enabled }}
     - name: Download THAPI build cache
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v7
       with:
         name: thapi_cache
         path: ${{ env.CACHE_DIR }}
@@ -158,7 +184,7 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v6
-    - uses: geekyeggo/delete-artifact@v5
+    - uses: geekyeggo/delete-artifact@v6
       with:
         name: |
           thapi_cache_legacy

.github/workflows/cron.yml

@@ -4,9 +4,19 @@ on:
     # At 00:00 on Sunday.”
     - cron: "0 0 * * 0"
   workflow_dispatch:
+env:
+  SPACK_OPT: "--show-log-on-error --fail-fast"
+  CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}"
 jobs:
-  myJob:
+  # Install with nothing pre-arranged: no `spack external find`, no system llvm,
+  # no build cache. Checks that THAPI installs on a bare machine.
+  from_scratch:
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ['thapi@master', 'thapi@develop']
     runs-on: ubuntu-latest
+    name: Install ${{ matrix.version }} (from scratch)
     steps:
     - name: Checkout
       uses: actions/checkout@v6
@@ -20,11 +30,32 @@ jobs:
       run: |
         . external/spack/share/spack/setup-env.sh
         spack repo add ./
-    - name: Install thapi master
+    - name: Install ${{ matrix.version }}
       run: |
         . external/spack/share/spack/setup-env.sh
-        spack install $SPACK_OPT thapi@master
-    - name: Install thapi develop
+        spack install $SPACK_OPT ${{ matrix.version }}
+
+  # Exercise the same fast path the PR CI uses (external find + apt llvm +
+  # hand-declared +python external). PR CI only runs when someone opens a PR, so
+  # without this the fast path can rot unnoticed during a quiet week.
+  fast_path:
+    runs-on: ubuntu-latest
+    name: Install thapi@develop (fast path)
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+    - name: Checkout dependencies (spack,...)
+      uses: ./.github/actions/setup-thapi-deps
+      with:
+        cache-dir: ${{ env.CACHE_DIR }}
+        install-llvm: 'true'
+    - name: Check llvm resolves to the external
+      run: |
+        . external/spack/share/spack/setup-env.sh
+        spack spec -j thapi@develop ^llvm@${LLVM_FULL_VER} | grep -v '^==>' \
+          | jq -e '.spec.nodes[] | select(.name == "llvm") | .external.path' \
+          || { echo "llvm did not resolve to the external"; exit 1; }
+    - name: Install thapi@develop
       run: |
         . external/spack/share/spack/setup-env.sh
-        spack install $SPACK_OPT thapi@develop
+        spack install $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER}

.github/workflows/lint.yml

@@ -22,6 +22,6 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v6
     - name: ruff format check
-      uses: astral-sh/ruff-action@v3
+      uses: astral-sh/ruff-action@v4.1.0
       with:
         args: "format --check --diff"

README.md

@@ -29,32 +29,65 @@ spack load thapi
 avoid building them when building THAPI. Use the following command before installing THAPI to find external
 packages available on the system:
 ```bash
-spack external find --all
+spack external find --all --exclude llvm
 ```
+Always exclude `llvm`: the entry detection writes cannot satisfy `h2yaml`. Declare that one with
+the helper instead -- see [Reusing a system LLVM](#reusing-a-system-llvm).
+
 Make sure to `module load` the packages you want Spack to find (or set other environment variables like `PATH`)
-before running `spack external find --all`.
+before running it.
 
 Some packages when found using `spack external find` are known to cause build failures. If you run into such
 cases, use `spack external find --exclude <pkg>` so that Spack will build them instead of using the system
 installed versions. For example:
 ```bash
-spack external find --all --exclude bzip2 --exclude xz --exclude curl
+spack external find --all --exclude bzip2 --exclude xz --exclude curl --exclude llvm
 ```
 
-#### `spack install -j<core> <spec>`
+#### Reusing a system LLVM
 
-Depending on the number of available cores on your platform, you can specify number of parallel build
-processes to be used during the build of `spec` using `-j` option to `spack` (e.g., `-j16`).
+`thapi@0.0.14:` needs `h2yaml`, which needs `llvm@18:+clang+python`. That `+python` means the LLVM
+prefix must contain clang's Python bindings (`clang/cindex.py`), not just `libclang.so`. Building
+LLVM from source is by far the longest step of a THAPI install, so it is worth reusing a system one.
 
-#### `--concurrent-packages`
+Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/llvm-*`,
+`/opt/llvm*`, or your site software tree) and apply what it prints:
 
-One may also be able to reduce the time to install THAPI by using `--concurrent-packages` option in `spack install`
-as below:
 ```bash
-spack install --concurrent-packages 2 thapi
+# The python Spack will run h2yaml with, so the overlay is laid out to match.
+PYVER=$(spack spec -j h2yaml | grep -v '^==>' \
+  | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")]
+           | unique | .[0]')
+
+python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml
+spack config add -f llvm-external.yaml
 ```
-Depending on the available number of CPU cores and how parallelizable the dependency graph is for the particular
-specification, one may be able to specify more concurrent packages (as compared to `2` in the above example).
+
+The script needs only Python 3, no dependencies. If the LLVM already has the bindings it points the
+external straight at it. Otherwise -- a site LLVM is usually read-only, so they cannot just be
+dropped in next to it -- it builds an *overlay* in `--overlay-dir` (default `~/.spack`): a tree of
+symlinks to the real install, plus the bindings, downloaded from the matching LLVM release if the
+system has none. The external points into that directory, so keep it.
+
+If there is no LLVM 18+ on the system at all, skip this: Spack will build one.
+
+Check that it took -- LLVM should show `[e]` rather than `-`:
+
+```bash
+spack spec -I thapi | grep llvm
+```
+
+> [!IMPORTANT]
+> The external has to be declared this way rather than by `spack external find`, which picks variants
+> from executables only and never looks for the bindings. It therefore always records LLVM as
+> `~python` -- bindings present or not -- and such an entry can never satisfy `h2yaml`: Spack ignores
+> it and, unless something else provides `+python`, builds LLVM from source. Always pass
+> `--exclude llvm` when running it.
+
+#### `spack install -j<core> <spec>`
+
+Depending on the number of available cores on your platform, you can specify number of parallel build
+processes to be used during the build of `spec` using `-j` option to `spack` (e.g., `-j16`).
 
 ## Building THAPI Manually