diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 8b1d6d64..04e08098 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -12,7 +12,7 @@ runs: - name: Install apt-get packages uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libnuma-dev + packages: libnuma-dev re2c protobuf-compiler libprotobuf-dev elfutils libelf-dev libdw-dev - name: Checkout Spack uses: actions/checkout@v6 with: @@ -30,9 +30,57 @@ runs: # Find external packages known to spack (avoid bzip2 and xz since # they cause build failures in elfutils, and libxml2 since externals # in system paths don't get added to PKG_CONFIG_PATH). + # + # llvm is declared by hand in a later step, because neither of these + # commands records the `+python` variant h2yaml needs. Any entry they + # leave behind would shadow it, so drop theirs. `compiler find` has no + # --exclude, hence the `rm`. . external/spack/share/spack/setup-env.sh spack compiler find - spack external find --all --exclude bzip2 --exclude xz --exclude libxml2 + spack compiler rm -a llvm || true + spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 + - name: Pin the target microarchitecture + shell: bash + run: | + # Spack puts the microarchitecture in the dag hash, and the runner pool + # is a mix of cpu generations, so a package built on one runner does not + # match the same package on another and the build cache never hits. + # + # `targets:granularity:generic` is not enough: it still picks the *best* + # generic target the host supports, so an avx512 runner (icelake, + # sapphire rapids) concretizes to x86_64_v4 while a zen2/zen3 one gets + # x86_64_v3 -- the pool just splits two ways instead of five. Requiring + # an explicit target is what actually makes the hashes stable. + # + # x86_64_v3 (avx2) is the highest level every runner in the pool + # supports. If one ever does not, `host_compatible: true` makes the + # solve fail loudly rather than emit illegal instructions. + . external/spack/share/spack/setup-env.sh + spack config add "packages:all:require:target=x86_64_v3" + - name: Declare the apt re2c, protobuf and elfutils as externals + shell: bash + run: | + # The three most expensive packages in the dependency jobs (~17, ~17 + # and ~6 min) that apt already ships. `spack external find` cannot see + # them -- no recipe declares `executables` or `libraries` -- so declare + # them by hand. + # + # This does pin PR CI to Noble's older protobuf and elfutils; the + # weekly cron `from_scratch` job installs with no externals and covers + # the current ones. + . external/spack/share/spack/setup-env.sh + for pkg_tool in re2c:re2c protobuf:protoc elfutils:eu-readelf; do + pkg=${pkg_tool%:*} + tool=${pkg_tool#*:} + python3 scripts/gen-external.py "${pkg}" --version-from "${tool}" \ + > ${{ github.workspace }}/${pkg}-external.yaml + spack config add -f ${{ github.workspace }}/${pkg}-external.yaml + # The script only writes config, so check Spack took it: an + # external node has a `.external.path`, everything else null. + spack spec -j "${pkg}" | grep -v '^==>' \ + | jq -e --arg p "${pkg}" '.spec.nodes[] | select(.name == $p) | .external.path' \ + || { echo "${pkg} did not resolve to the external"; exit 1; } + done - name: Setup spack mirror shell: bash run: | @@ -48,12 +96,20 @@ runs: # Source spack . external/spack/share/spack/setup-env.sh - # Find the llvm major version in the thapi@develop. We need to filter out - # the fetch lines generated by `spack spec` when it is run for the first time. - # These lines start with `==>`. - LLVM_VER=$( spack spec -j thapi@develop | grep -v '==>' | jq '.spec.nodes[] | select(.name == "llvm") | .version' ) - # Extract major version (21) from "\"21.1.4\"". - LLVM_VER=$( echo $LLVM_VER | awk '{ split($0,a,"[\".]"); print a[2] }' ) + # The llvm major version thapi@develop wants. `spack spec -j` writes its + # status lines ("==> Using cached archive: ...") to stdout alongside the + # json, so they have to be stripped before jq sees them. + # + # A spec can hold several llvm nodes (same version, different variants), + # so collapse them to the distinct majors and insist on exactly one -- + # two would otherwise be concatenated into an unusable "22\n22". + LLVM_VER=$( spack spec -j thapi@develop | grep -v '^==>' \ + | jq -r '[.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]] + | unique | join(" ")' ) + if [ "$( echo "${LLVM_VER}" | wc -w )" -ne 1 ]; then + echo "expected exactly one llvm major version, got: '${LLVM_VER}'" + exit 1 + fi # Add llvm to apt repository. wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -65,24 +121,35 @@ runs: sudo apt-get install libllvm${LLVM_VER} llvm-${LLVM_VER} llvm-${LLVM_VER}-dev sudo apt-get install clang-${LLVM_VER} libclang-${LLVM_VER}-dev python3-clang-${LLVM_VER} - # Let spack find newly installed llvm/clang. Keep the same exclusions, - # otherwise this run re-adds them as externals. + # Pick up what apt pulled in, with the same exclusions as above. spack compiler find - spack external find --all --exclude bzip2 --exclude xz --exclude libxml2 + spack compiler rm -a llvm || true + spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 LLVM_FULL_VER=`llvm-config-${LLVM_VER} --version` echo "LLVM_FULL_VER=${LLVM_FULL_VER}" >> ${GITHUB_ENV} - - name: Workaround for providing llvm+python in GitHub CI runners + # The next step needs the major version too. + echo "LLVM_VER=${LLVM_VER}" >> ${GITHUB_ENV} + - name: Declare the apt llvm as an external with +python shell: bash if: ${{ inputs.install-llvm == 'true' }} run: | - # Symlink python3-clang package to site-packages so h2yaml won't fail on - # `import clang`. This way, our llvm+python trick will work without an issue. - python_major=$(python -c "import sys; print(f'{sys.version_info.major}')") - python_minor=$(python -c "import sys; print(f'{sys.version_info.minor}')") - SITE_PACKAGES=/usr/lib/python${python_major}.${python_minor}/site-packages - sudo mkdir -p ${SITE_PACKAGES} - sudo ln -s /usr/lib/python${python_major}/dist-packages/clang ${SITE_PACKAGES}/clang - - # Trick spack to think that llvm found in packages.yml has +python variant: - sed -i 's/llvm@[0-9][0-9]\.[0-9]\.[0-9]/&+python/' ~/.spack/packages.yaml + # `spack external find` never records +python (it picks variants from + # executables and never looks for clang's python bindings), so h2yaml's + # llvm@18:+clang+python would go unsatisfied and Spack would rebuild LLVM. + # The helper below declares the external properly; apt splits the bindings + # into /usr/lib/python3/dist-packages, which it finds and links in. + # Same script the README tells users to run. + . external/spack/share/spack/setup-env.sh + PYVER=$( spack spec -j h2yaml | grep -v '^==>' \ + | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")] + | unique | join(" ")' ) + if [ "$( echo "${PYVER}" | wc -w )" -ne 1 ]; then + echo "expected exactly one python version, got: '${PYVER}'" + exit 1 + fi + python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \ + --python-version "${PYVER}" \ + --overlay-dir ${{ github.workspace }} \ + > ${{ github.workspace }}/llvm-external.yaml + spack config add -f ${{ github.workspace }}/llvm-external.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 346822e2..4a101ff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ on: required: false default: false env: - SPACK_OPT: "--concurrent-packages 2 --show-log-on-error --fail-fast" + SPACK_OPT: "--show-log-on-error --fail-fast" CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}" CACHE_DIR_LEGACY: "${{ github.workspace }}/thapi_cache_legacy_${{ github.run_id }}" jobs: @@ -35,10 +35,23 @@ jobs: - name: Install dependencies run: | # Install the dependencies of the latest legacy version. + # --include-build-deps: with a mirror configured, the installer keeps + # build deps out of the graph and only pulls them in after a package + # misses the cache. `--only dependencies` never installs thapi itself, + # so that miss never happens and thapi's build-only deps (re2c, + # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed -- + # leaving every downstream job to build them from source. . external/spack/share/spack/setup-env.sh - spack install --only dependencies $SPACK_OPT thapi@0.0.12 + spack install --only dependencies --include-build-deps $SPACK_OPT thapi@0.0.12 + - name: Index the build cache + run: | + # `--autopush` uploads packages but writes no index, and a mirror + # without one is skipped during concretization ("no index found"), + # leaving the downstream jobs to rebuild what is already here. + . external/spack/share/spack/setup-env.sh + spack buildcache update-index ${{ env.CACHE_DIR_LEGACY }} - name: Upload THAPI build cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: thapi_cache_legacy include-hidden-files: true @@ -61,7 +74,7 @@ jobs: uses: mxschmitt/action-tmate@v3 if: ${{ inputs.debug_enabled }} - name: Download THAPI build cache - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: thapi_cache_legacy path: ${{ env.CACHE_DIR_LEGACY }} @@ -98,9 +111,22 @@ jobs: # We have to use the exact llvm version from apt package as a direct dependency # as the versions in apt and spack doesn't agree to the patch version. The # latter will force spack to build llvm from scratch. - spack install --only dependencies $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} + # --include-build-deps: with a mirror configured, the installer keeps + # build deps out of the graph and only pulls them in after a package + # misses the cache. `--only dependencies` never installs thapi itself, + # so that miss never happens and thapi's build-only deps (re2c, + # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed -- + # leaving every downstream job to build them from source. + spack install --only dependencies --include-build-deps $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} + - name: Index the build cache + run: | + # `--autopush` uploads packages but writes no index, and a mirror + # without one is skipped during concretization ("no index found"), + # leaving the downstream jobs to rebuild what is already here. + . external/spack/share/spack/setup-env.sh + spack buildcache update-index ${{ env.CACHE_DIR }} - name: Upload THAPI build cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: thapi_cache include-hidden-files: true @@ -125,7 +151,7 @@ jobs: uses: mxschmitt/action-tmate@v3 if: ${{ inputs.debug_enabled }} - name: Download THAPI build cache - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: thapi_cache path: ${{ env.CACHE_DIR }} @@ -158,7 +184,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: geekyeggo/delete-artifact@v5 + - uses: geekyeggo/delete-artifact@v6 with: name: | thapi_cache_legacy diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 831e5ab1..90dc5732 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -4,9 +4,19 @@ on: # At 00:00 on Sunday.” - cron: "0 0 * * 0" workflow_dispatch: +env: + SPACK_OPT: "--show-log-on-error --fail-fast" + CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}" jobs: - myJob: + # Install with nothing pre-arranged: no `spack external find`, no system llvm, + # no build cache. Checks that THAPI installs on a bare machine. + from_scratch: + strategy: + fail-fast: false + matrix: + version: ['thapi@master', 'thapi@develop'] runs-on: ubuntu-latest + name: Install ${{ matrix.version }} (from scratch) steps: - name: Checkout uses: actions/checkout@v6 @@ -20,11 +30,32 @@ jobs: run: | . external/spack/share/spack/setup-env.sh spack repo add ./ - - name: Install thapi master + - name: Install ${{ matrix.version }} run: | . external/spack/share/spack/setup-env.sh - spack install $SPACK_OPT thapi@master - - name: Install thapi develop + spack install $SPACK_OPT ${{ matrix.version }} + + # Exercise the same fast path the PR CI uses (external find + apt llvm + + # hand-declared +python external). PR CI only runs when someone opens a PR, so + # without this the fast path can rot unnoticed during a quiet week. + fast_path: + runs-on: ubuntu-latest + name: Install thapi@develop (fast path) + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Checkout dependencies (spack,...) + uses: ./.github/actions/setup-thapi-deps + with: + cache-dir: ${{ env.CACHE_DIR }} + install-llvm: 'true' + - name: Check llvm resolves to the external + run: | + . external/spack/share/spack/setup-env.sh + spack spec -j thapi@develop ^llvm@${LLVM_FULL_VER} | grep -v '^==>' \ + | jq -e '.spec.nodes[] | select(.name == "llvm") | .external.path' \ + || { echo "llvm did not resolve to the external"; exit 1; } + - name: Install thapi@develop run: | . external/spack/share/spack/setup-env.sh - spack install $SPACK_OPT thapi@develop + spack install $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1e3b88a7..b707fc70 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,6 +22,6 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: ruff format check - uses: astral-sh/ruff-action@v3 + uses: astral-sh/ruff-action@v4.1.0 with: args: "format --check --diff" diff --git a/README.md b/README.md index 9bcbb124..32a7d74d 100644 --- a/README.md +++ b/README.md @@ -29,32 +29,65 @@ spack load thapi avoid building them when building THAPI. Use the following command before installing THAPI to find external packages available on the system: ```bash -spack external find --all +spack external find --all --exclude llvm ``` +Always exclude `llvm`: the entry detection writes cannot satisfy `h2yaml`. Declare that one with +the helper instead -- see [Reusing a system LLVM](#reusing-a-system-llvm). + Make sure to `module load` the packages you want Spack to find (or set other environment variables like `PATH`) -before running `spack external find --all`. +before running it. Some packages when found using `spack external find` are known to cause build failures. If you run into such cases, use `spack external find --exclude ` so that Spack will build them instead of using the system installed versions. For example: ```bash -spack external find --all --exclude bzip2 --exclude xz --exclude curl +spack external find --all --exclude bzip2 --exclude xz --exclude curl --exclude llvm ``` -#### `spack install -j ` +#### Reusing a system LLVM -Depending on the number of available cores on your platform, you can specify number of parallel build -processes to be used during the build of `spec` using `-j` option to `spack` (e.g., `-j16`). +`thapi@0.0.14:` needs `h2yaml`, which needs `llvm@18:+clang+python`. That `+python` means the LLVM +prefix must contain clang's Python bindings (`clang/cindex.py`), not just `libclang.so`. Building +LLVM from source is by far the longest step of a THAPI install, so it is worth reusing a system one. -#### `--concurrent-packages` +Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/llvm-*`, +`/opt/llvm*`, or your site software tree) and apply what it prints: -One may also be able to reduce the time to install THAPI by using `--concurrent-packages` option in `spack install` -as below: ```bash -spack install --concurrent-packages 2 thapi +# The python Spack will run h2yaml with, so the overlay is laid out to match. +PYVER=$(spack spec -j h2yaml | grep -v '^==>' \ + | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")] + | unique | .[0]') + +python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml +spack config add -f llvm-external.yaml ``` -Depending on the available number of CPU cores and how parallelizable the dependency graph is for the particular -specification, one may be able to specify more concurrent packages (as compared to `2` in the above example). + +The script needs only Python 3, no dependencies. If the LLVM already has the bindings it points the +external straight at it. Otherwise -- a site LLVM is usually read-only, so they cannot just be +dropped in next to it -- it builds an *overlay* in `--overlay-dir` (default `~/.spack`): a tree of +symlinks to the real install, plus the bindings, downloaded from the matching LLVM release if the +system has none. The external points into that directory, so keep it. + +If there is no LLVM 18+ on the system at all, skip this: Spack will build one. + +Check that it took -- LLVM should show `[e]` rather than `-`: + +```bash +spack spec -I thapi | grep llvm +``` + +> [!IMPORTANT] +> The external has to be declared this way rather than by `spack external find`, which picks variants +> from executables only and never looks for the bindings. It therefore always records LLVM as +> `~python` -- bindings present or not -- and such an entry can never satisfy `h2yaml`: Spack ignores +> it and, unless something else provides `+python`, builds LLVM from source. Always pass +> `--exclude llvm` when running it. + +#### `spack install -j ` + +Depending on the number of available cores on your platform, you can specify number of parallel build +processes to be used during the build of `spec` using `-j` option to `spack` (e.g., `-j16`). ## Building THAPI Manually diff --git a/packages/h2yaml/package.py b/packages/h2yaml/package.py index 6b55b567..a42bcb0e 100644 --- a/packages/h2yaml/package.py +++ b/packages/h2yaml/package.py @@ -15,6 +15,27 @@ def find_libclang(root, lib_ext): return os.path.join(root, file) +def find_clang_bindings(root): + """Return the site-packages dir holding clang's python bindings (clang/cindex.py). + + LLVM installs these under lib/python/site-packages, but the python + version in that path is whichever python LLVM was built against, which need + not be the one h2yaml runs with. External installs vary further: lib64 on + RHEL-likes, dist-packages on Debian/Ubuntu. So search instead of guessing. + """ + for libdir in ("lib", "lib64"): + base = os.path.join(root, libdir) + if not os.path.isdir(base): + continue + for entry in sorted(os.listdir(base)): + if not entry.startswith("python"): + continue + for pkgs in ("site-packages", "dist-packages"): + sp = os.path.join(base, entry, pkgs) + if os.path.isfile(os.path.join(sp, "clang", "cindex.py")): + return sp + + class H2yaml(PythonPackage): """Matrices describing affine transformation of the plane.""" @@ -63,10 +84,25 @@ def setup_run_environment(self, env): lib_so = join_path(self.spec["llvm"].prefix.lib, f"libclang.{lib_ext}") if not os.path.isfile(lib_so): lib_so = find_libclang(lib_path, lib_ext) - env.set("LIBCLANG_LIBRARY_FILE", join_path(self.spec["llvm"].prefix.lib, lib_so)) + if not lib_so: + raise InstallError( + f"h2yaml needs libclang.{lib_ext}, but none was found under " + f"{self.spec['llvm'].prefix}. If this is an external llvm, check that " + "its prefix really contains a libclang shared library." + ) + # lib_so is already absolute; do not re-join it onto prefix.lib. + env.set("LIBCLANG_LIBRARY_FILE", lib_so) # Set PYTHONPATH so that `import clang` will work without an issue. - env.append_path("PYTHONPATH", join_path(self.spec["llvm"].prefix.lib, f"python{self.spec['python'].version.up_to(2)}", "site-packages")) + site_packages = find_clang_bindings(self.spec["llvm"].prefix) + if not site_packages: + raise InstallError( + "h2yaml needs clang's python bindings (clang/cindex.py), but none were " + f"found under {self.spec['llvm'].prefix}. Note that external LLVM installs " + "frequently omit them even though they ship libclang. See the " + "'Reusing a system LLVM' section of the THAPI-spack README." + ) + env.append_path("PYTHONPATH", site_packages) def setup_test_environment(self, env): if self.version < Version("0.4.1"): diff --git a/scripts/gen-external.py b/scripts/gen-external.py new file mode 100755 index 00000000..fe273344 --- /dev/null +++ b/scripts/gen-external.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +r"""Declare a system package as a Spack external, with the version read from a tool. + +``spack external find`` only detects packages whose recipe declares ``executables`` +or ``libraries`` -- 172 of ~8900 in the builtin repo. Everything else has to be +declared by hand, even when it is plainly installed: ``re2c`` and ``protobuf`` are +both apt packages that Spack will otherwise spend ~15 minutes each rebuilding. + +This writes the ``packages.yaml`` entry for one such package. The version comes from +running the tool itself, so the config cannot drift from what is actually installed:: + + python3 scripts/gen-external.py re2c --version-from re2c + python3 scripts/gen-external.py protobuf --version-from protoc + +The config is printed on stdout, so it can be redirected or piped into Spack:: + + python3 scripts/gen-external.py re2c --version-from re2c > ext.yaml + spack config add -f ext.yaml + +Pass ``--version`` instead of ``--version-from`` when there is no tool to ask, and +``--prefix`` when the package does not live in ``/usr``. + +Note this only writes the config. Whether Spack actually picks the external up +depends on the version satisfying what depends on it -- check with ``spack spec``, +which marks an external ``[e]``. + +For LLVM use ``gen-llvm-external.py`` instead: h2yaml needs a ``+python`` variant +whose bindings distributions scatter outside the prefix, which needs real work +beyond writing a version and a path. +""" + +import argparse +import re +import subprocess +import sys +import textwrap + + +def die(msg): + sys.exit(f"error: {msg}") + + +def version_from_tool(tool): + """First dotted version in `` --version`` output. + + Covers the usual shapes without a per-tool rule: "re2c 3.1", + "libprotoc 3.21.12", "cmake version 3.28.3". + """ + try: + out = subprocess.check_output([tool, "--version"], universal_newlines=True, stderr=subprocess.STDOUT) + except OSError as e: + die(f"cannot run {tool} --version ({e}) -- is it installed and on PATH?") + except subprocess.CalledProcessError as e: + die(f"{tool} --version failed with status {e.returncode}:\n{e.output}") + m = re.search(r"\d+(?:\.\d+)+", out) + if not m: + die(f"no version found in {tool} --version output:\n{out}") + return m.group(0) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("package", help="Spack package name, e.g. re2c") + g = p.add_mutually_exclusive_group(required=True) + g.add_argument("--version-from", metavar="TOOL", help="read the version from `TOOL --version`") + g.add_argument("--version", help="use this version verbatim") + p.add_argument("--prefix", default="/usr", help="install prefix (default: /usr)") + p.add_argument( + "--buildable", + action="store_true", + help="let Spack build the package too. By default it may not, so a version that stops" + " satisfying its dependents fails the solve instead of silently rebuilding from source", + ) + p.add_argument( + "--variants", + default="", + help="variants to append to the spec, e.g. '+shared'. Only add what is actually true of" + " the installed package: Spack trusts the spec and will not verify it", + ) + args = p.parse_args() + + version = args.version or version_from_tool(args.version_from) + spec = f"{args.package}@{version}{args.variants}" + + yaml = textwrap.dedent(f"""\ + packages: + {args.package}: + externals: + - spec: {spec} + prefix: {args.prefix} + buildable: {str(args.buildable).lower()} + """) + # The config goes to stdout so it can be piped straight into Spack; everything + # else is progress info and belongs on stderr. + print(f"{spec}: prefix {args.prefix}", file=sys.stderr) + print(yaml, end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/gen-llvm-external.py b/scripts/gen-llvm-external.py new file mode 100755 index 00000000..8018b0fe --- /dev/null +++ b/scripts/gen-llvm-external.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +r"""Declare a system LLVM as a Spack external that satisfies h2yaml. + +h2yaml needs ``llvm@18:+clang+python``, where ``+python`` means the prefix ships +clang's Python bindings (``clang/cindex.py``) next to ``libclang``. Two things make +this awkward to set up by hand: + +* ``spack external find`` picks variants from executables and never looks for the + bindings, so it always records LLVM as ``~python`` -- which cannot satisfy h2yaml, + leaving the system LLVM unused. +* Distributions split the pieces up (Debian keeps the bindings in + ``/usr/lib/python3/dist-packages``, far from ``/usr/lib/llvm-N``) and site installs + are usually read-only, so the bindings often cannot be dropped in place. + +This script resolves both. It locates the bindings, and if they live outside the LLVM +prefix -- or are missing entirely, in which case they are fetched from the matching +LLVM release, since they are pure Python -- it builds an *overlay* prefix: a tree of +symlinks to the real install plus a ``site-packages`` holding the bindings. It then +prints the ``packages.yaml`` config pointing at whichever prefix is usable. + +The config is printed on stdout, so it can be redirected or piped straight into Spack:: + + python3 scripts/gen-llvm-external.py /usr/lib/llvm-18 # just look at it + python3 scripts/gen-llvm-external.py /usr/lib/llvm-18 > ext.yaml && spack config add -f ext.yaml + spack config add -f <(python3 scripts/gen-llvm-external.py /usr/lib/llvm-18) + +Pass ``--python-version`` to match the python Spack will run h2yaml with; it defaults +to the running python. Any overlay is created under ``--overlay-dir`` and must be +kept, since the external points into it. +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import textwrap +import urllib.error +import urllib.request +from pathlib import Path + +# The bindings are two or three plain .py files. enumerations.py was dropped after +# LLVM 18, so it is fetched only if the tag actually has it. +BINDINGS_REQUIRED = ("__init__.py", "cindex.py") +BINDINGS_OPTIONAL = ("enumerations.py",) +RAW_URL = "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{version}/clang/bindings/python/clang/{name}" + +# Overlays must outlive the command, since the Spack external points into one. +# ~/.spack is where the packages.yaml referring to it lives, so they travel together. +DEFAULT_OVERLAY_DIR = Path("~/.spack") + + +def die(msg): + sys.exit(f"error: {msg}") + + +def llvm_version(prefix): + """Full x.y.z version of the LLVM at *prefix*, via its own llvm-config.""" + llvm_config = prefix / "bin" / "llvm-config" + if not llvm_config.is_file(): + die(f"no llvm-config in {prefix}/bin -- is that really an LLVM prefix?") + raw = subprocess.check_output([str(llvm_config), "--version"], universal_newlines=True).strip() + # Strip suffixes such as "19.1.7git", which are not release tags. + m = re.match(r"^(\d+)\.(\d+)\.(\d+)", raw) + if not m: + die(f"cannot parse llvm-config --version output {raw!r}") + return m.group(0), int(m.group(1)) + + +def find_bindings(*roots): + """Directory containing ``clang/cindex.py``, searched over usual layouts.""" + for root in roots: + if root is None or not root.is_dir(): + continue + for libdir in ("lib", "lib64"): + base = root / libdir + if not base.is_dir(): + continue + for pydir in sorted(base.glob("python*")): + for pkgs in ("site-packages", "dist-packages"): + if (pydir / pkgs / "clang" / "cindex.py").is_file(): + return pydir / pkgs + # Debian: /usr/lib/python3/dist-packages, with no python level. + for pkgs in root.glob("lib/python3/dist-packages"): + if (pkgs / "clang" / "cindex.py").is_file(): + return pkgs + return None + + +def fetch_bindings(version, dest): + """Download clang's pure-python bindings for *version* into ``dest/clang``.""" + out = dest / "clang" + out.mkdir(parents=True, exist_ok=True) + for name in BINDINGS_REQUIRED + BINDINGS_OPTIONAL: + url = RAW_URL.format(version=version, name=name) + try: + with urllib.request.urlopen(url) as r: + (out / name).write_bytes(r.read()) + except (urllib.error.URLError, OSError) as e: + if name in BINDINGS_OPTIONAL: + continue + die(f"cannot fetch {name} from llvmorg-{version} ({e}) -- is that a real release tag?") + return out.parent + + +def build_overlay(llvm, overlay, bindings, py_version): + """Symlink *llvm* into *overlay*, with *bindings* exposed on Spack's python path. + + The real install is usually read-only, so nothing is copied into it: every entry + is symlinked, except ``lib`` which is recreated as a real directory so the + bindings can be added alongside the symlinked libraries. + """ + if overlay.exists(): + shutil.rmtree(overlay) + (overlay / "lib").mkdir(parents=True) + for entry in llvm.iterdir(): + if entry.name != "lib": + (overlay / entry.name).symlink_to(entry) + for entry in (llvm / "lib").iterdir(): + (overlay / "lib" / entry.name).symlink_to(entry) + + site = overlay / "lib" / f"python{py_version}" / "site-packages" + site.parent.mkdir(parents=True, exist_ok=True) + if bindings is None: + version, _ = llvm_version(llvm) + site.mkdir() + fetch_bindings(version, site) + else: + site.symlink_to(bindings) + return overlay + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("prefix", type=Path, help="system LLVM prefix, e.g. /usr/lib/llvm-18") + p.add_argument( + "--overlay-dir", + type=Path, + default=DEFAULT_OVERLAY_DIR, + help=f"where to create the overlay, if one is needed. It must keep existing for as long as Spack uses the external (default: {DEFAULT_OVERLAY_DIR})", + ) + p.add_argument("--python-version", help="python X.Y Spack runs h2yaml with (default: this interpreter)") + args = p.parse_args() + + llvm = args.prefix.resolve() + version, major = llvm_version(llvm) + if major < 18: + die(f"h2yaml needs llvm@18: but {llvm} is {version}") + + py_version = args.python_version or f"{sys.version_info.major}.{sys.version_info.minor}" + + # Bindings already inside the prefix: usable as-is, no overlay. + bindings = find_bindings(llvm) + if bindings and str(bindings).startswith(str(llvm) + os.sep): + prefix, note = llvm, f"bindings found in {bindings}" + else: + # Otherwise look where distributions put them, then fall back to fetching. + bindings = find_bindings(Path("/usr"), Path("/usr/local")) + overlay = args.overlay_dir.expanduser() / f"llvm-{version}-overlay" + prefix = build_overlay(llvm, overlay.resolve(), bindings, py_version) + note = f"bindings from {bindings}" if bindings else f"bindings fetched from llvmorg-{version}" + + if not find_bindings(prefix): + die(f"no clang/cindex.py under {prefix} after setup -- refusing to write a +python external that would fail later") + + # `llvm+clang` provides c/cxx, so Spack may pick this external as the compiler + # for the whole build. That only works if the entry carries the compiler paths -- + # spack's own llvm package treats `compilers` as mandatory for a detected llvm. + # Without them Spack still selects it, then builds with SPACK_CC=None and every + # compile dies with "C compiler cannot create executables". + # + # Which compiler to build THAPI with is a separate decision, left to whoever runs + # the install (e.g. `spack install thapi %gcc`); this file only makes the external + # usable and correct. + cc, cxx = prefix / "bin" / "clang", prefix / "bin" / "clang++" + for path in (cc, cxx): + if not path.exists(): + die(f"{path} is missing -- an llvm external with +clang must provide it") + + yaml = textwrap.dedent(f"""\ + packages: + llvm: + externals: + - spec: llvm@{version}+clang+python + prefix: {prefix} + extra_attributes: + compilers: + c: {cc} + cxx: {cxx} + buildable: false + """) + # The config goes to stdout so it can be piped straight into Spack; everything + # else is progress info and belongs on stderr. + print(f"llvm@{version}: {note}", file=sys.stderr) + if prefix != llvm: + print(f"created overlay {prefix} (keep it: the external points here)", file=sys.stderr) + print(yaml, end="") + + +if __name__ == "__main__": + main()