From 1f58414a73216a57712e7197186e7a4afc4f57aa Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Tue, 8 Sep 2026 22:55:59 +0000 Subject: [PATCH 01/11] Declare system LLVM as +python external instead of rebuilding it h2yaml needs llvm@18:+clang+python, where +python means the prefix ships clang's Python bindings (clang/cindex.py) next to libclang. `spack external find` picks variants from executables and never looks for the bindings, so it always records LLVM as ~python -- bindings present or not. Such an entry can never satisfy h2yaml, so a usable system LLVM goes unused and Spack builds one from source, by far the longest step of a THAPI install. Add scripts/gen-llvm-external.py, which prints the packages.yaml config for a given LLVM prefix. If the bindings are already there it points the external at the prefix directly; otherwise it builds an overlay (symlinks to the real install, since site LLVMs are read-only) and supplies the bindings, either from where the distribution put them or by downloading them from the matching LLVM release -- they are pure Python, so nothing is compiled. It verifies clang/cindex.py is reachable before emitting, so a wrong +python claim fails during setup rather than mid-build. Stdlib only, no dependencies. The entry carries extra_attributes:compilers. llvm+clang provides c/cxx, so Spack may select the external as the compiler; spack's own llvm package treats that attribute as mandatory, and without it the build runs with SPACK_CC=None and every compile fails with "C compiler cannot create executables". Use it from both the README and CI, replacing the sed that rewrote packages.yaml in place, and drop the separate symlink workaround it needed. h2yaml: search for the bindings rather than assuming lib/python/site-packages. That path is named for whichever python LLVM was built against, which need not be the one Spack runs h2yaml with, and externals may use lib64 or Debian's dist-packages. Also stop re-joining an already-absolute libclang path onto prefix.lib, and raise a pointed error naming the searched prefix instead of failing later with a bare ModuleNotFoundError: clang. cron.yml: keep the from-scratch install (the only job checking THAPI builds on a bare machine) and add a job exercising the fast path PR CI uses, which would otherwise go untested during a quiet week. It was also missing the env block, so SPACK_OPT expanded empty and failures produced no log. Co-Authored-By: Claude Opus 5 (1M context) --- .github/actions/setup-thapi-deps/action.yml | 27 +-- .github/workflows/cron.yml | 42 +++- README.md | 47 ++++- packages/h2yaml/package.py | 40 +++- scripts/gen-llvm-external.py | 202 ++++++++++++++++++++ 5 files changed, 337 insertions(+), 21 deletions(-) create mode 100755 scripts/gen-llvm-external.py diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 8b1d6d64..1e686f3e 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -72,17 +72,22 @@ runs: LLVM_FULL_VER=`llvm-config-${LLVM_VER} --version` echo "LLVM_FULL_VER=${LLVM_FULL_VER}" >> ${GITHUB_ENV} - - name: Workaround for providing llvm+python in GitHub CI runners + # The next step needs the major version too. + echo "LLVM_VER=${LLVM_VER}" >> ${GITHUB_ENV} + - name: Declare the apt llvm as an external with +python shell: bash if: ${{ inputs.install-llvm == 'true' }} run: | - # Symlink python3-clang package to site-packages so h2yaml won't fail on - # `import clang`. This way, our llvm+python trick will work without an issue. - python_major=$(python -c "import sys; print(f'{sys.version_info.major}')") - python_minor=$(python -c "import sys; print(f'{sys.version_info.minor}')") - SITE_PACKAGES=/usr/lib/python${python_major}.${python_minor}/site-packages - sudo mkdir -p ${SITE_PACKAGES} - sudo ln -s /usr/lib/python${python_major}/dist-packages/clang ${SITE_PACKAGES}/clang - - # Trick spack to think that llvm found in packages.yml has +python variant: - sed -i 's/llvm@[0-9][0-9]\.[0-9]\.[0-9]/&+python/' ~/.spack/packages.yaml + # `spack external find` never records +python (it picks variants from + # executables and never looks for clang's python bindings), so h2yaml's + # llvm@18:+clang+python would go unsatisfied and Spack would rebuild LLVM. + # The helper below declares the external properly; apt splits the bindings + # into /usr/lib/python3/dist-packages, which it finds and links in. + # Same script the README tells users to run. + . external/spack/share/spack/setup-env.sh + PYVER=$(spack spec h2yaml | sed -n 's/.*\^python@\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1) + python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \ + --python-version "${PYVER}" \ + --overlay-dir ${{ github.workspace }} \ + > ${{ github.workspace }}/llvm-external.yaml + spack config add -f ${{ github.workspace }}/llvm-external.yaml diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 831e5ab1..bd466b53 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -4,9 +4,19 @@ on: # At 00:00 on Sunday.” - cron: "0 0 * * 0" workflow_dispatch: +env: + SPACK_OPT: "--concurrent-packages 2 --show-log-on-error --fail-fast" + CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}" jobs: - myJob: + # Install with nothing pre-arranged: no `spack external find`, no system llvm, + # no build cache. Checks that THAPI installs on a bare machine. + from_scratch: + strategy: + fail-fast: false + matrix: + version: ['thapi@master', 'thapi@develop'] runs-on: ubuntu-latest + name: Install ${{ matrix.version }} (from scratch) steps: - name: Checkout uses: actions/checkout@v6 @@ -20,11 +30,33 @@ jobs: run: | . external/spack/share/spack/setup-env.sh spack repo add ./ - - name: Install thapi master + - name: Install ${{ matrix.version }} run: | . external/spack/share/spack/setup-env.sh - spack install $SPACK_OPT thapi@master - - name: Install thapi develop + spack install $SPACK_OPT ${{ matrix.version }} + + # Exercise the same fast path the PR CI uses (external find + apt llvm + + # hand-declared +python external). PR CI only runs when someone opens a PR, so + # without this the fast path can rot unnoticed during a quiet week. + fast_path: + runs-on: ubuntu-latest + name: Install thapi@develop (fast path) + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Checkout dependencies (spack,...) + uses: ./.github/actions/setup-thapi-deps + with: + cache-dir: ${{ env.CACHE_DIR }} + install-llvm: 'true' + - name: Install thapi@develop + run: | + . external/spack/share/spack/setup-env.sh + spack install $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} + - name: Check llvm was reused, not rebuilt run: | + # The point of the fast path: llvm must resolve to the external ([e]), + # not be built. `spack find` lists externals too, so match on the marker. . external/spack/share/spack/setup-env.sh - spack install $SPACK_OPT thapi@develop + spack spec -I thapi@develop ^llvm@${LLVM_FULL_VER} | grep -E '^\s*\[e\].*\^llvm@' \ + || { echo "llvm did not resolve to the external"; exit 1; } diff --git a/README.md b/README.md index 9bcbb124..234ccbfa 100644 --- a/README.md +++ b/README.md @@ -29,18 +29,59 @@ spack load thapi avoid building them when building THAPI. Use the following command before installing THAPI to find external packages available on the system: ```bash -spack external find --all +spack external find --all --exclude llvm ``` +Always exclude `llvm`: the entry detection writes cannot satisfy `h2yaml`. Declare that one with +the helper instead -- see [Reusing a system LLVM](#reusing-a-system-llvm). + Make sure to `module load` the packages you want Spack to find (or set other environment variables like `PATH`) -before running `spack external find --all`. +before running it. Some packages when found using `spack external find` are known to cause build failures. If you run into such cases, use `spack external find --exclude ` so that Spack will build them instead of using the system installed versions. For example: ```bash -spack external find --all --exclude bzip2 --exclude xz --exclude curl +spack external find --all --exclude bzip2 --exclude xz --exclude curl --exclude llvm +``` + +#### Reusing a system LLVM + +`thapi@0.0.14:` needs `h2yaml`, which needs `llvm@18:+clang+python`. That `+python` means the LLVM +prefix must contain clang's Python bindings (`clang/cindex.py`), not just `libclang.so`. Building +LLVM from source is by far the longest step of a THAPI install, so it is worth reusing a system one. + +Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/llvm-*`, +`/opt/llvm*`, or your site software tree) and apply what it prints: + +```bash +# The python Spack will run h2yaml with, so the overlay is laid out to match. +PYVER=$(spack spec h2yaml | sed -n 's/.*\^python@\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1) + +python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml +spack config add -f llvm-external.yaml ``` +The script needs only Python 3, no dependencies. If the LLVM already has the bindings it points the +external straight at it. Otherwise -- a site LLVM is usually read-only, so they cannot just be +dropped in next to it -- it builds an *overlay* in `--overlay-dir` (default `~/.spack`): a tree of +symlinks to the real install, plus the bindings, downloaded from the matching LLVM release if the +system has none. The external points into that directory, so keep it. + +If there is no LLVM 18+ on the system at all, skip this: Spack will build one. + +Check that it took -- LLVM should show `[e]` rather than `-`: + +```bash +spack spec -I thapi | grep llvm +``` + +> [!IMPORTANT] +> The external has to be declared this way rather than by `spack external find`, which picks variants +> from executables only and never looks for the bindings. It therefore always records LLVM as +> `~python` -- bindings present or not -- and such an entry can never satisfy `h2yaml`: Spack ignores +> it and, unless something else provides `+python`, builds LLVM from source. Always pass +> `--exclude llvm` when running it. + #### `spack install -j ` Depending on the number of available cores on your platform, you can specify number of parallel build diff --git a/packages/h2yaml/package.py b/packages/h2yaml/package.py index 6b55b567..a42bcb0e 100644 --- a/packages/h2yaml/package.py +++ b/packages/h2yaml/package.py @@ -15,6 +15,27 @@ def find_libclang(root, lib_ext): return os.path.join(root, file) +def find_clang_bindings(root): + """Return the site-packages dir holding clang's python bindings (clang/cindex.py). + + LLVM installs these under lib/python/site-packages, but the python + version in that path is whichever python LLVM was built against, which need + not be the one h2yaml runs with. External installs vary further: lib64 on + RHEL-likes, dist-packages on Debian/Ubuntu. So search instead of guessing. + """ + for libdir in ("lib", "lib64"): + base = os.path.join(root, libdir) + if not os.path.isdir(base): + continue + for entry in sorted(os.listdir(base)): + if not entry.startswith("python"): + continue + for pkgs in ("site-packages", "dist-packages"): + sp = os.path.join(base, entry, pkgs) + if os.path.isfile(os.path.join(sp, "clang", "cindex.py")): + return sp + + class H2yaml(PythonPackage): """Matrices describing affine transformation of the plane.""" @@ -63,10 +84,25 @@ def setup_run_environment(self, env): lib_so = join_path(self.spec["llvm"].prefix.lib, f"libclang.{lib_ext}") if not os.path.isfile(lib_so): lib_so = find_libclang(lib_path, lib_ext) - env.set("LIBCLANG_LIBRARY_FILE", join_path(self.spec["llvm"].prefix.lib, lib_so)) + if not lib_so: + raise InstallError( + f"h2yaml needs libclang.{lib_ext}, but none was found under " + f"{self.spec['llvm'].prefix}. If this is an external llvm, check that " + "its prefix really contains a libclang shared library." + ) + # lib_so is already absolute; do not re-join it onto prefix.lib. + env.set("LIBCLANG_LIBRARY_FILE", lib_so) # Set PYTHONPATH so that `import clang` will work without an issue. - env.append_path("PYTHONPATH", join_path(self.spec["llvm"].prefix.lib, f"python{self.spec['python'].version.up_to(2)}", "site-packages")) + site_packages = find_clang_bindings(self.spec["llvm"].prefix) + if not site_packages: + raise InstallError( + "h2yaml needs clang's python bindings (clang/cindex.py), but none were " + f"found under {self.spec['llvm'].prefix}. Note that external LLVM installs " + "frequently omit them even though they ship libclang. See the " + "'Reusing a system LLVM' section of the THAPI-spack README." + ) + env.append_path("PYTHONPATH", site_packages) def setup_test_environment(self, env): if self.version < Version("0.4.1"): diff --git a/scripts/gen-llvm-external.py b/scripts/gen-llvm-external.py new file mode 100755 index 00000000..8018b0fe --- /dev/null +++ b/scripts/gen-llvm-external.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +r"""Declare a system LLVM as a Spack external that satisfies h2yaml. + +h2yaml needs ``llvm@18:+clang+python``, where ``+python`` means the prefix ships +clang's Python bindings (``clang/cindex.py``) next to ``libclang``. Two things make +this awkward to set up by hand: + +* ``spack external find`` picks variants from executables and never looks for the + bindings, so it always records LLVM as ``~python`` -- which cannot satisfy h2yaml, + leaving the system LLVM unused. +* Distributions split the pieces up (Debian keeps the bindings in + ``/usr/lib/python3/dist-packages``, far from ``/usr/lib/llvm-N``) and site installs + are usually read-only, so the bindings often cannot be dropped in place. + +This script resolves both. It locates the bindings, and if they live outside the LLVM +prefix -- or are missing entirely, in which case they are fetched from the matching +LLVM release, since they are pure Python -- it builds an *overlay* prefix: a tree of +symlinks to the real install plus a ``site-packages`` holding the bindings. It then +prints the ``packages.yaml`` config pointing at whichever prefix is usable. + +The config is printed on stdout, so it can be redirected or piped straight into Spack:: + + python3 scripts/gen-llvm-external.py /usr/lib/llvm-18 # just look at it + python3 scripts/gen-llvm-external.py /usr/lib/llvm-18 > ext.yaml && spack config add -f ext.yaml + spack config add -f <(python3 scripts/gen-llvm-external.py /usr/lib/llvm-18) + +Pass ``--python-version`` to match the python Spack will run h2yaml with; it defaults +to the running python. Any overlay is created under ``--overlay-dir`` and must be +kept, since the external points into it. +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import textwrap +import urllib.error +import urllib.request +from pathlib import Path + +# The bindings are two or three plain .py files. enumerations.py was dropped after +# LLVM 18, so it is fetched only if the tag actually has it. +BINDINGS_REQUIRED = ("__init__.py", "cindex.py") +BINDINGS_OPTIONAL = ("enumerations.py",) +RAW_URL = "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{version}/clang/bindings/python/clang/{name}" + +# Overlays must outlive the command, since the Spack external points into one. +# ~/.spack is where the packages.yaml referring to it lives, so they travel together. +DEFAULT_OVERLAY_DIR = Path("~/.spack") + + +def die(msg): + sys.exit(f"error: {msg}") + + +def llvm_version(prefix): + """Full x.y.z version of the LLVM at *prefix*, via its own llvm-config.""" + llvm_config = prefix / "bin" / "llvm-config" + if not llvm_config.is_file(): + die(f"no llvm-config in {prefix}/bin -- is that really an LLVM prefix?") + raw = subprocess.check_output([str(llvm_config), "--version"], universal_newlines=True).strip() + # Strip suffixes such as "19.1.7git", which are not release tags. + m = re.match(r"^(\d+)\.(\d+)\.(\d+)", raw) + if not m: + die(f"cannot parse llvm-config --version output {raw!r}") + return m.group(0), int(m.group(1)) + + +def find_bindings(*roots): + """Directory containing ``clang/cindex.py``, searched over usual layouts.""" + for root in roots: + if root is None or not root.is_dir(): + continue + for libdir in ("lib", "lib64"): + base = root / libdir + if not base.is_dir(): + continue + for pydir in sorted(base.glob("python*")): + for pkgs in ("site-packages", "dist-packages"): + if (pydir / pkgs / "clang" / "cindex.py").is_file(): + return pydir / pkgs + # Debian: /usr/lib/python3/dist-packages, with no python level. + for pkgs in root.glob("lib/python3/dist-packages"): + if (pkgs / "clang" / "cindex.py").is_file(): + return pkgs + return None + + +def fetch_bindings(version, dest): + """Download clang's pure-python bindings for *version* into ``dest/clang``.""" + out = dest / "clang" + out.mkdir(parents=True, exist_ok=True) + for name in BINDINGS_REQUIRED + BINDINGS_OPTIONAL: + url = RAW_URL.format(version=version, name=name) + try: + with urllib.request.urlopen(url) as r: + (out / name).write_bytes(r.read()) + except (urllib.error.URLError, OSError) as e: + if name in BINDINGS_OPTIONAL: + continue + die(f"cannot fetch {name} from llvmorg-{version} ({e}) -- is that a real release tag?") + return out.parent + + +def build_overlay(llvm, overlay, bindings, py_version): + """Symlink *llvm* into *overlay*, with *bindings* exposed on Spack's python path. + + The real install is usually read-only, so nothing is copied into it: every entry + is symlinked, except ``lib`` which is recreated as a real directory so the + bindings can be added alongside the symlinked libraries. + """ + if overlay.exists(): + shutil.rmtree(overlay) + (overlay / "lib").mkdir(parents=True) + for entry in llvm.iterdir(): + if entry.name != "lib": + (overlay / entry.name).symlink_to(entry) + for entry in (llvm / "lib").iterdir(): + (overlay / "lib" / entry.name).symlink_to(entry) + + site = overlay / "lib" / f"python{py_version}" / "site-packages" + site.parent.mkdir(parents=True, exist_ok=True) + if bindings is None: + version, _ = llvm_version(llvm) + site.mkdir() + fetch_bindings(version, site) + else: + site.symlink_to(bindings) + return overlay + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("prefix", type=Path, help="system LLVM prefix, e.g. /usr/lib/llvm-18") + p.add_argument( + "--overlay-dir", + type=Path, + default=DEFAULT_OVERLAY_DIR, + help=f"where to create the overlay, if one is needed. It must keep existing for as long as Spack uses the external (default: {DEFAULT_OVERLAY_DIR})", + ) + p.add_argument("--python-version", help="python X.Y Spack runs h2yaml with (default: this interpreter)") + args = p.parse_args() + + llvm = args.prefix.resolve() + version, major = llvm_version(llvm) + if major < 18: + die(f"h2yaml needs llvm@18: but {llvm} is {version}") + + py_version = args.python_version or f"{sys.version_info.major}.{sys.version_info.minor}" + + # Bindings already inside the prefix: usable as-is, no overlay. + bindings = find_bindings(llvm) + if bindings and str(bindings).startswith(str(llvm) + os.sep): + prefix, note = llvm, f"bindings found in {bindings}" + else: + # Otherwise look where distributions put them, then fall back to fetching. + bindings = find_bindings(Path("/usr"), Path("/usr/local")) + overlay = args.overlay_dir.expanduser() / f"llvm-{version}-overlay" + prefix = build_overlay(llvm, overlay.resolve(), bindings, py_version) + note = f"bindings from {bindings}" if bindings else f"bindings fetched from llvmorg-{version}" + + if not find_bindings(prefix): + die(f"no clang/cindex.py under {prefix} after setup -- refusing to write a +python external that would fail later") + + # `llvm+clang` provides c/cxx, so Spack may pick this external as the compiler + # for the whole build. That only works if the entry carries the compiler paths -- + # spack's own llvm package treats `compilers` as mandatory for a detected llvm. + # Without them Spack still selects it, then builds with SPACK_CC=None and every + # compile dies with "C compiler cannot create executables". + # + # Which compiler to build THAPI with is a separate decision, left to whoever runs + # the install (e.g. `spack install thapi %gcc`); this file only makes the external + # usable and correct. + cc, cxx = prefix / "bin" / "clang", prefix / "bin" / "clang++" + for path in (cc, cxx): + if not path.exists(): + die(f"{path} is missing -- an llvm external with +clang must provide it") + + yaml = textwrap.dedent(f"""\ + packages: + llvm: + externals: + - spec: llvm@{version}+clang+python + prefix: {prefix} + extra_attributes: + compilers: + c: {cc} + cxx: {cxx} + buildable: false + """) + # The config goes to stdout so it can be piped straight into Spack; everything + # else is progress info and belongs on stderr. + print(f"llvm@{version}: {note}", file=sys.stderr) + if prefix != llvm: + print(f"created overlay {prefix} (keep it: the external points here)", file=sys.stderr) + print(yaml, end="") + + +if __name__ == "__main__": + main() From ced27f7671cc2fb3f961e5d5d39f0abed372b960 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 16:10:46 +0000 Subject: [PATCH 02/11] Keep spack off llvm, index the build cache, bump actions `spack external find` and `spack compiler find` both register the system llvm without `+python`, shadowing the entry gen-llvm-external.py writes. Exclude it from the former; `compiler find` has no such flag, so remove what it added. The mirror is created with `--autopush`, which uploads packages but never writes an index, and spack skips an unindexed mirror during concretization. Index it before upload so downstream jobs reuse it. Move the cron fast-path check ahead of the install: it only needs concretization, and running it afterwards means paying for the llvm build it exists to catch. upload/download-artifact v6->v7, delete-artifact v5->v6, ruff-action v3->v4.1.0, all now on node24. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 14 ++++++++---- .github/workflows/ci.yml | 24 ++++++++++++++++----- .github/workflows/cron.yml | 14 ++++++------ .github/workflows/lint.yml | 2 +- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 1e686f3e..38b93d5b 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -30,9 +30,15 @@ runs: # Find external packages known to spack (avoid bzip2 and xz since # they cause build failures in elfutils, and libxml2 since externals # in system paths don't get added to PKG_CONFIG_PATH). + # + # llvm is declared by hand in a later step, because neither of these + # commands records the `+python` variant h2yaml needs. Any entry they + # leave behind would shadow it, so drop theirs. `compiler find` has no + # --exclude, hence the `rm`. . external/spack/share/spack/setup-env.sh spack compiler find - spack external find --all --exclude bzip2 --exclude xz --exclude libxml2 + spack compiler rm -a llvm || true + spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 - name: Setup spack mirror shell: bash run: | @@ -65,10 +71,10 @@ runs: sudo apt-get install libllvm${LLVM_VER} llvm-${LLVM_VER} llvm-${LLVM_VER}-dev sudo apt-get install clang-${LLVM_VER} libclang-${LLVM_VER}-dev python3-clang-${LLVM_VER} - # Let spack find newly installed llvm/clang. Keep the same exclusions, - # otherwise this run re-adds them as externals. + # Pick up what apt pulled in, with the same exclusions as above. spack compiler find - spack external find --all --exclude bzip2 --exclude xz --exclude libxml2 + spack compiler rm -a llvm || true + spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 LLVM_FULL_VER=`llvm-config-${LLVM_VER} --version` echo "LLVM_FULL_VER=${LLVM_FULL_VER}" >> ${GITHUB_ENV} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 346822e2..fcc6a226 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,15 @@ jobs: # Install the dependencies of the latest legacy version. . external/spack/share/spack/setup-env.sh spack install --only dependencies $SPACK_OPT thapi@0.0.12 + - name: Index the build cache + run: | + # `--autopush` uploads packages but writes no index, and a mirror + # without one is skipped during concretization ("no index found"), + # leaving the downstream jobs to rebuild what is already here. + . external/spack/share/spack/setup-env.sh + spack buildcache update-index ${{ env.CACHE_DIR_LEGACY }} - name: Upload THAPI build cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: thapi_cache_legacy include-hidden-files: true @@ -61,7 +68,7 @@ jobs: uses: mxschmitt/action-tmate@v3 if: ${{ inputs.debug_enabled }} - name: Download THAPI build cache - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: thapi_cache_legacy path: ${{ env.CACHE_DIR_LEGACY }} @@ -99,8 +106,15 @@ jobs: # as the versions in apt and spack doesn't agree to the patch version. The # latter will force spack to build llvm from scratch. spack install --only dependencies $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} + - name: Index the build cache + run: | + # `--autopush` uploads packages but writes no index, and a mirror + # without one is skipped during concretization ("no index found"), + # leaving the downstream jobs to rebuild what is already here. + . external/spack/share/spack/setup-env.sh + spack buildcache update-index ${{ env.CACHE_DIR }} - name: Upload THAPI build cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: thapi_cache include-hidden-files: true @@ -125,7 +139,7 @@ jobs: uses: mxschmitt/action-tmate@v3 if: ${{ inputs.debug_enabled }} - name: Download THAPI build cache - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: thapi_cache path: ${{ env.CACHE_DIR }} @@ -158,7 +172,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: geekyeggo/delete-artifact@v5 + - uses: geekyeggo/delete-artifact@v6 with: name: | thapi_cache_legacy diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index bd466b53..3e7e0c6b 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -49,14 +49,14 @@ jobs: with: cache-dir: ${{ env.CACHE_DIR }} install-llvm: 'true' - - name: Install thapi@develop - run: | - . external/spack/share/spack/setup-env.sh - spack install $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} - - name: Check llvm was reused, not rebuilt + - name: Check llvm resolves to the external, not a build run: | - # The point of the fast path: llvm must resolve to the external ([e]), - # not be built. `spack find` lists externals too, so match on the marker. + # Concretization alone shows whether llvm is external ([e]). Checking it + # here fails in seconds; after the install it costs an llvm build. . external/spack/share/spack/setup-env.sh spack spec -I thapi@develop ^llvm@${LLVM_FULL_VER} | grep -E '^\s*\[e\].*\^llvm@' \ || { echo "llvm did not resolve to the external"; exit 1; } + - name: Install thapi@develop + run: | + . external/spack/share/spack/setup-env.sh + spack install $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1e3b88a7..b707fc70 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,6 +22,6 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: ruff format check - uses: astral-sh/ruff-action@v3 + uses: astral-sh/ruff-action@v4.1.0 with: args: "format --check --diff" From 4436483ca8c7961183929f88ff4e7a262829d970 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 16:18:15 +0000 Subject: [PATCH 03/11] Parse spack output as json instead of scraping text `spack spec -j` gives structured output, so version numbers and the external marker come from jq rather than sed/awk over the rendered tree. `.external.path` is null for a non-external node, which is what `jq -e` needs to set its exit code. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 13 ++++++------- .github/workflows/cron.yml | 7 +++---- README.md | 3 ++- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 38b93d5b..8720d687 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -54,12 +54,10 @@ runs: # Source spack . external/spack/share/spack/setup-env.sh - # Find the llvm major version in the thapi@develop. We need to filter out - # the fetch lines generated by `spack spec` when it is run for the first time. - # These lines start with `==>`. - LLVM_VER=$( spack spec -j thapi@develop | grep -v '==>' | jq '.spec.nodes[] | select(.name == "llvm") | .version' ) - # Extract major version (21) from "\"21.1.4\"". - LLVM_VER=$( echo $LLVM_VER | awk '{ split($0,a,"[\".]"); print a[2] }' ) + # The llvm major version thapi@develop wants. `spack spec` prefixes its + # json with `==>` fetch lines the first time it runs, hence the grep. + LLVM_VER=$( spack spec -j thapi@develop | grep -v '==>' \ + | jq -r '.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]' ) # Add llvm to apt repository. wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -91,7 +89,8 @@ runs: # into /usr/lib/python3/dist-packages, which it finds and links in. # Same script the README tells users to run. . external/spack/share/spack/setup-env.sh - PYVER=$(spack spec h2yaml | sed -n 's/.*\^python@\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1) + PYVER=$( spack spec -j h2yaml | grep -v '==>' \ + | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")' ) python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \ --python-version "${PYVER}" \ --overlay-dir ${{ github.workspace }} \ diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 3e7e0c6b..81418f24 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -49,12 +49,11 @@ jobs: with: cache-dir: ${{ env.CACHE_DIR }} install-llvm: 'true' - - name: Check llvm resolves to the external, not a build + - name: Check llvm resolves to the external run: | - # Concretization alone shows whether llvm is external ([e]). Checking it - # here fails in seconds; after the install it costs an llvm build. . external/spack/share/spack/setup-env.sh - spack spec -I thapi@develop ^llvm@${LLVM_FULL_VER} | grep -E '^\s*\[e\].*\^llvm@' \ + spack spec -j thapi@develop ^llvm@${LLVM_FULL_VER} | grep -v '==>' \ + | jq -e '.spec.nodes[] | select(.name == "llvm") | .external.path' \ || { echo "llvm did not resolve to the external"; exit 1; } - name: Install thapi@develop run: | diff --git a/README.md b/README.md index 234ccbfa..491cfe12 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/l ```bash # The python Spack will run h2yaml with, so the overlay is laid out to match. -PYVER=$(spack spec h2yaml | sed -n 's/.*\^python@\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1) +PYVER=$(spack spec -j h2yaml | grep -v '==>' \ + | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")') python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml spack config add -f llvm-external.yaml From b5f1926f4aae177a72f4d9bd1a1a54d1af0b1cd2 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 16:24:40 +0000 Subject: [PATCH 04/11] Anchor the ==> filter on spack json output Only status lines start with the marker; it can appear anywhere inside the json itself. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 9 +++++---- .github/workflows/cron.yml | 2 +- README.md | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 8720d687..36650724 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -54,9 +54,10 @@ runs: # Source spack . external/spack/share/spack/setup-env.sh - # The llvm major version thapi@develop wants. `spack spec` prefixes its - # json with `==>` fetch lines the first time it runs, hence the grep. - LLVM_VER=$( spack spec -j thapi@develop | grep -v '==>' \ + # The llvm major version thapi@develop wants. `spack spec -j` writes its + # status lines ("==> Using cached archive: ...") to stdout alongside the + # json, so they have to be stripped before jq sees them. + LLVM_VER=$( spack spec -j thapi@develop | grep -v '^==>' \ | jq -r '.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]' ) # Add llvm to apt repository. @@ -89,7 +90,7 @@ runs: # into /usr/lib/python3/dist-packages, which it finds and links in. # Same script the README tells users to run. . external/spack/share/spack/setup-env.sh - PYVER=$( spack spec -j h2yaml | grep -v '==>' \ + PYVER=$( spack spec -j h2yaml | grep -v '^==>' \ | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")' ) python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \ --python-version "${PYVER}" \ diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 81418f24..cf555c15 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -52,7 +52,7 @@ jobs: - name: Check llvm resolves to the external run: | . external/spack/share/spack/setup-env.sh - spack spec -j thapi@develop ^llvm@${LLVM_FULL_VER} | grep -v '==>' \ + spack spec -j thapi@develop ^llvm@${LLVM_FULL_VER} | grep -v '^==>' \ | jq -e '.spec.nodes[] | select(.name == "llvm") | .external.path' \ || { echo "llvm did not resolve to the external"; exit 1; } - name: Install thapi@develop diff --git a/README.md b/README.md index 491cfe12..e9c20b31 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/l ```bash # The python Spack will run h2yaml with, so the overlay is laid out to match. -PYVER=$(spack spec -j h2yaml | grep -v '==>' \ +PYVER=$(spack spec -j h2yaml | grep -v '^==>' \ | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")') python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml From f813db02f49b8d86c3782d9f60b82d12944c73eb Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 17:02:26 +0000 Subject: [PATCH 05/11] Take one version from spack json, not one per matching node A spec can carry more than one llvm or python node. Selecting them node-by-node yields a newline-separated list, which lands in the apt sources line as "llvm-toolchain-noble-22\n22 main" and apt rejects the file. Reduce to the distinct values and stop if more than one survives, so an ambiguous spec is a clear error rather than a malformed repo entry. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 18 ++++++++++++++++-- README.md | 3 ++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 36650724..37c2d526 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -57,8 +57,17 @@ runs: # The llvm major version thapi@develop wants. `spack spec -j` writes its # status lines ("==> Using cached archive: ...") to stdout alongside the # json, so they have to be stripped before jq sees them. + # + # A spec can hold several llvm nodes (same version, different variants), + # so collapse them to the distinct majors and insist on exactly one -- + # two would otherwise be concatenated into an unusable "22\n22". LLVM_VER=$( spack spec -j thapi@develop | grep -v '^==>' \ - | jq -r '.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]' ) + | jq -r '[.spec.nodes[] | select(.name == "llvm") | .version | split(".")[0]] + | unique | join(" ")' ) + if [ "$( echo "${LLVM_VER}" | wc -w )" -ne 1 ]; then + echo "expected exactly one llvm major version, got: '${LLVM_VER}'" + exit 1 + fi # Add llvm to apt repository. wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -91,7 +100,12 @@ runs: # Same script the README tells users to run. . external/spack/share/spack/setup-env.sh PYVER=$( spack spec -j h2yaml | grep -v '^==>' \ - | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")' ) + | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")] + | unique | join(" ")' ) + if [ "$( echo "${PYVER}" | wc -w )" -ne 1 ]; then + echo "expected exactly one python version, got: '${PYVER}'" + exit 1 + fi python3 scripts/gen-llvm-external.py /usr/lib/llvm-${LLVM_VER} \ --python-version "${PYVER}" \ --overlay-dir ${{ github.workspace }} \ diff --git a/README.md b/README.md index e9c20b31..901daa14 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ Point the helper at any LLVM 18+ on the system (`module avail llvm`, `/usr/lib/l ```bash # The python Spack will run h2yaml with, so the overlay is laid out to match. PYVER=$(spack spec -j h2yaml | grep -v '^==>' \ - | jq -r '.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")') + | jq -r '[.spec.nodes[] | select(.name == "python") | .version | split(".")[:2] | join(".")] + | unique | .[0]') python3 scripts/gen-llvm-external.py /path/to/system/llvm --python-version $PYVER > llvm-external.yaml spack config add -f llvm-external.yaml From b427dd35297915a2e9c289494868bb8c10907bc8 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 19:37:16 +0000 Subject: [PATCH 06/11] Concretize for a generic target so the build cache hits The runner pool mixes cpu generations. Spack defaults to concretizer:targets:granularity:microarchitectures, which puts the microarchitecture in the dag hash, so a package built on one runner does not match the same package on another. Every consumer job was rebuilding its whole dependency tree from source: the thapi@0.0.12 (legacy) log reported "no binary available" for all of protobuf, abseil-cpp, elfutils, babeltrace2 and lttng-tools, each of which the dependencies job had just built and pushed. Targeting x86_64_v3 keeps hashes stable across the pool. Binaries lose cpu-specific tuning, which does not matter for a build test. from_scratch installs without a build cache, so it keeps the default. Also drop --concurrent-packages 2. It dates from the old installer, where 0 meant serial. The new installer, the default since v1.0, reads 0 as no package limit and bounds concurrency with a jobserver sized to the core count, so the flag only caps parallelism below what the hardware allows. --- .github/actions/setup-thapi-deps/action.yml | 9 +++++++++ .github/workflows/ci.yml | 2 +- .github/workflows/cron.yml | 2 +- README.md | 10 ++++++---- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 37c2d526..ab925e79 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -39,6 +39,15 @@ runs: spack compiler find spack compiler rm -a llvm || true spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 + - name: Concretize for a generic target + shell: bash + run: | + # Spack puts the microarchitecture in the dag hash, and the runner pool + # is a mix of cpu generations, so a package built on one runner does not + # match the same package on another and the build cache never hits. + # Targeting x86_64_v3 instead keeps the hashes stable across the pool. + . external/spack/share/spack/setup-env.sh + spack config add concretizer:targets:granularity:generic - name: Setup spack mirror shell: bash run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcc6a226..5af8d8e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ on: required: false default: false env: - SPACK_OPT: "--concurrent-packages 2 --show-log-on-error --fail-fast" + SPACK_OPT: "--show-log-on-error --fail-fast" CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}" CACHE_DIR_LEGACY: "${{ github.workspace }}/thapi_cache_legacy_${{ github.run_id }}" jobs: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index cf555c15..90dc5732 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -5,7 +5,7 @@ on: - cron: "0 0 * * 0" workflow_dispatch: env: - SPACK_OPT: "--concurrent-packages 2 --show-log-on-error --fail-fast" + SPACK_OPT: "--show-log-on-error --fail-fast" CACHE_DIR: "${{ github.workspace }}/thapi_cache_${{ github.run_id }}" jobs: # Install with nothing pre-arranged: no `spack external find`, no system llvm, diff --git a/README.md b/README.md index 901daa14..9096af78 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,15 @@ processes to be used during the build of `spec` using `-j` option to `spack` (e. #### `--concurrent-packages` -One may also be able to reduce the time to install THAPI by using `--concurrent-packages` option in `spack install` -as below: +Spack's current installer builds independent packages concurrently on its own, bounded by the +number of available cores, so there is normally nothing to set. `--concurrent-packages N` caps how +many it will run at once; leaving it unset means no cap beyond that core count. + +Lowering it is useful when a build is memory-hungry rather than CPU-hungry, or on a shared node +where a full-width build would crowd out others: ```bash spack install --concurrent-packages 2 thapi ``` -Depending on the available number of CPU cores and how parallelizable the dependency graph is for the particular -specification, one may be able to specify more concurrent packages (as compared to `2` in the above example). ## Building THAPI Manually From 9ee00bc9cdbf743d7cbfd405f740e855e720545e Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 19:53:26 +0000 Subject: [PATCH 07/11] remove concurent --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index 9096af78..32a7d74d 100644 --- a/README.md +++ b/README.md @@ -89,18 +89,6 @@ spack spec -I thapi | grep llvm Depending on the number of available cores on your platform, you can specify number of parallel build processes to be used during the build of `spec` using `-j` option to `spack` (e.g., `-j16`). -#### `--concurrent-packages` - -Spack's current installer builds independent packages concurrently on its own, bounded by the -number of available cores, so there is normally nothing to set. `--concurrent-packages N` caps how -many it will run at once; leaving it unset means no cap beyond that core count. - -Lowering it is useful when a build is memory-hungry rather than CPU-hungry, or on a shared node -where a full-width build would crowd out others: -```bash -spack install --concurrent-packages 2 thapi -``` - ## Building THAPI Manually You can use Spack to install the required dependencies and set up the environment to build THAPI manually From ea0dcfa76651279cd150f14f429ed000385815de Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 21:01:42 +0000 Subject: [PATCH 08/11] One more try --- .github/actions/setup-thapi-deps/action.yml | 15 ++++++++++++--- .github/workflows/ci.yml | 16 ++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index ab925e79..e4204fa4 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -39,15 +39,24 @@ runs: spack compiler find spack compiler rm -a llvm || true spack external find --all --exclude llvm --exclude bzip2 --exclude xz --exclude libxml2 - - name: Concretize for a generic target + - name: Pin the target microarchitecture shell: bash run: | # Spack puts the microarchitecture in the dag hash, and the runner pool # is a mix of cpu generations, so a package built on one runner does not # match the same package on another and the build cache never hits. - # Targeting x86_64_v3 instead keeps the hashes stable across the pool. + # + # `targets:granularity:generic` is not enough: it still picks the *best* + # generic target the host supports, so an avx512 runner (icelake, + # sapphire rapids) concretizes to x86_64_v4 while a zen2/zen3 one gets + # x86_64_v3 -- the pool just splits two ways instead of five. Requiring + # an explicit target is what actually makes the hashes stable. + # + # x86_64_v3 (avx2) is the highest level every runner in the pool + # supports. If one ever does not, `host_compatible: true` makes the + # solve fail loudly rather than emit illegal instructions. . external/spack/share/spack/setup-env.sh - spack config add concretizer:targets:granularity:generic + spack config add "packages:all:require:target=x86_64_v3" - name: Setup spack mirror shell: bash run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5af8d8e1..4a101ff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,14 @@ jobs: - name: Install dependencies run: | # Install the dependencies of the latest legacy version. + # --include-build-deps: with a mirror configured, the installer keeps + # build deps out of the graph and only pulls them in after a package + # misses the cache. `--only dependencies` never installs thapi itself, + # so that miss never happens and thapi's build-only deps (re2c, + # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed -- + # leaving every downstream job to build them from source. . external/spack/share/spack/setup-env.sh - spack install --only dependencies $SPACK_OPT thapi@0.0.12 + spack install --only dependencies --include-build-deps $SPACK_OPT thapi@0.0.12 - name: Index the build cache run: | # `--autopush` uploads packages but writes no index, and a mirror @@ -105,7 +111,13 @@ jobs: # We have to use the exact llvm version from apt package as a direct dependency # as the versions in apt and spack doesn't agree to the patch version. The # latter will force spack to build llvm from scratch. - spack install --only dependencies $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} + # --include-build-deps: with a mirror configured, the installer keeps + # build deps out of the graph and only pulls them in after a package + # misses the cache. `--only dependencies` never installs thapi itself, + # so that miss never happens and thapi's build-only deps (re2c, + # ruby-metababel, ruby-nokogiri, ...) are never built nor pushed -- + # leaving every downstream job to build them from source. + spack install --only dependencies --include-build-deps $SPACK_OPT thapi@develop ^llvm@${LLVM_FULL_VER} - name: Index the build cache run: | # `--autopush` uploads packages but writes no index, and a mirror From e392f3dbd76b90396087e40afc4949a4c57c22cd Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 23:20:18 +0000 Subject: [PATCH 09/11] Declare apt re2c and protobuf as spack externals Both cost ~17 min in the dependency jobs and are already installed by apt, but `spack external find` cannot see them: neither recipe declares `executables` or `libraries`. Add scripts/gen-external.py, which reads the version from the installed tool and prints the packages.yaml entry, and use it for both. Same shape as gen-llvm-external.py. This pins PR CI to Noble's older protobuf; the weekly cron `from_scratch` job installs with no externals and covers the current one. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 25 ++++- scripts/gen-external.py | 104 ++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 scripts/gen-external.py diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index e4204fa4..728672a1 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -12,7 +12,7 @@ runs: - name: Install apt-get packages uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libnuma-dev + packages: libnuma-dev re2c protobuf-compiler libprotobuf-dev - name: Checkout Spack uses: actions/checkout@v6 with: @@ -57,6 +57,29 @@ runs: # solve fail loudly rather than emit illegal instructions. . external/spack/share/spack/setup-env.sh spack config add "packages:all:require:target=x86_64_v3" + - name: Declare the apt re2c and protobuf as externals + shell: bash + run: | + # These two cost ~17 min each in the dependency jobs and are both in + # apt. `spack external find` cannot see them -- neither recipe + # declares `executables` or `libraries` -- so declare them by hand. + # + # This does pin PR CI to Noble's old protobuf; the weekly cron + # `from_scratch` job installs with no externals and covers the + # current one. + . external/spack/share/spack/setup-env.sh + for pkg_tool in re2c:re2c protobuf:protoc; do + pkg=${pkg_tool%:*} + tool=${pkg_tool#*:} + python3 scripts/gen-external.py "${pkg}" --version-from "${tool}" \ + > ${{ github.workspace }}/${pkg}-external.yaml + spack config add -f ${{ github.workspace }}/${pkg}-external.yaml + # The script only writes config, so check Spack took it: an + # external node has a `.external.path`, everything else null. + spack spec -j "${pkg}" | grep -v '^==>' \ + | jq -e --arg p "${pkg}" '.spec.nodes[] | select(.name == $p) | .external.path' \ + || { echo "${pkg} did not resolve to the external"; exit 1; } + done - name: Setup spack mirror shell: bash run: | diff --git a/scripts/gen-external.py b/scripts/gen-external.py new file mode 100755 index 00000000..0cd67bca --- /dev/null +++ b/scripts/gen-external.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +r"""Declare a system package as a Spack external, with the version read from a tool. + +``spack external find`` only detects packages whose recipe declares ``executables`` +or ``libraries`` -- 172 of ~8900 in the builtin repo. Everything else has to be +declared by hand, even when it is plainly installed: ``re2c`` and ``protobuf`` are +both apt packages that Spack will otherwise spend ~15 minutes each rebuilding. + +This writes the ``packages.yaml`` entry for one such package. The version comes from +running the tool itself, so the config cannot drift from what is actually installed:: + + python3 scripts/gen-external.py re2c --version-from re2c + python3 scripts/gen-external.py protobuf --version-from protoc + +The config is printed on stdout, so it can be redirected or piped into Spack:: + + python3 scripts/gen-external.py re2c --version-from re2c > ext.yaml + spack config add -f ext.yaml + +Pass ``--version`` instead of ``--version-from`` when there is no tool to ask, and +``--prefix`` when the package does not live in ``/usr``. + +Note this only writes the config. Whether Spack actually picks the external up +depends on the version satisfying what depends on it -- check with ``spack spec``, +which marks an external ``[e]``. + +For LLVM use ``gen-llvm-external.py`` instead: h2yaml needs a ``+python`` variant +whose bindings distributions scatter outside the prefix, which needs real work +beyond writing a version and a path. +""" + +import argparse +import re +import subprocess +import sys +import textwrap + + +def die(msg): + sys.exit(f"error: {msg}") + + +def version_from_tool(tool): + """First dotted version in `` --version`` output. + + Covers the usual shapes without a per-tool rule: "re2c 3.1", + "libprotoc 3.21.12", "cmake version 3.28.3". + """ + try: + out = subprocess.check_output( + [tool, "--version"], universal_newlines=True, stderr=subprocess.STDOUT + ) + except OSError as e: + die(f"cannot run {tool} --version ({e}) -- is it installed and on PATH?") + except subprocess.CalledProcessError as e: + die(f"{tool} --version failed with status {e.returncode}:\n{e.output}") + m = re.search(r"\d+(?:\.\d+)+", out) + if not m: + die(f"no version found in {tool} --version output:\n{out}") + return m.group(0) + + +def main(): + p = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + p.add_argument("package", help="Spack package name, e.g. re2c") + g = p.add_mutually_exclusive_group(required=True) + g.add_argument("--version-from", metavar="TOOL", help="read the version from `TOOL --version`") + g.add_argument("--version", help="use this version verbatim") + p.add_argument("--prefix", default="/usr", help="install prefix (default: /usr)") + p.add_argument( + "--buildable", + action="store_true", + help="let Spack build the package too. By default it may not, so a version that stops" + " satisfying its dependents fails the solve instead of silently rebuilding from source", + ) + p.add_argument( + "--variants", + default="", + help="variants to append to the spec, e.g. '+shared'. Only add what is actually true of" + " the installed package: Spack trusts the spec and will not verify it", + ) + args = p.parse_args() + + version = args.version or version_from_tool(args.version_from) + spec = f"{args.package}@{version}{args.variants}" + + yaml = textwrap.dedent(f"""\ + packages: + {args.package}: + externals: + - spec: {spec} + prefix: {args.prefix} + buildable: {str(args.buildable).lower()} + """) + # The config goes to stdout so it can be piped straight into Spack; everything + # else is progress info and belongs on stderr. + print(f"{spec}: prefix {args.prefix}", file=sys.stderr) + print(yaml, end="") + + +if __name__ == "__main__": + main() From 6b4c32b2ea7321fe52ea70fe1ca2d89ffc46a2e6 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 23:24:11 +0000 Subject: [PATCH 10/11] Format gen-external.py with ruff line-length is 150, so the two wrapped calls fit on one line. Co-Authored-By: Claude Opus 5 --- scripts/gen-external.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/gen-external.py b/scripts/gen-external.py index 0cd67bca..fe273344 100755 --- a/scripts/gen-external.py +++ b/scripts/gen-external.py @@ -47,9 +47,7 @@ def version_from_tool(tool): "libprotoc 3.21.12", "cmake version 3.28.3". """ try: - out = subprocess.check_output( - [tool, "--version"], universal_newlines=True, stderr=subprocess.STDOUT - ) + out = subprocess.check_output([tool, "--version"], universal_newlines=True, stderr=subprocess.STDOUT) except OSError as e: die(f"cannot run {tool} --version ({e}) -- is it installed and on PATH?") except subprocess.CalledProcessError as e: @@ -61,9 +59,7 @@ def version_from_tool(tool): def main(): - p = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("package", help="Spack package name, e.g. re2c") g = p.add_mutually_exclusive_group(required=True) g.add_argument("--version-from", metavar="TOOL", help="read the version from `TOOL --version`") From 299462a7de61a663bea60044ddae9a986f34f838 Mon Sep 17 00:00:00 2001 From: tapplencourt Date: Wed, 9 Sep 2026 23:56:11 +0000 Subject: [PATCH 11/11] Declare the apt elfutils as an external too Same treatment as re2c and protobuf: ~6 min in the dependency jobs, apt already ships it, and the recipe has no detectors so `spack external find` cannot see it. Version comes from `eu-readelf --version`. Unlike the other two this is a link dependency (of babeltrace2, and of glib through the `elf` virtual), but `libelf.pc` lives in pkg-config's default search path, so it does not need the PKG_CONFIG_PATH handling that keeps libxml2 excluded from `spack external find`. Co-Authored-By: Claude Opus 5 --- .github/actions/setup-thapi-deps/action.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-thapi-deps/action.yml b/.github/actions/setup-thapi-deps/action.yml index 728672a1..04e08098 100644 --- a/.github/actions/setup-thapi-deps/action.yml +++ b/.github/actions/setup-thapi-deps/action.yml @@ -12,7 +12,7 @@ runs: - name: Install apt-get packages uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libnuma-dev re2c protobuf-compiler libprotobuf-dev + packages: libnuma-dev re2c protobuf-compiler libprotobuf-dev elfutils libelf-dev libdw-dev - name: Checkout Spack uses: actions/checkout@v6 with: @@ -57,18 +57,19 @@ runs: # solve fail loudly rather than emit illegal instructions. . external/spack/share/spack/setup-env.sh spack config add "packages:all:require:target=x86_64_v3" - - name: Declare the apt re2c and protobuf as externals + - name: Declare the apt re2c, protobuf and elfutils as externals shell: bash run: | - # These two cost ~17 min each in the dependency jobs and are both in - # apt. `spack external find` cannot see them -- neither recipe - # declares `executables` or `libraries` -- so declare them by hand. + # The three most expensive packages in the dependency jobs (~17, ~17 + # and ~6 min) that apt already ships. `spack external find` cannot see + # them -- no recipe declares `executables` or `libraries` -- so declare + # them by hand. # - # This does pin PR CI to Noble's old protobuf; the weekly cron - # `from_scratch` job installs with no externals and covers the - # current one. + # This does pin PR CI to Noble's older protobuf and elfutils; the + # weekly cron `from_scratch` job installs with no externals and covers + # the current ones. . external/spack/share/spack/setup-env.sh - for pkg_tool in re2c:re2c protobuf:protoc; do + for pkg_tool in re2c:re2c protobuf:protoc elfutils:eu-readelf; do pkg=${pkg_tool%:*} tool=${pkg_tool#*:} python3 scripts/gen-external.py "${pkg}" --version-from "${tool}" \