Add LLVM 23, drop LLVM 20, and stop emitting broken device debug info

.github/workflows/presubmit.yml

@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        llvm-version: [20]
+        llvm-version: [21]
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -127,7 +127,7 @@ jobs:
       matrix:
         include:
           - pocl-version: "7.1"
-            llvm-version: 20
+            llvm-version: 21
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -202,7 +202,7 @@ jobs:
           # PoCL backend with specific version combinations
           - backend: pocl
             pocl-version: "7.1"
-            llvm-version: 20
+            llvm-version: 21
             extra_exclude: ""
       fail-fast: false
     steps:

.github/workflows/test-llvm-patches.yml

@@ -28,10 +28,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - llvm-version: 20
-            build-id: "20"
-            install-suffix: ""
-            variant-flag: ""
           - llvm-version: 21
             build-id: "21"
             install-suffix: ""
@@ -44,6 +40,13 @@ jobs:
             build-id: "22-native"
             install-suffix: "-native"
             variant-flag: "--variant native"
+          # LLVM 23 needs only one build: chipStar patches in llvm#213052, so the
+          # SPIRV-target build also serves the translator and the producer is
+          # chosen per compilation by -f[no-]integrated-objemitter.
+          - llvm-version: 23
+            build-id: "23"
+            install-suffix: ""
+            variant-flag: "--variant native"
       fail-fast: true
     steps:
       - uses: actions/checkout@v3
@@ -82,9 +85,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - llvm-version: 20
-            build-id: "20"
-            install-suffix: ""
           - llvm-version: 21
             build-id: "21"
             install-suffix: ""
@@ -94,6 +94,9 @@ jobs:
           - llvm-version: 22
             build-id: "22-native"
             install-suffix: "-native"
+          - llvm-version: 23
+            build-id: "23"
+            install-suffix: ""
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -139,8 +142,30 @@ jobs:
           # install-only runtime headers such as omp.h, which made
           # TestHipccFopenmp fail with "'omp.h' file not found".
           LLVM_PREFIX="$HOME/install/llvm/${{ matrix.llvm-version }}.0${{ matrix.install-suffix }}"
-          LLVM_BIN="$HOME/llvm-stage/${{ matrix.build-id }}${LLVM_PREFIX}/bin"
+          LLVM_STAGE="$HOME/llvm-stage/${{ matrix.build-id }}${LLVM_PREFIX}"
+          LLVM_BIN="${LLVM_STAGE}/bin"
           test -f "${LLVM_BIN}/clang" || exit 1
+
+          # The staged binaries are thin drivers; all the driver logic lives in
+          # libclang-cpp.so / libLLVM.so. Their RUNPATH points at the *final*
+          # install prefix ($HOME/install/llvm/<ver>), not at the stage, so by
+          # default the staged clang loads the previously installed libraries
+          # and silently runs old code. That makes this lane test the installed
+          # toolchain rather than the one just built from llvm-patches/, which
+          # is precisely what this workflow exists to validate. Pin the loader
+          # to the stage, and persist it to the test steps via GITHUB_ENV since
+          # chipStar invokes clang again at test time.
+          export LD_LIBRARY_PATH="${LLVM_STAGE}/lib:${LLVM_STAGE}/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
+
+          # Fail loudly if the pin did not take effect, rather than silently
+          # validating the wrong toolchain again.
+          RESOLVED=$(ldd "${LLVM_BIN}/clang" 2>/dev/null | awk '/libclang-cpp/{print $3}')
+          case "$RESOLVED" in
+            "${LLVM_STAGE}"/*) echo "OK: clang resolves libclang-cpp from the stage" ;;
+            "") echo "NOTE: clang does not link libclang-cpp dynamically; nothing to pin" ;;
+            *) echo "ERROR: clang resolves libclang-cpp from '$RESOLVED', not the stage"; exit 1 ;;
+          esac
           test -f "${LLVM_BIN}/../lib/clang/${{ matrix.llvm-version }}/include/omp.h" \
             || { echo "ERROR: omp.h missing from LLVM install toolchain at ${LLVM_BIN}"; exit 1; }
           cd ${{ github.workspace }}
@@ -162,6 +187,9 @@ jobs:
           elif [ -f /etc/profile.d/lmod.sh ]; then source /etc/profile.d/lmod.sh &> /dev/null
           else source /etc/profile.d/modules.sh &> /dev/null; fi
           module use ~/modulefiles
+          # IGC 2.36.3/2.38.2 miscompile chipStar kernels under their
+          # optimizer (TestSnakeMiscompileO2); this build carries the fix.
+          module load igc/all-fixes-2026.07.30
           module load opencl/igpu
           cd ${{ github.workspace }}/build-llvm${{ matrix.build-id }}
           ../scripts/check.py ./ igpu opencl
@@ -175,6 +203,9 @@ jobs:
           elif [ -f /etc/profile.d/lmod.sh ]; then source /etc/profile.d/lmod.sh &> /dev/null
           else source /etc/profile.d/modules.sh &> /dev/null; fi
           module use ~/modulefiles
+          # IGC 2.36.3/2.38.2 miscompile chipStar kernels under their
+          # optimizer (TestSnakeMiscompileO2); this build carries the fix.
+          module load igc/all-fixes-2026.07.30
           module load level-zero/igpu
           cd ${{ github.workspace }}/build-llvm${{ matrix.build-id }}
           ../scripts/check.py ./ igpu level0
@@ -187,6 +218,9 @@ jobs:
           elif [ -f /etc/profile.d/lmod.sh ]; then source /etc/profile.d/lmod.sh &> /dev/null
           else source /etc/profile.d/modules.sh &> /dev/null; fi
           module use ~/modulefiles
+          # IGC 2.36.3/2.38.2 miscompile chipStar kernels under their
+          # optimizer (TestSnakeMiscompileO2); this build carries the fix.
+          module load igc/all-fixes-2026.07.30
           module load level-zero/dgpu
           cd ${{ github.workspace }}/build-llvm${{ matrix.build-id }}
           ../scripts/check.py ./ dgpu level0
@@ -199,6 +233,9 @@ jobs:
           elif [ -f /etc/profile.d/lmod.sh ]; then source /etc/profile.d/lmod.sh &> /dev/null
           else source /etc/profile.d/modules.sh &> /dev/null; fi
           module use ~/modulefiles
+          # IGC 2.36.3/2.38.2 miscompile chipStar kernels under their
+          # optimizer (TestSnakeMiscompileO2); this build carries the fix.
+          module load igc/all-fixes-2026.07.30
           module load opencl/dgpu
           cd ${{ github.workspace }}/build-llvm${{ matrix.build-id }}
           ../scripts/check.py ./ dgpu opencl
@@ -211,6 +248,9 @@ jobs:
           elif [ -f /etc/profile.d/lmod.sh ]; then source /etc/profile.d/lmod.sh &> /dev/null
           else source /etc/profile.d/modules.sh &> /dev/null; fi
           module use ~/modulefiles
+          # IGC 2.36.3/2.38.2 miscompile chipStar kernels under their
+          # optimizer (TestSnakeMiscompileO2); this build carries the fix.
+          module load igc/all-fixes-2026.07.30
           module load opencl/cpu
           cd ${{ github.workspace }}/build-llvm${{ matrix.build-id }}
           ../scripts/check.py ./ cpu opencl --num-tries=3
@@ -233,10 +273,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - llvm-version: 20
-            build-id: "20"
-            install-suffix: ""
-            module-name: "20.0"
           - llvm-version: 21
             build-id: "21"
             install-suffix: ""
@@ -249,6 +285,10 @@ jobs:
             build-id: "22-native"
             install-suffix: "-native"
             module-name: "22.0-native"
+          - llvm-version: 23
+            build-id: "23"
+            install-suffix: ""
+            module-name: "23.0"
       fail-fast: false
     steps:
       - name: Install LLVM ${{ matrix.build-id }}
@@ -415,11 +455,25 @@ jobs:
           # the build tree, which lacks install-only runtime headers such as
           # omp.h (breaks TestHipccFopenmp with "'omp.h' file not found").
           LLVM_PREFIX="$HOME/install/llvm/22.0-${{ matrix.variant }}"
-          LLVM_BIN="$HOME/llvm-stage/22-${{ matrix.variant }}${LLVM_PREFIX}/bin"
+          LLVM_STAGE="$HOME/llvm-stage/22-${{ matrix.variant }}${LLVM_PREFIX}"
+          LLVM_BIN="${LLVM_STAGE}/bin"
           test -f "${LLVM_BIN}/clang" || exit 1
           test -f "${LLVM_BIN}/../lib/clang/22/include/omp.h" \
             || { echo "ERROR: omp.h missing from LLVM install toolchain at ${LLVM_BIN}"; exit 1; }
 
+          # configure_llvm.sh builds with LLVM_LINK_LLVM_DYLIB=ON and
+          # CMAKE_INSTALL_RPATH pinned to the final install prefix, so a staged
+          # (DESTDIR) clang resolves libclang-cpp/libLLVM from the previously
+          # installed toolchain and runs old driver code. Pin the loader to the
+          # stage so this lane tests what it just built. DYLD_LIBRARY_PATH (not
+          # DYLD_FALLBACK_LIBRARY_PATH) is required: fallback is consulted only
+          # after @rpath resolution fails, and here it succeeds with the wrong
+          # library. Diagnostic only, not fatal, since the macOS linkage layout
+          # is not verified here the way the Linux one is.
+          export DYLD_LIBRARY_PATH="${LLVM_STAGE}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}"
+          echo "clang links against:"
+          otool -L "${LLVM_BIN}/clang" 2>/dev/null | grep -iE "clang-cpp|libLLVM" || true
+
           export PATH="/Users/paulius/.pyenv/shims:/opt/homebrew/bin:/opt/homebrew/sbin:$PATH"
           export CHIP_DEVICE_TYPE=pocl
           export OCL_ICD_VENDORS=/Users/paulius/install/pocl/etc/OpenCL/vendors/

.github/workflows/x86-intel-gpu-ci.yml

@@ -474,38 +474,6 @@ jobs:
   # debug builds. These run ONLY on a manual workflow_dispatch, not on every
   # PR, to keep automatic CI fast. Automatic PRs cover llvm-22 native-release
   # (Stage 1, above) and llvm-22 translator-release (below).
-  unit-tests-llvm-20-debug:
-    needs: build-and-test-libceed
-    if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' }}
-    runs-on: [self-hosted, Linux, X64]
-    env:
-      CHIP_MODULE_CACHE_DIR: ""
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          fetch-depth: 0
-          submodules: 'recursive'
-      - name: Run unit test checking script
-        run: ./scripts/unit_tests.sh debug llvm-20
-        shell: bash
-
-  unit-tests-llvm-20-release:
-    needs: build-and-test-libceed
-    if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' }}
-    runs-on: [self-hosted, Linux, X64]
-    env:
-      CHIP_MODULE_CACHE_DIR: ""
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          fetch-depth: 0
-          submodules: 'recursive'
-      - name: Run unit test checking script
-        run: ./scripts/unit_tests.sh release llvm-20
-        shell: bash
-
   unit-tests-llvm-21-debug:
     needs: build-and-test-libceed
     if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' }}

CMakeLists.txt

@@ -275,6 +275,7 @@ option(CHIP_VERBOSE "Verbose compilation" OFF)
 option(CHIP_BUILD_SHARED_LIBS "Build chipStar as a shared library" ON)
 option(CHIP_BUILD_DOCS "Build doxygen documentation" OFF)
 option(CHIP_LLVM_USE_INTERGRATED_SPIRV "Use LLVM's intergrated SPIR-V backend for emitting device binary instead of SPIR-V translator. Auto-detected from LLVM targets when LLVM >= 22." OFF)
+option(CHIP_KEEP_KERNEL_DEBUG_INFO "Keep device debug info in kernels. No SPIR-V producer emits debug info that validates, and only IGC on Intel Data Center GPU Max tolerates it, so this is off by default and should only be enabled for that device." OFF)
 option(CHIP_SET_RPATH "Add CMAKE_INSTALL_PREFIX/lib to the RPATH for chipStar executables" ON)
 option(CHIP_ENABLE_UNCOMPILABLE_TESTS "Enable tests which are known to not compile" OFF)
 option(CHIP_BUILD_TESTS "Enable build_tests target" ON)
@@ -381,6 +382,10 @@ endif()
 set(SPIRV_EMITTER_OPTS "")
 if(CHIP_LLVM_USE_INTERGRATED_SPIRV)
   set(SPIRV_EMITTER_OPTS "-fintegrated-objemitter")
+elseif(LLVM_VERSION_MAJOR GREATER_EQUAL 23)
+  # LLVM 23's HIPSPV toolchain defaults to the in-tree SPIR-V backend when it
+  # is built; explicitly request the llvm-spirv translator.
+  set(SPIRV_EMITTER_OPTS "-fno-integrated-objemitter")
 endif()
 
 if(CHIP_VERBOSE)

MacOS-Issues.md

@@ -110,7 +110,7 @@ For macOS development, use **Option A** (dynamic LLVM). The `configure_llvm.sh`
 
 ### Required Dependencies
 - Homebrew packages: `spirv-tools`, `cmake`, `ninja`
-- Environment modules: `llvm/20.0` (or newer), `ocl-icd-loader`, `pocl`
+- Environment modules: `llvm/22.0` (or newer), `ocl-icd-loader`, `pocl`
 
 ### Build Command
 ```bash

README.md

@@ -82,11 +82,11 @@ Release notes for [1.3](docs/release_notes/chipStar_1.3.rst), [1.2](docs/release
 ## Prerequisites
 
 * Cmake >= 3.20.0
-* Clang and LLVM 20, 21, 22 (plus an experimental `latest` option tracking the maintained [chipStar-llvm-23](https://github.com/CHIP-SPV/llvm-project/tree/chipStar-llvm-23) branch)
-  * Can be installed, for example, by adding the [LLVM's Debian/Ubuntu repository](https://apt.llvm.org/) and installing packages 'clang-20 llvm-20 clang-tools-20'.
+* Clang and LLVM 21, 22, 23 (plus an experimental `latest` option tracking the maintained [chipStar-llvm-23](https://github.com/CHIP-SPV/llvm-project/tree/chipStar-llvm-23) branch)
+  * Can be installed, for example, by adding the [LLVM's Debian/Ubuntu repository](https://apt.llvm.org/) and installing packages 'clang-21 llvm-21 clang-tools-21'.
   * For the best results, build Clang/LLVM with the chipStar patches applied (from `llvm-patches/llvm-<version>/`). See below for a scripted way to build and install the patched versions.
 * SPIRV-LLVM-Translator from a branch matching the LLVM major version:
-  (e.g. llvm\_release\_200 for LLVM 20, llvm\_release\_210 for LLVM 21)
+  (e.g. llvm\_release\_210 for LLVM 21, llvm\_release\_220 for LLVM 22)
 ,  [llvm-spirv](https://github.com/KhronosGroup/SPIRV-LLVM-Translator).
   * Make sure the built llvm-spirv binary is installed into the same path as clang binary, otherwise clang might find and use a different llvm-spirv, leading to errors.
 * SPIRV-Tools and SPIRV-Headers:
@@ -101,26 +101,26 @@ For this you can use a script included in the chipStar repository:
 ```bash
 ./scripts/configure_llvm.sh
 Usage: ./scripts/configure_llvm.sh --version <version> --install-dir <dir> --link-type static/dynamic(default) [--variant translator|native] [--with-binutils [path]] [--configure-only] [-N]
---version: LLVM version 20, 21, 22, or latest (experimental, tracks the maintained chipStar-llvm-23 branch, no patches)
+--version: LLVM version 21, 22, 23, or latest (experimental, tracks the maintained chipStar-llvm-23 branch, no patches)
 --install-dir: installation directory
 --link-type: static or dynamic (default: dynamic)
 
-./scripts/configure_llvm.sh --version 20 --install-dir /opt/install/llvm/20.0
-cd llvm-project/llvm/build_20
+./scripts/configure_llvm.sh --version 22 --install-dir /opt/install/llvm/22.0
+cd llvm-project/llvm/build_22
 make -j 16
 <sudo> make install
 ```
 
 Or you can do the steps manually (clone the release branches, apply the patches from `llvm-patches/llvm-<version>/`, then build):
 
 ```bash
-git clone --depth 1 https://github.com/llvm/llvm-project.git -b release/20.x
+git clone --depth 1 https://github.com/llvm/llvm-project.git -b release/22.x
 cd llvm-project
-for p in /path/to/chipStar/llvm-patches/llvm-20/llvm/*.patch; do git apply "$p"; done
+for p in /path/to/chipStar/llvm-patches/llvm-22/llvm/*.patch; do git apply "$p"; done
 cd llvm/projects
-git clone --depth 1 https://github.com/KhronosGroup/SPIRV-LLVM-Translator.git -b llvm_release_200
+git clone --depth 1 https://github.com/KhronosGroup/SPIRV-LLVM-Translator.git -b llvm_release_220
 cd SPIRV-LLVM-Translator
-for p in /path/to/chipStar/llvm-patches/llvm-20/spirv-translator/*.patch; do git apply "$p"; done
+for p in /path/to/chipStar/llvm-patches/llvm-22/spirv-translator/*.patch; do git apply "$p"; done
 cd ../../..
 
 # DLLVM_ENABLE_PROJECTS="clang;openmp" OpenMP is optional but many apps use it
@@ -131,7 +131,7 @@ cmake -S llvm -B build \
   -DCMAKE_BUILD_TYPE=Release \
   -DLLVM_ENABLE_PROJECTS="clang;openmp" \
   -DLLVM_TARGETS_TO_BUILD=X86 \
-  -DCMAKE_INSTALL_PREFIX=$HOME/local/llvm-20
+  -DCMAKE_INSTALL_PREFIX=$HOME/local/llvm-22
 make -C build -j8 all install
 ```
 

bitcode/CMakeLists.txt

@@ -205,8 +205,15 @@ function(embed_spirv_in_cpp
       OUTPUT "${OUTPUT_SOURCE}" "${OUTPUT_HEADER}"
       DEPENDS "${BC_SOURCE}"
       BYPRODUCTS "${SPIR_BINARY}"
+      # NOTE: the triple must NOT carry the "chipstar" OS component here.
+      # spirv64vX-unknown-chipstar selects the HIPSPV toolchain, whose -c
+      # action emits LLVM bitcode for the offload driver to link later; it
+      # exits 0, so the ".spv" silently contains "BC\xc0\xde" instead of
+      # SPIR-V and every clLinkProgram() of the rtdevlib fails with
+      # CL_LINK_PROGRAM_FAILURE / CL_INVALID_PROGRAM. The plain spirv64vX
+      # triple selects the SPIRV toolchain, which emits SPIR-V directly.
       COMMAND "${CMAKE_CXX_COMPILER}"
-      --target=spirv64v${MAX_SPIRV_VERSION}-unknown-chipstar
+      --target=spirv64v${MAX_SPIRV_VERSION}
       -mllvm -spirv-ext=${SPIRV_EXTENSIONS}
       -c -x ir
       "${BC_SOURCE}" -o "${SPIR_BINARY}"

cmake/LLVMCheck.cmake

@@ -1,10 +1,10 @@
 if((CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") OR
    (CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM"))
 
-  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 20.0.0)
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 21.0.0)
     message(FATAL_ERROR
       "Unsupported clang version '${CMAKE_CXX_COMPILER_VERSION}'. "
-      "chipStar requires clang/LLVM 20, 21, or 22 (or the experimental "
+      "chipStar requires clang/LLVM 21, 22, or 23 (or the experimental "
       "'latest' toolchain); see scripts/configure_llvm.sh.")
   endif()
 else()