ci_linux.yml

name: CI on Linux

on:
  workflow_dispatch:
  pull_request:
    paths-ignore:
      - "**.md"
  push:
    branches:
      - main
    paths-ignore:
      - "**.md"

env:
  RUST_LOG: info
  RUST_BACKTRACE: 1

jobs:
  build:
    name: Build / ${{ matrix.variance.name }}
    runs-on: ${{ matrix.variance.runner }}
    strategy:
      fail-fast: false
      matrix:
        variance:
          - name: Ubuntu-24.04 / CUDA-12.8.1 / x86_64
            image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
            runner: ubuntu-latest
            llvm: 7
          - name: Ubuntu-24.04 / CUDA-12.8.1 / ARM64
            image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
            runner: ubuntu-24.04-arm
            llvm: 7
          - name: Ubuntu-24.04 / CUDA-13.3.1 / x86_64
            image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
            runner: ubuntu-latest
            llvm: 7
          - name: Ubuntu-24.04 / CUDA-13.3.1 / ARM64
            image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
            runner: ubuntu-24.04-arm
            llvm: 7
          - name: RockyLinux-9 / CUDA-12.8.1 / x86_64
            image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda12:latest"
            runner: ubuntu-latest
            llvm: 7
          - name: RockyLinux-9 / CUDA-13.3.1 / x86_64
            image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda13:latest"
            runner: ubuntu-latest
            llvm: 7

          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / x86_64
            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
            runner: ubuntu-24.04
            llvm: 21
            artifact: linux-x86_64
          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / ARM64
            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
            runner: ubuntu-24.04-arm
            llvm: 21
            artifact: linux-aarch64

    steps:
      - name: Free up space
        # Without this the job will likely run out of disk space.
        run: |
          df -h

          # Remove Java
          sudo rm -rf /usr/lib/jvm

          # Remove .NET
          sudo rm -rf /usr/share/dotnet

          # Remove Swift
          sudo rm -rf /usr/share/swift

          # Remove Haskell
          sudo rm -rf /usr/local/.ghcup

          # Remove Julia
          sudo rm -rf /usr/local/julia*

          # Remove Android SDKs
          sudo rm -rf /usr/local/lib/android

          # Remove Chromium
          sudo rm -rf /usr/local/share/chromium

          # Remove Microsoft/Edge and Google Chrome builds
          sudo rm -rf /opt/microsoft /opt/google

          # Remove Azure CLI
          sudo rm -rf /opt/az

          # Remove PowerShell
          sudo rm -rf /usr/local/share/powershell

          # Remove CodeQL and other toolcaches
          sudo rm -rf /opt/hostedtoolcache

          docker system prune -af || true
          docker builder prune -af || true
          df -h

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Pull build image
        run: docker pull ${{ matrix.variance.image }}

      - name: Record host UID/GID
        run: |
          echo "HOST_UID=$(id -u)" >> $GITHUB_ENV
          echo "HOST_GID=$(id -g)" >> $GITHUB_ENV

      - name: Set container name
        run: echo "CONTAINER_NAME=ci-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.variance.runner }}" >> $GITHUB_ENV

      - name: Start build container
        run: |
          docker create --name "$CONTAINER_NAME" \
            -v "$PWD":/workspace \
            -w /workspace \
            -e RUST_LOG \
            -e RUST_BACKTRACE \
            ${{ matrix.variance.image }} \
            sleep infinity
          docker start "$CONTAINER_NAME"

      - name: Install LLVM 21 release and build dependencies
        if: matrix.variance.llvm == 21
        env:
          LLVM_ARCH: ${{ matrix.variance.artifact }}
          PREBUILT_LLVM_URL: ${{ vars.PREBUILT_LLVM_URL || 'https://github.com/Rust-GPU/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8' }}
        run: |
          mkdir -p llvm-prebuilt
          curl --fail --location --retry 3 \
            "${PREBUILT_LLVM_URL%/}/$LLVM_ARCH.tar.xz" \
            -o "llvm-prebuilt/$LLVM_ARCH.tar.xz"
          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            apt-get update
            DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
              build-essential libclang-dev python3 curl ca-certificates git pkg-config libssl-dev \
              libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev libzstd-dev xz-utils
            curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
              sh -s -- -y --profile minimal --default-toolchain none
          '

      - name: Verify CUDA, Rust installation
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            nvcc --version
            if [ -f /root/.cargo/env ]; then source /root/.cargo/env; fi
            rustup show
          '

      - name: Rustfmt
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            cargo fmt --all -- --check
          '

      - name: Build all bindings
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            cargo build --all-features -p cust_raw
          '

      - name: Build workspace
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            cargo build
          '

      - name: Clippy
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            export RUSTFLAGS=-Dwarnings
            cargo clippy
          '

      # Exclude crates with tests that require an NVIDIA GPU: blastoff, cudnn, cust.
      - name: Test
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            export RUSTFLAGS=-Dwarnings
            cargo test --workspace \
              --exclude blastoff \
              --exclude cudnn \
              --exclude cust
          '

      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
      # `--all-features` pass over the whole workspace. Doc those three crates with
      # default features (the LLVM 7 path the CI image already supports) and the rest
      # of the workspace with `--all-features`.
      - name: Check documentation
        if: matrix.variance.llvm == 7
        run: |
          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
            export RUSTDOCFLAGS=-Dwarnings
            cargo doc --workspace --all-features --document-private-items --no-deps \
              --exclude rustc_codegen_nvvm --exclude cuda_builder --exclude nvvm
            cargo doc -p rustc_codegen_nvvm -p cuda_builder -p nvvm \
              --document-private-items --no-deps
          '

      # Compile both the host and device code; execution needs an NVIDIA GPU.
      - name: Build vector-add with LLVM 21
        if: matrix.variance.llvm == 21
        env:
          LLVM_ARCH: ${{ matrix.variance.artifact }}
        run: |
          docker exec \
            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
            -e LLVM_LINK_STATIC=1 \
            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
            -e RUST_CUDA_EMIT_LLVM_IR=1 \
            "$CONTAINER_NAME" bash -lc 'set -euo pipefail
              source /root/.cargo/env
              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
              python3 tests/llvm-wrapper/run.py
              cargo build -p vecadd --features llvm21
            '

      - name: Upload LLVM 21 PTX and IR
        if: always() && matrix.variance.llvm == 21
        uses: actions/upload-artifact@v4
        with:
          name: vecadd-llvm21-${{ matrix.variance.artifact }}
          path: |
            target/**/kernels.ptx
            target/**/final-module.ll
          if-no-files-found: warn

      - name: Stop build container
        if: always()
        run: |
          docker rm -f "$CONTAINER_NAME" || true

  compiletest:
    name: Compile tests
    runs-on: ubuntu-latest
    container:
      image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Run cargo version
        run: cargo --version
      - name: Rustfmt compiletests
        shell: bash
        # Uses rustfmt directly, rather than via `cargo fmt`, because the compiletests .rs files
        # are not within a package.
        run: shopt -s globstar && rustfmt --check tests/compiletests/ui/**/*.rs
      - name: Compiletest
        run: cargo run -p compiletests --release --no-default-features -- --target-arch compute_61,compute_75,compute_90