From a18038f07b12a1c12242b8e566fe975e4189cb88 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 24 Jun 2026 13:55:16 +0200 Subject: [PATCH 1/8] Move from custom DALI conda recipe to upstream nvidia-dali-feedstock - uses nvidia-dali-feedstock recipe to build DALI in CI - removes DALI maintained recipe Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 105 ++++---- conda/config/conda_build_config.yaml | 15 -- .../activate.d/nvimgcodec_extensions_path.sh | 13 - conda/dali_native_libs/recipe/build.sh | 232 ------------------ .../nvimgcodec_extensions_path.sh | 4 - conda/dali_native_libs/recipe/meta.yaml | 129 ---------- conda/dali_python_bindings/recipe/build.sh | 116 --------- conda/dali_python_bindings/recipe/meta.yaml | 144 ----------- 8 files changed, 47 insertions(+), 711 deletions(-) delete mode 100644 conda/config/conda_build_config.yaml delete mode 100644 conda/dali_native_libs/recipe/activate.d/nvimgcodec_extensions_path.sh delete mode 100644 conda/dali_native_libs/recipe/build.sh delete mode 100644 conda/dali_native_libs/recipe/deactivate.d/nvimgcodec_extensions_path.sh delete mode 100644 conda/dali_native_libs/recipe/meta.yaml delete mode 100644 conda/dali_python_bindings/recipe/build.sh delete mode 100644 conda/dali_python_bindings/recipe/meta.yaml diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index 4c2f5663399..b434b5dfb5f 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -3,70 +3,59 @@ set -o xtrace set -e -# Adding conda-forge channel for dependencies -conda config --add channels conda-forge -conda config --add channels nvidia -conda config --add channels local +source /root/miniconda3/bin/activate -CONDA_BUILD_OPTIONS="--exclusive-config-file config/conda_build_config.yaml" +if [ -z "${CUDA_VERSION_MAJOR}" ]; then + echo "CUDA_VERSION_MAJOR must be set (e.g. 12 or 13)" >&2 + exit 1 +fi +if [ "${CUDA_VERSION_MAJOR}" -eq 13 ]; then + export METAL_YAML=linux_64_c_stdlib_version2.28cuda_compiler_version13.0.yaml +else + export METAL_YAML=linux_64_c_stdlib_version2.17cuda_compiler_version12.9.yaml +fi -CONDA_PREFIX=${CONDA_PREFIX:-/root/miniconda3} +cd /opt/ +git clone https://github.com/conda-forge/nvidia-dali-python-feedstock -export CUDA_TARGET_ARCHS=${CUDA_TARGET_ARCHS} -export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} -export BUILD_TEST=${BUILD_TEST:-ON} -export BUILD_BENCHMARK=${BUILD_BENCHMARK:-ON} -export BUILD_FUZZING=${BUILD_FUZZING:-OFF} -export BUILD_NVTX=${BUILD_NVTX} -export BUILD_LMDB=${BUILD_LMDB:-ON} -export BUILD_LIBJPEG_TURBO=${BUILD_LIBJPEG_TURBO:-ON} -export BUILD_NVJPEG=${BUILD_NVJPEG:-ON} -export BUILD_LIBTIFF=${BUILD_LIBTIFF:-ON} -export BUILD_LIBSND=${BUILD_LIBSND:-ON} -export BUILD_LIBTAR=${BUILD_LIBTAR:-ON} -export BUILD_FFTS=${BUILD_FFTS:-ON} -export BUILD_CFITSIO=${BUILD_CFITSIO:-ON} -export BUILD_NVOF=${BUILD_NVOF:-ON} -export BUILD_NVDEC=${BUILD_NVDEC:-ON} -export BUILD_NVML=${BUILD_NVML:-ON} -export BUILD_NVCOMP=${BUILD_NVCOMP:-ON} -export WITH_DYNAMIC_CUDA_TOOLKIT=${WITH_DYNAMIC_CUDA_TOOLKIT:-OFF} -export WITH_DYNAMIC_NVJPEG=${WITH_DYNAMIC_NVJPEG:-ON} -export WITH_DYNAMIC_CUFFT=${WITH_DYNAMIC_CUFFT:-ON} -export WITH_DYNAMIC_NPP=${WITH_DYNAMIC_NPP:-ON} -export WITH_DYNAMIC_NVCOMP=${WITH_DYNAMIC_NVCOMP:-ON} -export VERBOSE_LOGS=${VERBOSE_LOGS:-OFF} -export WERROR=${WERROR:-ON} -export BUILD_WITH_ASAN=${BUILD_WITH_ASAN:-OFF} -export BUILD_WITH_LSAN=${BUILD_WITH_LSAN:-OFF} -export BUILD_WITH_UBSAN=${BUILD_WITH_UBSAN:-OFF} -export NVIDIA_BUILD_ID=${NVIDIA_BUILD_ID:-0} -export GIT_SHA=${GIT_SHA} -export DALI_TIMESTAMP=${DALI_TIMESTAMP} -export NVIDIA_DALI_BUILD_FLAVOR=${NVIDIA_DALI_BUILD_FLAVOR} -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -export DALI_CONDA_BUILD_VERSION=$(cat ../VERSION)$(if [ "${NVIDIA_DALI_BUILD_FLAVOR}" != "" ]; then \ - echo .${DALI_TIMESTAMP}; \ - fi) -export CUDA_VERSION=$(echo $(ls /usr/local/cuda/lib64/libcudart.so*) | sed 's/.*\.\([0-9]\+\)\.\([0-9]\+\)\.\([0-9]\+\)/\1.\2/') -export CONDA_OVERRIDE_CUDA=${CUDA_VERSION} -# when building for any version >= 11.0 use CUDA compatibility mode and claim it is a CUDA 110 package -if [ "${CUDA_VERSION/./}" -ge 110 ]; then - export CUDA_VERSION="${CUDA_VERSION%?}0" +cd nvidia-dali-python-feedstock +if [ -n "${DALI_FEEDSTOCK_SHA}" ]; then + git checkout ${DALI_FEEDSTOCK_SHA} fi -# Building DALI core package -conda mambabuild ${CONDA_BUILD_OPTIONS} dali_native_libs/recipe +echo "Using nvidia-dali-python-feedstock at $(git rev-parse HEAD)" + +# Use the local DALI checkout instead of downloading the release tarball. +RECIPE=recipe/recipe.yaml +sed -i 's| - url: https://github.com/NVIDIA/DALI/archive/refs/tags/v${{ version }}.tar.gz| - path: /opt/dali|' "${RECIPE}" +grep -q 'path: /opt/dali' "${RECIPE}" || { echo "Recipe patch failed: path: /opt/dali not found" >&2; exit 1; } +grep -q 'github.com/NVIDIA/DALI/archive' "${RECIPE}" && \ + { echo "Recipe patch failed: DALI tarball URL still present" >&2; exit 1; } + +sed -i '/path: \/opt\/dali/a\ use_gitignore: false' "${RECIPE}" +grep -q 'use_gitignore: false' "${RECIPE}" || \ + { echo "Recipe patch failed: use_gitignore: false not found" >&2; exit 1; } + +# ToDo remove when feedstock is updated to DALI 2.2 +# Local checkout already contains upstream changes; drop feedstock-only patches until it moves to +# DALI 2.2 +sed -i '/ patches:/,/^ - url:/{ /^ - url:/!d; }' "${RECIPE}" +grep -q 'patches/0001-BLD' "${RECIPE}" && \ + { echo "Recipe patch failed: DALI patches still present" >&2; exit 1; } + +sed -i '0,/^ sha256: /{/^ sha256: /d;}' "${RECIPE}" + +BUILD_SCRIPT=recipe/build.sh +sed -i '/^cmake \${CMAKE_ARGS} \\$/,/^\$SRC_DIR$/ s/^ -GNinja \\$/ -GNinja \\\n -DBUILD_FOR_CONDA=ON \\/' \ + "${BUILD_SCRIPT}" +grep -q 'BUILD_FOR_CONDA=ON' "${BUILD_SCRIPT}" || \ + { echo "build.sh patch failed: BUILD_FOR_CONDA=ON not found" >&2; exit 1; } +# End of ToDo -# Building DALI python bindings package -conda mambabuild ${CONDA_BUILD_OPTIONS} \ - --variants="{python: [3.10, 3.11, 3.12, 3.13, 3.14], freethreading: ['no']}" \ - dali_python_bindings/recipe -# ToDo - enable when DALI deps fully support ree-threded python -# conda mambabuild ${CONDA_BUILD_OPTIONS} \ -# --variants="{python: [3.14], freethreading: ['yes']}" \ -# dali_python_bindings/recipe +rattler-build build --recipe $(pwd)/recipe -m $(pwd)/.ci_support/${METAL_YAML} \ + --target-platform linux-64 --extra-meta flow_run_id=${NVIDIA_BUILD_ID:-0} \ + --extra-meta remote_url= --extra-meta sha=${GIT_SHA} # Copying the artifacts from conda prefix mkdir -p artifacts -cp ${CONDA_PREFIX}/conda-bld/*/nvidia-dali*.tar.bz2 artifacts +cp output/linux-64/*.conda artifacts diff --git a/conda/config/conda_build_config.yaml b/conda/config/conda_build_config.yaml deleted file mode 100644 index e53556f9420..00000000000 --- a/conda/config/conda_build_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -c_compiler_version: # [linux] - - 14.2 # [linux] -cxx_compiler_version: # [linux] - - 14.2 # [linux] -# for FFMPEG c11 macros -c_stdlib: - - sysroot # [linux] -c_stdlib_version: # [linux] - - 2.17 # [linux] - -# DALI uses only numpy Python API (no C ABI) - no runtime pinning needed -numpy: - - "2" -ignore_version: - - numpy diff --git a/conda/dali_native_libs/recipe/activate.d/nvimgcodec_extensions_path.sh b/conda/dali_native_libs/recipe/activate.d/nvimgcodec_extensions_path.sh deleted file mode 100644 index c37300e5832..00000000000 --- a/conda/dali_native_libs/recipe/activate.d/nvimgcodec_extensions_path.sh +++ /dev/null @@ -1,13 +0,0 @@ -# Point libnvimgcodec at the conda-forge extension modules. -# libnvimgcodec resolves its default extensions path dynamically via dladdr: -# with libnvimgcodec.so installed at $CONDA_PREFIX/lib it strips the "lib" -# component and probes $CONDA_PREFIX/extensions, but the conda-forge feedstock -# installs the extension shared libraries under $CONDA_PREFIX/lib/extensions, -# so the computed default lands one directory off and every decode fails with -# NVIMGCODEC_PROCESSING_STATUS_*. Workaround: pin NVIMGCODEC_EXTENSIONS_PATH -# from the activate hook. Drop this once libnvimgcodec's default-path -# resolution (or the feedstock install layout) is fixed. -if [ -z "${NVIMGCODEC_EXTENSIONS_PATH:-}" ]; then - export _DALI_SET_NVIMGCODEC_EXTENSIONS_PATH=1 - export NVIMGCODEC_EXTENSIONS_PATH="${CONDA_PREFIX}/lib/extensions" -fi diff --git a/conda/dali_native_libs/recipe/build.sh b/conda/dali_native_libs/recipe/build.sh deleted file mode 100644 index 2a7ee4b9c77..00000000000 --- a/conda/dali_native_libs/recipe/build.sh +++ /dev/null @@ -1,232 +0,0 @@ -#!/bin/bash -# -# (C) Copyright IBM Corp. 2019. All Rights Reserved. -# (C) Copyright NVIDIA CORPORATION. 2019. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#Determine Architecture - -ARCH="$(arch)" -if [ ${ARCH} = "x86_64" ]; then - ARCH_LONGNAME="x86_64-conda_cos6" -elif [ ${ARCH} = "ppc64le" ]; then - ARCH_LONGNAME="powerpc64le-conda_cos7" -else - echo "Error: Unsupported Architecture. Expected: [x86_64|ppc64le] Actual: ${ARCH}" - exit 1 -fi - -# Create 'gcc' and 'g++' symlinks so nvcc can find it -ln -s $CC $BUILD_PREFIX/bin/gcc -ln -s $CXX $BUILD_PREFIX/bin/g++ - -# Force -std=c++20 in CXXFLAGS -export CXXFLAGS=${CXXFLAGS/-std=c++??/-std=c++20} - -# For some reason `aligned_alloc` is present when we use compiler version 5.4.x -# Adding NO_ALIGNED_ALLOC definition for cutt -export CXXFLAGS="${CXXFLAGS} -DNO_ALIGNED_ALLOC" -export PATH=/usr/local/cuda/bin:${PATH} - -CUDA_VERSION=$(echo $(nvcc --version) | sed 's/.*\(release \)\([0-9]\+\)\.\([0-9]\+\).*/\2\3/') -# make it on by default for CUDA 11.x -if [ "${CUDA_VERSION/./}" -ge 110 ]; then - export WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT=ON -else - export WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT=OFF -fi - -# Create build directory for cmake and enter it -mkdir $SRC_DIR/build_core -cd $SRC_DIR/build_core -# Build -cmake -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ - -DCUDA_rt_LIBRARY=$BUILD_PREFIX/${ARCH_LONGNAME}-linux-gnu/sysroot/usr/lib/librt.so \ - -DCUDA_CUDA_LIBRARY=/usr/local/cuda/targets/${ARCH}-linux/lib/stubs/libcuda.so \ - -DCUDA_TARGET_ARCHS=${CUDA_TARGET_ARCHS} \ - -DFFMPEG_ROOT_DIR=$PREFIX/lib \ - -DCMAKE_PREFIX_PATH="$PREFIX" \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \ - -DBUILD_TEST=${BUILD_TEST:-ON} \ - -DBUILD_BENCHMARK=${BUILD_BENCHMARK:-ON} \ - -DBUILD_NVTX=${BUILD_NVTX} \ - -DBUILD_PYTHON=OFF \ - -DBUILD_FOR_CONDA=ON \ - -DPREBUILD_DALI_LIBS=OFF \ - -DPYTHON_STUBGEN_INTERPRETER=${PYTHON} \ - -DBUILD_LMDB=${BUILD_LMDB:-ON} \ - -DBUILD_LIBJPEG_TURBO=${BUILD_LIBJPEG_TURBO:-ON} \ - -DBUILD_OPENCV=${BUILD_OPENCV:-ON} \ - -DBUILD_PROTOBUF=${BUILD_PROTOBUF:-ON} \ - -DBUILD_NVJPEG=${BUILD_NVJPEG:-ON} \ - -DBUILD_NVJPEG2K=${BUILD_NVJPEG2K:-ON} \ - -DBUILD_LIBTIFF=${BUILD_LIBTIFF:-ON} \ - -DBUILD_LIBSND=${BUILD_LIBSND:-ON} \ - -DBUILD_LIBTAR=${BUILD_LIBTAR:-ON} \ - -DBUILD_FFTS=${BUILD_FFTS:-ON} \ - -DBUILD_NVOF=${BUILD_NVOF:-ON} \ - -DBUILD_NVDEC=${BUILD_NVDEC:-ON} \ - -DBUILD_NVML=${BUILD_NVML:-ON} \ - -DBUILD_CUFILE=${BUILD_CUFILE:-ON} \ - -DBUILD_NVCOMP=${BUILD_NVCOMP:-ON} \ - -DBUILD_CVCUDA=${BUILD_CVCUDA:-ON} \ - -DBUILD_NVIMAGECODEC=${BUILD_NVIMAGECODEC:-ON} \ - -DLINK_LIBCUDA=${LINK_LIBCUDA:-OFF} \ - -DWITH_DYNAMIC_CUDA_TOOLKIT=${WITH_DYNAMIC_CUDA_TOOLKIT:-${WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT}}\ - -DWITH_DYNAMIC_CUFFT=${WITH_DYNAMIC_CUFFT:-ON} \ - -DWITH_DYNAMIC_NPP=${WITH_DYNAMIC_NPP:-ON} \ - -DWITH_DYNAMIC_NVIMGCODEC=${WITH_DYNAMIC_NVIMGCODEC:-ON} \ - -DWITH_DYNAMIC_NVCOMP=${WITH_DYNAMIC_NVCOMP:-ON } \ - -DVERBOSE_LOGS=${VERBOSE_LOGS:-OFF} \ - -DWERROR=${WERROR:-ON} \ - -DBUILD_WITH_ASAN=${BUILD_WITH_ASAN:-OFF} \ - -DBUILD_WITH_LSAN=${BUILD_WITH_LSAN:-OFF} \ - -DBUILD_WITH_UBSAN=${BUILD_WITH_UBSAN:-OFF} \ - -DDALI_BUILD_FLAVOR=${NVIDIA_DALI_BUILD_FLAVOR} \ - -DTIMESTAMP=${DALI_TIMESTAMP} -DGIT_SHA=${GIT_SHA-${GIT_FULL_HASH}} \ - .. -make -j"$(nproc --all)" - -fname_with_sha256() { - HASH=$(sha256sum $1 | cut -c1-8) - BASENAME=$(basename $1) - INITNAME=$(echo $BASENAME | cut -f1 -d".") - ENDNAME=$(echo $BASENAME | cut -f 2- -d".") - echo "$INITNAME-$HASH.$ENDNAME" -} - -DEPS_LIST=( - "lib/libcvcuda.so.0" - "lib/libnvcv_types.so.0" -) - -DEPS_PATH=/usr/local/ - -if [ "$BUILD_NVCOMP" = "ON" ] && [ "$WITH_DYNAMIC_NVCOMP" = "ON" ]; then - DEPS_LIST+=( - "${DEPS_PATH}/cuda/lib64/libnvcomp.so.5" - ) -fi - -PKGNAME_PATH=$PWD/dali/python/nvidia/dali -DEPS_LIB_DST_PATH=$PKGNAME_PATH/dali_deps_libs -mkdir -p $DEPS_LIB_DST_PATH - -# copy needed dependent .so files and tag them with their hash -original=() -patched=() - -copy_and_patch() { - local filepath=$1 - filename=$(basename $filepath) - - if [[ ! -f "$filepath" ]]; then - echo "Didn't find $filename, skipping..." - return - fi - patchedname=$(fname_with_sha256 $filepath) - patchedpath=$DEPS_LIB_DST_PATH/$patchedname - original+=("$filename") - patched+=("$patchedname") - - echo "Copying $filepath to $patchedpath" - cp $filepath $patchedpath - - echo "Patching DT_SONAME field in $patchedpath" - patchelf --set-soname $patchedname $patchedpath & -} - -echo "Patching DT_SONAMEs..." -for filepath in "${DEPS_LIST[@]}"; do - copy_and_patch $filepath -done -wait -echo "Patched DT_SONAMEs" - -patch_hashed_names() { - local sofile=$1 - local patch_cmd="" - needed_so_files=$(patchelf --print-needed $sofile) - for ((j=0;j<${#original[@]};++j)); do - origname=${original[j]} - patchedname=${patched[j]} - if [[ "$origname" != "$patchedname" ]]; then - set +e - echo $needed_so_files | grep $origname 2>&1 >/dev/null - ERRCODE=$? - set -e - if [ "$ERRCODE" -eq "0" ]; then - echo "patching $sofile entry $origname to $patchedname" - patch_cmd="$patch_cmd --replace-needed $origname $patchedname" - fi - fi - done - if [ -n "$patch_cmd" ]; then - echo "running $patch_cmd on $sofile" - patchelf $patch_cmd $sofile - fi -} -echo "Patching to fix the so names to the hashed names..." -# get list of files to iterate over -sofile_list=() -while IFS= read -r -d $'\0'; do - sofile_list+=("$REPLY") -done < <(find $PKGNAME_PATH -name '*.so*' -print0) -while IFS= read -r -d $'\0'; do - sofile_list+=("$REPLY") -done < <(find $DEPS_LIB_DST_PATH -name '*.so*' -print0) -while IFS= read -r -d $'\0'; do - sofile_list+=("$REPLY") -done < <(find $PKGNAME_PATH -name '*.bin' -print0) -for ((i=0;i<${#sofile_list[@]};++i)); do - sofile=${sofile_list[i]} - patch_hashed_names $sofile & -done -wait -echo "Fixed hashed names" - -# set RPATH of backend_impl.so and similar to $ORIGIN, $ORIGIN$UPDIRS, $ORIGIN$UPDIRS/dali_deps_libs -find $PKGNAME_PATH -type f -name "*.so*" -o -name "*.bin" | while read FILE; do - UPDIRS=$(dirname $(echo "$FILE" | sed "s|$PKGNAME_PATH||") | sed 's/[^\/][^\/]*/../g') - echo "Setting rpath of $FILE to '\$ORIGIN:\$ORIGIN$UPDIRS:\$ORIGIN$UPDIRS/dali_deps_libs'" - patchelf --set-rpath "\$ORIGIN:\$ORIGIN$UPDIRS:\$ORIGIN$UPDIRS/dali_deps_libs" $FILE - patchelf --print-rpath $FILE - if [[ "$FILE" == *".so"* ]]; then - cp $FILE $PREFIX/lib/; - fi - if [[ "$FILE" == *"dali_deps_libs"* ]]; then - mkdir -p $PREFIX/lib/dali_deps_libs/ - cp $FILE $PREFIX/lib/dali_deps_libs/; - fi - if [[ "$FILE" == *".bin"* ]]; then - cp $FILE $PREFIX/bin/; - fi -done - -# copy generated headers for the bindings build -find -iname *.pb.h | while read FILE; do - echo $FILE $PREFIX/include/$FILE - mkdir -p $(dirname $PREFIX/include/$FILE) - cp $FILE $PREFIX/include/$FILE -done - -# Install conda activate/deactivate scripts so libnvimgcodec finds the conda-forge -# extension modules. The default search path compiled into libnvimgcodec.so does not -# match the conda-forge install layout, so without this nvimgcodecDecoderCreate would -# succeed but every decode would fail with NVIMGCODEC_PROCESSING_STATUS_*. -mkdir -p "$PREFIX/etc/conda/activate.d" "$PREFIX/etc/conda/deactivate.d" -cp "$RECIPE_DIR/activate.d/"*.sh "$PREFIX/etc/conda/activate.d/" -cp "$RECIPE_DIR/deactivate.d/"*.sh "$PREFIX/etc/conda/deactivate.d/" diff --git a/conda/dali_native_libs/recipe/deactivate.d/nvimgcodec_extensions_path.sh b/conda/dali_native_libs/recipe/deactivate.d/nvimgcodec_extensions_path.sh deleted file mode 100644 index de8fc476bd6..00000000000 --- a/conda/dali_native_libs/recipe/deactivate.d/nvimgcodec_extensions_path.sh +++ /dev/null @@ -1,4 +0,0 @@ -if [ "${_DALI_SET_NVIMGCODEC_EXTENSIONS_PATH:-}" = "1" ]; then - unset NVIMGCODEC_EXTENSIONS_PATH - unset _DALI_SET_NVIMGCODEC_EXTENSIONS_PATH -fi diff --git a/conda/dali_native_libs/recipe/meta.yaml b/conda/dali_native_libs/recipe/meta.yaml deleted file mode 100644 index 766549a8b1a..00000000000 --- a/conda/dali_native_libs/recipe/meta.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# (C) Copyright IBM Corp. 2019. All Rights Reserved. -# Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -package: - name: nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} - version: {{ environ.get('DALI_CONDA_BUILD_VERSION', '') }} - -source: - # Beware: Only committed files are used - - git_url: ../../.. - -build: - script_env: - - CUDA_TARGET_ARCHS - - CMAKE_BUILD_TYPE - - BUILD_TEST - - BUILD_BENCHMARK - - BUILD_FUZZING - - BUILD_NVTX - - BUILD_LMDB - - BUILD_LIBJPEG_TURBO - - BUILD_NVJPEG - - BUILD_LIBTIFF - - BUILD_LIBSND - - BUILD_LIBTAR - - BUILD_FFTS - - BUILD_CFITSIO - - BUILD_NVOF - - BUILD_NVDEC - - BUILD_NVML - - BUILD_NVCOMP - - WITH_DYNAMIC_CUDA_TOOLKIT - - WITH_DYNAMIC_NVJPEG - - WITH_DYNAMIC_CUFFT - - WITH_DYNAMIC_NPP - - WITH_DYNAMIC_NVIMGCODEC - - WITH_DYNAMIC_NVCOMP - - VERBOSE_LOGS - - WERROR - - BUILD_WITH_ASAN - - BUILD_WITH_LSAN - - BUILD_WITH_UBSAN - - NVIDIA_BUILD_ID - - GIT_SHA - - DALI_TIMESTAMP - - NVIDIA_DALI_BUILD_FLAVOR - - LD_LIBRARY_PATH - - DALI_CONDA_BUILD_VERSION - - CUDA_VERSION - string: {{ environ.get('NVIDIA_BUILD_ID', '') }} - -requirements: - build: - skip: True # [not linux] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - pkg-config - - cmake >=3.25.2 - - make - - patchelf - - python-clang - - git-lfs - host: - - protobuf =7.35.1 - - libprotobuf-static =7.35.1 - # libprotobuf's CMake config exposes a ZLIB::ZLIB link interface dep, - # so zlib must be in host for find_package(Protobuf) to succeed. - - zlib - - libjpeg-turbo - - libopencv >=4.13,<5 - # FFmpeg 8.1.2 needs libjxl.so.0.11, but its metadata permits libjxl 0.12. - # Remove these pins once FFmpeg is rebuilt for the newer libjxl ABI. - - ffmpeg =8.1.2 - - libjxl <0.12 - # LMDB 1.0 cannot open the 0.9-format DALI_extra databases used by readers tests. - - lmdb <1.0 - - libtiff - - libsndfile - - libtar - - libvorbis =1.3.7 - - cfitsio - - libnvimgcodec-dev >=0.9.0,<0.10.0 - - libnvcomp - # libnvcomp depends on cuda, so make sure it uses the right version for the build - - cuda-version {{ environ.get('CUDA_VERSION', '').split('.')[0] }} - - aws-sdk-cpp - run: - - libjpeg-turbo - - libopencv >=4.13,<5 - - ffmpeg =8.1.2 - - libjxl <0.12 - - lmdb <1.0 - - libtiff - - libsndfile - - libvorbis =1.3.7 - # libprotobuf-static we link statically depends on libabseil so add libprotobuf here as a runtime - # dependency to install the right version on the libabseil (as protobuf depends on - # libprotobuf-static and a newer version of libprotobuf-static may be available than - # the protobuf was build with) - - libprotobuf =7.35.1 - - cfitsio - - libnvimgcodec >=0.9.0,<0.10.0 - - libnvimgcodec-libjpeg-turbo-ext >=0.9.0,<0.10.0 - - libnvimgcodec-libtiff-ext >=0.9.0,<0.10.0 - - libnvimgcodec-libopencv-ext >=0.9.0,<0.10.0 - - libnvcomp - # libnvcomp depends on cuda, so make sure it uses the right version for the build - - cuda-version {{ environ.get('CUDA_VERSION', '').split('.')[0] }} - - aws-sdk-cpp -about: - home: https://github.com/NVIDIA/DALI - license: Apache-2.0 license - license_family: APACHE - license_file: - - ../../../LICENSE - - ../../../COPYRIGHT - - ../../../Acknowledgements.txt diff --git a/conda/dali_python_bindings/recipe/build.sh b/conda/dali_python_bindings/recipe/build.sh deleted file mode 100644 index 96d5b4f2381..00000000000 --- a/conda/dali_python_bindings/recipe/build.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# -# (C) Copyright IBM Corp. 2019. All Rights Reserved. -# (C) Copyright NVIDIA CORPORATION. 2019. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#Determine Architecture - -ARCH="$(arch)" -if [ ${ARCH} = "x86_64" ]; then - ARCH_LONGNAME="x86_64-conda_cos6" -elif [ ${ARCH} = "ppc64le" ]; then - ARCH_LONGNAME="powerpc64le-conda_cos7" -else - echo "Error: Unsupported Architecture. Expected: [x86_64|ppc64le] Actual: ${ARCH}" - exit 1 -fi - -# Create 'gcc' and 'g++' symlinks so nvcc can find it -ln -s $CC $BUILD_PREFIX/bin/gcc -ln -s $CXX $BUILD_PREFIX/bin/g++ - -# Force -std=c++20 in CXXFLAGS -export CXXFLAGS=${CXXFLAGS/-std=c++??/-std=c++20} - -# For some reason `aligned_alloc` is present when we use compiler version 5.4.x -# Adding NO_ALIGNED_ALLOC definition for cutt -export CXXFLAGS="${CXXFLAGS} -DNO_ALIGNED_ALLOC" -export PATH=/usr/local/cuda/bin:${PATH} - -# make it on by default for CUDA 11.x -if [ "${CUDA_VERSION/./}" -ge 110 ]; then - export WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT=ON -else - export WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT=OFF -fi - - -export BUILD_NVCOMP=${BUILD_NVCOMP:-OFF} - -# Create build directory for cmake and enter it -mkdir $SRC_DIR/build_bindings -cd $SRC_DIR/build_bindings - -# allow DALI import all dependencies in the build env -export LD_LIBRARY_PATH="$PREFIX/lib:$LD_LIBRARY_PATH" - -# Build -cmake -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ - -DCUDA_rt_LIBRARY=$BUILD_PREFIX/${ARCH_LONGNAME}-linux-gnu/sysroot/usr/lib/librt.so \ - -DCUDA_CUDA_LIBRARY=/usr/local/cuda/targets/${ARCH}-linux/lib/stubs/libcuda.so \ - -DCUDA_TARGET_ARCHS=${CUDA_TARGET_ARCHS} \ - -DFFMPEG_ROOT_DIR=$PREFIX/lib \ - -DCMAKE_PREFIX_PATH="$PREFIX" \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \ - -DBUILD_TEST=${BUILD_TEST:-ON} \ - -DBUILD_BENCHMARK=${BUILD_BENCHMARK:-ON} \ - -DBUILD_NVTX=${BUILD_NVTX} \ - -DBUILD_PYTHON=ON \ - -DBUILD_FOR_CONDA=ON \ - -DPREBUILD_DALI_LIBS=ON \ - -DPYTHON_STUBGEN_INTERPRETER=${PYTHON} \ - -DBUILD_LMDB=${BUILD_LMDB:-ON} \ - -DBUILD_LIBJPEG_TURBO=${BUILD_LIBJPEG_TURBO:-ON} \ - -DBUILD_OPENCV=${BUILD_OPENCV:-ON} \ - -DBUILD_PROTOBUF=${BUILD_PROTOBUF:-ON} \ - -DBUILD_NVJPEG=${BUILD_NVJPEG:-ON} \ - -DBUILD_NVJPEG2K=${BUILD_NVJPEG2K:-ON} \ - -DBUILD_LIBTIFF=${BUILD_LIBTIFF:-ON} \ - -DBUILD_LIBSND=${BUILD_LIBSND:-ON} \ - -DBUILD_LIBTAR=${BUILD_LIBTAR:-ON} \ - -DBUILD_FFTS=${BUILD_FFTS:-ON} \ - -DBUILD_NVOF=${BUILD_NVOF:-ON} \ - -DBUILD_NVDEC=${BUILD_NVDEC:-ON} \ - -DBUILD_NVML=${BUILD_NVML:-ON} \ - -DBUILD_CUFILE=${BUILD_CUFILE:-ON} \ - -DBUILD_NVCOMP=${BUILD_NVCOMP:-ON} \ - -DBUILD_CVCUDA=${BUILD_CVCUDA:-ON} \ - -DBUILD_NVIMAGECODEC=${BUILD_NVIMAGECODEC:-ON} \ - -DLINK_LIBCUDA=${LINK_LIBCUDA:-OFF} \ - -DWITH_DYNAMIC_CUDA_TOOLKIT=${WITH_DYNAMIC_CUDA_TOOLKIT:-${WITH_DYNAMIC_CUDA_TOOLKIT_DEFAULT}}\ - -DWITH_DYNAMIC_CUFFT=${WITH_DYNAMIC_CUFFT:-ON} \ - -DWITH_DYNAMIC_NPP=${WITH_DYNAMIC_NPP:-ON} \ - -DWITH_DYNAMIC_NVIMGCODEC=${WITH_DYNAMIC_NVIMGCODEC:-ON} \ - -DWITH_DYNAMIC_NVCOMP=${WITH_DYNAMIC_NVCOMP:-ON } \ - -DVERBOSE_LOGS=${VERBOSE_LOGS:-OFF} \ - -DWERROR=${WERROR:-ON} \ - -DBUILD_WITH_ASAN=${BUILD_WITH_ASAN:-OFF} \ - -DBUILD_WITH_LSAN=${BUILD_WITH_LSAN:-OFF} \ - -DBUILD_WITH_UBSAN=${BUILD_WITH_UBSAN:-OFF} \ - -DDALI_BUILD_FLAVOR=${NVIDIA_DALI_BUILD_FLAVOR} \ - -DTIMESTAMP=${DALI_TIMESTAMP} -DGIT_SHA=${GIT_SHA-${GIT_FULL_HASH}} \ - .. -make -j"$(nproc --all)" dali_python python_function_plugin copy_post_build_target dali_python_generate_stubs install_headers - -# pip install -$PYTHON -m pip install --no-deps --ignore-installed -v dali/python - -DALI_PATH=$($PYTHON -c 'import nvidia.dali as dali; import os; print(os.path.dirname(dali.__file__))') -echo "DALI_PATH is ${DALI_PATH}" - -# Move tfrecord2idx to host env so it can be found at runtime -cp $SRC_DIR/tools/tfrecord2idx $PREFIX/bin diff --git a/conda/dali_python_bindings/recipe/meta.yaml b/conda/dali_python_bindings/recipe/meta.yaml deleted file mode 100644 index a51867d3bcb..00000000000 --- a/conda/dali_python_bindings/recipe/meta.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# (C) Copyright IBM Corp. 2019. All Rights Reserved. -# Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -package: - name: nvidia-dali{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} - version: {{ environ.get('DALI_CONDA_BUILD_VERSION', '') }} - -{% if freethreading is not defined %} -{% set freethreading = "no" %} -{% endif %} -{% set freethreading = freethreading | string | lower %} -{% if freethreading == "true" %} -{% set freethreading = "yes" %} -{% endif %} -{% set py_abi_suffix = "t" if freethreading == "yes" else "" %} - -source: - # Beware: Only committed files are used - - git_url: ../../.. - -build: - script_env: - - CUDA_TARGET_ARCHS - - CMAKE_BUILD_TYPE - - BUILD_TEST - - BUILD_BENCHMARK - - BUILD_FUZZING - - BUILD_NVTX - - BUILD_LMDB - - BUILD_LIBJPEG_TURBO - - BUILD_NVJPEG - - BUILD_LIBTIFF - - BUILD_LIBSND - - BUILD_LIBTAR - - BUILD_FFTS - - BUILD_CFITSIO - - BUILD_NVOF - - BUILD_NVDEC - - BUILD_NVML - - BUILD_NVCOMP - - WITH_DYNAMIC_CUDA_TOOLKIT - - WITH_DYNAMIC_NVJPEG - - WITH_DYNAMIC_CUFFT - - WITH_DYNAMIC_NPP - - WITH_DYNAMIC_NVIMGCODEC - - WITH_DYNAMIC_NVCOMP - - VERBOSE_LOGS - - WERROR - - BUILD_WITH_ASAN - - BUILD_WITH_LSAN - - BUILD_WITH_UBSAN - - NVIDIA_BUILD_ID - - GIT_SHA - - DALI_TIMESTAMP - - NVIDIA_DALI_BUILD_FLAVOR - - LD_LIBRARY_PATH - - DALI_CONDA_BUILD_VERSION - - CUDA_VERSION - string: py{{ python | replace(".","") }}{{ py_abi_suffix }}_{{ environ.get('NVIDIA_BUILD_ID', '') }} - -requirements: - build: - skip: True # [not linux] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - pkg-config - - cmake >=3.25.2 - - make - - patchelf - - python-clang - - git-lfs - - black =26.5.1 - - black-jupyter =26.5.1 - - bandit - host: - - libprotobuf-static =7.35.1 - # libprotobuf's CMake config exposes a ZLIB::ZLIB link interface dep, - # so zlib must be in host for find_package(Protobuf) to succeed. - - zlib - - libopencv >=4.13,<5 - - python - {% if freethreading == "yes" %} - - python-freethreading - {% elif python | float >= 3.13 %} - - python-gil - {% endif %} - - setuptools - - future - - astunparse >=1.6.0 - - gast >=0.3.3 - - optree - - packaging - - nvtx - - makefun - - numpy - - libcst ~=1.8 - - nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} ={{ environ.get('DALI_CONDA_BUILD_VERSION', '') }} - - libnvimgcodec-dev >=0.9.0,<0.10.0 - run: - - python - {% if freethreading == "yes" %} - - python-freethreading - {% elif python | float >= 3.13 %} - - python-gil - {% endif %} - # libprotobuf-static we link statically depends on libabseil so add libprotobuf here as a runtime - # dependency to install the right version on the libabseil (as protobuf depends on - # libprotobuf-static and a newer version of libprotobuf-static may be available than - # the protobuf was build with) - - libprotobuf =7.35.1 - - future - - astunparse >=1.6.0 - - gast >=0.3.3 - - optree - - packaging - - nvtx - - makefun - - numpy - - libcst ~=1.8 - - nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} ={{ environ.get('DALI_CONDA_BUILD_VERSION', '') }} - - libnvimgcodec >=0.9.0,<0.10.0 - - libnvimgcodec-libjpeg-turbo-ext >=0.9.0,<0.10.0 - - libnvimgcodec-libtiff-ext >=0.9.0,<0.10.0 - - libnvimgcodec-libopencv-ext >=0.9.0,<0.10.0 -about: - home: https://github.com/NVIDIA/DALI - license: Apache-2.0 license - license_family: APACHE - license_file: - - ../../../LICENSE - - ../../../COPYRIGHT - - ../../../Acknowledgements.txt From e9f1f65b3c600568566ceb2d755b794adfedfec1 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 24 Jun 2026 16:47:02 +0200 Subject: [PATCH 2/8] Fix feedstock source patch for local conda builds Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index b434b5dfb5f..1a41d928f5f 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -45,6 +45,15 @@ grep -q 'patches/0001-BLD' "${RECIPE}" && \ sed -i '0,/^ sha256: /{/^ sha256: /d;}' "${RECIPE}" +# The local DALI checkout already has initialized submodules. Feedstock adds these sources only +# because GitHub release tarballs do not contain submodule contents. +sed -i '\|https://github.com/cocodataset/cocoapi/archive/|,+2d' "${RECIPE}" +sed -i '\|https://github.com/JanuszL/ffts/archive/|,+2d' "${RECIPE}" +grep -q 'target_directory: third_party/cocoapi' "${RECIPE}" && \ + { echo "Recipe patch failed: cocoapi source still present" >&2; exit 1; } +grep -q 'target_directory: third_party/ffts' "${RECIPE}" && \ + { echo "Recipe patch failed: ffts source still present" >&2; exit 1; } + BUILD_SCRIPT=recipe/build.sh sed -i '/^cmake \${CMAKE_ARGS} \\$/,/^\$SRC_DIR$/ s/^ -GNinja \\$/ -GNinja \\\n -DBUILD_FOR_CONDA=ON \\/' \ "${BUILD_SCRIPT}" From ea05234bb2e96a2d10b6a93b42ac98bc11d6d694 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 24 Jun 2026 16:56:39 +0200 Subject: [PATCH 3/8] Fix feedstock symlink setup for local conda builds Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index 1a41d928f5f..14bbd3870a8 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -55,6 +55,17 @@ grep -q 'target_directory: third_party/ffts' "${RECIPE}" && \ { echo "Recipe patch failed: ffts source still present" >&2; exit 1; } BUILD_SCRIPT=recipe/build.sh +sed -i '/^ln -sf \$PREFIX\/include\/boost /i\rm -rf third_party/boost/preprocessor/include/boost' \ + "${BUILD_SCRIPT}" +sed -i '/^ln -sf \$PREFIX\/include\/dlpack /i\rm -rf third_party/dlpack/include/dlpack' \ + "${BUILD_SCRIPT}" +sed -i '/^ln -sf \$PREFIX\/include\/cute /i\rm -rf third_party/cutlass/include/cute' \ + "${BUILD_SCRIPT}" +sed -i '/^ln -sf \$PREFIX\/include\/cutlass /i\rm -rf third_party/cutlass/include/cutlass' \ + "${BUILD_SCRIPT}" +grep -q 'rm -rf third_party/boost/preprocessor/include/boost' "${BUILD_SCRIPT}" || \ + { echo "build.sh patch failed: boost symlink cleanup not found" >&2; exit 1; } + sed -i '/^cmake \${CMAKE_ARGS} \\$/,/^\$SRC_DIR$/ s/^ -GNinja \\$/ -GNinja \\\n -DBUILD_FOR_CONDA=ON \\/' \ "${BUILD_SCRIPT}" grep -q 'BUILD_FOR_CONDA=ON' "${BUILD_SCRIPT}" || \ From 80126c454801b210652801fd3176f56c502c3f9c Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 24 Jun 2026 17:05:35 +0200 Subject: [PATCH 4/8] Verify feedstock symlink cleanup patches Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index 14bbd3870a8..61169fb23d1 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -65,6 +65,12 @@ sed -i '/^ln -sf \$PREFIX\/include\/cutlass /i\rm -rf third_party/cutlass/includ "${BUILD_SCRIPT}" grep -q 'rm -rf third_party/boost/preprocessor/include/boost' "${BUILD_SCRIPT}" || \ { echo "build.sh patch failed: boost symlink cleanup not found" >&2; exit 1; } +grep -q 'rm -rf third_party/dlpack/include/dlpack' "${BUILD_SCRIPT}" || \ + { echo "build.sh patch failed: dlpack symlink cleanup not found" >&2; exit 1; } +grep -q 'rm -rf third_party/cutlass/include/cute' "${BUILD_SCRIPT}" || \ + { echo "build.sh patch failed: cute symlink cleanup not found" >&2; exit 1; } +grep -q 'rm -rf third_party/cutlass/include/cutlass' "${BUILD_SCRIPT}" || \ + { echo "build.sh patch failed: cutlass symlink cleanup not found" >&2; exit 1; } sed -i '/^cmake \${CMAKE_ARGS} \\$/,/^\$SRC_DIR$/ s/^ -GNinja \\$/ -GNinja \\\n -DBUILD_FOR_CONDA=ON \\/' \ "${BUILD_SCRIPT}" From 648da7a0ccaaa8042fda6da734c242c3f416ed4a Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 24 Jun 2026 17:11:57 +0200 Subject: [PATCH 5/8] Update feedstock nvCOMP pin for conda builds Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index 61169fb23d1..15871e77957 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -25,6 +25,14 @@ fi echo "Using nvidia-dali-python-feedstock at $(git rev-parse HEAD)" +# DALI uses nvCOMP APIs introduced after 5.1. Keep all feedstock variants aligned with +# the DALI source dependency requirement until the upstream feedstock catches up. +sed -i 's/5\.1\.0\.\*/5.2.0.*/g' .ci_support/*.yaml recipe/conda_build_config.yaml +grep -R -q '5\.1\.0\.\*' .ci_support recipe/conda_build_config.yaml && \ + { echo "Feedstock patch failed: libnvcomp 5.1 pin still present" >&2; exit 1; } +grep -R -q '5\.2\.0\.\*' .ci_support recipe/conda_build_config.yaml || \ + { echo "Feedstock patch failed: libnvcomp 5.2 pin not found" >&2; exit 1; } + # Use the local DALI checkout instead of downloading the release tarball. RECIPE=recipe/recipe.yaml sed -i 's| - url: https://github.com/NVIDIA/DALI/archive/refs/tags/v${{ version }}.tar.gz| - path: /opt/dali|' "${RECIPE}" From 25b0b23107f224e1e50c8ecc5bff5af1800d7eac Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Mon, 13 Jul 2026 18:53:10 +0200 Subject: [PATCH 6/8] Fix paths Signed-off-by: Janusz Lisiecki --- conda/build_conda_packages.sh | 77 +++++-------- conda/nvidia-dali-python-feedstock.patch | 135 +++++++++++++++++++++++ 2 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 conda/nvidia-dali-python-feedstock.patch diff --git a/conda/build_conda_packages.sh b/conda/build_conda_packages.sh index 15871e77957..32323e61b5e 100644 --- a/conda/build_conda_packages.sh +++ b/conda/build_conda_packages.sh @@ -15,6 +15,24 @@ else export METAL_YAML=linux_64_c_stdlib_version2.17cuda_compiler_version12.9.yaml fi +export DALI_VERSION="$(tr -d '\r\n' < /opt/dali/VERSION)" +export NVIDIA_DALI_BUILD_FLAVOR="${NVIDIA_DALI_BUILD_FLAVOR:-}" +export DALI_TIMESTAMP="${DALI_TIMESTAMP:-}" +export NVIDIA_BUILD_ID="${NVIDIA_BUILD_ID:-0}" + +if [[ ! "${DALI_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+[[:alnum:].]*$ ]]; then + echo "Invalid DALI version in /opt/dali/VERSION: ${DALI_VERSION}" >&2 + exit 1 +fi +if [[ -n "${NVIDIA_DALI_BUILD_FLAVOR}" && ! "${DALI_TIMESTAMP}" =~ ^[0-9]{8}$ ]]; then + echo "DALI_TIMESTAMP must use YYYYMMDD format when NVIDIA_DALI_BUILD_FLAVOR is set" >&2 + exit 1 +fi +if [[ ! "${NVIDIA_BUILD_ID}" =~ ^[0-9]+$ ]]; then + echo "NVIDIA_BUILD_ID must be a non-negative integer" >&2 + exit 1 +fi + cd /opt/ git clone https://github.com/conda-forge/nvidia-dali-python-feedstock @@ -25,13 +43,12 @@ fi echo "Using nvidia-dali-python-feedstock at $(git rev-parse HEAD)" -# DALI uses nvCOMP APIs introduced after 5.1. Keep all feedstock variants aligned with -# the DALI source dependency requirement until the upstream feedstock catches up. -sed -i 's/5\.1\.0\.\*/5.2.0.*/g' .ci_support/*.yaml recipe/conda_build_config.yaml -grep -R -q '5\.1\.0\.\*' .ci_support recipe/conda_build_config.yaml && \ - { echo "Feedstock patch failed: libnvcomp 5.1 pin still present" >&2; exit 1; } -grep -R -q '5\.2\.0\.\*' .ci_support recipe/conda_build_config.yaml || \ - { echo "Feedstock patch failed: libnvcomp 5.2 pin not found" >&2; exit 1; } +# Adapt the current feedstock for building from the local DALI checkout. The patch removes +# release-tarball-only sources and patches, clears vendored include directories before the +# feedstock creates conda include symlinks, enables DALI's conda build configuration, and +# documents the nvCOMP 5.2 requirement. +git apply --check /opt/dali/conda/nvidia-dali-python-feedstock.patch +git apply /opt/dali/conda/nvidia-dali-python-feedstock.patch # Use the local DALI checkout instead of downloading the release tarball. RECIPE=recipe/recipe.yaml @@ -44,52 +61,10 @@ sed -i '/path: \/opt\/dali/a\ use_gitignore: false' "${RECIPE}" grep -q 'use_gitignore: false' "${RECIPE}" || \ { echo "Recipe patch failed: use_gitignore: false not found" >&2; exit 1; } -# ToDo remove when feedstock is updated to DALI 2.2 -# Local checkout already contains upstream changes; drop feedstock-only patches until it moves to -# DALI 2.2 -sed -i '/ patches:/,/^ - url:/{ /^ - url:/!d; }' "${RECIPE}" -grep -q 'patches/0001-BLD' "${RECIPE}" && \ - { echo "Recipe patch failed: DALI patches still present" >&2; exit 1; } - -sed -i '0,/^ sha256: /{/^ sha256: /d;}' "${RECIPE}" - -# The local DALI checkout already has initialized submodules. Feedstock adds these sources only -# because GitHub release tarballs do not contain submodule contents. -sed -i '\|https://github.com/cocodataset/cocoapi/archive/|,+2d' "${RECIPE}" -sed -i '\|https://github.com/JanuszL/ffts/archive/|,+2d' "${RECIPE}" -grep -q 'target_directory: third_party/cocoapi' "${RECIPE}" && \ - { echo "Recipe patch failed: cocoapi source still present" >&2; exit 1; } -grep -q 'target_directory: third_party/ffts' "${RECIPE}" && \ - { echo "Recipe patch failed: ffts source still present" >&2; exit 1; } - -BUILD_SCRIPT=recipe/build.sh -sed -i '/^ln -sf \$PREFIX\/include\/boost /i\rm -rf third_party/boost/preprocessor/include/boost' \ - "${BUILD_SCRIPT}" -sed -i '/^ln -sf \$PREFIX\/include\/dlpack /i\rm -rf third_party/dlpack/include/dlpack' \ - "${BUILD_SCRIPT}" -sed -i '/^ln -sf \$PREFIX\/include\/cute /i\rm -rf third_party/cutlass/include/cute' \ - "${BUILD_SCRIPT}" -sed -i '/^ln -sf \$PREFIX\/include\/cutlass /i\rm -rf third_party/cutlass/include/cutlass' \ - "${BUILD_SCRIPT}" -grep -q 'rm -rf third_party/boost/preprocessor/include/boost' "${BUILD_SCRIPT}" || \ - { echo "build.sh patch failed: boost symlink cleanup not found" >&2; exit 1; } -grep -q 'rm -rf third_party/dlpack/include/dlpack' "${BUILD_SCRIPT}" || \ - { echo "build.sh patch failed: dlpack symlink cleanup not found" >&2; exit 1; } -grep -q 'rm -rf third_party/cutlass/include/cute' "${BUILD_SCRIPT}" || \ - { echo "build.sh patch failed: cute symlink cleanup not found" >&2; exit 1; } -grep -q 'rm -rf third_party/cutlass/include/cutlass' "${BUILD_SCRIPT}" || \ - { echo "build.sh patch failed: cutlass symlink cleanup not found" >&2; exit 1; } - -sed -i '/^cmake \${CMAKE_ARGS} \\$/,/^\$SRC_DIR$/ s/^ -GNinja \\$/ -GNinja \\\n -DBUILD_FOR_CONDA=ON \\/' \ - "${BUILD_SCRIPT}" -grep -q 'BUILD_FOR_CONDA=ON' "${BUILD_SCRIPT}" || \ - { echo "build.sh patch failed: BUILD_FOR_CONDA=ON not found" >&2; exit 1; } -# End of ToDo - rattler-build build --recipe $(pwd)/recipe -m $(pwd)/.ci_support/${METAL_YAML} \ --target-platform linux-64 --extra-meta flow_run_id=${NVIDIA_BUILD_ID:-0} \ --extra-meta remote_url= --extra-meta sha=${GIT_SHA} # Copying the artifacts from conda prefix -mkdir -p artifacts -cp output/linux-64/*.conda artifacts +mkdir -p /opt/dali/conda/artifacts/ +cp output/linux-64/*.conda /opt/dali/conda/artifacts/ diff --git a/conda/nvidia-dali-python-feedstock.patch b/conda/nvidia-dali-python-feedstock.patch new file mode 100644 index 00000000000..62b39d6e343 --- /dev/null +++ b/conda/nvidia-dali-python-feedstock.patch @@ -0,0 +1,135 @@ +diff --git a/recipe/build.sh b/recipe/build.sh +index 23d3cee..b1e9f12 100755 +--- a/recipe/build.sh ++++ b/recipe/build.sh +@@ -48,13 +48,17 @@ if [ -z "${targetsDir+x}" ]; then + fi + + mkdir -p third_party/boost/preprocessor/include ++rm -rf third_party/boost/preprocessor/include/boost + ln -sf $PREFIX/include/boost third_party/boost/preprocessor/include/ + + mkdir -p third_party/dlpack/include/ ++rm -rf third_party/dlpack/include/dlpack + ln -sf $PREFIX/include/dlpack third_party/dlpack/include/ + + mkdir -p third_party/cutlass/include/ ++rm -rf third_party/cutlass/include/cute + ln -sf $PREFIX/include/cute third_party/cutlass/include/ ++rm -rf third_party/cutlass/include/cutlass + ln -sf $PREFIX/include/cutlass third_party/cutlass/include/ + + export CXXFLAGS="$CXXFLAGS -isystem $PREFIX/include/opencv4" +@@ -105,6 +109,9 @@ fi + # https://docs.nvidia.com/deeplearning/dali/user-guide/docs/compilation.html#optional-cmake-build-parameters + cmake ${CMAKE_ARGS} \ + -GNinja \ ++ -DBUILD_FOR_CONDA=ON \ ++ -DDALI_BUILD_FLAVOR="${NVIDIA_DALI_BUILD_FLAVOR}" \ ++ -DTIMESTAMP="${DALI_TIMESTAMP}" \ + -DBUILD_AWSSDK=ON \ + -DBUILD_BENCHMARK=ON \ ++ -DBUILD_TEST=ON \ + -DBUILD_CFITSIO=ON \ +diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml +index d0bd8f8..4bbc9b0 100644 +--- a/recipe/conda_build_config.yaml ++++ b/recipe/conda_build_config.yaml +@@ -1,5 +1,6 @@ + libnvimgcodec: + - "0.8.*" ++# nvcompGetStatusString used by DALI requires nvCOMP 5.2 or newer. + libnvcomp: + - "5.2.0.*" + # https://github.com/NVIDIA/DALI/tree/main/third_party +diff --git a/recipe/recipe.yaml b/recipe/recipe.yaml +index 274759a..940d1b0 100644 +--- a/recipe/recipe.yaml ++++ b/recipe/recipe.yaml +@@ -1,7 +1,11 @@ + # TODO: Update package pinnings in conda_build_config for every release + schema_version: 1 + context: +- version: "2.2.0" ++ dali_version: ${{ env.get("DALI_VERSION") }} ++ dali_build_flavor: ${{ env.get("NVIDIA_DALI_BUILD_FLAVOR", default="") }} ++ dali_timestamp: ${{ env.get("DALI_TIMESTAMP", default="") }} ++ version: ${{ dali_version ~ ("." ~ dali_timestamp if dali_build_flavor else "") }} ++ build_number: ${{ env.get("NVIDIA_BUILD_ID", default="0") }} + + recipe: + name: nvidia-dali +@@ -9,20 +13,9 @@ recipe: + + source: + - url: https://github.com/NVIDIA/DALI/archive/refs/tags/v${{ version }}.tar.gz +- sha256: 2a3fda834ab0f767159996c3685f1981cc59ce6fd097483f8b23ff8fabe9331d +- patches: +- - patches/0001-BLD-Use-CUDA-target-include-directory-to-support-cro.patch +- - patches/0002-DOC-Patch-dlopen-failure-message-and-dlopen-search-p.patch +- - patches/0003-BLD-Use-shared-libtar-instead-of-static.patch +- - url: https://github.com/cocodataset/cocoapi/archive/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9.tar.gz +- sha256: 4dd3450bab2287d1c1b271cf4a1420db05294194f0ad9af4e3af592d6f2b4410 +- target_directory: third_party/cocoapi +- - url: https://github.com/JanuszL/ffts/archive/6ef8d818f46f679c95110ab199ae915fb04bfef5.tar.gz +- sha256: 1d157157828ce1c85d0a43a6de312ce168b3c5dc1223c6ad3f02e074c3806c15 +- target_directory: third_party/ffts + + build: +- number: 0 ++ number: ${{ build_number }} + skip: + - cuda_compiler_version == "None" + - win +@@ -32,7 +25,11 @@ outputs: + - staging: + name: core-build + build: +- script: build.sh ++ script: ++ file: build.sh ++ env: ++ NVIDIA_DALI_BUILD_FLAVOR: ${{ dali_build_flavor }} ++ DALI_TIMESTAMP: ${{ dali_timestamp }} + requirements: + build: + - ${{ compiler('cuda') }} +@@ -93,7 +90,11 @@ outputs: + from: core-build + run_exports: true + build: +- script: build.sh ++ script: ++ file: build.sh ++ env: ++ NVIDIA_DALI_BUILD_FLAVOR: ${{ dali_build_flavor }} ++ DALI_TIMESTAMP: ${{ dali_timestamp }} + files: + - lib/libdali*.so + requirements: +@@ -164,7 +165,11 @@ outputs: + from: core-build + run_exports: false + build: +- script: build.sh ++ script: ++ file: build.sh ++ env: ++ NVIDIA_DALI_BUILD_FLAVOR: ${{ dali_build_flavor }} ++ DALI_TIMESTAMP: ${{ dali_timestamp }} + files: + - include/dali/** + requirements: +@@ -211,7 +216,11 @@ outputs: + from: core-build + run_exports: false + build: +- script: build.sh ++ script: ++ file: build.sh ++ env: ++ NVIDIA_DALI_BUILD_FLAVOR: ${{ dali_build_flavor }} ++ DALI_TIMESTAMP: ${{ dali_timestamp }} + requirements: + ignore_run_exports: + by_name: From 63fbf2ada5250d91a0ff89b0c8ecb987566d6381 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 15 Jul 2026 11:23:48 +0200 Subject: [PATCH 7/8] Fix feedstock patch hunk length Signed-off-by: Janusz Lisiecki --- conda/nvidia-dali-python-feedstock.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/nvidia-dali-python-feedstock.patch b/conda/nvidia-dali-python-feedstock.patch index 62b39d6e343..5cae9d47743 100644 --- a/conda/nvidia-dali-python-feedstock.patch +++ b/conda/nvidia-dali-python-feedstock.patch @@ -20,7 +20,7 @@ index 23d3cee..b1e9f12 100755 ln -sf $PREFIX/include/cutlass third_party/cutlass/include/ export CXXFLAGS="$CXXFLAGS -isystem $PREFIX/include/opencv4" -@@ -105,6 +109,9 @@ fi +@@ -105,6 +109,10 @@ fi # https://docs.nvidia.com/deeplearning/dali/user-guide/docs/compilation.html#optional-cmake-build-parameters cmake ${CMAKE_ARGS} \ -GNinja \ From 72e4bd3d80d9857f33e7c84276d0aedbca459e55 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Wed, 15 Jul 2026 12:27:10 +0200 Subject: [PATCH 8/8] Rebase feedstock patch on current recipe Signed-off-by: Janusz Lisiecki --- conda/nvidia-dali-python-feedstock.patch | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/conda/nvidia-dali-python-feedstock.patch b/conda/nvidia-dali-python-feedstock.patch index 5cae9d47743..8d8c46c9796 100644 --- a/conda/nvidia-dali-python-feedstock.patch +++ b/conda/nvidia-dali-python-feedstock.patch @@ -20,7 +20,7 @@ index 23d3cee..b1e9f12 100755 ln -sf $PREFIX/include/cutlass third_party/cutlass/include/ export CXXFLAGS="$CXXFLAGS -isystem $PREFIX/include/opencv4" -@@ -105,6 +109,10 @@ fi +@@ -105,6 +105,9 @@ fi # https://docs.nvidia.com/deeplearning/dali/user-guide/docs/compilation.html#optional-cmake-build-parameters cmake ${CMAKE_ARGS} \ -GNinja \ @@ -28,9 +28,18 @@ index 23d3cee..b1e9f12 100755 + -DDALI_BUILD_FLAVOR="${NVIDIA_DALI_BUILD_FLAVOR}" \ + -DTIMESTAMP="${DALI_TIMESTAMP}" \ -DBUILD_AWSSDK=ON \ - -DBUILD_BENCHMARK=ON \ -+ -DBUILD_TEST=ON \ +- -DBUILD_BENCHMARK=OFF \ ++ -DBUILD_BENCHMARK=ON \ -DBUILD_CFITSIO=ON \ +@@ -122,7 +125,7 @@ cmake ${CMAKE_ARGS} \ + -DBUILD_NVOF=ON \ + -DBUILD_NVTX=ON \ + -DBUILD_OPENCV=ON \ +- -DBUILD_TEST=OFF \ ++ -DBUILD_TEST=ON \ + -DBUILD_WITH_ASAN=OFF \ + -DBUILD_WITH_LSAN=OFF \ + -DBUILD_WITH_UBSAN=OFF \ diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index d0bd8f8..4bbc9b0 100644 --- a/recipe/conda_build_config.yaml