diff --git a/.github/workflows/build-steps.yml b/.github/workflows/build-steps.yml index 1ea0184b8..ecec1f4d3 100644 --- a/.github/workflows/build-steps.yml +++ b/.github/workflows/build-steps.yml @@ -181,6 +181,24 @@ jobs: with: path: ~/llvm/llvm-build key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} + - name: deps-cache-key + # Windows CI builds several dependencies from source, which takes + # most of the job's time. Cache the installed results. + id: deps-cache-key + if: runner.os == 'Windows' + shell: bash + run: | + key=$(src/build-scripts/gh-win-deps-cache-key.bash deps-${{inputs.nametag}}) + echo "key=$key" >> $GITHUB_OUTPUT + - name: deps-cache-restore + id: deps-cache-restore + if: runner.os == 'Windows' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/ext/dist + ext/dist + key: ${{ steps.deps-cache-key.outputs.key }} - name: Install gcc if: inputs.gcc_action_ver != '' run: | @@ -194,6 +212,8 @@ jobs: python-version: ${{ inputs.python_action_ver }} - name: Dependencies shell: bash + env: + OSL_DEPS_CACHE_HIT: ${{ steps.deps-cache-restore.outputs.cache-hit }} run: | eval "$DEPCMDS" if [[ "$RUNNER_OS" == "Linux" ]]; then @@ -217,6 +237,14 @@ jobs: with: path: ~/llvm/llvm-build key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} + - name: deps-cache-save + if: runner.os == 'Windows' && steps.deps-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/ext/dist + ext/dist + key: ${{ steps.deps-cache-key.outputs.key }} - name: Install sonar-scanner and build-wrapper if: inputs.sonar == '1' uses: sonarsource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f # v8.2.1 diff --git a/src/build-scripts/gh-win-deps-cache-key.bash b/src/build-scripts/gh-win-deps-cache-key.bash new file mode 100755 index 000000000..22d9c6ef1 --- /dev/null +++ b/src/build-scripts/gh-win-deps-cache-key.bash @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright Contributors to the Open Shading Language project. +# SPDX-License-Identifier: BSD-3-Clause +# https://github.com/AcademySoftwareFoundation/OpenShadingLanguage + +# Print a cache key for the dependencies that gh-win-installdeps.bash +# builds from source. The key changes when anything that affects those +# builds changes: the dependency versions, the build settings, or the +# build scripts. Branch names (like OIIO's "release") are resolved to their +# current commit, so the deps are rebuilt when the branch moves. +# +# Usage: gh-win-deps-cache-key.bash + +set -e + +# Resolve a branch name to its current commit. Tags and commit hashes are +# printed unchanged. +resolve_ref () { + local repo=$1 ref=$2 + if [[ "$ref" == "" ]] ; then + return + fi + # If ls-remote fails, fail the script rather than keying on a branch + # name, which could keep restoring stale builds. + local refs + refs=$(git ls-remote "$repo" "refs/heads/$ref") || exit 1 + local sha=$(echo "$refs" | cut -f1) + echo "${sha:-$ref}" +} + +openexr=$(resolve_ref ${OPENEXR_REPO:-https://github.com/AcademySoftwareFoundation/openexr.git} ${OPENEXR_VERSION}) +opencolorio=$(resolve_ref ${OPENCOLORIO_REPO:-https://github.com/AcademySoftwareFoundation/OpenColorIO.git} ${OPENCOLORIO_VERSION}) +openimageio=$(resolve_ref https://github.com/${OPENIMAGEIO_REPO:-AcademySoftwareFoundation/OpenImageIO} ${OPENIMAGEIO_VERSION}) +scripts=$(cat src/build-scripts/gh-win-installdeps.bash src/build-scripts/build_*.bash | sha256sum) + +desc=$( + echo "zlib=${ZLIB_VERSION}" + echo "libpng=${LIBPNG_VERSION}" + echo "pybind11=${PYBIND11_VERSION}" + echo "pugixml=${PUGIXML_VERSION}" + echo "openexr=${openexr}" + echo "opencolorio=${opencolorio}" + echo "openimageio=${openimageio}" + echo "OpenImageIO_BUILD_MISSING_DEPS=${OpenImageIO_BUILD_MISSING_DEPS}" + echo "OPENIMAGEIO_UNITY=${OPENIMAGEIO_UNITY}" + echo "CMAKE_GENERATOR=${CMAKE_GENERATOR}" + echo "CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}" + echo "PYTHON_VERSION=${PYTHON_VERSION}" + echo "USE_SIMD=${USE_SIMD}" + echo "scripts=${scripts}" +) + +# Show the inputs in the log, to help explain cache misses +echo "$desc" | sed 's/^/ /' >&2 + +echo "$1-$(echo "$desc" | sha256sum | cut -c1-16)" diff --git a/src/build-scripts/gh-win-installdeps.bash b/src/build-scripts/gh-win-installdeps.bash index a842967bc..3c214e807 100755 --- a/src/build-scripts/gh-win-installdeps.bash +++ b/src/build-scripts/gh-win-installdeps.bash @@ -71,15 +71,30 @@ vcpkg list ######################################################################## # Dependency method #2: Build from source ourselves # +# If the CI workflow restored the built dependencies from its cache (see +# gh-win-deps-cache-key.bash), OSL_DEPS_CACHE_HIT is "true" and we skip +# the builds, only setting the variables that the build scripts export. # -src/build-scripts/build_zlib.bash -export ZLIB_ROOT=$PWD/ext/dist +# build_dep SCRIPT ROOTVAR : source a dependency build script, or on a +# cache hit, just set ROOTVAR to where the build would have installed it. +build_dep () { + if [[ "$OSL_DEPS_CACHE_HIT" == "true" ]] ; then + echo "Using cached build from $1" + export $2=$LOCAL_DEPS_DIR/dist + else + source src/build-scripts/$1 + fi +} -src/build-scripts/build_libpng.bash +if [[ "$OSL_DEPS_CACHE_HIT" != "true" ]] ; then + src/build-scripts/build_zlib.bash + src/build-scripts/build_libpng.bash +fi +export ZLIB_ROOT=$PWD/ext/dist export PNG_ROOT=$PWD/ext/dist -source src/build-scripts/build_pybind11.bash +build_dep build_pybind11.bash pybind11_ROOT export pybind11_ROOT=$PWD/ext/dist # nanobind is only needed for the non-default python binding backends. Install @@ -90,16 +105,16 @@ fi if [[ "$OPENEXR_VERSION" != "" ]] ; then - source src/build-scripts/build_openexr.bash + build_dep build_openexr.bash OpenEXR_ROOT fi if [[ "$PUGIXML_VERSION" != "" ]] ; then - source src/build-scripts/build_pugixml.bash + build_dep build_pugixml.bash pugixml_ROOT export OSL_CMAKE_FLAGS+=" -DUSE_EXTERNAL_PUGIXML=1 " fi if [[ "$OPENCOLORIO_VERSION" != "" ]] ; then - source src/build-scripts/build_opencolorio.bash + build_dep build_opencolorio.bash OpenColorIO_ROOT fi cp $DEP_DIR/lib/*.lib $DEP_DIR/bin || true @@ -126,7 +141,8 @@ if [[ "$OPENIMAGEIO_VERSION" != "" ]] ; then # only a savings in CI where there are only 1-2 cores available.) export OPENIMAGEIO_CMAKE_FLAGS+=" -DCMAKE_UNITY_BUILD=ON -DCMAKE_UNITY_BUILD_MODE=BATCH" fi - source src/build-scripts/build_openimageio.bash + build_dep build_openimageio.bash OpenImageIO_ROOT + export PYTHONPATH=$OpenImageIO_ROOT/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH fi cp $DEP_DIR/lib/*.lib $DEP_DIR/bin || true