diff --git a/.ci/hpc/build.sh.j2 b/.ci/hpc/build.sh.j2 index b472dc038..4fce0f928 100644 --- a/.ci/hpc/build.sh.j2 +++ b/.ci/hpc/build.sh.j2 @@ -1,57 +1,21 @@ -#!/bin/bash - -#SBATCH --qos=nf -#SBATCH --gres=ssdtmp:20G -#SBATCH --time=01:30:00 -#SBATCH --nodes=1 -#SBATCH --ntasks=8 - -{% for m in modules %} -module {{ m }} -{% endfor %} - -echo "Using: $(command -v {{ cc }}) ($($(command -v {{ cc }}) --version | head -1))" -echo "Using: $(command -v {{ fc }}) ($($(command -v {{ fc }}) --version | head -1))" - -cmake -S "$CI_SOURCE_DIR" -B "${TMPDIR:-/tmp}/build" \ - -GNinja \ - -DCMAKE_BUILD_TYPE={{ build_type }} \ - -DCMAKE_C_COMPILER="$(command -v {{ cc }})" \ - -DCMAKE_CXX_COMPILER="$(command -v {{ cxx }})" \ - -DCMAKE_Fortran_COMPILER="$(command -v {{ fc }})" \ - -DENABLE_EXTRA_TESTS=1 \ - -DENABLE_ECCODES_OMP_THREADS=1 \ -{% if options == "eckit-geo" %} - -DENABLE_GEOGRAPHY=1 \ - -DENABLE_ECKIT_GEO=1 \ -{% endif %} - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ - -DCMAKE_INSTALL_PREFIX="$CI_INSTALL_PREFIX" -cmake --build "${TMPDIR:-/tmp}/build" --parallel "${SLURM_NTASKS:-8}" -# The C++ tests resolve definitions through the prefix compiled into the library; -# the .sh ones source include.ctest.sh, which points them at the build tree. This -# recipe never populates that prefix -- it stages the install under DESTDIR on -# node-local disk (below) -- so point the C++ tests at the build tree too, exactly -# as include.ctest.sh does. Without this, eccodes_t_grib_{from,to}_gridspec abort -# on "Unable to find boot.def". -export ECCODES_DEFINITION_PATH="${TMPDIR:-/tmp}/build/share/eccodes/definitions" -export ECCODES_SAMPLES_PATH="${TMPDIR:-/tmp}/build/share/eccodes/samples" -# Fetch the test data first. A download failure otherwise surfaces as dozens of -# unrelated test failures: -ctest --test-dir "${TMPDIR:-/tmp}/build" --output-on-failure \ - -L download_data -j 6 -ctest --test-dir "${TMPDIR:-/tmp}/build" --output-on-failure \ - -LE download_data -j "${SLURM_NTASKS:-8}" - -# Install to node-local SSD, then copy the tree across in one pass. eccodes -# installs ~24k small definition files; doing that as individual `cmake -# --install` operations straight onto shared scratch is metadata-bound and has -# taken anywhere from 7 to >35 minutes depending on filesystem load. +{# SPDX-FileCopyrightText: 2005- European Centre for Medium-Range Weather Forecasts (ECMWF) + SPDX-License-Identifier: Apache-2.0 #} +{% extends "ci-infrastructure/cmake-build.sh.j2" %} +{% block test %} +# The install prefix is never populated here (see install below), so point the +# C++ tests at the build tree, as include.ctest.sh does for the .sh ones. Without +# this, eccodes_t_grib_{from,to}_gridspec abort on "Unable to find boot.def". +export ECCODES_DEFINITION_PATH="$build/share/eccodes/definitions" +export ECCODES_SAMPLES_PATH="$build/share/eccodes/samples" +# Fetch the test data first; a download failure otherwise surfaces as dozens of +# unrelated test failures. +ctest --test-dir "$build" --output-on-failure -L download_data -j 6 +ctest --test-dir "$build" --output-on-failure -LE download_data -j "$jobs" +{% endblock %} +{% block install %} +# ~24k small definition files: installing straight onto shared scratch is +# metadata-bound (7 to >35 min), so stage on node-local SSD and tar from there. stage="${TMPDIR:-/tmp}/stage" -DESTDIR="$stage" cmake --install "${TMPDIR:-/tmp}/build" -# The fetcher takes the artifact from CI_INSTALL_ARCHIVE, not from the install -# tree; .part + mv so it only ever appears complete. -mkdir -p "$(dirname "$CI_INSTALL_ARCHIVE")" -tar -cf - -C "$stage$CI_INSTALL_PREFIX" . | zstd -T0 -q -o "$CI_INSTALL_ARCHIVE.part" -mv "$CI_INSTALL_ARCHIVE.part" "$CI_INSTALL_ARCHIVE" +DESTDIR="$stage" cmake --install "$build" +install_root="$stage$CI_INSTALL_PREFIX" +{% endblock %} diff --git a/.ci/manifest.toml b/.ci/manifest.toml index 9bc92dc6b..7bef0d432 100644 --- a/.ci/manifest.toml +++ b/.ci/manifest.toml @@ -3,7 +3,7 @@ name = "eccodes" prefix = "eccodes" repo = "ecmwf/eccodes" visibility = "public" -compiler-inputs = ["cxx-compiler", "fortran-compiler"] +compiler-inputs = ["cxx-compiler"] [[deps]] repo = "ecmwf/ecbuild" @@ -24,45 +24,46 @@ ref = "develop" compiler-inputs = ["cxx-compiler"] when = { options = ["eckit-geo"] } +# No fortran-compiler: the clang image ships no Fortran, because gfortran-N +# depends on gcc-N. The Fortran interface is covered by the g++-13 legs and HPC. +[matrix.build.defaults] +build-type = "RelWithDebInfo" +runs-on = "arc-runner-very-large" + [[matrix.build.include]] cxx-compiler = "clang++-18" -build-type = "Release" -fortran-compiler = "gfortran-13" -runs-on = "arc-runner-very-large" -container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-clang18-gfortran13:latest" +c-compiler = "clang-18" +container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-clang18:latest" platform = "ubuntu-24.04" [[matrix.build.include]] cxx-compiler = "g++-13" -build-type = "Release" +c-compiler = "gcc-13" fortran-compiler = "gfortran-13" -runs-on = "arc-runner-very-large" -container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-clang18-gfortran13:latest" +container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-gcc13-gfortran13:latest" platform = "ubuntu-24.04" [[matrix.build.include]] cxx-compiler = "g++-13" -build-type = "Release" +c-compiler = "gcc-13" fortran-compiler = "gfortran-13" options = "eckit-geo" -runs-on = "arc-runner-very-large" -container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-clang18-gfortran13:latest" +container = "eccr.ecmwf.int/public-ci-images/ubuntu24.04-gcc13-gfortran13:latest" platform = "ubuntu-24.04" [matrix.build] triggers = ["upstream-change", "rebuild-request"] action = "./.github/actions/build-eccodes" -forwarded-inputs = ["cxx-compiler", "fortran-compiler", "build-type", "options"] +forwarded-inputs = ["c-compiler", "cxx-compiler", "fortran-compiler", "build-type", "options"] forwarded-deps-outputs = ["cmake-prefix-path"] needs = ["ecbuild/build", "stack-deps/build", "eckit/build"] ctest = true ctest-args = '-j "$(nproc)"' -[[matrix.build-hpc.include]] +[matrix.build-hpc.defaults] cxx-compiler = "g++-8" fortran-compiler = "gfortran-8" -build-type = "Release" -options = "" +build-type = "RelWithDebInfo" runs-on = "hpc-submit" container = "eccr.ecmwf.int/private-ci-images/ubuntu24.04-internal-tools:latest" site = "hpc-batch" @@ -73,20 +74,13 @@ modules = ["load prgenv/gnu", "unload gcc", "load gcc/old", "load cmake", "load cc = "gcc" cxx = "g++" fc = "gfortran" +time = "01:30:00" + +[[matrix.build-hpc.include]] +options = "" [[matrix.build-hpc.include]] -cxx-compiler = "g++-8" -fortran-compiler = "gfortran-8" -build-type = "Release" options = "eckit-geo" -runs-on = "hpc-submit" -container = "eccr.ecmwf.int/private-ci-images/ubuntu24.04-internal-tools:latest" -site = "hpc-batch" -platform = "hpc-atos-gnu" -modules = ["load prgenv/gnu", "unload gcc", "load gcc/old", "load cmake", "load ninja"] -cc = "gcc" -cxx = "g++" -fc = "gfortran" [matrix.build-hpc] execution = "hpc" diff --git a/.github/actions/build-eccodes/action.yml b/.github/actions/build-eccodes/action.yml index f7f60c0a8..bb1e10430 100644 --- a/.github/actions/build-eccodes/action.yml +++ b/.github/actions/build-eccodes/action.yml @@ -10,6 +10,13 @@ inputs: cmake-prefix-path: description: 'Semicolon-separated prefixes of the resolved deps' required: true + c-compiler: + description: > + C compiler binary including version (e.g. clang-18, gcc-13). Must be the + same toolchain family as cxx-compiler: left unset CMake discovers + /usr/bin/cc, which is the image's gcc whatever the C++ compiler is, and a + mixed build then gets two OpenMP runtimes and two sets of flag spellings. + required: true cxx-compiler: description: 'C++ compiler binary including version (e.g. clang++-18, g++-13)' required: true @@ -19,16 +26,21 @@ inputs: builds its Fortran interface by default (FEATURE FORTRAN, DEFAULT ON) and the CI images ship only versioned binaries — there is no unversioned `gfortran` for CMake to discover — so this must be passed explicitly. - required: true + Empty builds without the Fortran interface, for images that ship no Fortran + compiler at all: the clang image does not, because gfortran-N depends on + gcc-N and would put an unnamed GNU C compiler in it. + required: false + default: '' build-type: description: 'CMake build type (e.g. Release, Debug, RelWithDebInfo)' required: true options: description: > - Named feature configuration for this leg. Empty is the default build. - "eckit-geo" turns on the eckit-backed geography backend, and is the only - configuration for which eckit is a dependency at all (see the `when` - predicate on the eckit [[deps]] entry in .ci/manifest.toml). + Named feature configuration for this leg: the configure preset in + CMakePresets.json to use. Empty selects the `ci` preset. "eckit-geo" turns + on the eckit-backed geography backend, and is the only configuration for + which eckit is a dependency at all (see the `when` predicate on the eckit + [[deps]] entry in .ci/manifest.toml). required: false default: '' @@ -57,23 +69,6 @@ runs: echo "build-dir=$RUNNER_TEMP/_eccodes-build" } >> "$GITHUB_OUTPUT" - - name: Map options to CMake flags - id: opts - shell: bash - env: - OPTIONS: ${{ inputs.options }} - run: | - set -euo pipefail - case "$OPTIONS" in - '') flags='' ;; - 'eckit-geo') flags='-DENABLE_GEOGRAPHY=1 -DENABLE_ECKIT_GEO=1' ;; - *) - echo "::error::unknown options value '$OPTIONS' for eccodes; add it here and to .ci/manifest.toml" >&2 - exit 1 - ;; - esac - echo "cmake-flags=$flags" >> "$GITHUB_OUTPUT" - - name: Set up sccache if: env.SCCACHE_BUCKET != '' uses: ecmwf/ci-infrastructure/actions/setup-sccache@main @@ -96,23 +91,31 @@ runs: shell: bash env: CMAKE_PREFIX_PATH_IN: ${{ inputs.cmake-prefix-path }} + C_COMPILER: ${{ inputs.c-compiler }} CXX_COMPILER: ${{ inputs.cxx-compiler }} FC_COMPILER: ${{ inputs.fortran-compiler }} BUILD_TYPE: ${{ inputs.build-type }} INSTALL_PATH: ${{ steps.set-paths.outputs.install-path }} BUILD_DIR: ${{ steps.set-paths.outputs.build-dir }} - EXTRA_FLAGS: ${{ steps.opts.outputs.cmake-flags }} + OPTIONS: ${{ inputs.options }} LAUNCHER: ${{ steps.launcher.outputs.flags }} run: | set -euo pipefail - cmake -S "$GITHUB_WORKSPACE" \ + if [ -n "$FC_COMPILER" ]; then + fortran=( -DCMAKE_Fortran_COMPILER="$FC_COMPILER" ) + else + fortran=( -DENABLE_FORTRAN=OFF ) + fi + cmake --preset "${OPTIONS:-ci}" \ + -S "$GITHUB_WORKSPACE" \ -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ + -DCMAKE_C_COMPILER="$C_COMPILER" \ -DCMAKE_CXX_COMPILER="$CXX_COMPILER" \ - -DCMAKE_Fortran_COMPILER="$FC_COMPILER" \ + "${fortran[@]}" \ -DCMAKE_INSTALL_PREFIX="$INSTALL_PATH" \ -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH_IN" \ - $EXTRA_FLAGS $LAUNCHER + $LAUNCHER - name: Build and install eccodes shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9ff1de5..56e8ee36d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,7 @@ jobs: uses: ./.github/actions/build-eccodes with: cmake-prefix-path: ${{ steps.deps.outputs.cmake-prefix-path }} + c-compiler: ${{ matrix.c-compiler }} cxx-compiler: ${{ matrix.cxx-compiler }} fortran-compiler: ${{ matrix.fortran-compiler }} build-type: ${{ matrix.build-type }} diff --git a/.github/workflows/cross-repo-trigger.yml b/.github/workflows/cross-repo-trigger.yml index e83f54250..f6467c843 100644 --- a/.github/workflows/cross-repo-trigger.yml +++ b/.github/workflows/cross-repo-trigger.yml @@ -168,13 +168,15 @@ jobs: fi printf '%s' "$val" } + c_compiler=$(require '."c-compiler"' c-compiler) cxx_compiler=$(require '."cxx-compiler"' cxx-compiler) - fortran_compiler=$(require '."fortran-compiler"' fortran-compiler) + fortran_compiler=$(jq -r '."fortran-compiler" // ""' <<<"$leg") build_type=$(require '."build-type"' build-type) options=$(jq -r '."options" // ""' <<<"$leg") deps_json=$(require '._resolved.deps | tojson' '_resolved.deps') own_artifact_name=$(require '._resolved."own-artifact-name"' '_resolved.own-artifact-name') { + echo "c-compiler=${c_compiler}" echo "cxx-compiler=${cxx_compiler}" echo "fortran-compiler=${fortran_compiler}" echo "build-type=${build_type}" @@ -193,6 +195,7 @@ jobs: uses: ./.github/actions/build-eccodes with: cmake-prefix-path: ${{ steps.deps.outputs.cmake-prefix-path }} + c-compiler: ${{ steps.m.outputs.c-compiler }} cxx-compiler: ${{ steps.m.outputs.cxx-compiler }} fortran-compiler: ${{ steps.m.outputs.fortran-compiler }} build-type: ${{ steps.m.outputs.build-type }} diff --git a/CMakeLists.txt b/CMakeLists.txt index fb2bc304c..91ce48b95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,7 +229,8 @@ ecbuild_add_option( FEATURE INSTALL_ECCODES_SAMPLES ecbuild_add_option( FEATURE MEMORY_MANAGEMENT DESCRIPTION "Enable memory management" DEFAULT OFF ADVANCED ) ecbuild_add_option( FEATURE ALIGN_MEMORY DESCRIPTION "Enable memory alignment" DEFAULT OFF ADVANCED ) ecbuild_add_option( FEATURE ECCODES_THREADS DESCRIPTION "Enable thread-safety using POSIX threads" DEFAULT OFF ADVANCED ) -ecbuild_add_option( FEATURE ECCODES_OMP_THREADS DESCRIPTION "Enable thread-safety using OpenMP threads" DEFAULT OFF ADVANCED ) +ecbuild_add_option( FEATURE ECCODES_OMP_THREADS DESCRIPTION "Enable thread-safety using OpenMP threads" DEFAULT OFF ADVANCED + REQUIRED_PACKAGES "OpenMP COMPONENTS C CXX" ) ecbuild_add_option( FEATURE EXTRA_TESTS DESCRIPTION "Enable extended regression testing" DEFAULT OFF ADVANCED ) ############################################################################### @@ -319,7 +320,9 @@ if( HAVE_ECCODES_THREADS ) set( GRIB_LINUX_PTHREADS 1 ) endif() elseif(HAVE_ECCODES_OMP_THREADS) - ecbuild_enable_omp() + if( NOT TARGET OpenMP::OpenMP_CXX ) + ecbuild_critical("ENABLE_ECCODES_OMP_THREADS is ON but no OpenMP C++ toolchain was found") + endif() set( GRIB_OMP_THREADS 1 ) endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 000000000..cd1f929a4 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,25 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "ci", + "cacheVariables": { + "ENABLE_EXTRA_TESTS": "ON", + "ENABLE_ECCODES_OMP_THREADS": "ON" + } + }, + { + "name": "eckit-geo", + "inherits": "ci", + "cacheVariables": { + "ENABLE_GEOGRAPHY": "ON", + "ENABLE_ECKIT_GEO": "ON" + } + } + ] +} diff --git a/src/eccodes/CMakeLists.txt b/src/eccodes/CMakeLists.txt index ebcd4a66b..55a581234 100644 --- a/src/eccodes/CMakeLists.txt +++ b/src/eccodes/CMakeLists.txt @@ -448,6 +448,7 @@ ecbuild_add_library( ${CMAKE_THREAD_LIBS_INIT} ${PNG_LIBRARIES} $<$:libaec::aec> + $<$:OpenMP::OpenMP_CXX> PUBLIC_LIBS ${CMATH_LIBRARIES} ${THREADS_LIBRARIES} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f2e4fb913..582762732 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -562,10 +562,17 @@ if( HAVE_BUILD_TOOLS ) if( ENABLE_EXTRA_TESTS AND (HAVE_ECCODES_THREADS OR HAVE_ECCODES_OMP_THREADS) ) + # These carry their own #pragma omp, so they need the flag on their own + # compile line; linking eccodes PRIVATE does not propagate it. + set( eccodes_thread_test_libs eccodes ${CMAKE_THREAD_LIBS_INIT} ) + if( HAVE_ECCODES_OMP_THREADS ) + list( APPEND eccodes_thread_test_libs OpenMP::OpenMP_CXX ) + endif() + ecbuild_add_executable( TARGET grib_encode_threads NOINSTALL SOURCES grib_encode_threads.cc - LIBS eccodes ${CMAKE_THREAD_LIBS_INIT} ) + LIBS ${eccodes_thread_test_libs} ) ecbuild_add_test( TARGET eccodes_t_grib_encode_threads TYPE SCRIPT COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_threads.sh ) @@ -574,7 +581,7 @@ if( HAVE_BUILD_TOOLS ) ecbuild_add_executable( TARGET ${test} NOINSTALL SOURCES ${test}.cc - LIBS eccodes ${CMAKE_THREAD_LIBS_INIT} ) + LIBS ${eccodes_thread_test_libs} ) endforeach() ecbuild_add_test( TARGET eccodes_t_grib_encode_threads2 TYPE SCRIPT