diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d89f474ec00..9386bef8c6e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -26,7 +26,6 @@ permissions: contents: read env: - MARGS: "-j2" CFLAGS: "-g" # This will need updating as the ubuntu-latest image changes: PHP_FPM: "/usr/sbin/php-fpm8.3" @@ -39,13 +38,112 @@ jobs: build: strategy: fail-fast: false + # https://infra.apache.org/github-actions-policy.html requires a + # job concurrency level of 20 or less. + max-parallel: 20 matrix: include: # ------------------------------------------------------------------------- - - name: Empty APLOGNO() test + - name: Shared MPMs (event), pool-debug, SSL/TLS variants + config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=event env: | - SKIP_TESTING=1 - TEST_LOGNO=1 + APR_VERSION=1.7.x + APR_CONFIG="--enable-pool-debug" + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_MALLOC=1 + TEST_SSL=1 + # ------------------------------------------------------------------------- + - name: Shared MPMs (worker), pool-debug, SSL/TLS variants + config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=worker + env: | + APR_VERSION=1.7.x + APR_CONFIG="--enable-pool-debug" + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_MALLOC=1 + TEST_SSL=1 + # ------------------------------------------------------------------------- + - name: Shared MPMs (prefork), pool-debug, SSL/TLS variants + config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=prefork + env: | + APR_VERSION=1.7.x + APR_CONFIG="--enable-pool-debug" + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_MALLOC=1 + TEST_SSL=1 + # ------------------------------------------------------------------------- + - name: UBSan + notest-cflags: -fsanitize=undefined -fno-sanitize-recover=undefined + config: --enable-mods-shared=reallyall --disable-http2 + env: | + NOTEST_LIBS=-lubsan + TEST_UBSAN=1 + TEST_PYTEST=1 + # ------------------------------------------------------------------------- + # The ASan jobs are the slowest in the matrix and so set the + # wall-clock time for the whole workflow; running the Perl + # framework and the pytest suites in separate jobs halves that. + # Both halves build the same dependencies, so they share one + # ~/root cache entry. + - name: ASan + notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer + config: --enable-mods-shared=reallyall + env: | + APR_VERSION=1.7.x + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_ASAN=1 + # ------------------------------------------------------------------------- + - name: ASan, pytest + notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer + config: --enable-mods-shared=reallyall + pkgs: nghttp2-client + env: | + APR_VERSION=1.7.x + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_ASAN=1 + NO_TEST_FRAMEWORK=1 + TEST_PYTEST=1 + # ------------------------------------------------------------------------- + - name: ASan, pool-debug + notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer -O2 + config: --enable-mods-shared=reallyall --enable-maintainer-mode + env: | + APR_VERSION=1.7.x + APR_CONFIG="--enable-pool-debug" + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_ASAN=1 + # ------------------------------------------------------------------------- + - name: ASan, pool-debug, pytest + notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer -O2 + config: --enable-mods-shared=reallyall --enable-maintainer-mode + pkgs: nghttp2-client + env: | + APR_VERSION=1.7.x + APR_CONFIG="--enable-pool-debug" + APU_VERSION=1.7.x + APU_CONFIG="--with-crypto --with-ldap" + TEST_ASAN=1 + NO_TEST_FRAMEWORK=1 + TEST_PYTEST=1 + # ------------------------------------------------------------------------- + # Runs every pytest-based test suite (pytest_suite/ + all + # test/modules/*/ pyhttpd suites except modules/md, which needs a + # local ACME/pebble server that isn't available here) via `make + # check-all-pytest`. See TEST_PYTEST in test/travis_run_linux.sh. + - name: Python pytest test suites + config: --enable-mods-shared=reallyall --with-mpm=event --enable-mpms-shared=all + pkgs: nghttp2-client + env: | + APR_VERSION=1.7.6 + APU_VERSION=1.6.5 + APU_CONFIG="--with-crypto" + NO_TEST_FRAMEWORK=1 + TEST_PYTEST=1 # ------------------------------------------------------------------------- - name: Default # ------------------------------------------------------------------------- @@ -117,7 +215,6 @@ jobs: APR_VERSION=1.7.x APU_VERSION=1.6.x APU_CONFIG="--with-crypto --with-ldap" - CLEAR_CACHE=1 # ------------------------------------------------------------------------- - name: APR 1.8.x, APR-util 1.7.x config: --enable-mods-shared=reallyall @@ -125,7 +222,6 @@ jobs: APR_VERSION=1.8.x APU_VERSION=1.7.x APU_CONFIG="--with-crypto --with-ldap" - CLEAR_CACHE=1 # ------------------------------------------------------------------------- - name: APR 1.7.x, APR-util 1.7.x, LDAP config: --enable-mods-shared=reallyall @@ -139,7 +235,6 @@ jobs: TEST_ARGS="-defines LDAP" TESTS="t/modules/" TEST_PYTEST=1 - CLEAR_CACHE=1 # ------------------------------------------------------------------------- - name: APR trunk thread debugging config: --enable-mods-shared=reallyall --with-mpm=event @@ -155,40 +250,6 @@ jobs: APU_VERSION=1.7.x APU_CONFIG="--with-crypto --with-ldap" TEST_MALLOC=1 - CLEAR_CACHE=1 - # ------------------------------------------------------------------------- - - name: Shared MPMs (event), pool-debug, SSL/TLS variants - config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=event - env: | - APR_VERSION=1.7.x - APR_CONFIG="--enable-pool-debug" - APU_VERSION=1.7.x - APU_CONFIG="--with-crypto --with-ldap" - TEST_MALLOC=1 - TEST_SSL=1 - CLEAR_CACHE=1 - # ------------------------------------------------------------------------- - - name: Shared MPMs (worker), pool-debug, SSL/TLS variants - config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=worker - env: | - APR_VERSION=1.7.x - APR_CONFIG="--enable-pool-debug" - APU_VERSION=1.7.x - APU_CONFIG="--with-crypto --with-ldap" - TEST_MALLOC=1 - TEST_SSL=1 - CLEAR_CACHE=1 - # ------------------------------------------------------------------------- - - name: Shared MPMs (prefork), pool-debug, SSL/TLS variants - config: --enable-mods-shared=reallyall --enable-mpms-shared=all --with-mpm=prefork - env: | - APR_VERSION=1.7.x - APR_CONFIG="--enable-pool-debug" - APU_VERSION=1.7.x - APU_CONFIG="--with-crypto --with-ldap" - TEST_MALLOC=1 - TEST_SSL=1 - CLEAR_CACHE=1 # ------------------------------------------------------------------------- - name: litmus WebDAV tests config: --enable-dav --enable-dav-fs @@ -215,53 +276,6 @@ jobs: BUILDCONFIG="--with-regen-expr" MFLAGS= # ------------------------------------------------------------------------- - - name: UBSan - notest-cflags: -fsanitize=undefined -fno-sanitize-recover=undefined - config: --enable-mods-shared=reallyall --disable-http2 - env: | - NOTEST_LIBS=-lubsan - TEST_UBSAN=1 - TEST_PYTEST=1 - # ------------------------------------------------------------------------- - - name: ASan - notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer - config: --enable-mods-shared=reallyall - pkgs: nghttp2-client - env: | - APR_VERSION=1.7.x - APU_VERSION=1.7.x - APU_CONFIG="--with-crypto --with-ldap" - TEST_ASAN=1 - TEST_PYTEST=1 - CLEAR_CACHE=1 - # ------------------------------------------------------------------------- - - name: ASan, pool-debug - notest-cflags: -ggdb -fsanitize=address -fno-sanitize-recover=address -fno-omit-frame-pointer -O2 - config: --enable-mods-shared=reallyall --enable-maintainer-mode - pkgs: nghttp2-client - env: | - APR_VERSION=1.7.x - APR_CONFIG="--enable-pool-debug" - APU_VERSION=1.7.x - APU_CONFIG="--with-crypto --with-ldap" - TEST_ASAN=1 - TEST_PYTEST=1 - CLEAR_CACHE=1 - # ------------------------------------------------------------------------- - # Runs every pytest-based test suite (pytest_suite/ + all - # test/modules/*/ pyhttpd suites except modules/md, which needs a - # local ACME/pebble server that isn't available here) via `make - # check-all-pytest`. See TEST_PYTEST in test/travis_run_linux.sh. - - name: Python pytest test suites - config: --enable-mods-shared=reallyall --with-mpm=event --enable-mpms-shared=all - pkgs: nghttp2-client - env: | - APR_VERSION=1.7.6 - APU_VERSION=1.6.5 - APU_CONFIG="--with-crypto" - NO_TEST_FRAMEWORK=1 - TEST_PYTEST=1 - # ------------------------------------------------------------------------- ### TODO: if: *condition_not_24x - name: Configured w/reduced exports config: --enable-reduced-exports --enable-maintainer-mode --enable-systemd @@ -293,7 +307,6 @@ jobs: APR_VERSION=1.7.6 APU_VERSION=1.6.5 APU_CONFIG="--without-crypto" - pkgs: subversion # ------------------------------------------------------------------------- - name: OpenSSL 3.4 -Werror config: --enable-mods-shared=most --enable-maintainer-mode --disable-md --disable-http2 --disable-ldap --disable-crypto @@ -303,7 +316,6 @@ jobs: APR_VERSION=1.7.6 APU_VERSION=1.6.5 APU_CONFIG="--without-crypto" - pkgs: subversion # ------------------------------------------------------------------------- - name: OpenSSL 3.4 no-engine config: --enable-mods-shared=most --enable-maintainer-mode --disable-md --disable-http2 --disable-ldap --disable-crypto @@ -313,7 +325,6 @@ jobs: APR_VERSION=1.7.6 APU_VERSION=1.6.5 APU_CONFIG="--without-crypto" - pkgs: subversion # ------------------------------------------------------------------------- - name: OpenSSL 3.5 no-engine -Werror config: --enable-mods-shared=most --enable-maintainer-mode --disable-md --disable-http2 --disable-ldap --disable-crypto @@ -324,7 +335,6 @@ jobs: APR_VERSION=1.7.6 APU_VERSION=1.6.5 APU_CONFIG="--without-crypto" - pkgs: subversion # ------------------------------------------------------------------------- - name: OpenSSL 4.0 config: --enable-mods-shared=most --enable-maintainer-mode --disable-md --disable-http2 --disable-ldap --disable-crypto @@ -335,7 +345,6 @@ jobs: APR_VERSION=1.7.6 APU_VERSION=1.6.5 APU_CONFIG="--without-crypto" - pkgs: subversion # ------------------------------------------------------------------------- runs-on: ${{ matrix.os == '' && 'ubuntu-latest' || matrix.os }} timeout-minutes: 30 @@ -344,22 +353,33 @@ jobs: CONFIG: ${{ matrix.config }} name: ${{ matrix.name }} steps: - # JOBID is used in the cache keys, created here as a hash of all - # properties of the environment, including the image OS version, - # compiler flags and any job-specific properties. + # JOBID is used to name the artifacts uploaded on failure. IMAGEOS + # is exported so that it is visible to the env context, which does + # not see the runner's own $ImageOS. - name: Set environment variables run: | echo "${{ matrix.env }}" >> $GITHUB_ENV + echo "IMAGEOS=$ImageOS" >> $GITHUB_ENV + # The scripts run "make $MFLAGS"; default it to the number of + # CPUs the runner has, unless the matrix entry set it already + # (the ap_expr job sets it empty on purpose). + grep -q '^MFLAGS=' $GITHUB_ENV || echo "MFLAGS=-j`nproc`" >> $GITHUB_ENV echo JOBID=`echo "OS=$ImageOS ${{ matrix.notest-cflags }} ${{ matrix.env }} ${{ matrix.config }}" \ | md5sum - | sed 's/ .*//'` >> $GITHUB_ENV - name: apt refresh run: sudo apt-get -o Acquire::Retries=5 update + # cpanminus and perl-doc are only used to build the CPAN modules for + # the Perl test framework, and perl-doc drags in around a hundred + # packages. gdb is only used to extract a backtrace from any core + # dumped during testing, which includes the pytest suites. - name: Install prerequisites run: sudo apt-get install -o Acquire::Retries=5 - cpanminus libtool-bin libapr1-dev libaprutil1-dev + libtool-bin libapr1-dev libaprutil1-dev liblua5.3-dev libbrotli-dev libcurl4-openssl-dev - libnghttp2-dev libjansson-dev libpcre2-dev gdb - perl-doc libsasl2-dev curl pipx ${{ matrix.pkgs }} check + libnghttp2-dev libjansson-dev libpcre2-dev + ${{ ! ( env.SKIP_TESTING || env.NO_TEST_FRAMEWORK ) && 'cpanminus perl-doc' || '' }} + ${{ ! env.SKIP_TESTING && 'gdb' || '' }} + libsasl2-dev curl pipx ${{ matrix.pkgs }} check - uses: actions/checkout@v6 - uses: actions/checkout@v6 with: @@ -367,19 +387,42 @@ jobs: path: test/perl-framework submodules: true if: ${{ ! ( env.SKIP_TESTING || env.NO_TEST_FRAMEWORK ) }} - - name: Cache installed libraries - uses: actions/cache@v5 + # Sets $ROOTID, identifying the contents of ~/root, and leaves it + # unset for a job which builds no dependencies from source. + - name: Resolve dependency versions + run: ./test/gha-resolve-deps.sh + # Restored here but saved only once the dependencies have been built + # successfully, below: a job which fails part-way through building + # them would otherwise save a half-built tree under a key which can + # never be updated. Bump the serial in the key to discard the cached + # builds and start afresh. + - name: Restore installed libraries + uses: actions/cache/restore@v5 + id: cache-libs + if: env.ROOTID != '' with: path: ~/root - key: cache-libs-${{ env.JOBID }} + key: cache-libs-1-${{ env.IMAGEOS }}-${{ runner.arch }}-${{ env.ROOTID }} + # ~/perl5 does not vary by job: the module list is fixed and the + # modules are always built with CC=gcc. It varies only by the perl + # ABI (hence the image and architecture) and by the module list + # itself, so one cache is shared by every job. Bump the serial in + # the key to force a rebuild against current CPAN. - name: Cache CPAN modules uses: actions/cache@v5 + if: ${{ ! ( env.SKIP_TESTING || env.NO_TEST_FRAMEWORK ) }} with: path: ~/perl5 - key: cache-cpan-${{ env.JOBID }} + key: cache-cpan-1-${{ env.IMAGEOS }}-${{ runner.arch }}-${{ hashFiles('test/travis_before_linux.sh') }} - name: Configure environment run: ./test/travis_before_linux.sh timeout-minutes: 15 + - name: Save installed libraries + uses: actions/cache/save@v5 + if: success() && env.ROOTID != '' && steps.cache-libs.outputs.cache-hit != 'true' + with: + path: ~/root + key: ${{ steps.cache-libs.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 if: failure() with: diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 00000000000..4f6076db180 --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,43 @@ +# Quick checks against the source which need nothing built, and so do +# not belong in the build matrix: there they would spend twenty seconds +# installing packages to run a one second check. + +name: Sanity + +on: + push: + branches: [ "*" ] + paths-ignore: + - 'docs/**' + - STATUS + - CHANGES + - README* + - '**.md' + - changes-entries/* + pull_request: + branches: [ "trunk", "2.4.x" ] + paths-ignore: + - 'docs/**' + - STATUS + - CHANGES + - README* + - '**.md' + - changes-entries/* + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + aplogno: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - name: Check APLOGNO() macros + run: ./test/travis_run_linux.sh + env: + TEST_LOGNO: 1 diff --git a/test/README.ci b/test/README.ci index f9e25505aba..715b46f2402 100644 --- a/test/README.ci +++ b/test/README.ci @@ -22,7 +22,10 @@ The CI scripts use the following environment variables: * BUILDCONFIG - arguments to pass when running httpd's ./buildconf script -* MFLAGS - arguments to pass when running "make" for httpd. +* MFLAGS - arguments to pass when running "make", both for httpd and + for any dependency built from source. The Linux workflow defaults + this to "-j"; a matrix entry can override it, e.g. + setting it empty to force a serial build. * SKIP_TESTING - if set, no testing is done at all @@ -99,42 +102,66 @@ The CI scripts use the following environment variables: included in the pytest test suites. Set globally in the workflow (e.g. "/usr/sbin/php-fpm8.3"). -* CLEAR_CACHE - if set, the cached $HOME/root is removed before each build - Caching ------- Caching was designed in Travis then migrated to GitHub Actions, and should probably be redone from scratch. -Caches in GHA are immutable. Currently the ~/perl5 and ~/root -directories are cached (separately) for each unique job configuration, -which is identified by the $JOBID variable. $JOBID is a hash of the -variables defined in the matrix. - -Hence, if e.g. the APR_VERSION or APR_CONFIG changes, a different -cache key will be used for future builds, but while it remains the -same the cached install can be used across builds. - -This does not work optimally for e.g. APR_VERSION=trunk - in this -case, a trunk build of APR is built and cached (assuming the job -succeeds). Once the trunk revision changes, install_apx() will see the -cached trunk install is stale and throw it away for every single -subsequent job; the cache is never updated. - -(Similarly for CPAN - the cached version of ~/perl5 for each job will -increasingly become stale over time as the version in CPAN changes -from whatever was cached for the first build.) - -The solution (TBD) is likely to incorporate the versions of whatever -is cached into the keys as in the example documentation: -https://github.com/actions/cache/blob/main/examples.md - -If APR_VERSION and APU_VERSION are both set to 1.x versions, then -CLEAR_CACHE should also be set to disable APR* caching. APR-util can -only be rebuilt if an APR checkout is present, so a APR-util cannot be -built from source alone. (i.e. the scripts do not handle the case of -cached, fresh APR plus a cached but stale APR-util) +The root cause of most of what follows is that Travis caches were +mutable - restored at the start of a build and re-uploaded under the +same key at the end - so it was fine for the scripts to do their own +staleness checks and update the cached tree in place. Caches in GHA +are immutable: on an exact key hit the post-job save is skipped +entirely, so any rebuild the scripts do is discarded and repeated on +every subsequent run. Invalidation therefore has to live in the cache +key, not in the script. + +Two directories are cached. + +~/root (key "cache-libs----$ROOTID") holds the +APR, APR-util and OpenSSL installs. gha-resolve-deps.sh computes +$ROOTID before the cache is restored, hashing only what determines +those installs: the versions, their configure arguments and $CC. The +job's own configuration is deliberately not included, so jobs building +the same dependencies share one entry however differently they +configure or test httpd - the five pool-debug jobs, for instance. A job +which builds no dependencies gets no $ROOTID and does not use the cache +at all. + +For anything built from a branch rather than a tag - APR_VERSION=trunk +or 1.7.x, and TEST_OPENSSL3_BRANCH - the version alone does not identify +what gets built, so the branch is resolved to a commit and that goes +into $ROOTID too. The commit is exported as +$APR_COMMIT/$APU_COMMIT/$OPENSSL_COMMIT so that the build uses exactly +what the key names rather than resolving the branch a second time, +minutes later, by which point it may have moved. + +It is restored before the dependencies are built and saved afterwards, +only if they built successfully - a failed job would otherwise write a +half-built tree to a key which can never be updated. + +~/perl5 (key "cache-cpan----") holds the +CPAN modules. Nothing in it varies per job: the module list is fixed +and the modules are always built with CC=gcc, so the content depends +only on the perl ABI - which is determined by the image and the +architecture - and on the module list itself, hashed from +travis_before_linux.sh. One cache is therefore shared by every job. The .perlver check in the script remains +as a backstop for a perl upgrade within the same image. + +The cached CPAN modules still become gradually stale as versions in +CPAN move on from whatever was installed for the first build. Bump +the serial in the key ("cache-cpan-1-" -> "cache-cpan-2-") to discard +them and rebuild against current CPAN. + +APR and APR-util must stay in a single cache entry. APR-util's +buildconf needs APR's *source* tree, which lives in $HOME/build and is +not cached, so a cached but stale APR-util alongside a cached and fresh +APR cannot be rebuilt from the cache alone. This used to be handled by +setting CLEAR_CACHE on such jobs to disable caching altogether; +install_apx() now fetches the APR source on demand in that case +instead, so no job needs to opt out of caching. + Travis to Github Actions Migration TODO --------------------------------------- diff --git a/test/gha-resolve-deps.sh b/test/gha-resolve-deps.sh new file mode 100755 index 00000000000..61297e7fbae --- /dev/null +++ b/test/gha-resolve-deps.sh @@ -0,0 +1,84 @@ +#!/bin/bash -e +# +# Work out what identifies the contents of ~/root, and write it to +# $GITHUB_ENV as $ROOTID for the cache key to use. +# +# ~/root holds only the dependencies built from source, so the key +# covers those alone - not the job's own configuration. Jobs which +# build the same dependencies then share one cache entry even where they +# configure or test httpd differently. $ROOTID is left unset for a job +# which builds no dependencies at all, and the cache is skipped. +# +# Anything built from a branch is resolved to a commit here and the +# commit exported, so that the build uses exactly what the key names; +# resolving the branch again at build time would race with it moving. +# Tags and release tarballs are immutable, so the version is enough. + +root= + +# $1 = name, $2 = version, $3 = configure arguments, $4 = variable to +# export the resolved commit into. +function add_apx() { + local name=$1 + local version=$2 + local config=$3 + local var=$4 + local sha + + if test -z "$version"; then + return 0 + fi + + case "$version" in + trunk|*.x) + sha=`git ls-remote https://github.com/apache/${name}.git refs/heads/${version} | cut -f1` + if test -z "$sha"; then + : Could not resolve ${name} branch ${version} - check branch is valid? + exit 1 + fi + echo "${var}=${sha}" >> $GITHUB_ENV ;; + *) sha=${version} ;; + esac + + root="${root} ${name}-${version}-${sha}-${config}" +} + +add_apx apr "${APR_VERSION-}" "${APR_CONFIG-}" APR_COMMIT +add_apx apr-util "${APU_VERSION-}" "${APU_CONFIG-}" APU_COMMIT + +if test -n "${TEST_OPENSSL3-}"; then + if test -n "${TEST_OPENSSL3_BRANCH-}"; then + sha=`git ls-remote https://github.com/openssl/openssl refs/heads/${TEST_OPENSSL3_BRANCH} | cut -f1` + if test -z "$sha"; then + : Could not resolve openssl branch ${TEST_OPENSSL3_BRANCH} - check branch is valid? + exit 1 + fi + echo "OPENSSL_COMMIT=${sha}" >> $GITHUB_ENV + root="${root} openssl-${TEST_OPENSSL3_BRANCH}-${sha}-${OPENSSL_CONFIG-}" + else + root="${root} openssl-${TEST_OPENSSL3}-${OPENSSL_CONFIG-}" + fi +fi + +if test -n "${TEST_NGHTTP2-}"; then + root="${root} nghttp2-${TEST_NGHTTP2}" +fi + +if test -n "${RUSTLS_VERSION-}"; then + root="${root} rustls-${RUSTLS_VERSION}" +fi + +if test -n "$root"; then + # The compiler and every flag which reaches the build are baked into + # the libraries, NOTEST_* included: APR adds those to the build too, + # it just does not use them for ./configure. + root="${root} CC=${CC-} CFLAGS=${CFLAGS-} CPPFLAGS=${CPPFLAGS-}" + root="${root} LDFLAGS=${LDFLAGS-} LIBS=${LIBS-}" + root="${root} NOTEST_CFLAGS=${NOTEST_CFLAGS-} NOTEST_CPPFLAGS=${NOTEST_CPPFLAGS-}" + root="${root} NOTEST_LDFLAGS=${NOTEST_LDFLAGS-} NOTEST_LIBS=${NOTEST_LIBS-}" + root="${root}${TEST_APR_TARBALL+ tarball}" + : Dependencies:${root} + echo ROOTID=`echo "$root" | md5sum - | sed 's/ .*//'` >> $GITHUB_ENV +else + : No dependencies are built from source, so ~/root is not cached. +fi diff --git a/test/travis_before_linux.sh b/test/travis_before_linux.sh index eb12b0f5d27..432af0512d6 100755 --- a/test/travis_before_linux.sh +++ b/test/travis_before_linux.sh @@ -1,9 +1,5 @@ #!/bin/bash -xe -if test -v CLEAR_CACHE; then - rm -rf $HOME/root -fi - : Travis tag = ${TRAVIS_TAG} : Travis branch = ${TRAVIS_BRANCH} @@ -21,61 +17,111 @@ if grep ip6-localhost /etc/hosts; then cat /etc/hosts fi -function install_apx() { +# Echo the object ID (hash) of the commit to build for $1 at version +# $2, or "tarball" if a release tarball is used instead of a checkout. +function resolve_apx() { local name=$1 local version=$2 - local prefix=${HOME}/root/${name}-${version} - local build=${HOME}/build/${name}-${version} - local giturl=https://github.com/apache/${name}.git - local config=$3 - local buildconf=$4 - local commit=tarball + local ref commit - mkdir -p ${HOME}/build + if test -v TEST_APR_TARBALL; then + echo tarball + return 0 + fi - if ! test -v TEST_APR_TARBALL; then - case $version in - trunk|*.x) ref=refs/heads/${version} ;; - *) ref=refs/tags/${version} ;; - esac - - # Fetch the object ID (hash) of latest commit - commit=`git ls-remote ${giturl} ${ref} | cut -f1` - if test -z "$commit"; then - : Could not determine latest commit hash for ${ref} in ${giturl} - check branch is valid? - exit 1 - fi + # For a branch, prefer the commit already resolved by + # gha-resolve-deps.sh for the cache key; resolving it again here + # would race with the branch moving in the meantime. Tags are + # immutable so they are always resolved below. + case $name in + apr) commit=${APR_COMMIT-} ;; + apr-util) commit=${APU_COMMIT-} ;; + esac + if test -n "$commit"; then + echo ${commit} + return 0 fi - # Blow away the cached install root if the cached install is stale - # or doesn't match the expected configuration. - grep -q "${version} ${commit} ${config} CC=$CC" ${HOME}/root/.key-${name} || rm -rf ${prefix} + case $version in + trunk|*.x) ref=refs/heads/${version} ;; + *) ref=refs/tags/${version} ;; + esac - if test -d ${prefix}; then + commit=`git ls-remote https://github.com/apache/${name}.git ${ref} | cut -f1` + if test -z "$commit"; then + : Could not determine latest commit hash for ${ref} in ${name} - check branch is valid? + exit 1 + fi + + echo ${commit} +} + +# Unpack the source for $1 at version $2, commit $3, into +# $HOME/build/$1-$2, running ./buildconf with the arguments in $4. Does +# nothing if the source tree is already present. Note $HOME/build is +# not cached, unlike the install root in $HOME/root. +function fetch_apx() { + local name=$1 + local version=$2 + local commit=$3 + local buildconf=$4 + local build=${HOME}/build/${name}-${version} + + if test -d ${build}; then return 0 fi + mkdir -p ${HOME}/build + if test -v TEST_APR_TARBALL; then curl https://archive.apache.org/dist/apr/${name}-${version}.tar.gz > apx.tar.gz tar -C ${HOME}/build -xzf apx.tar.gz rm apx.tar.gz - pushd ${build} else git init -q ${build} pushd $build # Clone and checkout the commit identified above. - git remote add origin ${giturl} + git remote add origin https://github.com/apache/${name}.git git fetch -q --depth=1 origin ${commit} git checkout ${commit} ./buildconf ${buildconf} + popd fi +} +function install_apx() { + local name=$1 + local version=$2 + local prefix=${HOME}/root/${name}-${version} + local build=${HOME}/build/${name}-${version} + local config=$3 + local buildconf=$4 + local commit + + # The cache key covers the version, the resolved commit for a branch + # build, the configuration and $CC, and is only written once the + # build has succeeded, so anything restored here is usable as-is. + if test -d ${prefix}; then + return 0 + fi + + commit=`resolve_apx ${name} ${version}` + + # apr-util's buildconf needs APR's source tree, which is not cached. + # If APR itself was restored from the cache it was never built here, + # so fetch the source now - otherwise a stale APR-util alongside a + # fresh APR cannot be rebuilt. + if test ${name} = apr-util -a ! -d ${HOME}/build/apr-${APR_VERSION}; then + fetch_apx apr ${APR_VERSION} `resolve_apx apr ${APR_VERSION}` + fi + + fetch_apx ${name} ${version} ${commit} "${buildconf}" + + pushd ${build} ./configure --prefix=${prefix} ${config} make -j2 make install popd - - echo ${version} ${commit} "${config}" "CC=${CC}" > ${HOME}/root/.key-${name} } # Allow to load $HOME/build/apache/httpd/.gdbinit @@ -84,9 +130,6 @@ echo "add-auto-load-safe-path $HOME/work/httpd/httpd/.gdbinit" >> $HOME/.gdbinit # Unless either SKIP_TESTING or NO_TEST_FRAMEWORK are set, install # CPAN modules required to run the Perl test framework. if ! test -v SKIP_TESTING -o -v NO_TEST_FRAMEWORK; then - # Clear CPAN cache if necessary - if [ -v CLEAR_CACHE ]; then rm -rf ~/perl5; fi - if ! perl -V > perlver; then : Perl binary broken perl -V @@ -138,26 +181,20 @@ fi # Build the requested version of OpenSSL if it's not already installed # in the cached ~/root if test -v TEST_OPENSSL3; then - # For a branch, rebuild if the remote branch has updated. - if test -v TEST_OPENSSL3_BRANCH -a -f $HOME/root/openssl-is-${TEST_OPENSSL3}; then - latest=`git ls-remote https://github.com/openssl/openssl refs/heads/${TEST_OPENSSL3_BRANCH} | cut -f1` - : Got branch latest commit ${latest} - if grep -q ^${latest} $HOME/root/openssl-is-${TEST_OPENSSL3}; then - : Cached repos already at ${latest} - else - : Forcing rebuild - rm -f $HOME/root/openssl-is-${TEST_OPENSSL3} - fi - fi - - if ! test -f $HOME/root/openssl-is-${TEST_OPENSSL3}; then - # Remove any previous install. - rm -rf $HOME/root/openssl3 - + # The cache key covers $TEST_OPENSSL3, $OPENSSL_CONFIG and, for a + # branch build, the resolved commit, so an install found here is + # current. + if ! test -d $HOME/root/openssl3; then mkdir -p build/openssl pushd build/openssl if test -v TEST_OPENSSL3_BRANCH; then git clone --depth=1 -b $TEST_OPENSSL3_BRANCH -q https://github.com/openssl/openssl openssl-${TEST_OPENSSL3} + # Build the commit named in the cache key, not whatever + # the branch tip has become since it was resolved. + if test -n "${OPENSSL_COMMIT-}"; then + git -C openssl-${TEST_OPENSSL3} fetch -q --depth=1 origin ${OPENSSL_COMMIT} + git -C openssl-${TEST_OPENSSL3} checkout -q ${OPENSSL_COMMIT} + fi else curl -L "https://github.com/openssl/openssl/releases/download/openssl-${TEST_OPENSSL3}/openssl-${TEST_OPENSSL3}.tar.gz" | tar -xzf - @@ -169,12 +206,6 @@ if test -v TEST_OPENSSL3; then '-Wl,-rpath=$(LIBRPATH)' make $MFLAGS make install_sw - if test -d .git; then - : Caching git commit hash: - git rev-parse HEAD | tee $HOME/root/openssl-is-${TEST_OPENSSL3} - else - touch $HOME/root/openssl-is-${TEST_OPENSSL3} - fi popd fi