From a9433454c5158be8ccc0426fe2a93fc10bcd372c Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Mon, 30 Oct 2023 11:07:25 +0000 Subject: [PATCH 1/4] Fix CI housekeep --- .github/workflows/ci.yml | 67 +++++++++++++++++++++------------------- ci/CMakeLists.txt | 2 ++ ci/build-and-test | 6 ++++ 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4ea826d..8705bce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: branches: [develop] pull_request: branches: [develop] + concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} env: REPO: ${{ github.event.repository.name }} @@ -16,13 +19,12 @@ jobs: name: gnu 11.3.1 (UKMO) if: github.repository_owner == 'MetOffice' runs-on: jedi-self-hosted-rg - + steps: - - name: pre-submit cleanup + - name: Pre-submit cleanup run: | - ls -la ./ rm -fr ./${{ env.BUNDLE }} || true - ls -la ./ + ls -lgo ./ - name: checkout current uses: actions/checkout@v4 with: @@ -30,7 +32,7 @@ jobs: - name: initiate bundle run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/ - - name: checkout jedicmake + - name: Checkout jedicmake uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/jedicmake @@ -38,45 +40,48 @@ jobs: submodules: true token: ${{ secrets.GH_PAT }} - - name: checkout oops + - name: Checkout oops uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt oops ${{ github.head_ref }} + - name: Checkout matching branch for oops ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops - - name: checkout ioda + - name: Checkout ioda uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ioda ${{ github.head_ref }} + - name: Checkout matching branch for ioda ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda - - name: checkout ufo + - name: Checkout ufo uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ufo ${{ github.head_ref }} + - name: Checkout matching branch for ufo ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo - - name: build and test + - name: Build and Test run: | az acr login --name ngmssboxjediacr docker run --rm \ --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-test \ --workdir=/usr/local/src/${REPO}/${BUNDLE} \ --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \ - 'ngmssboxjediacr.azurecr.io/jedibase:alma9' \ - && rm -fr ${{github.workspace}}/${{env.BUNDLE}} + 'ngmssboxjediacr.azurecr.io/jedibase:alma9' + + - name: Cleanup workspace + working-directory: ${{github.workspace}} + run: rm -fr ${{ env.BUNDLE }} || true build2: name: gnu 9.4.0 (JCSDA) @@ -84,14 +89,14 @@ jobs: runs-on: ubuntu-latest steps: - - name: checkout current + - name: Checkout current uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/${{ env.REPO }} - name: initiate bundle run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/ - - name: checkout jedicmake + - name: Checkout jedicmake uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/jedicmake @@ -99,33 +104,33 @@ jobs: submodules: true token: ${{ secrets.GH_PAT }} - - name: checkout oops + - name: Checkout oops uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt oops ${{ github.head_ref }} + - name: Checkout matching branch for oops ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops - - name: checkout ioda + - name: Checkout ioda uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ioda ${{ github.head_ref }} + - name: Checkout matching branch for ioda ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda - - name: checkout ufo + - name: Checkout ufo uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ufo ${{ github.head_ref }} + - name: Checkout matching branch for ufo ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo @@ -143,14 +148,14 @@ jobs: runs-on: ubuntu-latest steps: - - name: checkout current + - name: Checkout current uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/${{ env.REPO }} - name: initiate bundle run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/ - - name: checkout jedicmake + - name: Checkout jedicmake uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/jedicmake @@ -158,37 +163,37 @@ jobs: submodules: true token: ${{ secrets.GH_PAT }} - - name: checkout oops + - name: Checkout oops uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt oops ${{ github.head_ref }} + - name: Checkout matching branch for oops ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops - - name: checkout ioda + - name: Checkout ioda uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ioda ${{ github.head_ref }} + - name: Checkout matching branch for ioda ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda - - name: checkout ufo + - name: Checkout ufo uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: matching branch checkout attempt ufo ${{ github.head_ref }} + - name: Checkout matching branch for ufo ${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo - - name: build and test + - name: Build and Test run: | docker run --rm \ --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-test \ diff --git a/ci/CMakeLists.txt b/ci/CMakeLists.txt index f1f0decc..c5874cf5 100644 --- a/ci/CMakeLists.txt +++ b/ci/CMakeLists.txt @@ -7,6 +7,8 @@ project( opsinputs-ci VERSION 0.1 LANGUAGES C CXX Fortran ) set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) set( ENABLE_OMP ON CACHE BOOL "Compile with OpenMP" ) +option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) +option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/jedicmake") if(NOT DEFINED jedicmake_DIR) diff --git a/ci/build-and-test b/ci/build-and-test index 975c5c93..81b94676 100755 --- a/ci/build-and-test +++ b/ci/build-and-test @@ -2,6 +2,8 @@ # # (C) Crown Copyright 2023, the Met Office. All rights reserved. # +# shellcheck disable=SC2317 +# shellcheck source=/dev/null set -euo pipefail finally() { @@ -11,6 +13,10 @@ finally() { cd / rm -fr "${WORKD}" fi + if [[ -d ${BASE:-} ]]; then + cd / + rm -fr "$BASE" 2>/dev/null || true + fi } # -- HERE is /usr/local/src//pr-<#> (cf ../.github/workflow/ci.yml) From 70c3e7e4d47a3c218d2f1f0e136a0cd345e6cbf8 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Mon, 30 Oct 2023 11:18:24 +0000 Subject: [PATCH 2/4] fix workflow syntax --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8705bce7..8fdba910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,8 +80,8 @@ jobs: 'ngmssboxjediacr.azurecr.io/jedibase:alma9' - name: Cleanup workspace - working-directory: ${{github.workspace}} - run: rm -fr ${{ env.BUNDLE }} || true + working-directory: ${{github.workspace}} + run: rm -fr ${{ env.BUNDLE }} || true build2: name: gnu 9.4.0 (JCSDA) From 64934bcfd5a9b0840fb67f3f17257e471d4903b8 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Mon, 30 Oct 2023 11:25:31 +0000 Subject: [PATCH 3/4] fix workflow syntaxt --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fdba910..f5345c1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,9 +6,10 @@ on: branches: [develop] pull_request: branches: [develop] - concurrency: - group: ${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} env: REPO: ${{ github.event.repository.name }} From fad3abf11d1f7fa89fc40982a26aba981e41d44a Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Mon, 30 Oct 2023 11:31:17 +0000 Subject: [PATCH 4/4] tidy --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5345c1f..ce1e6b78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,11 @@ jobs: run: | rm -fr ./${{ env.BUNDLE }} || true ls -lgo ./ - - name: checkout current + - name: Checkout current uses: actions/checkout@v4 with: path: ./${{ env.BUNDLE }}/${{ env.REPO }} - - name: initiate bundle + - name: Initiate bundle run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/ - name: Checkout jedicmake @@ -47,7 +47,7 @@ jobs: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for oops ${{ github.head_ref }} + - name: Checkout attempt for oops@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops @@ -57,7 +57,7 @@ jobs: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ioda ${{ github.head_ref }} + - name: Checkout attempt for ioda@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda @@ -67,7 +67,7 @@ jobs: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ufo ${{ github.head_ref }} + - name: Checkout attempt for ufo@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo @@ -111,7 +111,7 @@ jobs: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for oops ${{ github.head_ref }} + - name: Checkout attempt for oops@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops @@ -121,7 +121,7 @@ jobs: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ioda ${{ github.head_ref }} + - name: Checkout attempt for ioda@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda @@ -131,7 +131,7 @@ jobs: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ufo ${{ github.head_ref }} + - name: Checkout attempt for ufo@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo @@ -170,7 +170,7 @@ jobs: path: ./${{ env.BUNDLE }}/oops repository: JCSDA-internal/oops token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for oops ${{ github.head_ref }} + - name: Checkout attempt for oops@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops @@ -180,7 +180,7 @@ jobs: path: ./${{ env.BUNDLE }}/ioda repository: JCSDA-internal/ioda token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ioda ${{ github.head_ref }} + - name: Checkout attempt for ioda@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda @@ -190,7 +190,7 @@ jobs: path: ./${{ env.BUNDLE }}/ufo repository: JCSDA-internal/ufo token: ${{ secrets.GH_PAT }} - - name: Checkout matching branch for ufo ${{ github.head_ref }} + - name: Checkout attempt for ufo@${{ github.head_ref }} if: github.event_name == 'pull_request' run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo