From 3cc42129eeed2928c881d22812e1f2f10c4f5f08 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Fri, 29 Sep 2023 23:29:51 +0100 Subject: [PATCH 1/3] add clang container to CI workflow --- .github/workflows/ci.yml | 75 +++++++++++++++++++++++++++++++++++----- ci/build-and-test | 7 ++-- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e759fd9..b4ea826d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,19 +2,21 @@ name: CI on: + push: + branches: [develop] pull_request: branches: [develop] +env: + REPO: ${{ github.event.repository.name }} + BUNDLE: pr-${{ github.event.number }} + jobs: build1: name: gnu 11.3.1 (UKMO) if: github.repository_owner == 'MetOffice' runs-on: jedi-self-hosted-rg - - env: - REPO: ${{ github.event.repository.name }} - BUNDLE: pr-${{ github.event.number }} - + steps: - name: pre-submit cleanup run: | @@ -81,10 +83,6 @@ jobs: if: github.repository_owner == 'MetOffice' runs-on: ubuntu-latest - env: - REPO: ${{ github.event.repository.name }} - BUNDLE: pr-${{ github.event.number }} - steps: - name: checkout current uses: actions/checkout@v4 @@ -138,3 +136,62 @@ jobs: --workdir=/usr/local/src/${REPO}/${BUNDLE} \ --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \ 'jcsda/docker-gnu-openmpi-dev:latest' + + build3: + name: clang 10.0.0 (JCSDA) + if: github.repository_owner == 'MetOffice' + runs-on: ubuntu-latest + + steps: + - name: checkout current + uses: actions/checkout@v4 + with: + path: ./${{ env.BUNDLE }}/${{ env.REPO }} + - name: initiate bundle + run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/ + + - name: checkout jedicmake + uses: actions/checkout@v4 + with: + path: ./${{ env.BUNDLE }}/jedicmake + repository: JCSDA-internal/jedi-cmake + submodules: true + token: ${{ secrets.GH_PAT }} + + - name: checkout oops + uses: actions/checkout@v4 + with: + path: ./${{ env.BUNDLE }}/oops + repository: JCSDA-internal/oops + token: ${{ secrets.GH_PAT }} + - name: matching branch checkout attempt oops ${{ github.head_ref }} + if: github.event_name == 'pull_request' + run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops + + - name: checkout ioda + uses: actions/checkout@v4 + with: + path: ./${{ env.BUNDLE }}/ioda + repository: JCSDA-internal/ioda + token: ${{ secrets.GH_PAT }} + - name: matching branch checkout attempt ioda ${{ github.head_ref }} + if: github.event_name == 'pull_request' + run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda + + - name: checkout ufo + uses: actions/checkout@v4 + with: + path: ./${{ env.BUNDLE }}/ufo + repository: JCSDA-internal/ufo + token: ${{ secrets.GH_PAT }} + - name: matching branch checkout attempt ufo ${{ github.head_ref }} + if: github.event_name == 'pull_request' + run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo + + - name: build and test + run: | + docker run --rm \ + --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-test \ + --workdir=/usr/local/src/${REPO}/${BUNDLE} \ + --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \ + 'jcsda/docker-clang-mpich-dev:latest' diff --git a/ci/build-and-test b/ci/build-and-test index 2320be3e..08d7b0b0 100755 --- a/ci/build-and-test +++ b/ci/build-and-test @@ -13,12 +13,15 @@ finally() { fi } +# HERE is /usr/local/src//pr-<#> (cf ../.github/workflow/ci.yml) HERE="$(cd "$(dirname "$0")" && pwd)" THIS="$(basename "$0")" NPROC=${NPROC:-$(nproc)} WORKD="$(mktemp -d "${THIS}-XXXXXX" -t)" GENERATOR=Unix\ Makefiles if command -v ninja &>/dev/null; then GENERATOR=Ninja; fi +BASE="${HERE%/*}" +TESTDIR="${BASE##*/}" trap finally ERR EXIT @@ -39,8 +42,8 @@ cmake --build . -j "${NPROC}" # -- Test env OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ - ctest -j"${NPROC}" --test-dir './opsinputs' --output-on-failure ||\ + ctest -j"${NPROC}" --test-dir "${TESTDIR}" || \ env OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ - ctest --test-dir './opsinputs' --rerun-failed -V + ctest --test-dir "${TESTDIR}" --rerun-failed -V exit From 5cfd2cc27182e53f59c0015a62be8809cb144114 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Sat, 30 Sep 2023 00:15:30 +0100 Subject: [PATCH 2/3] set oversubscribe args only for OpenMPI --- ci/build-and-test | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/build-and-test b/ci/build-and-test index 08d7b0b0..5394ca7a 100755 --- a/ci/build-and-test +++ b/ci/build-and-test @@ -22,6 +22,7 @@ GENERATOR=Unix\ Makefiles if command -v ninja &>/dev/null; then GENERATOR=Ninja; fi BASE="${HERE%/*}" TESTDIR="${BASE##*/}" +export OMPI_MCA_rmaps_base_oversubscribe=true trap finally ERR EXIT @@ -33,9 +34,7 @@ if [[ -f /opt/spack-environment/activate.sh ]]; then fi # -- Configure -cmake -B . -S "${HERE}" -G "${GENERATOR}" \ - -DCMAKE_BUILD_TYPE=Debug \ - -DMPI_ARGS="--oversubscribe" +cmake -B . -S "${HERE}" -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=Debug # -- Build cmake --build . -j "${NPROC}" From 59e7ff6cc1aa3fabddb4a8887c2793fa1b5179d2 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan Date: Tue, 3 Oct 2023 11:31:43 +0100 Subject: [PATCH 3/3] disable trace from ci build script --- ci/build-and-test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build-and-test b/ci/build-and-test index 5394ca7a..975c5c93 100755 --- a/ci/build-and-test +++ b/ci/build-and-test @@ -2,7 +2,7 @@ # # (C) Crown Copyright 2023, the Met Office. All rights reserved. # -set -euxo pipefail +set -euo pipefail finally() { trap '' ERR @@ -13,7 +13,7 @@ finally() { fi } -# HERE is /usr/local/src//pr-<#> (cf ../.github/workflow/ci.yml) +# -- HERE is /usr/local/src//pr-<#> (cf ../.github/workflow/ci.yml) HERE="$(cd "$(dirname "$0")" && pwd)" THIS="$(basename "$0")" NPROC=${NPROC:-$(nproc)} @@ -28,7 +28,7 @@ trap finally ERR EXIT cd "${WORKD}" -# If using JCSDA Docker container +# -- Activate spack env if using JCSDA Docker container if [[ -f /opt/spack-environment/activate.sh ]]; then source /opt/spack-environment/activate.sh fi