Add jopa environment

environments/jopa/README.md

@@ -0,0 +1,62 @@
+# JOPA Container
+
+Contains the dependencies required to build mo-bundle and run JOPA
+
+https://github.com/MetOffice/mo-bundle
+
+# Building JOPA / mo-bundle
+
+## NCI, with a container
+
+1. Load the container
+
+```
+module use /g/data/access/ngm
+module load envs/jopa
+```
+
+2. Build Jopa inside the container
+
+```
+cd mo-bundle
+
+imagerun cmake --preset=bom-container --workflow
+```
+
+## Met Office VDI, no container
+
+1. Install Spack, and activate with `$SPACK/share/spack/setup-env.sh`
+
+2. Copy `configs/meto_vdi/*` to the Spack configuration directory (e.g. `~/.spack` or `$SPACK/etc/spack`)
+
+3. Install and setup a compiler, e.g. GCC 9
+
+```
+spack install gcc@9
+spack compiler find $(spack find --format='{prefix}' gcc@9)
+```
+
+4. Install the environment with GCC and MPICH
+
+```
+export SPACK_COMPILER="gcc@9"
+export SPACK_MPI="mpich +slurm"
+
+srun -q -n1 -c10 --mem=40G --time=120 env -u SLURM_NODELIST ./bin/install.sh jopa-v0
+```
+
+5. Activate the environment
+
+```
+spack load $SPACK_COMPILER
+spack env activate jopa-v0
+export SPACK_ENV_VIEW=$SPACK_ENV/.spack-env/view
+```
+
+6. Build mo-bundle
+
+```
+cd mo-bundle
+
+cmake --preset=bom-container --workflow
+```

environments/jopa/etc/env.activate.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export CPATH="$SPACK_VIEW/include_r8:${CPATH:-}"
+export LIBRARY_PATH="$SPACK_VIEW/lib64:$SPACK_VIEW/lib:${LIBRARY_PATH:-}"
+export LD_LIBRARY_PATH="$SPACK_VIEW/lib64:$SPACK_VIEW/lib:${LD_LIBRARY_PATH:-}"

environments/jopa/post-install.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f ${BASH_SOURCE[0]})" )" &> /dev/null && pwd )
+
+# Add FMS
+echo "CPATH=$(spack find --format '{prefix}' fms)/include_r4:\$CPATH" >> $SPACK_ROOT/bin/activate-full.sh

environments/jopa/spack.yaml

@@ -0,0 +1,40 @@
+spack:
+    specs:
+        - [email protected] compression=lz4
+        - [email protected] +fckit +trans +tesselation +fftw
+        - [email protected]
+        - [email protected] +fftw
+        - fiat
+        - odc
+
+        - openblas
+        - boost
+        - cmake
+        - ecbuild
+        - eigen
+        - git-lfs
+        - gsl-lite
+        - jedi-cmake
+        - lapack
+        - netcdf-c
+        - netcdf-cxx4
+        - netcdf-fortran
+        - ninja
+        - nlohmann-json-schema-validator
+        - [email protected]
+        - py-pybind11
+        - shumlib
+        - udunits
+        - zstd
+        - [email protected] precision=32,64 +deprecated_io +pic
+        - nccmp
+        - py-pycodestyle
+        - parallel-netcdf
+        - parallelio
+        - bufr
+
+    packages:
+        git-lfs:
+            require: '%gcc'
+        go:
+            require: '%gcc'

site/nci/envrun

@@ -10,4 +10,7 @@ export FCM_KEYWORDS=/g/data/hr22/apps/etc/fcm/mosrs/keyword.cfg
 # Add central OpenMPI to cflags
 export FFLAGS="-I /apps/openmpi/4.1.5/include/Intel ${FFLAGS:-}"
 
+# Add system libraries to LD_LIBRARY_PATH for MPI
+export SINGULARITYENV_LD_LIBRARY_PATH="/apps/intel-oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/system/lib64:/system/lib"
+
 /opt/singularity/bin/singularity run "${MOUNT_ARGS[@]}" "$ENV_DIR/etc/apptainer.sif" "$@"

site/nci/install.sh

@@ -5,6 +5,7 @@ set -o pipefail
 
 SITE_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )
 
+
 export ENVIRONMENT="$1"
 
 : "${NGMOENVS_BASEDIR:="/scratch/$PROJECT/$USER/ngmo-envs"}"

site/nci/spack-packages.yaml

@@ -23,3 +23,13 @@ packages:
               # Fix up runtime environment
               LD_LIBRARY_PATH: /apps/intel-oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/system/lib64
     buildable: False
+  git:
+    externals:
+        - spec: "[email protected]"
+          prefix: /apps/git/2.39.2
+    buildable: False
+  git-lfs:
+      externals:
+          - spec: "[email protected]"
+            prefix: /apps/git/2.39.2
+      buildable: False

spack/packages/shumlib/package.py

@@ -0,0 +1,31 @@
+class Shumlib(MakefilePackage):
+    homepage = "https://code.metoffice.gov.uk/trac/utils/wiki/shumlib"
+    parallel = False
+
+    url = "https://github.com/metomi/shumlib/archive/refs/tags/um13.0.tar.gz"
+
+    version(
+        "um13.0",
+        sha256="50f43a2f8980e8fbeafd053376612503bcb17c34948297f19b2c95ce0642b340",
+        url="https://github.com/metomi/shumlib/archive/refs/tags/um13.0.tar.gz",
+    )
+
+    def edit(self, spec, prefix):
+        env["PLATFORM"] = "spack"
+        env["FPP"] = "cpp"
+        env[
+            "FPPFLAGS"
+        ] = "-C -P -undef -nostdinc -DEVAL_NAN_BY_BITS -DEVAL_DENORMAL_BY_BITS"
+        env["FCFLAGS_OPENMP"] = "-fopenmp"
+        env["FCFLAGS_PIC"] = "-fPIC"
+        env["FCFLAGS_SHARED"] = "-shared"
+        env["CCFLAGS_OPENMP"] = "-fopenmp"
+        env["CCFLAGS_PIC"] = "-fPIC"
+        env["AR"] = "ar -rc"
+
+        env["DIR_ROOT"] = self.stage.source_path
+        env["LIBDIR_OUT"] = prefix
+        env["SHUM_USE_C_OPENMP_VIA_THREAD_UTILS"] = "false"
+
+    def install(self, spec, prefix):
+        pass

tests/envs/jopa.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )
+
+
+cat <<EOF
+Running jopa test in $BASEDIR
+
+EOF
+
+pushd "$BASEDIR"
+
+export jedi_cmake_ROOT=$(envrun spack find --format '{prefix}' jedi-cmake)
+
+if [[ "${TEST_STAGE:-}" != "two" ]]; then
+    # Download sources
+
+    if [[ ! -d "$BASEDIR/jedi-bundle" ]]; then
+        envrun git clone -b 8.0.0 https://github.com/JCSDA/jedi-bundle
+    fi
+
+    #sed -i jedi-bundle/CMakeLists.txt -e '/mom6/D'
+    #sed -i jedi-bundle/CMakeLists.txt -e '/soca/s/UPDATE )/UPDATE RECURSIVE )/'
+
+    mkdir -p build
+    pushd build
+
+
+    # Download bundle packages
+    envrun ecbuild -- ../jedi-bundle
+
+    # Fixup FMS linking
+    sed -i ../jedi-bundle/mom6/CMakeLists.txt \
+        -e 's/target_link_libraries(\(.*\) fms)/target_link_libraries(\1 FMS::fms_r8)/'
+    sed -i ../jedi-bundle/fv3-jedi-lm/src/CMakeLists.txt \
+        -e 's/list( APPEND FV3JEDILM_LIB_DEP fms )/list( APPEND FV3JEDILM_LIB_DEP FMS::fms_r8 )/'
+    sed -i ../jedi-bundle/fv3/model/fv_control.F90 \
+        -e 's/,\s*INPUT_STR_LENGTH//'
+    sed -i ../jedi-bundle/fv3/CMakeLists.txt \
+        -e 's/PUBLIC_LIBS\s\+fms/PUBLIC_LIBS FMS::fms_r8/'
+    sed -i ../jedi-bundle/soca/src/soca/CMakeLists.txt \
+        -e 's/target_link_libraries(\(.*\) fms\s*)/target_link_libraries(\1 FMS::fms_r8)/'
+
+    # Reprocess after patches
+    # envrun ecbuild -- ../jedi-bundle
+
+    popd
+fi
+
+
+if [[ "${TEST_STAGE:-}" != "one" ]]; then
+    pushd build
+
+    # Build package
+    envrun make VERBOSE=1 -j ${PBS_NCPUS:-4}
+
+    popd
+fi

tests/site/nci.sh

@@ -2,23 +2,46 @@
 #
 # Run an environment test case at NCI
 #
-#     tests/run.sh $ENVIRONMENT
+#     tests/run.sh $NGMO_ENVIRONMENT
 
 set -eu
 set -o pipefail
+set -x
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )
-
-ENVIRONMENT="$1"
+: ${NGMO_ENVIRONMENT:="$1"}
+: ${BASEDIR:="$TMPDIR/ngmo-envs-test/$NGMO_ENVIRONMENT"}
+export NGMO_ENVIRONMENT
+export BASEDIR
 
 # Load the environment
 module purge
 module use "/scratch/$PROJECT/$USER/ngmo-envs/modules"
-module load "$ENVIRONMENT"
+module load "$NGMO_ENVIRONMENT"
+
+if [[ ! -v PBS_ENVIRONMENT ]]; then
+    # Only works outside of PBS
+    SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )
+    export SCRIPT_DIR
+
+    # Not running in the queue
+    mkdir -p "$BASEDIR"
+
+    # Run stage one
+    export TEST_STAGE="one"
+    /bin/bash "$SCRIPT_DIR/../envs/${NGMO_ENVIRONMENT}.sh"
+
+    # Submit stage two
+    qsub \
+        -P $PROJECT \
+        -N "ngmoenvs-test-$NGMO_ENVIRONMENT" \
+        -l ncpus=8,mem=64gb,walltime=1:00:00,jobfs=20gb,wd \
+        -v BASEDIR,NGMO_ENVIRONMENT,SCRIPT_DIR \
+        -j oe \
+        "$SCRIPT_DIR/nci.sh"
 
-# Set up run directory
-export BASEDIR="$TMPDIR/ngmo-envs-test/lfric"
-mkdir -p "$BASEDIR"
+else
+    # Running stage two in the queue
+    export TEST_STAGE="two"
+    /bin/bash "$SCRIPT_DIR/../envs/${NGMO_ENVIRONMENT}.sh"
+fi
 
-# Run the test
-/bin/bash "$SCRIPT_DIR/../envs/${ENVIRONMENT}.sh"

utils/install-stage-one.sh

@@ -138,8 +138,10 @@ export NGMOENVS_ENVDIR="$ENVDIR"
 export NGMOENVS_COMPILER="$NGMOENVS_COMPILER"
 export NGMOENVS_MPI="$NGMOENVS_MPI"
 
-spack env activate "\$NGMOENVS_ENVDIR/spack"
-eval "\$(conda shell.bash activate "\$NGMOENVS_ENVDIR/conda")"
+[[ -f "\$NGMOENVS_ENVDIR/spack" ]] || spack env activate "\$NGMOENVS_ENVDIR/spack"
+[[ -f "\$NGMOENVS_ENVDIR/conda" ]] || eval "\$(conda shell.bash activate "\$NGMOENVS_ENVDIR/conda")"
+
+export SPACK_VIEW="$SPACK_ENV/.spack-env/view/"
 
 if [[ -f "\$NGMOENVS_ENVDIR/etc/env.activate.sh" ]]; then
 	source "\$NGMOENVS_ENVDIR/etc/env.activate.sh"