Import jopa container

environments/jopa/README.md

@@ -0,0 +1,62 @@
+# JOPA Container
+
+Contains the dependencies required to build mo-bundle and run JOPA
+
+https://github.com/MetOffice/mo-bundle
+
+# Building JOPA / mo-bundle
+
+## NCI, with a container
+
+1. Load the container
+
+```
+module use /g/data/access/ngm
+module load envs/jopa
+```
+
+2. Build Jopa inside the container
+
+```
+cd mo-bundle
+
+imagerun cmake --preset=bom-container --workflow
+```
+
+## Met Office VDI, no container
+
+1. Install Spack, and activate with `$SPACK/share/spack/setup-env.sh`
+
+2. Copy `configs/meto_vdi/*` to the Spack configuration directory (e.g. `~/.spack` or `$SPACK/etc/spack`)
+
+3. Install and setup a compiler, e.g. GCC 9
+
+```
+spack install gcc@9
+spack compiler find $(spack find --format='{prefix}' gcc@9)
+```
+
+4. Install the environment with GCC and MPICH
+
+```
+export SPACK_COMPILER="gcc@9"
+export SPACK_MPI="mpich +slurm"
+
+srun -q -n1 -c10 --mem=40G --time=120 env -u SLURM_NODELIST ./bin/install.sh jopa-v0
+```
+
+5. Activate the environment
+
+```
+spack load $SPACK_COMPILER
+spack env activate jopa-v0
+export SPACK_ENV_VIEW=$SPACK_ENV/.spack-env/view
+```
+
+6. Build mo-bundle
+
+```
+cd mo-bundle
+
+cmake --preset=bom-container --workflow
+```

environments/jopa/post-install.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+#  Copyright 2023 Bureau of Meteorology
+#  Author Scott Wales
+
+set -eu
+set -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f ${BASH_SOURCE[0]})" )" &> /dev/null && pwd )
+
+# Add FMS
+echo "CPATH=$(spack find --format '{prefix}' fms)/include_r4:\$CPATH" >> $SPACK_ROOT/bin/activate-full.sh

environments/jopa/spack.yaml

@@ -0,0 +1,39 @@
+spack:
+    specs:
+        - [email protected] compression=lz4
+        - [email protected]
+        - [email protected]
+        - [email protected]
+        - [email protected]
+        - odc
+
+        - openblas
+        - boost
+        - '[email protected]:'
+        - [email protected]
+        - eigen
+        - git-lfs
+        - gsl-lite
+        - hdf5-plugins
+        - jedi-cmake
+        - lapack
+        - netcdf-c
+        - netcdf-cxx4
+        - netcdf-fortran
+        - ninja
+        - nlohmann-json-schema-validator
+        - nlohmann-json
+        - py-pybind11
+        - shumlib
+        - udunits
+        - zstd
+        - [email protected] # Must match mamba
+        - fms@release-jcsda
+
+    packages:
+        git-lfs:
+            require: '%gcc'
+        go:
+            require: '%gcc'
+        openmpi:
+            require: '~static +cxx'

spack/packages/ecbuild/package.py

@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Ecbuild(CMakePackage):
+    """ecBuild is the ECMWF build system. It is built on top of CMake and
+    consists of a set of macros as well as a wrapper around CMake"""
+
+    homepage = "https://github.com/ecmwf/ecbuild"
+    url = "https://github.com/ecmwf/ecbuild/archive/refs/tags/3.6.1.tar.gz"
+
+    maintainers("skosukhin")
+
+    version(
+        "3.7.2",
+        sha256="7a2d192cef1e53dc5431a688b2e316251b017d25808190faed485903594a3fb9",
+    )
+    version(
+        "3.6.5",
+        sha256="98bff3d3c269f973f4bfbe29b4de834cd1d43f15b1c8d1941ee2bfe15e3d4f7f",
+    )
+    version(
+        "3.6.1",
+        sha256="796ccceeb7af01938c2f74eab0724b228e9bf1978e32484aa3e227510f69ac59",
+    )
+
+    depends_on("[email protected]:", type=("build", "run"))
+
+    # See https://github.com/ecmwf/ecbuild/issues/35
+    depends_on("cmake@:3.19", type=("build", "run"), when="@:3.6.1")
+
+    # Some of the installed scripts require running Perl:
+    depends_on("perl", type=("build", "run"))

spack/packages/eckit/package.py

@@ -0,0 +1,200 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Eckit(CMakePackage):
+    """ecKit is a cross-platform c++ toolkit that supports development of tools
+    and applications at ECMWF."""
+
+    homepage = "https://github.com/ecmwf/eckit"
+    url = "https://github.com/ecmwf/eckit/archive/refs/tags/1.16.0.tar.gz"
+
+    maintainers("skosukhin")
+
+    version(
+        "1.24.4",
+        sha256="b6129eb4f7b8532aa6905033e4cf7d09aadc8547c225780fea3db196e34e4671",
+    )
+    version(
+        "1.24.0",
+        sha256="a25440011c66decbfb4908e5f4b7d0025b2166acfb9db941abdeb38d477c45b3",
+    )
+    version(
+        "1.20.2",
+        sha256="9c11ddaaf346e40d11312b81ca7f1b510017f26618f4c0f5c5c59c37623fbac8",
+    )
+    version(
+        "1.19.0",
+        sha256="a5fef36b4058f2f0aac8daf5bcc9740565f68da7357ddd242de3a5eed4765cc7",
+    )
+    version(
+        "1.16.3",
+        sha256="d2aae7d8030e2ce39e5d04e36dd6aa739f3c8dfffe32c61c2a3127c36b573485",
+    )
+    version(
+        "1.16.0",
+        sha256="9e09161ea6955df693d3c9ac70131985eaf7cf24a9fa4d6263661c6814ebbaf1",
+    )
+
+    variant("tools", default=True, description="Build the command line tools")
+    variant("mpi", default=True, description="Enable MPI support")
+    variant(
+        "admin", default=True, description="Build utilities for administration tools"
+    )
+    variant("sql", default=True, description="Build SQL engine")
+    variant(
+        "linalg",
+        values=any_combination_of("eigen", "armadillo", "mkl", "lapack"),
+        description="List of supported linear algebra backends",
+    )
+    variant(
+        "compression",
+        values=any_combination_of("bzip2", "snappy", "lz4", "aec"),
+        description="List of supported compression backends",
+    )
+    variant("xxhash", default=True, description="Enable xxHash support for hashing")
+    variant(
+        "ssl", default=False, description="Enable MD4 and SHA1 support with OpenSSL"
+    )
+    variant("curl", default=False, description="Enable URL data transferring with cURL")
+    variant(
+        "jemalloc", default=False, description="Link against jemalloc memory allocator"
+    )
+    variant(
+        "unicode",
+        default=True,
+        description="Enable support for Unicode characters in Yaml/JSON" "parsers",
+    )
+    variant("aio", default=True, description="Enable asynchronous IO")
+
+    depends_on("[email protected]:", type="build")
+    depends_on("[email protected]:", type="build")
+
+    depends_on("mpi", when="+mpi")
+
+    depends_on("yacc", type="build", when="+admin")
+    depends_on("flex", type="build", when="+admin")
+    depends_on("ncurses", when="+admin")
+
+    depends_on("yacc", type="build", when="+sql")
+    depends_on("flex", type="build", when="+sql")
+    depends_on("python@3", type="build")
+
+    depends_on("eigen", when="linalg=eigen")
+    depends_on("armadillo", when="linalg=armadillo")
+    depends_on("mkl", when="linalg=mkl")
+    depends_on("lapack", when="linalg=lapack")
+
+    depends_on("bzip2", when="compression=bzip2")
+    depends_on("snappy", when="compression=snappy")
+    depends_on("lz4", when="compression=lz4")
+    depends_on("libaec", when="compression=aec")
+
+    depends_on("openssl", when="+ssl")
+
+    depends_on("curl", when="+curl")
+
+    depends_on("jemalloc", when="+jemalloc")
+
+    # The package enables LAPACK backend (together with MKL backend)
+    # when='linalg=mkl'. This leads to two identical installations when:
+    #   eckit linalg=mkl
+    #   eckit linalg=mkl,lapack
+    # We prevent that by introducing the following conflict:
+    conflicts(
+        "linalg=lapack",
+        when="linalg=mkl",
+        msg='"linalg=lapack" is implied when "linalg=mkl" and '
+        "must not be specified additionally",
+    )
+
+    def cmake_args(self):
+        args = [
+            # Some features that we want to build are experimental:
+            self.define("ENABLE_EXPERIMENTAL", self._enable_experimental),
+            self.define_from_variant("ENABLE_BUILD_TOOLS", "tools"),
+            # We let ecBuild find the MPI library. We could help it by setting
+            # CMAKE_C_COMPILER to mpicc but that might give CMake a wrong
+            # impression that no additional flags are needed to link to
+            # libpthread, which will lead to problems with libraries that are
+            # linked with the C++ compiler. We could additionally set
+            # CMAKE_CXX_COMPILER to mpicxx. That would solve the problem with
+            # libpthread but lead to overlinking to MPI libraries, which we
+            # currently prefer to avoid since ecBuild does the job in all known
+            # cases.
+            self.define_from_variant("ENABLE_MPI", "mpi"),
+            self.define_from_variant("ENABLE_ECKIT_CMD", "admin"),
+            self.define_from_variant("ENABLE_ECKIT_SQL", "sql"),
+            self.define("ENABLE_EIGEN", "linalg=eigen" in self.spec),
+            self.define("ENABLE_ARMADILLO", "linalg=armadillo" in self.spec),
+            self.define("ENABLE_MKL", "linalg=mkl" in self.spec),
+            self.define("ENABLE_BZIP2", "compression=bzip2" in self.spec),
+            self.define("ENABLE_SNAPPY", "compression=snappy" in self.spec),
+            self.define("ENABLE_LZ4", "compression=lz4" in self.spec),
+            self.define("ENABLE_AEC", "compression=aec" in self.spec),
+            self.define_from_variant("ENABLE_XXHASH", "xxhash"),
+            self.define_from_variant("ENABLE_SSL", "ssl"),
+            self.define_from_variant("ENABLE_CURL", "curl"),
+            self.define_from_variant("ENABLE_JEMALLOC", "jemalloc"),
+            self.define_from_variant("ENABLE_UNICODE", "unicode"),
+            self.define_from_variant("ENABLE_AIO", "aio"),
+            self.define("ENABLE_TESTS", self.run_tests),
+            # Unconditionally disable additional unit/performance tests, since
+            # they download additional data (~1.6GB):
+            self.define("ENABLE_EXTRA_TESTS", False),
+            # No reason to check for doxygen and generate the documentation
+            # since it is not installed:
+            self.define("ENABLE_DOCS", False),
+            # Disable features that are currently not needed:
+            self.define("ENABLE_CUDA", False),
+            self.define("ENABLE_VIENNACL", False),
+            # Ceph/Rados storage support requires https://github.com/ceph/ceph
+            # and will be added later:
+            self.define("ENABLE_RADOS", False),
+            # rsync support requires https://github.com/librsync/librsync and
+            # will be added later:
+            self.define("ENABLE_RSYNC", False),
+            # Disable "prototyping code that may never see the light of day":
+            self.define("ENABLE_SANDBOX", False),
+            self.define(
+                "PYTHON_EXECUTABLE", self.spec["python"].prefix + "/bin/python3"
+            ),
+        ]
+
+        if "linalg=mkl" not in self.spec:
+            # ENABLE_LAPACK is ignored if MKL backend is enabled
+            # (the LAPACK backend is still built though):
+            args.append(self.define("ENABLE_LAPACK", "linalg=lapack" in self.spec))
+
+        if "+admin" in self.spec and "+termlib" in self.spec["ncurses"]:
+            # Make sure that libeckit_cmd is linked to a library that resolves 'setupterm',
+            # 'tputs', etc. That is either libncurses (when 'ncurses~termlib') or libtinfo (when
+            # 'ncurses+termlib'). CMake considers the latter only if CURSES_NEED_NCURSES is set to
+            # TRUE. Note that the installation of eckit does not fail without this but the building
+            # of a dependent package (e.g. fdb) might fail due to the undefined references.
+            args.append(self.define("CURSES_NEED_NCURSES", True))
+
+        return args
+
+    def check(self):
+        ctest_args = ["-j", str(make_jobs)]
+
+        broken_tests = []
+        if self._enable_experimental:
+            # The following test quasi-randomly fails not because it reveals a bug in the library
+            # but because its implementation has a bug (static initialization order fiasco):
+            broken_tests.append("eckit_test_experimental_singleton_singleton")
+
+        if broken_tests:
+            ctest_args.extend(["-E", "|".join(broken_tests)])
+
+        with working_dir(self.build_directory):
+            ctest(*ctest_args)
+
+    @property
+    def _enable_experimental(self):
+        return "linalg=armadillo" in self.spec

spack/packages/ecmwf-atlas/a27fc304525607c16cae7a44a2424c328df34d30.patch

@@ -0,0 +1,22 @@
+From a27fc304525607c16cae7a44a2424c328df34d30 Mon Sep 17 00:00:00 2001
+From: Willem Deconinck <[email protected]>
+Date: Wed, 10 May 2023 13:44:58 +0200
+Subject: [PATCH] Fix compilation of atlas_io due to missing header <cstdint>
+
+---
+ atlas_io/src/atlas_io/Data.h | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/atlas_io/src/atlas_io/Data.h b/atlas_io/src/atlas_io/Data.h
+index 7431fd7f4..69c51096f 100644
+--- a/atlas_io/src/atlas_io/Data.h
++++ b/atlas_io/src/atlas_io/Data.h
+@@ -10,6 +10,8 @@
+
+ #pragma once
+
++#include <cstdint>
++
+ #include "eckit/io/Buffer.h"
+
+ namespace atlas {