Merge pull request #307 from davidhassell/h5-read

h5netcdf read

Changelog.rst

@@ -1,13 +1,29 @@
-Version NEXTRELEASE
+Version NEXTVERSION
 -------------------
 
 **2024-??-??**
 
 * Upgrades to allow cfdm to work with Python 3.12
   (https://github.com/NCAS-CMS/cfdm/issues/302)
+* New function `cfdm.netcdf_flattener` that replaces the import of
+  `netcdf_flattener` (https://github.com/NCAS-CMS/cfdm/issues/286)
+* New function `cfdm.netcdf_indexer` that applies netCDF masking and
+  unpacking to arbitrary arrays
+  (https://github.com/NCAS-CMS/cfdm/issues/285)
+* Allow access to netCDF-4 files in S3 object stores
+  (https://github.com/NCAS-CMS/cfdm/issues/285)
+* Allow a choice of netCDF engines
+  (https://github.com/NCAS-CMS/cfdm/issues/285)
 * Fix bug that caused `cfdm.write` to fail when a parametric Z
   dimension coordinate did not have a ``computed_standard_name``
   attribute (https://github.com/NCAS-CMS/cfdm/issues/303)
+* New class `cfdm.H5netcdfArray`
+* New class `cfdm.NetCDF4Array`
+* New dependency: ``h5netcdf>=1.3.0``
+* New dependency: ``h5py>=3.10.0``
+* New dependency: ``s3fs>=2024.6.0``
+* New dependency: ``dask>=2024.6.0``
+* Removed dependency: ``netcdf_flattener``
 
 ----
 

README.md

@@ -73,7 +73,8 @@ inspecting it:
 
 The ``cfdm`` package can:
 
-* read field and domain constructs from netCDF and CDL datasets,
+* read field and domain constructs from netCDF and CDL datasets with a
+  choice of netCDF backends,
 * create new field and domain constructs in memory,
 * write and append field and domain constructs to netCDF datasets on disk,
 * read, write, and manipulate UGRID mesh topologies,

cfdm/__init__.py

@@ -48,49 +48,105 @@
 __cf_version__ = core.__cf_version__
 __version__ = core.__version__
 
-_requires = ("cftime", "netcdf_flattener", "scipy")
+_requires = core._requires + (
+    "cftime",
+    "netCDF4",
+    "scipy",
+    "h5netcdf",
+    "s3fs",
+)
 
 _error0 = f"cfdm requires the modules {', '.join(_requires)}. "
 
+# Check the version of cftime
 try:
     import cftime
 except ImportError as error1:
     raise ImportError(_error0 + str(error1))
 
-# Check the version of cftime
 _minimum_vn = "1.6.0"
 if Version(cftime.__version__) < Version(_minimum_vn):
     raise ValueError(
         f"Bad cftime version: cfdm requires cftime>={_minimum_vn}. "
         f"Got {cftime.__version__} at {cftime.__file__}"
     )
 
+# Check the version of netCDF4
 try:
-    import netcdf_flattener
+    import netCDF4
 except ImportError as error1:
     raise ImportError(_error0 + str(error1))
 
-# Check the version of netcdf_flattener
-_minimum_vn = "1.2.0"
-if Version(netcdf_flattener.__version__) < Version(_minimum_vn):
+_minimum_vn = "1.5.4"
+if Version(netCDF4.__version__) < Version(_minimum_vn):
     raise ValueError(
-        f"Bad netcdf_flattener version: cfdm requires "
-        f"netcdf_flattener>={_minimum_vn}. Got {netcdf_flattener.__version__} "
-        f"at {netcdf_flattener.__file__}"
+        f"Bad netCDF4 version: cfdm requires netCDF4>={_minimum_vn}. "
+        f"Got {netCDF4.__version__} at {netCDF4.__file__}"
     )
 
+# Check the version of h5netcdf
 try:
-    import scipy
+    import h5netcdf
 except ImportError as error1:
     raise ImportError(_error0 + str(error1))
 
+_minimum_vn = "1.3.0"
+if Version(h5netcdf.__version__) < Version(_minimum_vn):
+    raise ValueError(
+        f"Bad h5netcdf version: cfdm requires h5netcdf>={_minimum_vn}. "
+        f"Got {h5netcdf.__version__} at {h5netcdf.__file__}"
+    )
+
+# Check the version of h5py
+try:
+    import h5py
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
+_minimum_vn = "3.10.0"
+if Version(h5py.__version__) < Version(_minimum_vn):
+    raise ValueError(
+        f"Bad h5py version: cfdm requires h5py>={_minimum_vn}. "
+        f"Got {h5py.__version__} at {h5py.__file__}"
+    )
+
+# Check the version of s3fs
+try:
+    import s3fs
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
+_minimum_vn = "2024.6.0"
+if Version(s3fs.__version__) < Version(_minimum_vn):
+    raise ValueError(
+        f"Bad s3fs version: cfdm requires s3fs>={_minimum_vn}. "
+        f"Got {s3fs.__version__} at {s3fs.__file__}"
+    )
+
 # Check the version of scipy
+try:
+    import scipy
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
 _minimum_vn = "1.10.0"
 if Version(scipy.__version__) < Version(_minimum_vn):
     raise ValueError(
-        f"Bad scipy version: cfdm requires "
-        f"scipy>={_minimum_vn}. Got {scipy.__version__} "
-        f"at {scipy.__file__}"
+        f"Bad scipy version: cfdm requires scipy>={_minimum_vn}. "
+        f"Got {scipy.__version__} at {scipy.__file__}"
+    )
+
+# Check the version of dask
+try:
+    import dask
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
+_minimum_vn = "2024.6.0"
+if Version(dask.__version__) < Version(_minimum_vn):
+    raise ValueError(
+        f"Bad scipy version: cfdm requires dask>={_minimum_vn}. "
+        f"Got {dask.__version__} at {dask.__file__}"
     )
 
 from .constants import masked
@@ -140,7 +196,10 @@
     CompressedArray,
     Data,
     GatheredArray,
+    H5netcdfArray,
     NetCDFArray,
+    NetCDF4Array,
+    netcdf_indexer,
     NumpyArray,
     PointTopologyArray,
     RaggedArray,
@@ -196,6 +255,7 @@
 from .cfdmimplementation import CFDMImplementation, implementation
 
 from .read_write import read, write
+from .read_write.netcdf.flatten import netcdf_flatten
 
 from .examplefield import example_field, example_fields, example_domain
 

cfdm/abstract/implementation.py

@@ -54,7 +54,7 @@ def classes(self):
          'Index',
          'InteriorRing',
          'List',
-         'NetCDFArray',
+         'NetCDF4Array',
          'NodeCountProperties',
          'PartNodeCountProperties',
          'RaggedContiguousArray',

cfdm/cfdmimplementation.py

@@ -30,7 +30,8 @@
     CellConnectivityArray,
     Data,
     GatheredArray,
-    NetCDFArray,
+    H5netcdfArray,
+    NetCDF4Array,
     PointTopologyArray,
     RaggedContiguousArray,
     RaggedIndexedArray,
@@ -1358,10 +1359,10 @@ def get_data_maximum(self, parent):
 
         :Returns:
 
-            Data instance
+            Scalar `Data` instance
 
         """
-        return parent.data.maximum()
+        return parent.data.maximum(squeeze=True)
 
     def get_data_sum(self, parent):
         """Return the sum of the data.
@@ -1372,10 +1373,10 @@ def get_data_sum(self, parent):
 
         :Returns:
 
-            Data instance
+            Scalar `Data` instance
 
         """
-        return parent.data.sum()
+        return parent.data.sum(squeeze=True)
 
     def get_count(self, construct):
         """Return the count variable of compressed data.
@@ -2291,67 +2292,41 @@ def initialise_TiePointIndex(self):
         cls = self.get_class("TiePointIndex")
         return cls()
 
-    def initialise_NetCDFArray(
-        self,
-        filename=None,
-        address=None,
-        dtype=None,
-        shape=None,
-        mask=True,
-        units=False,
-        calendar=None,
-        missing_values=None,
-    ):
-        """Return a netCDF array instance.
+    def initialise_NetCDF4Array(self, **kwargs):
+        """Return a `NetCDF4Array` instance.
 
         :Parameters:
 
-            filename: `str`
-
-            address: `str`
-
-            dytpe: `numpy.dtype`
+            kwargs: optional
+                Initialisation parameters to pass to the new instance.
 
-            shape: sequence of `int`, optional
+                .. versionadded:: (cfdm) NEXTVERSION
 
-            mask: `bool`, optional
+        :Returns:
 
-            units: `str` or `None` or False, optional
-                The units of the netCDF variable. Set to `None` to
-                indicate that there are no units. If False (the
-                default) then the units are considered unset.
+            `NetCDF4Array`
 
-                .. versionadded:: (cfdm) 1.10.0.2
+        """
+        cls = self.get_class("NetCDF4Array")
+        return cls(**kwargs)
 
-            calendar: `str` or `None`, optional
-                The calendar of the netCDF variable. By default, or if
-                set to `None`, then the CF default calendar is
-                assumed, if applicable.
+    def initialise_H5netcdfArray(self, **kwargs):
+        """Return a `H5netcdfArray` instance.
 
-                .. versionadded:: (cfdm) 1.10.0.2
+        .. versionadded:: (cfdm) NEXTVERSION
 
-            missing_values: `dict`, optional
-                The missing value indicators defined by the netCDF
-                variable attributes.
+        :Parameters:
 
-                .. versionadded:: (cfdm) 1.10.0.3
+            kwargs: optional
+                Initialisation parameters to pass to the new instance.
 
         :Returns:
 
-            `NetCDFArray`
+            `H5netcdfArray`
 
         """
-        cls = self.get_class("NetCDFArray")
-        return cls(
-            filename=filename,
-            address=address,
-            dtype=dtype,
-            shape=shape,
-            mask=mask,
-            units=units,
-            calendar=calendar,
-            missing_values=missing_values,
-        )
+        cls = self.get_class("H5netcdfArray")
+        return cls(**kwargs)
 
     def initialise_BoundsFromNodesArray(self, **kwargs):
         """Return a node bounds array.
@@ -3707,7 +3682,8 @@ def squeeze(self, construct, axes=None):
     Data=Data,
     BoundsFromNodesArray=BoundsFromNodesArray,
     GatheredArray=GatheredArray,
-    NetCDFArray=NetCDFArray,
+    H5netcdfArray=H5netcdfArray,
+    NetCDF4Array=NetCDF4Array,
     PointTopologyArray=PointTopologyArray,
     RaggedContiguousArray=RaggedContiguousArray,
     RaggedIndexedArray=RaggedIndexedArray,
@@ -3750,7 +3726,8 @@ def implementation():
      'Datum': <class 'cfdm.datum.Datum'>,
      'Data': <class 'cfdm.data.data.Data'>,
      'GatheredArray': <class 'cfdm.data.gatheredarray.GatheredArray'>,
-     'NetCDFArray': <class 'cfdm.data.netcdfarray.NetCDFArray'>,
+     'H5netcdfArray': <class 'cfdm.data.h5netcdfarray.H5netcdfArray'>,
+     'NetCDF4Array': <class 'cfdm.data.netcdf4array.NetCDF4Array'>,
      'PointTopologyArray': <class 'cfdm.data.pointtopologyarray.PointTopologyArray'>,
      'RaggedContiguousArray': <class 'cfdm.data.raggedcontiguousarray.RaggedContiguousArray'>,
      'RaggedIndexedArray': <class 'cfdm.data.raggedindexedarray.RaggedIndexedArray'>,

cfdm/core/__init__.py

@@ -11,30 +11,20 @@
 
 """
 
-__date__ = "2024-03-01"
+__date__ = "2024-??-??"
 __cf_version__ = "1.11"
-__version__ = "1.11.1.0"
+__version__ = "1.11.2.0"
 
 from packaging import __version__ as _packaging_ver
 from packaging import __file__ as _packaging_file
 from packaging.version import Version
 
 import platform
 
-_requires = ("numpy", "netCDF4", "packaging")
+_requires = ("numpy", "packaging")
 
 _error0 = f"cfdm.core requires the modules {', '.join(_requires)}. "
 
-try:
-    import netCDF4
-except ImportError as error1:
-    raise ImportError(_error0 + str(error1))
-
-try:
-    import numpy as np
-except ImportError as error1:
-    raise ImportError(_error0 + str(error1))
-
 # Check the version of python
 _minimum_vn = "3.8.0"
 if Version(platform.python_version()) < Version(_minimum_vn):
@@ -44,22 +34,24 @@
     )
 
 # Check the version of packaging
+try:
+    import packaging
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
 _minimum_vn = "20.0"
 if Version(_packaging_ver) < Version(_minimum_vn):
     raise ValueError(
-        f"Bad packaging version: cfdm requires packaging>={_minimum_vn}. "
+        f"Bad packaging version: cfdm.core requires packaging>={_minimum_vn}. "
         f"Got {_packaging_ver} at {_packaging_file}"
     )
 
-# Check the version of netCDF4
-_minimum_vn = "1.5.4"
-if Version(netCDF4.__version__) < Version(_minimum_vn):
-    raise ValueError(
-        f"Bad netCDF4 version: cfdm.core requires netCDF4>={_minimum_vn}. "
-        f"Got {netCDF4.__version__} at {netCDF4.__file__}"
-    )
-
 # Check the version of numpy
+try:
+    import numpy as np
+except ImportError as error1:
+    raise ImportError(_error0 + str(error1))
+
 _minimum_vn = "1.15"
 if Version(np.__version__) < Version(_minimum_vn):
     raise ValueError(