Adding HadEX3 CMORizer

esmvaltool/cmorizers/data/cmor_config/HadEX3.yml

@@ -0,0 +1,66 @@
+---
+filename: 'HadEX3_{raw_variable}_{raw_frequency}.nc'
+
+# Common global attributes for CMORizer output
+attributes:
+  dataset_id: HadEX3
+  version: v3.0.4
+  tier: 2
+  modeling_realm: ground
+  project_id: OBS
+  source: https://www.metoffice.gov.uk/hadobs/hadex3/download.html
+  reference: 'hadex3'
+  comment: 'ETCCDI (Expert Team on Climate Change Detection and Indices)'
+
+# Variables to cmorize
+variables:
+  txx_yr:
+    short_name: txx
+    raw: TXx
+    raw_units: celsius
+    raw_frequency: ANN
+    mip: yr
+  txx_mon:
+    short_name: txx
+    raw: TXx
+    raw_units: celsius
+    raw_frequency: MON
+    mip: Amon
+  tnn_yr:
+    short_name: tnn
+    raw: TNn
+    raw_units: celsius
+    raw_frequency: ANN
+    mip: yr
+  tnn_mon:
+    short_name: tnn
+    raw: TNn
+    raw_units: celsius
+    raw_frequency: MON
+    mip: Amon
+  # For precitation extremes technically raw units are mm, but since 
+  # the pr CMOR units are kg m-2 s-1, we use kg m-2 for raw units
+  rx1day_yr:
+    short_name: rx1day
+    raw: Rx1day
+    raw_units: mm
+    raw_frequency: ANN
+    mip: yr
+  rx1day_mon:
+    short_name: rx1day
+    raw: Rx1day
+    raw_units: mm
+    raw_frequency: MON
+    mip: Amon
+  rx5day_yr:
+    short_name: rx5day
+    raw: Rx5day
+    raw_units: mm
+    raw_frequency: ANN
+    mip: yr
+  rx5day_mon:
+    short_name: rx5day
+    raw: Rx5day
+    raw_units: mm
+    raw_frequency: MON
+    mip: Amon

esmvaltool/cmorizers/data/datasets.yml

@@ -731,6 +731,16 @@ datasets:
         climatology
             [Source]/absolute_v5.nc
 
+  HadEX3:
+    tier: 2
+    source: https://www.metoffice.gov.uk/hadobs/hadex3/download_etccdi.html
+    last_access: 2024-05-29
+    info: |
+      Download and unzip "HadEX3_TXx_MON.nc.gz", "HadEX3_TXx_ANN.nc.gz", 
+      "HadEX3_TNn_MON.nc.gz", "HadEX3_TNn_ANN.nc.gz", 
+      "HadEX3_Rx1day_MON.nc.gz", "HadEX3_Rx1day_ANN.nc.gz",
+      "HadEX3_Rx5day_MON.nc.gz" and "HadEX3_Rx5day_ANN.nc.gz".
+
   HadISST:
     tier: 2
     source: http://www.metoffice.gov.uk/hadobs/hadisst/data/download.html

esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py

@@ -0,0 +1,63 @@
+"""Script to download HadEX3 from its webpage."""
+import logging
+import os
+
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
+from esmvaltool.cmorizers.data.utilities import unpack_files_in_folder
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(config, dataset, dataset_info, start_date, end_date,
+                     overwrite):
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    downloader = WGetDownloader(
+        config=config,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+
+    os.makedirs(downloader.local_folder, exist_ok=True)
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TXx_MON.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TXx_ANN.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TNn_MON.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TNn_ANN.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx1day_MON.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx1day_ANN.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx5day_MON.nc.gz",
+        wget_options=[])
+    downloader.download_file(
+        "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx5day_ANN.nc.gz",
+        wget_options=[])
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/data/formatters/datasets/hadex3.py

@@ -0,0 +1,105 @@
+"""ESMValTool CMORizer for HadEX3 data.
+
+Tier
+    Tier 2: other freely-available dataset.
+
+Source
+    https://www.metoffice.gov.uk/hadobs/hadex3/download.html
+
+Last access
+    2024-05-29
+
+Download and processing instructions
+    Download the following files:
+        Annual.nc.gz and Monthly.nc.gz for TXx, TNn, Rx1day and Rx5day
+    No registration is required for downloading the data.
+
+When using the dataset in a paper, the following are possible citations to use.
+
+Dunn, R. J. H., et al. (2020), Development of an updated global land in-situ-based
+dataset of temperature and precipitation extremes: HadEX3 JGR-A
+
+Dunn, R. J. H., Donat, M. G., Alexander, L. V., al. (2014), Investigating uncertainties
+in global gridded datasets of climate extremes, Climate of the Past, 10, 2171-2199
+
+Donat, M. G., Alexander, L. V., .... Dunn, R. J. H., et al. (2013), Updated analyses
+of temperature and precipitation extreme indices since the beginning of the twentieth century:
+The HadEX2 dataset, J. Geophys. Res. Atmos., 118, 2098-2118
+"""
+
+import logging
+import os
+
+import iris
+from cf_units import Unit
+import cftime
+
+import ...utilities as utils
+
+logger = logging.getLogger(__name__)
+
+
+def _fix_time_coord(cube):
+    """Convert the time to the gregorian calendar. """
+    time_coord = cube.coord('time')
+    time_coord.guess_bounds()
+    new_unit = Unit('days since 1850-01-01 00:00:00', calendar='gregorian')
+    new_time_points = cftime.num2pydate(time_coord.points,
+                                        time_coord.units.origin,
+                                        time_coord.units.calendar)
+    time_points = cftime.date2num(new_time_points, new_unit.origin,
+                                  calendar=new_unit.calendar)
+    new_time_bounds = cftime.num2pydate(time_coord.bounds,
+                                        time_coord.units.origin,
+                                        time_coord.units.calendar)
+    time_bounds = cftime.date2num(new_time_bounds, new_unit.origin,
+                                  calendar=new_unit.calendar)
+
+    cube.coord('time').points = time_points
+    cube.coord('time').bounds = time_bounds
+    cube.coord('time').units = new_unit
+
+
+def _extract_variable(var, var_info, cmor_info, attrs, filepath, out_dir):
+    """Extract variable."""
+    raw_var = var_info.get('raw', var)
+    var = cmor_info.short_name
+    cube = iris.load_cube(filepath, utils.var_name_constraint(raw_var))
+    # Fix units
+    cube.units = var_info.get('raw_units', var)
+    cube.convert_units(cmor_info.units)
+
+    _fix_time_coord(cube)
+    utils.fix_var_metadata(cube, cmor_info)
+    utils.convert_timeunits(cube, 1950)
+    utils.fix_coords(cube)
+    utils.set_global_atts(cube, attrs)
+    utils.save_variable(cube,
+                        var,
+                        out_dir,
+                        attrs,
+                        unlimited_dimensions=['time'])
+
+
+def cmorization(in_dir, out_dir, cfg, _):
+    """Cmorization function call."""
+    glob_attrs = cfg['attributes']
+    cmor_table = cfg['cmor_table']
+    filename = cfg['filename']
+
+    # Run the cmorization
+    for (var, var_info) in cfg['variables'].items():
+        var_name = var_info['short_name']
+        var_mip = var_info['mip']
+        filepath = os.path.join(in_dir, filename.format(var_info))
+        if os.path.isfile(filepath):
+            logger.info(f"Found input file {filepath}")
+        else:
+            raise ValueError(f"Couldn't find {filepath}")
+        logger.info(f"CMORizing variable {var_name} in {var_mip} table")
+        glob_attrs['mip'] = var_info['mip']
+        cmor_info = cmor_table.get_variable(var_info['mip'], var)
+        _extract_variable(var, var_info, cmor_info, glob_attrs, filepath,
+                          out_dir)
+        logger.info(f"CMORization of {var_name} in {var_mip} table was "
+                                                                "successful")

esmvaltool/references/hadex3.bibtex

@@ -0,0 +1,12 @@
+@article{hadex3,
+  title = {Development of an updated global land in situ-based data set of temperature and precipitation extremes: HadEX3},
+  author = {Dunn, Robert JH and Alexander, Lisa V and Donat, Markus G and Zhang, Xuebin and Bador, Margot and Herold, Nicholas and Lippmann, Tanya and Allan, Rob and Aguilar, Enric and Barry, Abdoul Aziz and others},
+  doi = {10.1029/2019jd032263},
+  url = {https://doi.org/10.1029/2019JD032263},
+  journal = {Journal of Geophysical Research: Atmospheres},
+  volume = {125},
+  number = {16},
+  pages = {e2019JD032263},
+  year = {2020},
+  publisher = {Wiley Online Library}
+}