From 34a552cb6b42cbdd2f9b1a3665b87ca459266f8e Mon Sep 17 00:00:00 2001 From: Elizaveta Malinina Date: Wed, 29 May 2024 15:04:39 +0000 Subject: [PATCH] add HadEX3 cmorizer --- .../cmorizers/data/cmor_config/HadEX3.yml | 66 +++++++++++ esmvaltool/cmorizers/data/datasets.yml | 10 ++ .../data/downloaders/datasets/hadex3.py | 63 +++++++++++ .../data/formatters/datasets/hadex3.py | 105 ++++++++++++++++++ esmvaltool/references/hadex3.bibtex | 12 ++ 5 files changed, 256 insertions(+) create mode 100644 esmvaltool/cmorizers/data/cmor_config/HadEX3.yml create mode 100644 esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py create mode 100644 esmvaltool/cmorizers/data/formatters/datasets/hadex3.py create mode 100644 esmvaltool/references/hadex3.bibtex diff --git a/esmvaltool/cmorizers/data/cmor_config/HadEX3.yml b/esmvaltool/cmorizers/data/cmor_config/HadEX3.yml new file mode 100644 index 0000000000..f52c2d3c29 --- /dev/null +++ b/esmvaltool/cmorizers/data/cmor_config/HadEX3.yml @@ -0,0 +1,66 @@ +--- +filename: 'HadEX3_{raw_variable}_{raw_frequency}.nc' + +# Common global attributes for CMORizer output +attributes: + dataset_id: HadEX3 + version: v3.0.4 + tier: 2 + modeling_realm: ground + project_id: OBS + source: https://www.metoffice.gov.uk/hadobs/hadex3/download.html + reference: 'hadex3' + comment: 'ETCCDI (Expert Team on Climate Change Detection and Indices)' + +# Variables to cmorize +variables: + txx_yr: + short_name: txx + raw: TXx + raw_units: celsius + raw_frequency: ANN + mip: yr + txx_mon: + short_name: txx + raw: TXx + raw_units: celsius + raw_frequency: MON + mip: Amon + tnn_yr: + short_name: tnn + raw: TNn + raw_units: celsius + raw_frequency: ANN + mip: yr + tnn_mon: + short_name: tnn + raw: TNn + raw_units: celsius + raw_frequency: MON + mip: Amon + # For precitation extremes technically raw units are mm, but since + # the pr CMOR units are kg m-2 s-1, we use kg m-2 for raw units + rx1day_yr: + short_name: rx1day + raw: Rx1day + raw_units: mm + raw_frequency: ANN + mip: yr + rx1day_mon: + short_name: rx1day + raw: Rx1day + raw_units: mm + raw_frequency: MON + mip: Amon + rx5day_yr: + short_name: rx5day + raw: Rx5day + raw_units: mm + raw_frequency: ANN + mip: yr + rx5day_mon: + short_name: rx5day + raw: Rx5day + raw_units: mm + raw_frequency: MON + mip: Amon \ No newline at end of file diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 47632da653..d0cffb5235 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -731,6 +731,16 @@ datasets: climatology [Source]/absolute_v5.nc + HadEX3: + tier: 2 + source: https://www.metoffice.gov.uk/hadobs/hadex3/download_etccdi.html + last_access: 2024-05-29 + info: | + Download and unzip "HadEX3_TXx_MON.nc.gz", "HadEX3_TXx_ANN.nc.gz", + "HadEX3_TNn_MON.nc.gz", "HadEX3_TNn_ANN.nc.gz", + "HadEX3_Rx1day_MON.nc.gz", "HadEX3_Rx1day_ANN.nc.gz", + "HadEX3_Rx5day_MON.nc.gz" and "HadEX3_Rx5day_ANN.nc.gz". + HadISST: tier: 2 source: http://www.metoffice.gov.uk/hadobs/hadisst/data/download.html diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py new file mode 100644 index 0000000000..2374c4b1e9 --- /dev/null +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py @@ -0,0 +1,63 @@ +"""Script to download HadEX3 from its webpage.""" +import logging +import os + +from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader +from esmvaltool.cmorizers.data.utilities import unpack_files_in_folder + +logger = logging.getLogger(__name__) + + +def download_dataset(config, dataset, dataset_info, start_date, end_date, + overwrite): + """Download dataset. + + Parameters + ---------- + config : dict + ESMValTool's user configuration + dataset : str + Name of the dataset + dataset_info : dict + Dataset information from the datasets.yml file + start_date : datetime + Start of the interval to download + end_date : datetime + End of the interval to download + overwrite : bool + Overwrite already downloaded files + """ + downloader = WGetDownloader( + config=config, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) + + os.makedirs(downloader.local_folder, exist_ok=True) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TXx_MON.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TXx_ANN.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TNn_MON.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_TNn_ANN.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx1day_MON.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx1day_ANN.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx5day_MON.nc.gz", + wget_options=[]) + downloader.download_file( + "https://www.metoffice.gov.uk/hadobs/hadex3/data/HadEX3_Rx5day_ANN.nc.gz", + wget_options=[]) + + unpack_files_in_folder(downloader.local_folder) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/hadex3.py b/esmvaltool/cmorizers/data/formatters/datasets/hadex3.py new file mode 100644 index 0000000000..b2b74974a6 --- /dev/null +++ b/esmvaltool/cmorizers/data/formatters/datasets/hadex3.py @@ -0,0 +1,105 @@ +"""ESMValTool CMORizer for HadEX3 data. + +Tier + Tier 2: other freely-available dataset. + +Source + https://www.metoffice.gov.uk/hadobs/hadex3/download.html + +Last access + 2024-05-29 + +Download and processing instructions + Download the following files: + Annual.nc.gz and Monthly.nc.gz for TXx, TNn, Rx1day and Rx5day + No registration is required for downloading the data. + +When using the dataset in a paper, the following are possible citations to use. + +Dunn, R. J. H., et al. (2020), Development of an updated global land in-situ-based +dataset of temperature and precipitation extremes: HadEX3 JGR-A + +Dunn, R. J. H., Donat, M. G., Alexander, L. V., al. (2014), Investigating uncertainties +in global gridded datasets of climate extremes, Climate of the Past, 10, 2171-2199 + +Donat, M. G., Alexander, L. V., .... Dunn, R. J. H., et al. (2013), Updated analyses +of temperature and precipitation extreme indices since the beginning of the twentieth century: +The HadEX2 dataset, J. Geophys. Res. Atmos., 118, 2098-2118 +""" + +import logging +import os + +import iris +from cf_units import Unit +import cftime + +import ...utilities as utils + +logger = logging.getLogger(__name__) + + +def _fix_time_coord(cube): + """Convert the time to the gregorian calendar. """ + time_coord = cube.coord('time') + time_coord.guess_bounds() + new_unit = Unit('days since 1850-01-01 00:00:00', calendar='gregorian') + new_time_points = cftime.num2pydate(time_coord.points, + time_coord.units.origin, + time_coord.units.calendar) + time_points = cftime.date2num(new_time_points, new_unit.origin, + calendar=new_unit.calendar) + new_time_bounds = cftime.num2pydate(time_coord.bounds, + time_coord.units.origin, + time_coord.units.calendar) + time_bounds = cftime.date2num(new_time_bounds, new_unit.origin, + calendar=new_unit.calendar) + + cube.coord('time').points = time_points + cube.coord('time').bounds = time_bounds + cube.coord('time').units = new_unit + + +def _extract_variable(var, var_info, cmor_info, attrs, filepath, out_dir): + """Extract variable.""" + raw_var = var_info.get('raw', var) + var = cmor_info.short_name + cube = iris.load_cube(filepath, utils.var_name_constraint(raw_var)) + # Fix units + cube.units = var_info.get('raw_units', var) + cube.convert_units(cmor_info.units) + + _fix_time_coord(cube) + utils.fix_var_metadata(cube, cmor_info) + utils.convert_timeunits(cube, 1950) + utils.fix_coords(cube) + utils.set_global_atts(cube, attrs) + utils.save_variable(cube, + var, + out_dir, + attrs, + unlimited_dimensions=['time']) + + +def cmorization(in_dir, out_dir, cfg, _): + """Cmorization function call.""" + glob_attrs = cfg['attributes'] + cmor_table = cfg['cmor_table'] + filename = cfg['filename'] + + # Run the cmorization + for (var, var_info) in cfg['variables'].items(): + var_name = var_info['short_name'] + var_mip = var_info['mip'] + filepath = os.path.join(in_dir, filename.format(var_info)) + if os.path.isfile(filepath): + logger.info(f"Found input file {filepath}") + else: + raise ValueError(f"Couldn't find {filepath}") + logger.info(f"CMORizing variable {var_name} in {var_mip} table") + glob_attrs['mip'] = var_info['mip'] + cmor_info = cmor_table.get_variable(var_info['mip'], var) + _extract_variable(var, var_info, cmor_info, glob_attrs, filepath, + out_dir) + logger.info(f"CMORization of {var_name} in {var_mip} table was " + "successful") \ No newline at end of file diff --git a/esmvaltool/references/hadex3.bibtex b/esmvaltool/references/hadex3.bibtex new file mode 100644 index 0000000000..2946bbe0eb --- /dev/null +++ b/esmvaltool/references/hadex3.bibtex @@ -0,0 +1,12 @@ +@article{hadex3, + title = {Development of an updated global land in situ-based data set of temperature and precipitation extremes: HadEX3}, + author = {Dunn, Robert JH and Alexander, Lisa V and Donat, Markus G and Zhang, Xuebin and Bador, Margot and Herold, Nicholas and Lippmann, Tanya and Allan, Rob and Aguilar, Enric and Barry, Abdoul Aziz and others}, + doi = {10.1029/2019jd032263}, + url = {https://doi.org/10.1029/2019JD032263}, + journal = {Journal of Geophysical Research: Atmospheres}, + volume = {125}, + number = {16}, + pages = {e2019JD032263}, + year = {2020}, + publisher = {Wiley Online Library} +} \ No newline at end of file