From caf735a28139658745d9d9a0c1eccd4d5c4a702d Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 19 Mar 2024 10:01:31 +0100 Subject: [PATCH 01/40] add downloader --- .../data/downloaders/datasets/esacci_sst.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py new file mode 100644 index 0000000000..a2e68b8589 --- /dev/null +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -0,0 +1,49 @@ +"""Script to download ESACCI-SOILMOISTURE.""" +from datetime import datetime + +from dateutil import relativedelta + +from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader + + +def download_dataset(config, dataset, dataset_info, start_date, end_date, + overwrite): + """Download dataset. + + Parameters + ---------- + config : dict + ESMValTool's user configuration + dataset : str + Name of the dataset + dataset_info : dict + Dataset information from the datasets.yml file + start_date : datetime + Start of the interval to download + end_date : datetime + End of the interval to download + overwrite : bool + Overwrite already downloaded files + """ + if start_date is None: + start_date = datetime(2003, 1, 1) + if end_date is None: + end_date = datetime(2007,12, 31) + + loop_date = start_date + + downloader = CCIDownloader( + config=config, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) + downloader.ftp_name = 'sst' + downloader.connect() + downloader.set_cwd('CDR_v2/Analysis/L4/v2.1/') + while loop_date <= end_date: + year = loop_date.year + month = loop_date.strftime("%m") + day = loop_date.strftime("%d") + downloader.download_folder(f'./{year}/{month}/{day}/') + loop_date += relativedelta.relativedelta(days=1) From 765825423479a350239750a746d17c42465b5032 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 19 Mar 2024 11:26:20 +0100 Subject: [PATCH 02/40] first try with daily data --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 36 ++++++++++--------- .../data/formatters/datasets/esacci_sst.py | 35 ++++++++++-------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 49d4731681..50c9cdae83 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -1,29 +1,33 @@ --- # Common global attributes for Cmorizer output -filename: '{year}{month}15_regridded_sst.nc' +#filename: '{year}{month}15_regridded_sst.nc' attributes: dataset_id: ESACCI-SST - version: '2.2' + version: '2.1' tier: 2 modeling_realm: sat - project_id: OBS - source: 'http://surftemp.net/regridding/index.html' + project_id: OBS6 + source: 'dx.doi.org/10.5285/62c0f97b1eac4e0197a674870afe1ee6' reference: ["esacci-sst", "esacci-sst-bias-correction"] comment: "Note that the variable tsStderr is an uncertainty not a standard error." # Variables to cmorize (here use only filename prefix) variables: - ts: - mip: Amon - raw: sst - file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB - tsStderr: - mip: Amon - raw: sst_uncertainty - file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB + #ts: + # mip: Amon + # raw: sst + # file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB + #tsStderr: + # mip: Amon + # raw: sst_uncertainty + # file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB # uncomment this part to produce sst cmorized data for ocean realm (Omon, tos) -# tos: -# mip: Omon -# raw: sst -# file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB + tos_day: + short_name: tos + mip: Oday + raw: analysed_sst + frequency: day + filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc + start_year: 2003 + end_year: 2003 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 8e55296f9e..3f9e0ad8a3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -32,10 +32,12 @@ 20201214-lauer_axel: approved. """ +import glob import logging import os import iris +from esmvalcore.cmor.fixes import get_time_bounds from esmvalcore.preprocessor import concatenate from ...utilities import ( @@ -62,7 +64,7 @@ def extract_variable(var_info, raw_info, attrs, year): # Fix cube fix_var_metadata(cube, var_info) convert_timeunits(cube, year) - fix_coords(cube) + fix_coords(cube, overwrite_time_bounds=False) set_global_atts(cube, attrs) return cube @@ -73,23 +75,28 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): glob_attrs = cfg['attributes'] # run the cmorization - for var, vals in cfg['variables'].items(): + for var_name, vals in cfg['variables'].items(): + var = vals['short_name'] var_info = cmor_table.get_variable(vals['mip'], var) glob_attrs['mip'] = vals['mip'] - raw_info = {'name': vals['raw'], 'file': vals['file']} - inpfile = os.path.join(in_dir, cfg['filename']) - logger.info("CMORizing var %s from file type %s", var, inpfile) - years = range(1982, 2020) - months = ["0" + str(mo) for mo in range(1, 10)] + ["10", "11", "12"] - for year in years: - monthly_cubes = [] - for month in months: - raw_info['file'] = inpfile.format(year=year, month=month) + raw_info = {'name': vals['raw']} + #inpfile = os.path.join(in_dir, cfg['filename']) + inpfile_pattern = os.path.join(in_dir, '{year}*'+vals['filename']) + logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) + for year in range(vals['start_year'], vals['end_year'] + 1): + data_cubes = [] + year_inpfile_pattern = inpfile_pattern.format(year=year) + inpfiles = sorted(glob.glob(year_inpfile_pattern)) + for inpfile in inpfiles: + raw_info['file'] = inpfile logger.info("CMORizing var %s from file type %s", var, raw_info['file']) - cube = extract_variable(var_info, raw_info, glob_attrs, year) - monthly_cubes.append(cube) - yearly_cube = concatenate(monthly_cubes) + data_cubes.append( + extract_variable(var_info, raw_info, glob_attrs, year)) + yearly_cube = concatenate(data_cubes) + # Fix monthly time bounds + time = yearly_cube.coord('time') + time.bounds = get_time_bounds(time, vals['frequency']) save_variable(yearly_cube, var, out_dir, From 88450d9c52bb4a8f1c931e664b43816a23d26a69 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 20 Mar 2024 12:19:13 +0100 Subject: [PATCH 03/40] modify downloader for daily data --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 4 ++-- esmvaltool/recipes/examples/recipe_check_obs.yml | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 50c9cdae83..26b1c67950 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -29,5 +29,5 @@ variables: raw: analysed_sst frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc - start_year: 2003 - end_year: 2003 + start_year: 2004 + end_year: 2007 diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 70368c87dc..0fd39be94f 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -290,12 +290,16 @@ diagnostics: ESACCI-SST: description: ESACCI-SST check variables: - ts: - tsStderr: + #ts: + #tsStderr: + tos: additional_datasets: - - {dataset: ESACCI-SST, project: OBS, mip: Amon, tier: 2, - type: sat, version: 2.2, - start_year: 1982, end_year: 2019} + #- {dataset: ESACCI-SST, project: OBS, mip: Amon, tier: 2, + # type: sat, version: 2.2, + # start_year: 1982, end_year: 2019} + - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 2, + type: sat, version: 2.1, + start_year: 2003, end_year: 2003} scripts: null From 8678f73ca55ba615fd34b7a03866020451d37fda Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 27 Mar 2024 12:27:03 +0100 Subject: [PATCH 04/40] add fix type --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 3f9e0ad8a3..e32bc439cb 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -38,6 +38,7 @@ import iris from esmvalcore.cmor.fixes import get_time_bounds +from esmvaltool.cmorizers.data import utilities as utils from esmvalcore.preprocessor import concatenate from ...utilities import ( @@ -61,6 +62,8 @@ def extract_variable(var_info, raw_info, attrs, year): raise ValueError(f"No data available for variable {rawvar}" f"and year {year}") from constraint_error + #Fix dtype + utils.fix_dtype(cube) # Fix cube fix_var_metadata(cube, var_info) convert_timeunits(cube, year) From 3d132039a30626051ef6eab8c9ebb09334611557 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 30 May 2024 12:53:36 +0200 Subject: [PATCH 05/40] fix cmorizer --- .../data/formatters/datasets/esacci_sst.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index e32bc439cb..b94d1fab2b 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -40,6 +40,7 @@ from esmvalcore.cmor.fixes import get_time_bounds from esmvaltool.cmorizers.data import utilities as utils from esmvalcore.preprocessor import concatenate +from esmvalcore.preprocessor import regrid from ...utilities import ( convert_timeunits, @@ -62,6 +63,8 @@ def extract_variable(var_info, raw_info, attrs, year): raise ValueError(f"No data available for variable {rawvar}" f"and year {year}") from constraint_error + # regridding from 0.05x0.05 to 0.5x0.5 + cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') #Fix dtype utils.fix_dtype(cube) # Fix cube @@ -87,21 +90,27 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): inpfile_pattern = os.path.join(in_dir, '{year}*'+vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) for year in range(vals['start_year'], vals['end_year'] + 1): - data_cubes = [] - year_inpfile_pattern = inpfile_pattern.format(year=year) - inpfiles = sorted(glob.glob(year_inpfile_pattern)) - for inpfile in inpfiles: - raw_info['file'] = inpfile - logger.info("CMORizing var %s from file type %s", var, - raw_info['file']) - data_cubes.append( - extract_variable(var_info, raw_info, glob_attrs, year)) - yearly_cube = concatenate(data_cubes) - # Fix monthly time bounds - time = yearly_cube.coord('time') - time.bounds = get_time_bounds(time, vals['frequency']) - save_variable(yearly_cube, - var, - out_dir, - glob_attrs, - unlimited_dimensions=['time']) + logger.info("Processing year %s", year) + for month in range(1,13): + data_cubes = [] + #year_inpfile_pattern = inpfile_pattern.format(year=year) + month_inpfile_pattern = inpfile_pattern.format(year=str(year)+"{:02}".format(month)) + logger.info("Pattern: %s", month_inpfile_pattern) + inpfiles = sorted(glob.glob(month_inpfile_pattern)) + #inpfiles = sorted(glob.glob(year_inpfile_pattern)) + logger.info("Found input files: %s", inpfiles) + for inpfile in inpfiles: + raw_info['file'] = inpfile + logger.info("CMORizing var %s from file type %s", var, + raw_info['file']) + data_cubes.append( + extract_variable(var_info, raw_info, glob_attrs, year)) + yearly_cube = concatenate(data_cubes) + # Fix monthly time bounds + time = yearly_cube.coord('time') + time.bounds = get_time_bounds(time, vals['frequency']) + save_variable(yearly_cube, + var, + out_dir, + glob_attrs, + unlimited_dimensions=['time']) From a03421ec669a490bedf5522c2491d8c260680dd9 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 30 May 2024 12:55:14 +0200 Subject: [PATCH 06/40] first try downloader v3.0 --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 7 ++- .../data/downloaders/datasets/esacci_sst.py | 61 +++++++++++++++++-- esmvaltool/cmorizers/data/downloaders/ftp.py | 8 ++- .../recipes/examples/recipe_check_obs.yml | 4 +- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 26b1c67950..a301f2d9f8 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -3,8 +3,8 @@ #filename: '{year}{month}15_regridded_sst.nc' attributes: dataset_id: ESACCI-SST - version: '2.1' - tier: 2 + version: '3.0-L4-analysis' + tier: 3 modeling_realm: sat project_id: OBS6 source: 'dx.doi.org/10.5285/62c0f97b1eac4e0197a674870afe1ee6' @@ -28,6 +28,7 @@ variables: mip: Oday raw: analysed_sst frequency: day - filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc + #filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc + filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc start_year: 2004 end_year: 2007 diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index a2e68b8589..2ad7fccbf3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -1,9 +1,14 @@ """Script to download ESACCI-SOILMOISTURE.""" +# Import required python modules +import ftplib +import os + from datetime import datetime from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader +from esmvaltool.cmorizers.data.downloaders.ftp import FTPDownloader def download_dataset(config, dataset, dataset_info, start_date, end_date, @@ -26,21 +31,69 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Overwrite already downloaded files """ if start_date is None: - start_date = datetime(2003, 1, 1) + start_date = datetime(2004, 1, 1) if end_date is None: end_date = datetime(2007,12, 31) loop_date = start_date - downloader = CCIDownloader( + ## Define the local directory name to put data in + #Ddir="C:\\datadir" + + ## If directory doesn't exist make it + #If not os.path.isdir(ddir): + # os.mkdir(ddir) + + ## Change the local directory to where you want to put the data + #Os.chdir(ddir) + + #user = os.environ.get("ceda-user") + #if user is None: + # user = str(input("CEDA user name? ")) + # if user == "": + # errmsg = ("A CEDA account is required to download CCI SST data." + # " Please visit https://rda.ucar.edu/login/register/" + # " to create an account at the Research Data Archive" + # " (RDA) if needed.") + # logger.error(errmsg) + # raise ValueError + + #passwd = os.environ.get("ceda-passwd") + #if passwd is None: + # passwd = str(input("CEDA-password? ")) + + # login to FTP + #f=ftplib.FTP("ftp3.ceda.ac.uk", user=user, passwd=passwd) + #f=ftplib.FTP("ftp3.ceda.ac.uk", user="lbock", passwd="~?s`Cb`@E8Vf") + #f=ftplib.FTP("ftp3.ceda.ac.uk", user="lbock", passwd="Pasw4ceda!") + + downloader = FTPDownloader( config=config, + server='ftp3.ceda.ac.uk', dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, ) - downloader.ftp_name = 'sst' + + ##os.makedirs(downloader.local_folder, exist_ok=True) + + + ##options = ["-O", "Authentication.log", "--save-cookies=auth.rda_ucar_edu", + ## f"--post-data=\"email={user}&passwd={passwd}&action=login\""] + + ### login to Research Data Archive (RDA) + + ##downloader.login("https://www.ceda.com/login", options) + + #downloader.ftp_name = 'eocis' + ##downloader.ftp_name = 'sst' downloader.connect() - downloader.set_cwd('CDR_v2/Analysis/L4/v2.1/') + ##downloader.set_cwd('CDR_v2/Analysis/L4/v2.1/') + downloader.set_cwd('neodc/eocis/data/global_and_regional/sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/') + #downloader.set_cwd('') + + #f.cwd = 'neodc/eocis/data/global_and_regional/sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/' + while loop_date <= end_date: year = loop_date.year month = loop_date.strftime("%m") diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 9f0cd5e8f9..8667cdcbc3 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -44,7 +44,7 @@ def connect(self): """Connect to the FTP server.""" self._client = ftplib.FTP(self.server) logger.info(self._client.getwelcome()) - self._client.login() + self._client.login(user="lbock", passwd="~?s`Cb`@E8Vf") def set_cwd(self, path): """Set current working directory in the remote. @@ -178,7 +178,7 @@ class CCIDownloader(FTPDownloader): Overwrite already downloaded files """ def __init__(self, config, dataset, dataset_info, overwrite): - super().__init__(config, 'anon-ftp.ceda.ac.uk', dataset, dataset_info, + super().__init__(config, 'ftp3.ceda.ac.uk', dataset, dataset_info, overwrite) self.ftp_name = self.dataset_name[7:] @@ -192,7 +192,9 @@ def set_cwd(self, path): path : str Remote path to set as current working directory. """ - cwd = f'/neodc/esacci/{self.ftp_name}/data/{path}' + #cwd = f'/neodc/esacci/{self.ftp_name}/data/{path}' + #cwd = f'/neodc/{self.ftp_name}/data/{path}' + cwd = f'/neodc/eocis' super().set_cwd(cwd) @property diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index fc5c6ba1e2..01f03497f4 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -297,8 +297,8 @@ diagnostics: # type: sat, version: 2.2, # start_year: 1982, end_year: 2019} - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 2, - type: sat, version: 2.1, - start_year: 2003, end_year: 2003} + type: sat, version: 3.0-L4-analysis, timerange: '200301/200712'} + # start_year: 2004, end_year: 2004} scripts: null From 4c279fb113b124d28520608c0b777cf1848a787a Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Fri, 28 Jun 2024 10:31:56 +0200 Subject: [PATCH 07/40] clean downloader --- .../data/downloaders/datasets/esacci_sst.py | 63 +++++-------------- esmvaltool/cmorizers/data/downloaders/ftp.py | 16 +++-- 2 files changed, 27 insertions(+), 52 deletions(-) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index 2ad7fccbf3..824ae9b6b7 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -37,35 +37,19 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, loop_date = start_date - ## Define the local directory name to put data in - #Ddir="C:\\datadir" - - ## If directory doesn't exist make it - #If not os.path.isdir(ddir): - # os.mkdir(ddir) - - ## Change the local directory to where you want to put the data - #Os.chdir(ddir) - - #user = os.environ.get("ceda-user") - #if user is None: - # user = str(input("CEDA user name? ")) - # if user == "": - # errmsg = ("A CEDA account is required to download CCI SST data." - # " Please visit https://rda.ucar.edu/login/register/" - # " to create an account at the Research Data Archive" - # " (RDA) if needed.") - # logger.error(errmsg) - # raise ValueError - - #passwd = os.environ.get("ceda-passwd") - #if passwd is None: - # passwd = str(input("CEDA-password? ")) - - # login to FTP - #f=ftplib.FTP("ftp3.ceda.ac.uk", user=user, passwd=passwd) - #f=ftplib.FTP("ftp3.ceda.ac.uk", user="lbock", passwd="~?s`Cb`@E8Vf") - #f=ftplib.FTP("ftp3.ceda.ac.uk", user="lbock", passwd="Pasw4ceda!") + user = os.environ.get("ceda-user") + if user is None: + user = str(input("CEDA user name? ")) + if user == "": + errmsg = ("A CEDA account is required to download CCI SST data." + " Please visit https://services.ceda.ac.uk/cedasite/register/info/" + " to create an account at CEDA if needed.") + logger.error(errmsg) + raise ValueError + + passwd = os.environ.get("ceda-passwd") + if passwd is None: + passwd = str(input("CEDA-password? ")) downloader = FTPDownloader( config=config, @@ -73,27 +57,14 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, + user=user, + passwd=passwd, ) - ##os.makedirs(downloader.local_folder, exist_ok=True) - - - ##options = ["-O", "Authentication.log", "--save-cookies=auth.rda_ucar_edu", - ## f"--post-data=\"email={user}&passwd={passwd}&action=login\""] - - ### login to Research Data Archive (RDA) - - ##downloader.login("https://www.ceda.com/login", options) - - #downloader.ftp_name = 'eocis' - ##downloader.ftp_name = 'sst' downloader.connect() - ##downloader.set_cwd('CDR_v2/Analysis/L4/v2.1/') - downloader.set_cwd('neodc/eocis/data/global_and_regional/sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/') - #downloader.set_cwd('') + downloader.set_cwd('neodc/eocis/data/global_and_regional/' + 'sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/') - #f.cwd = 'neodc/eocis/data/global_and_regional/sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/' - while loop_date <= end_date: year = loop_date.year month = loop_date.strftime("%m") diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 8667cdcbc3..b40a1d569e 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -35,16 +35,22 @@ class FTPDownloader(BaseDownloader): overwrite : bool Overwrite already downloaded files """ - def __init__(self, config, server, dataset, dataset_info, overwrite): + def __init__(self, config, server, dataset, dataset_info, overwrite, + user=None, passwd=None): super().__init__(config, dataset, dataset_info, overwrite) self._client = None self.server = server + self.user = user + self.passwd = passwd def connect(self): """Connect to the FTP server.""" self._client = ftplib.FTP(self.server) logger.info(self._client.getwelcome()) - self._client.login(user="lbock", passwd="~?s`Cb`@E8Vf") + if self.user == None: + self._client.login() + else: + self._client.login(user=self.user, passwd=self.passwd) def set_cwd(self, path): """Set current working directory in the remote. @@ -178,7 +184,7 @@ class CCIDownloader(FTPDownloader): Overwrite already downloaded files """ def __init__(self, config, dataset, dataset_info, overwrite): - super().__init__(config, 'ftp3.ceda.ac.uk', dataset, dataset_info, + super().__init__(config, 'anon-ftp.ceda.ac.uk', dataset, dataset_info, overwrite) self.ftp_name = self.dataset_name[7:] @@ -192,9 +198,7 @@ def set_cwd(self, path): path : str Remote path to set as current working directory. """ - #cwd = f'/neodc/esacci/{self.ftp_name}/data/{path}' - #cwd = f'/neodc/{self.ftp_name}/data/{path}' - cwd = f'/neodc/eocis' + cwd = f'/neodc/esacci/{self.ftp_name}/data/{path}' super().set_cwd(cwd) @property From 04e1d62ea5b06f6fc555f53f9cb935c3e8114076 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Fri, 28 Jun 2024 15:43:29 +0200 Subject: [PATCH 08/40] add monthly sst --- .../data/formatters/datasets/esacci_sst.py | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index b94d1fab2b..6db7a620fc 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -1,32 +1,30 @@ """ESMValTool CMORizer for ESACCI-SST data. Tier - Tier 2: other freely-available dataset. + Tier 3: need to register at CEDA Source - http://surftemp.net/regridding/index.html + https://catalogue.ceda.ac.uk/uuid/4a9654136a7148e39b7feb56f8bb02d2 Last access - 20201214 + 20240628 Download and processing instructions - Download the following files: - Go to http://surftemp.net/regridding/index.html - and request regridded data with the following options: - Time Resolution: monthly - Longitude Resolution: 0.5 - Latitude Resolution: 0.5 - Start Date: 1982-01-01 - End Date: 2019-12-31 - Exclude data above sea ice threshold: True - (Threshold: 100 %) - Include post-hoc SST bias adjustments: True - Output Absolute or Anomaly SST: absolute - Generate Sea Ice Fraction: True - Error Correlation in Time (Days): 7 - Error Correlation In Space (Degrees): 3.0 + A donwnloader is provided by ESMValTool. First you need + to register. + Go to https://services.ceda.ac.uk/cedasite/register/info/ + and create an account at CEDA if needed. + +? Exclude data above sea ice threshold: True +? (Threshold: 100 %) +? Include post-hoc SST bias adjustments: True +? Output Absolute or Anomaly SST: absolute +? Generate Sea Ice Fraction: True +? Error Correlation in Time (Days): 7 +? Error Correlation In Space (Degrees): 3.0 Modification history + 20240618-bock_lisa: update for v3.0 20201204-roberts_charles: written. 20201214-predoi_valeriu: approved. 20201214-lauer_axel: approved. @@ -40,7 +38,7 @@ from esmvalcore.cmor.fixes import get_time_bounds from esmvaltool.cmorizers.data import utilities as utils from esmvalcore.preprocessor import concatenate -from esmvalcore.preprocessor import regrid +from esmvalcore.preprocessor import regrid, monthly_statistics from ...utilities import ( convert_timeunits, @@ -86,18 +84,17 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): var_info = cmor_table.get_variable(vals['mip'], var) glob_attrs['mip'] = vals['mip'] raw_info = {'name': vals['raw']} - #inpfile = os.path.join(in_dir, cfg['filename']) inpfile_pattern = os.path.join(in_dir, '{year}*'+vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) for year in range(vals['start_year'], vals['end_year'] + 1): logger.info("Processing year %s", year) - for month in range(1,13): + mon_cubes = [] + #for month in range(1,13): + for month in range(1,4): data_cubes = [] - #year_inpfile_pattern = inpfile_pattern.format(year=year) month_inpfile_pattern = inpfile_pattern.format(year=str(year)+"{:02}".format(month)) logger.info("Pattern: %s", month_inpfile_pattern) inpfiles = sorted(glob.glob(month_inpfile_pattern)) - #inpfiles = sorted(glob.glob(year_inpfile_pattern)) logger.info("Found input files: %s", inpfiles) for inpfile in inpfiles: raw_info['file'] = inpfile @@ -105,12 +102,22 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): raw_info['file']) data_cubes.append( extract_variable(var_info, raw_info, glob_attrs, year)) - yearly_cube = concatenate(data_cubes) + monthly_cube = concatenate(data_cubes) # Fix monthly time bounds - time = yearly_cube.coord('time') + time = monthly_cube.coord('time') time.bounds = get_time_bounds(time, vals['frequency']) - save_variable(yearly_cube, + mon_cubes.append(monthly_cube) + #mon_cubes.append(monthly_cube.collapsed('time', iris.analysis.MEAN)) + save_variable(monthly_cube, var, out_dir, glob_attrs, unlimited_dimensions=['time']) + yearly_cube = monthly_statistics(concatenate(mon_cubes), operator='mean') + #yearly_cube = concatenate(mon_cubes) + glob_attrs['mip'] = 'Omon' + save_variable(yearly_cube, + var, + out_dir, + glob_attrs, + unlimited_dimensions=['time']) From f3dd97dac997488fbcc7b4786aebc0edb5f07739 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 10:14:09 +0200 Subject: [PATCH 09/40] update cmor_config --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 31 +++++++++---------- esmvaltool/cmorizers/data/datasets.yml | 8 ++--- esmvaltool/references/esacci-sst.bibtex | 18 +++++------ 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index a301f2d9f8..ccc8b8593e 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -1,34 +1,31 @@ --- # Common global attributes for Cmorizer output -#filename: '{year}{month}15_regridded_sst.nc' attributes: dataset_id: ESACCI-SST version: '3.0-L4-analysis' tier: 3 modeling_realm: sat project_id: OBS6 - source: 'dx.doi.org/10.5285/62c0f97b1eac4e0197a674870afe1ee6' - reference: ["esacci-sst", "esacci-sst-bias-correction"] + source: 'dx.doi.org/10.5285/4a9654136a7148e39b7feb56f8bb02d2' + reference: ["esacci-sst"] comment: "Note that the variable tsStderr is an uncertainty not a standard error." # Variables to cmorize (here use only filename prefix) variables: - #ts: - # mip: Amon - # raw: sst - # file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB - #tsStderr: - # mip: Amon - # raw: sst_uncertainty - # file: ESACCI-SST_sat_L4-GHRSST-SSTdepth-OSTIA-GLOB - -# uncomment this part to produce sst cmorized data for ocean realm (Omon, tos) - tos_day: + tos: short_name: tos - mip: Oday + mip: [Oday, Omon] raw: analysed_sst frequency: day - #filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc start_year: 2004 - end_year: 2007 + end_year: 2004 + + # tosStderr: + # short_name: tosStderr + # mip: [Oday] + # raw: analysed_sst_uncertainty + # frequency: day + # filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc + # start_year: 2004 + # end_year: 2004 diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 759dc6177e..d3fd376753 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -549,12 +549,12 @@ datasets: Put all files under a single directory (no subdirectories with years). ESACCI-SST: - tier: 2 - source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/sst/data/ - last_access: 2019-02-01 + tier: 3 + source: ftp3.ceda.ac.uk/neodc/eocis/data/global_and_regional/sea_surface_temperature/ + last_access: 2024-07-01 info: | Download the data from: - lt/Analysis/L4/v01.1/ + CDR_v3/Analysis/L4/v3.0.1/ Put all files under a single directory (no subdirectories with years). ESACCI-WATERVAPOUR: diff --git a/esmvaltool/references/esacci-sst.bibtex b/esmvaltool/references/esacci-sst.bibtex index 30eafc7756..4d49c5e2f4 100644 --- a/esmvaltool/references/esacci-sst.bibtex +++ b/esmvaltool/references/esacci-sst.bibtex @@ -1,13 +1,9 @@ @article{esacci-sst, - doi = {10.1038/s41597-019-0236-x}, - url = {https://doi.org/10.1038/s41597-019-0236-x}, - year = 2019, - month = {oct}, - publisher = {Springer Nature}, - volume = {6}, - number = {1}, - pages = {223}, - author = {Christopher J. Merchant and Owen Embury and Claire E. Bulgin and Thomas Block and Gary K. Corlett and Emma Fiedler and Simon A. Good and Jonathan Mittaz and Nick A. Rayner and David Berry and Steinar Eastwood and Michael Taylor and Yoko Tsushima and Alison Waterfall and Ruth Wilson and Craig Donlon}, - title = {Satellite-based time-series of sea-surface temperature since 1981 for climate applications ({SST} {CCI})}, - journal = {Scientific Data} + doi = {10.5285/4a9654136a7148e39b7feb56f8bb02d2}, + url = {https://dx.doi.org/10.5285/4a9654136a7148e39b7feb56f8bb02d2}, + year = 2024, + publisher = {NERC EDS Centre for Environmental Data Analysis}, + author = {Good, S.A. and Embury, O.}, + title = {{ESA} Sea Surface Temperature Climate Change Initiative + ({SST}_cci): Level 4 Analysis product, version 3.0}, } From 1104156a3adddcf808dc457324bf4d4bfe19abca Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 12:37:22 +0200 Subject: [PATCH 10/40] update reference --- esmvaltool/references/esacci-sst.bibtex | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/esmvaltool/references/esacci-sst.bibtex b/esmvaltool/references/esacci-sst.bibtex index 4d49c5e2f4..278e41966e 100644 --- a/esmvaltool/references/esacci-sst.bibtex +++ b/esmvaltool/references/esacci-sst.bibtex @@ -1,9 +1,12 @@ @article{esacci-sst, - doi = {10.5285/4a9654136a7148e39b7feb56f8bb02d2}, - url = {https://dx.doi.org/10.5285/4a9654136a7148e39b7feb56f8bb02d2}, + doi = {10.1038/s41597-024-03147-w}, + url = {https://doi.org/10.1038/s41597-024-03147-w}, year = 2024, - publisher = {NERC EDS Centre for Environmental Data Analysis}, - author = {Good, S.A. and Embury, O.}, - title = {{ESA} Sea Surface Temperature Climate Change Initiative - ({SST}_cci): Level 4 Analysis product, version 3.0}, + journal = {Sci Data}, + volume = {11}, + number = {1}, + pages = {326}, + publisher = {{Nature Publishing Group}}, + author = {Embury, O. and Merchant, C.J. and Good, S.A. and Rayner, N.A. and Høyer, J.L. and Atkinson, C. and Block, T. and Alerskans, E. and Pearson, K.J. and Worsfold, M. and McCarroll, N. and Donlon, C}, + title = {Satellite-based time-series of sea-surface temperature since 1980 for climate applications}, } From f3af2f7ba55001fac50705119327e9000eb55613 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 13:28:26 +0200 Subject: [PATCH 11/40] save monthly data --- .../data/formatters/datasets/esacci_sst.py | 41 +++++++++++-------- .../recipes/examples/recipe_check_obs.yml | 2 +- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 6db7a620fc..bd5f1edb58 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -15,14 +15,6 @@ Go to https://services.ceda.ac.uk/cedasite/register/info/ and create an account at CEDA if needed. -? Exclude data above sea ice threshold: True -? (Threshold: 100 %) -? Include post-hoc SST bias adjustments: True -? Output Absolute or Anomaly SST: absolute -? Generate Sea Ice Fraction: True -? Error Correlation in Time (Days): 7 -? Error Correlation In Space (Degrees): 3.0 - Modification history 20240618-bock_lisa: update for v3.0 20201204-roberts_charles: written. @@ -81,20 +73,25 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # run the cmorization for var_name, vals in cfg['variables'].items(): var = vals['short_name'] - var_info = cmor_table.get_variable(vals['mip'], var) - glob_attrs['mip'] = vals['mip'] + var_info = cmor_table.get_variable(vals['mip'][0], var) + glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} inpfile_pattern = os.path.join(in_dir, '{year}*'+vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) for year in range(vals['start_year'], vals['end_year'] + 1): logger.info("Processing year %s", year) mon_cubes = [] - #for month in range(1,13): - for month in range(1,4): + for month in range(1,13): + #for month in range(1,3): data_cubes = [] - month_inpfile_pattern = inpfile_pattern.format(year=str(year)+"{:02}".format(month)) + month_inpfile_pattern = inpfile_pattern.format( + year=str(year)+"{:02}".format(month)) logger.info("Pattern: %s", month_inpfile_pattern) inpfiles = sorted(glob.glob(month_inpfile_pattern)) + if inpfiles == []: + logger.error("Could not find any files with this pattern %s", + month_inpfile_pattern) + raise ValueError logger.info("Found input files: %s", inpfiles) for inpfile in inpfiles: raw_info['file'] = inpfile @@ -106,16 +103,24 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # Fix monthly time bounds time = monthly_cube.coord('time') time.bounds = get_time_bounds(time, vals['frequency']) - mon_cubes.append(monthly_cube) - #mon_cubes.append(monthly_cube.collapsed('time', iris.analysis.MEAN)) + # Save daily data save_variable(monthly_cube, var, out_dir, glob_attrs, unlimited_dimensions=['time']) - yearly_cube = monthly_statistics(concatenate(mon_cubes), operator='mean') - #yearly_cube = concatenate(mon_cubes) - glob_attrs['mip'] = 'Omon' + # Calculate monthly mean + logger.info("Calculating monthly mean") + iris.coord_categorisation.add_month_number(monthly_cube, 'time') + iris.coord_categorisation.add_year(monthly_cube, 'time') + monthly_cube = monthly_cube.aggregated_by(['month_number', 'year'], + iris.analysis.MEAN) + monthly_cube.remove_coord('month_number') + monthly_cube.remove_coord('year') + mon_cubes.append(monthly_cube) + # Save monthly data + yearly_cube = concatenate(mon_cubes) + glob_attrs['mip'] = vals['mip'][1] save_variable(yearly_cube, var, out_dir, diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index b3834638ba..e9b4eea5f3 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -318,7 +318,7 @@ diagnostics: #- {dataset: ESACCI-SST, project: OBS, mip: Amon, tier: 2, # type: sat, version: 2.2, # start_year: 1982, end_year: 2019} - - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 2, + - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, type: sat, version: 3.0-L4-analysis, timerange: '200301/200712'} # start_year: 2004, end_year: 2004} scripts: null From 756b409e0257f9031b84fcfa0e7b6cdf41eccf56 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 14:04:28 +0200 Subject: [PATCH 12/40] fix codacy --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 30 +++++++-------- .../data/downloaders/datasets/esacci_sst.py | 13 +++---- esmvaltool/cmorizers/data/downloaders/ftp.py | 2 +- .../data/formatters/datasets/esacci_sst.py | 37 ++++++++++--------- .../recipes/examples/recipe_check_obs.yml | 11 ++---- 5 files changed, 45 insertions(+), 48 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index ccc8b8593e..083caae6d9 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -12,20 +12,20 @@ attributes: # Variables to cmorize (here use only filename prefix) variables: - tos: - short_name: tos - mip: [Oday, Omon] - raw: analysed_sst + #tos: + # short_name: tos + # mip: [Oday, Omon] + # raw: analysed_sst + # frequency: day + # filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc + # start_year: 2004 + # end_year: 2004 + + tosStderr: + short_name: tosStderr + mip: [Oday] + raw: analysed_sst_uncertainty frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - start_year: 2004 - end_year: 2004 - - # tosStderr: - # short_name: tosStderr - # mip: [Oday] - # raw: analysed_sst_uncertainty - # frequency: day - # filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - # start_year: 2004 - # end_year: 2004 + start_year: 2003 + end_year: 2003 diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index 824ae9b6b7..e29bf50821 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -1,13 +1,12 @@ -"""Script to download ESACCI-SOILMOISTURE.""" +"""Script to download ESACCI-SST.""" # Import required python modules -import ftplib +import logging import os from datetime import datetime from dateutil import relativedelta -from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader from esmvaltool.cmorizers.data.downloaders.ftp import FTPDownloader @@ -33,7 +32,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, if start_date is None: start_date = datetime(2004, 1, 1) if end_date is None: - end_date = datetime(2007,12, 31) + end_date = datetime(2007, 12, 31) loop_date = start_date @@ -42,8 +41,8 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, user = str(input("CEDA user name? ")) if user == "": errmsg = ("A CEDA account is required to download CCI SST data." - " Please visit https://services.ceda.ac.uk/cedasite/register/info/" - " to create an account at CEDA if needed.") + " Please visit https://services.ceda.ac.uk/cedasite/" + "register/info/ to create an account at CEDA if needed.") logger.error(errmsg) raise ValueError @@ -64,7 +63,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, downloader.connect() downloader.set_cwd('neodc/eocis/data/global_and_regional/' 'sea_surface_temperature/CDR_v3/Analysis/L4/v3.0.1/') - + while loop_date <= end_date: year = loop_date.year month = loop_date.strftime("%m") diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index b40a1d569e..7f9d4e6492 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -47,7 +47,7 @@ def connect(self): """Connect to the FTP server.""" self._client = ftplib.FTP(self.server) logger.info(self._client.getwelcome()) - if self.user == None: + if self.user is None: self._client.login() else: self._client.login(user=self.user, passwd=self.passwd) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index bd5f1edb58..1a7c2be309 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -14,7 +14,7 @@ to register. Go to https://services.ceda.ac.uk/cedasite/register/info/ and create an account at CEDA if needed. - + Modification history 20240618-bock_lisa: update for v3.0 20201204-roberts_charles: written. @@ -28,9 +28,9 @@ import iris from esmvalcore.cmor.fixes import get_time_bounds +from esmvalcore.preprocessor import regrid from esmvaltool.cmorizers.data import utilities as utils from esmvalcore.preprocessor import concatenate -from esmvalcore.preprocessor import regrid, monthly_statistics from ...utilities import ( convert_timeunits, @@ -55,7 +55,7 @@ def extract_variable(var_info, raw_info, attrs, year): # regridding from 0.05x0.05 to 0.5x0.5 cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') - #Fix dtype + # Fix dtype utils.fix_dtype(cube) # Fix cube fix_var_metadata(cube, var_info) @@ -76,21 +76,20 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): var_info = cmor_table.get_variable(vals['mip'][0], var) glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} - inpfile_pattern = os.path.join(in_dir, '{year}*'+vals['filename']) + inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) for year in range(vals['start_year'], vals['end_year'] + 1): logger.info("Processing year %s", year) mon_cubes = [] - for month in range(1,13): - #for month in range(1,3): + for month in range(1, 13): data_cubes = [] month_inpfile_pattern = inpfile_pattern.format( - year=str(year)+"{:02}".format(month)) + year=str(year)+"{:02}".format(month)) logger.info("Pattern: %s", month_inpfile_pattern) inpfiles = sorted(glob.glob(month_inpfile_pattern)) if inpfiles == []: - logger.error("Could not find any files with this pattern %s", - month_inpfile_pattern) + logger.error("Could not find any files with this" + " pattern %s", month_inpfile_pattern) raise ValueError logger.info("Found input files: %s", inpfiles) for inpfile in inpfiles: @@ -111,18 +110,20 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): unlimited_dimensions=['time']) # Calculate monthly mean logger.info("Calculating monthly mean") - iris.coord_categorisation.add_month_number(monthly_cube, 'time') + iris.coord_categorisation.add_month_number(monthly_cube, + 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') monthly_cube = monthly_cube.aggregated_by(['month_number', 'year'], - iris.analysis.MEAN) + iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) # Save monthly data - yearly_cube = concatenate(mon_cubes) - glob_attrs['mip'] = vals['mip'][1] - save_variable(yearly_cube, - var, - out_dir, - glob_attrs, - unlimited_dimensions=['time']) + if 'Stderr' not in var_name: + yearly_cube = concatenate(mon_cubes) + glob_attrs['mip'] = vals['mip'][1] + save_variable(yearly_cube, + var, + out_dir, + glob_attrs, + unlimited_dimensions=['time']) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index e9b4eea5f3..7564851b18 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -311,16 +311,13 @@ diagnostics: ESACCI-SST: description: ESACCI-SST check variables: - #ts: - #tsStderr: tos: + tosStderr: additional_datasets: - #- {dataset: ESACCI-SST, project: OBS, mip: Amon, tier: 2, - # type: sat, version: 2.2, - # start_year: 1982, end_year: 2019} + - {dataset: ESACCI-SST, project: OBS6, mip: Omon, tier: 3, + type: sat, version: 3.0-L4-analysis, timerange: '200401/200412'} - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200301/200712'} - # start_year: 2004, end_year: 2004} + type: sat, version: 3.0-L4-analysis, timerange: '200401/200412'} scripts: null From faa6ffb75f3990e433c056ec9f5fc9ee94d5ef75 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 14:10:59 +0200 Subject: [PATCH 13/40] fix codacy --- .../cmorizers/data/downloaders/datasets/esacci_sst.py | 2 ++ .../cmorizers/data/formatters/datasets/esacci_sst.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index e29bf50821..dbf8e0b84c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -9,6 +9,8 @@ from esmvaltool.cmorizers.data.downloaders.ftp import FTPDownloader +logger = logging.getLogger(__name__) + def download_dataset(config, dataset, dataset_info, start_date, end_date, overwrite): diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 1a7c2be309..970c26300c 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -113,13 +113,14 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): iris.coord_categorisation.add_month_number(monthly_cube, 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') - monthly_cube = monthly_cube.aggregated_by(['month_number', 'year'], - iris.analysis.MEAN) + monthly_cube = monthly_cube.aggregated_by(['month_number', + 'year'], + iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) # Save monthly data - if 'Stderr' not in var_name: + if 'Stderr' not in var_name: yearly_cube = concatenate(mon_cubes) glob_attrs['mip'] = vals['mip'][1] save_variable(yearly_cube, From 576dc8a7ea6f40467ef928c3dc69c64d37ad37e8 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 15:35:15 +0200 Subject: [PATCH 14/40] tosStderr added --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 18 ++++---- .../data/formatters/datasets/esacci_sst.py | 41 +++++++++++-------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 083caae6d9..990dd521ab 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -10,19 +10,17 @@ attributes: reference: ["esacci-sst"] comment: "Note that the variable tsStderr is an uncertainty not a standard error." -# Variables to cmorize (here use only filename prefix) +# Variables to cmorize (here use only filename ending) variables: - #tos: - # short_name: tos - # mip: [Oday, Omon] - # raw: analysed_sst - # frequency: day - # filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - # start_year: 2004 - # end_year: 2004 + tos: + mip: [Oday, Omon] + raw: analysed_sst + frequency: day + filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc + start_year: 2004 + end_year: 2004 tosStderr: - short_name: tosStderr mip: [Oday] raw: analysed_sst_uncertainty frequency: day diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 970c26300c..d9bc898b07 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -47,11 +47,16 @@ def extract_variable(var_info, raw_info, attrs, year): """Extract to all vars.""" rawvar = raw_info['name'] constraint = iris.NameConstraint(var_name=rawvar) - try: - cube = iris.load_cube(raw_info['file'], constraint) - except iris.exceptions.ConstraintMismatchError as constraint_error: - raise ValueError(f"No data available for variable {rawvar}" - f"and year {year}") from constraint_error + if rawvar == 'analysed_sst_uncertainty': + tmp_cube = iris.load_cube(raw_info['file'], iris.NameConstraint(var_name='analysed_sst')) + ancillary_var = tmp_cube.ancillary_variable('sea_water_temperature standard_error') + cube = tmp_cube.copy(ancillary_var.core_data()) + else: + try: + cube = iris.load_cube(raw_info['file'], constraint) + except iris.exceptions.ConstraintMismatchError as constraint_error: + raise ValueError(f"No data available for variable {rawvar}" + f" and year {year}") from constraint_error # regridding from 0.05x0.05 to 0.5x0.5 cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') @@ -71,8 +76,8 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): glob_attrs = cfg['attributes'] # run the cmorization - for var_name, vals in cfg['variables'].items(): - var = vals['short_name'] + for var, vals in cfg['variables'].items(): + #var = vals['short_name'] var_info = cmor_table.get_variable(vals['mip'][0], var) glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} @@ -109,18 +114,18 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): glob_attrs, unlimited_dimensions=['time']) # Calculate monthly mean - logger.info("Calculating monthly mean") - iris.coord_categorisation.add_month_number(monthly_cube, - 'time') - iris.coord_categorisation.add_year(monthly_cube, 'time') - monthly_cube = monthly_cube.aggregated_by(['month_number', - 'year'], - iris.analysis.MEAN) - monthly_cube.remove_coord('month_number') - monthly_cube.remove_coord('year') - mon_cubes.append(monthly_cube) + if 'Stderr' not in var: + logger.info("Calculating monthly mean") + iris.coord_categorisation.add_month_number(monthly_cube, + 'time') + iris.coord_categorisation.add_year(monthly_cube, 'time') + monthly_cube = monthly_cube.aggregated_by(['month_number', + 'year'], iris.analysis.MEAN) + monthly_cube.remove_coord('month_number') + monthly_cube.remove_coord('year') + mon_cubes.append(monthly_cube) # Save monthly data - if 'Stderr' not in var_name: + if 'Stderr' not in var: yearly_cube = concatenate(mon_cubes) glob_attrs['mip'] = vals['mip'][1] save_variable(yearly_cube, From 2922a3a88d93dc374de0c9ad58d8479680cf115e Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 15:36:54 +0200 Subject: [PATCH 15/40] update doc table --- doc/sphinx/source/input.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index 20a417cfc6..bad1234f8d 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -310,7 +310,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-SOILMOISTURE | dos, dosStderr, sm, smStderr (Lmon) | 2 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| ESACCI-SST | ts, tsStderr (Amon) | 2 | NCL | +| ESACCI-SST | tos, tosStderr (Omon, Oday) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-WATERVAPOUR | prw (Amon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ From d7d64f99ba933f838cdc5e333f67eb1b2d3d1ef4 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 16:09:07 +0200 Subject: [PATCH 16/40] update time period --- esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 990dd521ab..164f23f1ab 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -17,8 +17,8 @@ variables: raw: analysed_sst frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - start_year: 2004 - end_year: 2004 + start_year: 2003 + end_year: 2007 tosStderr: mip: [Oday] @@ -26,4 +26,4 @@ variables: frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc start_year: 2003 - end_year: 2003 + end_year: 2007 From 5cbbf57ccbe5d12c529f4df0b8b78c9f923f41ee Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 16:15:09 +0200 Subject: [PATCH 17/40] fix codacy --- .../cmorizers/data/formatters/datasets/esacci_sst.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index d9bc898b07..94e36d7d62 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -48,8 +48,10 @@ def extract_variable(var_info, raw_info, attrs, year): rawvar = raw_info['name'] constraint = iris.NameConstraint(var_name=rawvar) if rawvar == 'analysed_sst_uncertainty': - tmp_cube = iris.load_cube(raw_info['file'], iris.NameConstraint(var_name='analysed_sst')) - ancillary_var = tmp_cube.ancillary_variable('sea_water_temperature standard_error') + tmp_cube = iris.load_cube(raw_info['file'], + iris.NameConstraint(var_name='analysed_sst')) + ancillary_var = tmp_cube.ancillary_variable('sea_water_temperature' + ' standard_error') cube = tmp_cube.copy(ancillary_var.core_data()) else: try: @@ -120,7 +122,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') monthly_cube = monthly_cube.aggregated_by(['month_number', - 'year'], iris.analysis.MEAN) + 'year'], iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) From 820e4dad9ec6fd5260b1f1f95a170d43d5f5af52 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 16:21:46 +0200 Subject: [PATCH 18/40] fix codacy --- .../cmorizers/data/formatters/datasets/esacci_sst.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 94e36d7d62..d5ae6c50bb 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -49,7 +49,7 @@ def extract_variable(var_info, raw_info, attrs, year): constraint = iris.NameConstraint(var_name=rawvar) if rawvar == 'analysed_sst_uncertainty': tmp_cube = iris.load_cube(raw_info['file'], - iris.NameConstraint(var_name='analysed_sst')) + iris.NameConstraint(var_name='analysed_sst')) ancillary_var = tmp_cube.ancillary_variable('sea_water_temperature' ' standard_error') cube = tmp_cube.copy(ancillary_var.core_data()) @@ -79,7 +79,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # run the cmorization for var, vals in cfg['variables'].items(): - #var = vals['short_name'] var_info = cmor_table.get_variable(vals['mip'][0], var) glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} @@ -121,8 +120,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): iris.coord_categorisation.add_month_number(monthly_cube, 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') - monthly_cube = monthly_cube.aggregated_by(['month_number', - 'year'], iris.analysis.MEAN) + monthly_cube = + monthly_cube.aggregated_by(['month_number', 'year'], + iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) From 7080d8dfab9f41d0e6ea93db11cb1bbb9127a1e1 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 1 Jul 2024 16:27:13 +0200 Subject: [PATCH 19/40] fix --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index d5ae6c50bb..90f930e599 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -120,9 +120,8 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): iris.coord_categorisation.add_month_number(monthly_cube, 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') - monthly_cube = - monthly_cube.aggregated_by(['month_number', 'year'], - iris.analysis.MEAN) + monthly_cube = monthly_cube.aggregated_by(['month_number', + 'year'], iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) From f04e9cfcb2d852f6445f597b3c59dc940fd21b48 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 13:36:11 +0200 Subject: [PATCH 20/40] fix memory issue --- .../data/formatters/datasets/esacci_sst.py | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 90f930e599..fc08a0430d 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) -def extract_variable(var_info, raw_info, attrs, year): +def extract_variable(raw_info, year): """Extract to all vars.""" rawvar = raw_info['name'] constraint = iris.NameConstraint(var_name=rawvar) @@ -60,6 +60,36 @@ def extract_variable(var_info, raw_info, attrs, year): raise ValueError(f"No data available for variable {rawvar}" f" and year {year}") from constraint_error + # Remove ancillary data + for ancillary_variable in cube.ancillary_variables(): + cube.remove_ancillary_variable(ancillary_variable) + ## regridding from 0.05x0.05 to 0.5x0.5 + #cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') + return cube + + +def get_monthly_cube(var, vals, raw_info, var_info, attrs, + inpfile_pattern, year, month): + data_cubes = [] + month_inpfile_pattern = inpfile_pattern.format( + year=str(year)+"{:02}".format(month)) + logger.info("Pattern: %s", month_inpfile_pattern) + inpfiles = sorted(glob.glob(month_inpfile_pattern)) + if inpfiles == []: + logger.error("Could not find any files with this" + " pattern %s", month_inpfile_pattern) + raise ValueError + logger.info("Found input files: %s", inpfiles) + + for inpfile in inpfiles: + raw_info['file'] = inpfile + logger.info("CMORizing var %s from file type %s", var, + raw_info['file']) + data_cubes.append( + extract_variable(raw_info, year)) + + cube = concatenate(data_cubes) + # regridding from 0.05x0.05 to 0.5x0.5 cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') # Fix dtype @@ -69,8 +99,12 @@ def extract_variable(var_info, raw_info, attrs, year): convert_timeunits(cube, year) fix_coords(cube, overwrite_time_bounds=False) set_global_atts(cube, attrs) - return cube + # Fix monthly time bounds + time = cube.coord('time') + time.bounds = get_time_bounds(time, vals['frequency']) + return cube + def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" @@ -84,30 +118,13 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): raw_info = {'name': vals['raw']} inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) + mon_cubes = [] for year in range(vals['start_year'], vals['end_year'] + 1): logger.info("Processing year %s", year) - mon_cubes = [] for month in range(1, 13): - data_cubes = [] - month_inpfile_pattern = inpfile_pattern.format( - year=str(year)+"{:02}".format(month)) - logger.info("Pattern: %s", month_inpfile_pattern) - inpfiles = sorted(glob.glob(month_inpfile_pattern)) - if inpfiles == []: - logger.error("Could not find any files with this" - " pattern %s", month_inpfile_pattern) - raise ValueError - logger.info("Found input files: %s", inpfiles) - for inpfile in inpfiles: - raw_info['file'] = inpfile - logger.info("CMORizing var %s from file type %s", var, - raw_info['file']) - data_cubes.append( - extract_variable(var_info, raw_info, glob_attrs, year)) - monthly_cube = concatenate(data_cubes) - # Fix monthly time bounds - time = monthly_cube.coord('time') - time.bounds = get_time_bounds(time, vals['frequency']) + monthly_cube = get_monthly_cube(var, vals, raw_info, var_info, + glob_attrs, inpfile_pattern, + year, month) # Save daily data save_variable(monthly_cube, var, @@ -134,3 +151,4 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): out_dir, glob_attrs, unlimited_dimensions=['time']) + mon_cubes.clear() From cf05609029a5e4e8c943346e0561e6c95b15b02b Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:16:07 +0200 Subject: [PATCH 21/40] rm hardcoded years --- esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml | 4 ++-- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 164f23f1ab..2ae26752d5 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -25,5 +25,5 @@ variables: raw: analysed_sst_uncertainty frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - start_year: 2003 - end_year: 2007 + start_year: 1980 + end_year: 2023 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index fc08a0430d..8b6d1e42c0 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -110,6 +110,10 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" cmor_table = cfg['cmor_table'] glob_attrs = cfg['attributes'] + if not start_date: + start_date = vals['start_year'] + if not end_date: + end_date = vals['end_year'] # run the cmorization for var, vals in cfg['variables'].items(): @@ -119,7 +123,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) mon_cubes = [] - for year in range(vals['start_year'], vals['end_year'] + 1): + for year in range(start_year, end_year + 1): logger.info("Processing year %s", year) for month in range(1, 13): monthly_cube = get_monthly_cube(var, vals, raw_info, var_info, From 1ef0ab8816b3b17ba72ed7b496cab7893a08537c Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:18:09 +0200 Subject: [PATCH 22/40] fix --- esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 2ae26752d5..07898ff306 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -17,8 +17,8 @@ variables: raw: analysed_sst frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc - start_year: 2003 - end_year: 2007 + start_year: 1980 + end_year: 2023 tosStderr: mip: [Oday] From c785eb83e64da147cefa4e388ef440033010b7e3 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:21:07 +0200 Subject: [PATCH 23/40] fix --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 8b6d1e42c0..b7457fe0b8 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -111,9 +111,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): cmor_table = cfg['cmor_table'] glob_attrs = cfg['attributes'] if not start_date: - start_date = vals['start_year'] + start_year = vals['start_year'] if not end_date: - end_date = vals['end_year'] + end_year = vals['end_year'] # run the cmorization for var, vals in cfg['variables'].items(): From 28866f27a46bfa7b027eb0804b57b2c34350e035 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:50:39 +0200 Subject: [PATCH 24/40] fix --- .../cmorizers/data/formatters/datasets/esacci_sst.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index b7457fe0b8..4083d37699 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -110,20 +110,20 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" cmor_table = cfg['cmor_table'] glob_attrs = cfg['attributes'] - if not start_date: - start_year = vals['start_year'] - if not end_date: - end_year = vals['end_year'] # run the cmorization for var, vals in cfg['variables'].items(): + if not start_date: + start_date = vals['start_year'] + if not end_date: + end_date = vals['end_year'] var_info = cmor_table.get_variable(vals['mip'][0], var) glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) mon_cubes = [] - for year in range(start_year, end_year + 1): + for year in range(start_date, end_date + 1): logger.info("Processing year %s", year) for month in range(1, 13): monthly_cube = get_monthly_cube(var, vals, raw_info, var_info, From 02d0aef96f11598f2bfeb645b567555d6f10c27e Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:52:17 +0200 Subject: [PATCH 25/40] fix date --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 4083d37699..7ca20ea2d9 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -123,7 +123,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) mon_cubes = [] - for year in range(start_date, end_date + 1): + for year in range(start_date.year, end_date.year + 1): logger.info("Processing year %s", year) for month in range(1, 13): monthly_cube = get_monthly_cube(var, vals, raw_info, var_info, From 82c47f7872a10e60e00a8c72ae840f78740d5148 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 10 Jul 2024 14:59:14 +0200 Subject: [PATCH 26/40] fix date --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 7ca20ea2d9..9320f52fe6 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -27,6 +27,7 @@ import os import iris +from datetime import datetime from esmvalcore.cmor.fixes import get_time_bounds from esmvalcore.preprocessor import regrid from esmvaltool.cmorizers.data import utilities as utils @@ -114,9 +115,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # run the cmorization for var, vals in cfg['variables'].items(): if not start_date: - start_date = vals['start_year'] + start_date = datetime(vals['start_year'], 1, 1) if not end_date: - end_date = vals['end_year'] + end_date = datetime(vals['end_year'], 12, 31) var_info = cmor_table.get_variable(vals['mip'][0], var) glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} From c358b7251cb81b34030bad37290b544d1405de77 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 11 Jul 2024 09:15:44 +0200 Subject: [PATCH 27/40] fix style --- .../cmorizers/data/formatters/datasets/esacci_sst.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 9320f52fe6..9c088a225b 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -64,8 +64,6 @@ def extract_variable(raw_info, year): # Remove ancillary data for ancillary_variable in cube.ancillary_variables(): cube.remove_ancillary_variable(ancillary_variable) - ## regridding from 0.05x0.05 to 0.5x0.5 - #cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') return cube @@ -105,7 +103,7 @@ def get_monthly_cube(var, vals, raw_info, var_info, attrs, time.bounds = get_time_bounds(time, vals['frequency']) return cube - + def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" @@ -142,8 +140,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): iris.coord_categorisation.add_month_number(monthly_cube, 'time') iris.coord_categorisation.add_year(monthly_cube, 'time') - monthly_cube = monthly_cube.aggregated_by(['month_number', - 'year'], iris.analysis.MEAN) + monthly_cube = monthly_cube.aggregated_by( + ['month_number', 'year'], + iris.analysis.MEAN) monthly_cube.remove_coord('month_number') monthly_cube.remove_coord('year') mon_cubes.append(monthly_cube) From 96dc7a541661847dbef310e50f44166b91f500d9 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 11 Jul 2024 09:16:12 +0200 Subject: [PATCH 28/40] Update doc/sphinx/source/input.rst Co-authored-by: Axel Lauer --- doc/sphinx/source/input.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index bad1234f8d..906a1b901e 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -310,7 +310,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-SOILMOISTURE | dos, dosStderr, sm, smStderr (Lmon) | 2 | NCL | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ -| ESACCI-SST | tos, tosStderr (Omon, Oday) | 3 | Python | +| ESACCI-SST | tos (Omon, Oday), tosStderr (Oday) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-WATERVAPOUR | prw (Amon) | 3 | Python | +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ From b3d9e75c177733d7d679c0a8e9b2d084095f2ee6 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 11 Jul 2024 09:17:21 +0200 Subject: [PATCH 29/40] Update years Co-authored-by: Axel Lauer --- esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 07898ff306..a428febd1c 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -18,7 +18,7 @@ variables: frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc start_year: 1980 - end_year: 2023 + end_year: 2021 tosStderr: mip: [Oday] From 4590c1be6bc2562f3ea7ed7afb5c33eb67c39ab6 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Thu, 11 Jul 2024 09:17:40 +0200 Subject: [PATCH 30/40] Update years Co-authored-by: Axel Lauer --- esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index a428febd1c..0754c751bd 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -26,4 +26,4 @@ variables: frequency: day filename: ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR3.0-v02.0-fv01.0.nc start_year: 1980 - end_year: 2023 + end_year: 2021 From 0ee1513bdfd8ed252d4119a159ec8e7aa175b33a Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Mon, 15 Jul 2024 13:05:35 +0200 Subject: [PATCH 31/40] specify global attrs --- esmvaltool/cmorizers/data/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/utilities.py b/esmvaltool/cmorizers/data/utilities.py index 82da07c12e..23087dad06 100644 --- a/esmvaltool/cmorizers/data/utilities.py +++ b/esmvaltool/cmorizers/data/utilities.py @@ -425,7 +425,7 @@ def set_global_atts(cube, attrs): # Additional attributes glob_dict.update(attrs) - cube.attributes = glob_dict + cube.attributes.global = glob_dict def fix_bounds(cube, dim_coord): From 53df0055c04ddad5d7a72b1c3107214bc058c465 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 08:30:13 +0200 Subject: [PATCH 32/40] fix syntax --- esmvaltool/cmorizers/data/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/utilities.py b/esmvaltool/cmorizers/data/utilities.py index 23087dad06..853ebd8526 100644 --- a/esmvaltool/cmorizers/data/utilities.py +++ b/esmvaltool/cmorizers/data/utilities.py @@ -425,7 +425,7 @@ def set_global_atts(cube, attrs): # Additional attributes glob_dict.update(attrs) - cube.attributes.global = glob_dict + cube.attributes.globals = glob_dict def fix_bounds(cube, dim_coord): From d3829fc4e9f21a30be4c4433c69ed9e9e1806806 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 08:31:07 +0200 Subject: [PATCH 33/40] changes regarding the review --- .../cmorizers/data/cmor_config/ESACCI-SST.yml | 4 +- .../data/formatters/datasets/esacci_sst.py | 52 +++++++++++-------- .../recipes/examples/recipe_check_obs.yml | 7 +-- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml index 0754c751bd..61e54c0a2b 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SST.yml @@ -1,14 +1,14 @@ --- # Common global attributes for Cmorizer output + attributes: dataset_id: ESACCI-SST - version: '3.0-L4-analysis' + version: 3.0-L4-analysis tier: 3 modeling_realm: sat project_id: OBS6 source: 'dx.doi.org/10.5285/4a9654136a7148e39b7feb56f8bb02d2' reference: ["esacci-sst"] - comment: "Note that the variable tsStderr is an uncertainty not a standard error." # Variables to cmorize (here use only filename ending) variables: diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 9c088a225b..53e9485bf3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -15,13 +15,9 @@ Go to https://services.ceda.ac.uk/cedasite/register/info/ and create an account at CEDA if needed. -Modification history - 20240618-bock_lisa: update for v3.0 - 20201204-roberts_charles: written. - 20201214-predoi_valeriu: approved. - 20201214-lauer_axel: approved. """ +import copy import glob import logging import os @@ -44,7 +40,7 @@ logger = logging.getLogger(__name__) -def extract_variable(raw_info, year): +def extract_variable(raw_info): """Extract to all vars.""" rawvar = raw_info['name'] constraint = iris.NameConstraint(var_name=rawvar) @@ -58,8 +54,8 @@ def extract_variable(raw_info, year): try: cube = iris.load_cube(raw_info['file'], constraint) except iris.exceptions.ConstraintMismatchError as constraint_error: - raise ValueError(f"No data available for variable {rawvar}" - f" and year {year}") from constraint_error + raise ValueError(f"No data available for variable {rawvar} in file" + f" {raw_info['file']}") from constraint_error # Remove ancillary data for ancillary_variable in cube.ancillary_variables(): @@ -67,7 +63,7 @@ def extract_variable(raw_info, year): return cube -def get_monthly_cube(var, vals, raw_info, var_info, attrs, +def get_monthly_cube(cfg, var, vals, raw_info, attrs, inpfile_pattern, year, month): data_cubes = [] month_inpfile_pattern = inpfile_pattern.format( @@ -84,31 +80,42 @@ def get_monthly_cube(var, vals, raw_info, var_info, attrs, raw_info['file'] = inpfile logger.info("CMORizing var %s from file type %s", var, raw_info['file']) - data_cubes.append( - extract_variable(raw_info, year)) + data_cubes.append(extract_variable(raw_info)) cube = concatenate(data_cubes) # regridding from 0.05x0.05 to 0.5x0.5 cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') + + #if var == 'tos': cube.attributes.pop('comment', None) + # Fix dtype utils.fix_dtype(cube) - # Fix cube - fix_var_metadata(cube, var_info) - convert_timeunits(cube, year) - fix_coords(cube, overwrite_time_bounds=False) - set_global_atts(cube, attrs) + # Fix units + cmor_info = cfg['cmor_table'].get_variable(vals['mip'][0], var) + cube.convert_units(cmor_info.units) + # Fix metadata + fix_var_metadata(cube, cmor_info) + # Fix coordinates + #convert_timeunits(cube, year) + #fix_coords(cube, overwrite_time_bounds=False) + fix_coords(cube) + cube.coord('time').long_name = 'time' + cube.coord('latitude').long_name = 'latitude' + cube.coord('longitude').long_name = 'longitude' # Fix monthly time bounds time = cube.coord('time') time.bounds = get_time_bounds(time, vals['frequency']) + # set global attributes + set_global_atts(cube, attrs) + return cube def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" - cmor_table = cfg['cmor_table'] - glob_attrs = cfg['attributes'] + glob_attrs = copy.deepcopy(cfg['attributes']) # run the cmorization for var, vals in cfg['variables'].items(): @@ -116,18 +123,19 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): start_date = datetime(vals['start_year'], 1, 1) if not end_date: end_date = datetime(vals['end_year'], 12, 31) - var_info = cmor_table.get_variable(vals['mip'][0], var) - glob_attrs['mip'] = vals['mip'][0] raw_info = {'name': vals['raw']} inpfile_pattern = os.path.join(in_dir, '{year}*' + vals['filename']) logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) mon_cubes = [] for year in range(start_date.year, end_date.year + 1): logger.info("Processing year %s", year) - for month in range(1, 13): - monthly_cube = get_monthly_cube(var, vals, raw_info, var_info, + glob_attrs['mip'] = vals['mip'][0] + for month in range(start_date.month, end_date.month + 1): + monthly_cube = get_monthly_cube(cfg, var, vals, raw_info, glob_attrs, inpfile_pattern, year, month) + logger.info(glob_attrs) + #set_global_atts(monthly_cube, glob_attrs) # Save daily data save_variable(monthly_cube, var, diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 7564851b18..071de5ab13 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -312,12 +312,13 @@ diagnostics: description: ESACCI-SST check variables: tos: + additional_datasets: + - {dataset: ESACCI-SST, project: OBS6, mip: Omon, tier: 3, + type: sat, version: 3.0-L4-analysis, timerange: '200301/200312'} tosStderr: additional_datasets: - - {dataset: ESACCI-SST, project: OBS6, mip: Omon, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200401/200412'} - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200401/200412'} + type: sat, version: 3.0-L4-analysis, timerange: '200301/200312'} scripts: null From fb299140db3220bb29eb7f666467c81f77d0c76c Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 08:47:58 +0200 Subject: [PATCH 34/40] recipes/examples/recipe_check_obs.yml --- esmvaltool/recipes/examples/recipe_check_obs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 1c3210f76d..4cff8c89c9 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -320,11 +320,11 @@ diagnostics: tos: additional_datasets: - {dataset: ESACCI-SST, project: OBS6, mip: Omon, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200301/200312'} + type: sat, version: 3.0-L4-analysis, timerange: '200301/202112'} tosStderr: additional_datasets: - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200301/200312'} + type: sat, version: 3.0-L4-analysis, timerange: '200301/202112'} scripts: null From cf88af8780402f64d71fb33e928ac1ae29e84492 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 08:51:58 +0200 Subject: [PATCH 35/40] clear --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 53e9485bf3..4b884ca763 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -87,8 +87,6 @@ def get_monthly_cube(cfg, var, vals, raw_info, attrs, # regridding from 0.05x0.05 to 0.5x0.5 cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') - #if var == 'tos': cube.attributes.pop('comment', None) - # Fix dtype utils.fix_dtype(cube) # Fix units @@ -97,8 +95,6 @@ def get_monthly_cube(cfg, var, vals, raw_info, attrs, # Fix metadata fix_var_metadata(cube, cmor_info) # Fix coordinates - #convert_timeunits(cube, year) - #fix_coords(cube, overwrite_time_bounds=False) fix_coords(cube) cube.coord('time').long_name = 'time' cube.coord('latitude').long_name = 'latitude' From fe950bebe02f29121178a1fb904906c7f784b0a8 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 09:18:30 +0200 Subject: [PATCH 36/40] add comment --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 4b884ca763..552125387f 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -105,6 +105,10 @@ def get_monthly_cube(cfg, var, vals, raw_info, attrs, # set global attributes set_global_atts(cube, attrs) + # add comment to tosStderr + if var == 'tosStderr': + cube.attributes['comment'] = ('Note that the variable tsStderr is an ' + 'uncertainty not a standard error.') return cube @@ -130,8 +134,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): monthly_cube = get_monthly_cube(cfg, var, vals, raw_info, glob_attrs, inpfile_pattern, year, month) - logger.info(glob_attrs) - #set_global_atts(monthly_cube, glob_attrs) # Save daily data save_variable(monthly_cube, var, From c13daad188939315bb3c24a4f7784eac11dd772d Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 09:46:41 +0200 Subject: [PATCH 37/40] rm import --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index 552125387f..c612c8a3ef 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -30,7 +30,6 @@ from esmvalcore.preprocessor import concatenate from ...utilities import ( - convert_timeunits, fix_coords, fix_var_metadata, save_variable, From 77de1fec40e5cdfe06e12f618500dee336d24602 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Tue, 16 Jul 2024 12:42:27 +0200 Subject: [PATCH 38/40] adjust years --- esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py | 4 ++-- esmvaltool/recipes/examples/recipe_check_obs.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index dbf8e0b84c..9143287d87 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -32,9 +32,9 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Overwrite already downloaded files """ if start_date is None: - start_date = datetime(2004, 1, 1) + start_date = datetime(1980, 1, 1) if end_date is None: - end_date = datetime(2007, 12, 31) + end_date = datetime(2021, 12, 31) loop_date = start_date diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 4cff8c89c9..0486fd7eaa 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -320,11 +320,11 @@ diagnostics: tos: additional_datasets: - {dataset: ESACCI-SST, project: OBS6, mip: Omon, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200301/202112'} + type: sat, version: 3.0-L4-analysis, timerange: '198001/202112'} tosStderr: additional_datasets: - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '200301/202112'} + type: sat, version: 3.0-L4-analysis, timerange: '198001/202112'} scripts: null From 6e9fc0deab202557e9dc01cef2ba8d0bd8179bca Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 17 Jul 2024 12:18:47 +0200 Subject: [PATCH 39/40] rm regridding of uncertainty field --- esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py index c612c8a3ef..16f4679991 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_sst.py @@ -83,8 +83,9 @@ def get_monthly_cube(cfg, var, vals, raw_info, attrs, cube = concatenate(data_cubes) - # regridding from 0.05x0.05 to 0.5x0.5 - cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') + # regridding from 0.05x0.05 to 0.5x0.5 (not for uncertainty field + if 'Stderr' not in var: + cube = regrid(cube, target_grid='0.5x0.5', scheme='area_weighted') # Fix dtype utils.fix_dtype(cube) From 6346b30e43aa1aed45ab55ebb256a2d6e27b5377 Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Wed, 17 Jul 2024 12:27:22 +0200 Subject: [PATCH 40/40] reduce time period for tosStderr in recipe_check_obs.yml --- esmvaltool/recipes/examples/recipe_check_obs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 0486fd7eaa..6f3116a955 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -324,7 +324,7 @@ diagnostics: tosStderr: additional_datasets: - {dataset: ESACCI-SST, project: OBS6, mip: Oday, tier: 3, - type: sat, version: 3.0-L4-analysis, timerange: '198001/202112'} + type: sat, version: 3.0-L4-analysis, timerange: '202001/202112'} scripts: null