From 1752706aecf1aa5133a6ffd7a8306e1f04ca41ce Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 12:38:05 -0400 Subject: [PATCH 1/6] adding multiple timeseries feature --- src/eva/data/data_collections.py | 22 +-- src/eva/eva_driver.py | 174 ++++++++++-------- ...aObsSpaceAmsuaN19_Multiple_TimeSeries.yaml | 119 ++++++++++++ .../data/ctrl_amsua_n19.20230726T030000Z.nc4 | 3 + .../data/ctrl_amsua_n19.20230726T090000Z.nc4 | 3 + .../data/exp_amsua_n19.20230726T030000Z.nc4 | 3 + .../data/exp_amsua_n19.20230726T090000Z.nc4 | 3 + src/eva/time_series/time_series.py | 8 +- 8 files changed, 242 insertions(+), 93 deletions(-) create mode 100644 src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml create mode 100644 src/eva/tests/data/ctrl_amsua_n19.20230726T030000Z.nc4 create mode 100644 src/eva/tests/data/ctrl_amsua_n19.20230726T090000Z.nc4 create mode 100644 src/eva/tests/data/exp_amsua_n19.20230726T030000Z.nc4 create mode 100644 src/eva/tests/data/exp_amsua_n19.20230726T090000Z.nc4 diff --git a/src/eva/data/data_collections.py b/src/eva/data/data_collections.py index 0b8a711..105851c 100644 --- a/src/eva/data/data_collections.py +++ b/src/eva/data/data_collections.py @@ -65,9 +65,9 @@ def create_or_add_to_collection(self, collection_name, collection, concat_dimens """ # If time_series collection name must also be time_series - if self.time_series and collection_name != 'time_series': - self.logger.abort('In create_or_add_to_collection: time_series collection must ' + - 'be \'time_series\'') + if self.time_series and 'time_series' not in collection_name: + self.logger.abort('In get_variable_data: time_series collection must ' + + 'have name containing \'time_series\'') # Collections should only be xarray datasets if not isinstance(collection, Dataset): @@ -158,9 +158,9 @@ def add_variable_to_collection(self, collection_name, group_name, variable_name, """ # If time_series collection name must also be time_series - if self.time_series and collection_name != 'time_series': - self.logger.abort('In add_variable_to_collection: time_series collection must ' + - 'be \'time_series\'') + if self.time_series and 'time_series' not in collection_name: + self.logger.abort('In get_variable_data: time_series collection must ' + + 'have name containing \'time_series\'') # Assert that new variable is an xarray Dataarray if not isinstance(variable, DataArray): @@ -211,9 +211,9 @@ def get_variable_data_array(self, collection_name, group_name, variable_name, """ # If time_series collection name must also be time_series - if self.time_series and collection_name != 'time_series': - self.logger.abort('In get_variable_data_array: time_series collection must ' + - 'be \'time_series\'') + if self.time_series and 'time_series' not in collection_name: + self.logger.abort('In get_variable_data: time_series collection must ' + + 'have name containing \'time_series\'') group_variable_name = group_name + '::' + variable_name data_array = self._collections[collection_name][group_variable_name] @@ -293,9 +293,9 @@ def get_variable_data(self, collection_name, group_name, variable_name, """ # If time_series collection name must also be time_series - if self.time_series and collection_name != 'time_series': + if self.time_series and 'time_series' not in collection_name: self.logger.abort('In get_variable_data: time_series collection must ' + - 'be \'time_series\'') + 'have name containing \'time_series\'') variable_array = self.get_variable_data_array(collection_name, group_name, variable_name, channels, levels, datatypes) diff --git a/src/eva/eva_driver.py b/src/eva/eva_driver.py index 7c738a0..2a9dec9 100644 --- a/src/eva/eva_driver.py +++ b/src/eva/eva_driver.py @@ -12,6 +12,7 @@ from datetime import datetime import argparse import os +from collections import defaultdict from eva.utilities.config import get from eva.utilities.logger import Logger @@ -94,85 +95,102 @@ def read_transform_time_series(logger, timing, eva_dict, data_collections): None """ - # Check for required keys - # ----------------------- - required_keys = [ - 'begin_date', - 'final_date', - 'interval', - 'collection', - 'variables', - ] - for key in required_keys: - logger.assert_abort(key in eva_dict['time_series'], 'If running Eva in time series ' + - f'mode the time series config must contain "{key}"') - - # Write message that this is a time series run - logger.info('This instance of Eva is being used to accumulate a time series.') + # Iterate through list of time series dictionaries + for time_series_config in eva_dict['time_series']: + + # Check for required keys + # ----------------------- + required_keys = [ + 'begin_date', + 'final_date', + 'interval', + 'collection', + 'variables', + ] + for key in required_keys: + logger.assert_abort(key in time_series_config, 'If running Eva in time series ' + + f'mode the time series config must contain "{key}"') + + # Write message that this is a time series run + logger.info('This instance of Eva is being used to accumulate a time series.') + + # Optionally suppress the display of the collection + suppress_collection_display = get(eva_dict, logger, 'suppress_collection_display', False) + + # Extract the dates of the time series + begin_date = time_series_config['begin_date'] + final_date = time_series_config['final_date'] + interval = time_series_config['interval'] + + # Convert begin and end dates from ISO strings to datetime objects + begin_date = datetime.fromisoformat(begin_date) + final_date = datetime.fromisoformat(final_date) + + # Convert interval ISO string to timedelta object + interval = iso_duration_to_timedelta(logger, interval) + + # Make list of dates from begin to end with interval + dates = [] + date = begin_date + count = 0 + while date <= final_date: + dates.append(date) + date += interval + count += 1 + # Abort if count hits one million + logger.assert_abort(count < 1e6, 'You are planning to read more than one million ' + + 'time steps. This is likely an error. Please check your ' + + 'configuration.') + + # Get all datasets configuration + all_datasets = get(eva_dict, logger, 'datasets') + + # Find all dataset_configs with collection name + datasets_config = [] + for dataset in all_datasets: + if dataset['name'] == time_series_config["collection"]: + datasets_config.append(dataset) + + # Save transforms to transform_dict based on collection + transform_dict = defaultdict(list) + for transform in get(eva_dict, logger, 'transforms'): + # Get collection name + name = transform['new name'].split('::')[0] + if name == time_series_config['collection']: + transform_dict['transforms'].append(transform) + + # Assert that datasets_config is the same length as dates + logger.assert_abort(len(datasets_config) == len(dates), 'When running in time ' + + 'series mode the number of datasets must be the same as the ' + + 'number of dates.') + + # Loop over datasets reading each one in turn, internally appending the data_collections + for ind, dataset_config in enumerate(datasets_config): + + # Create a temporary collection for this time step + data_collections_tmp = DataCollections() + + # Prepare diagnostic data + logger.info('Running data driver') + timing.start('DataDriverExecute') + data_driver(dataset_config, data_collections_tmp, timing, logger) + timing.stop('DataDriverExecute') + + # Perform any transforms on the fly + if 'transforms' in transform_dict: + logger.info(f'Running transform driver') + timing.start('TransformDriverExecute') + transform_driver(transform_dict, data_collections_tmp, timing, logger) + timing.stop('TransformDriverExecute') + + # Collapse data into time series + date = dates[ind] + collapse_collection_to_time_series(logger, ind, date, time_series_config, + data_collections, data_collections_tmp) - # Optionally suppress the display of the collection - suppress_collection_display = get(eva_dict, logger, 'suppress_collection_display', False) - - # Get the datasets configuration - time_series_config = eva_dict['time_series'] - - # Extract the dates of the time series - begin_date = time_series_config['begin_date'] - final_date = time_series_config['final_date'] - interval = time_series_config['interval'] - - # Convert begin and end dates from ISO strings to datetime objects - begin_date = datetime.fromisoformat(begin_date) - final_date = datetime.fromisoformat(final_date) - - # Convert interval ISO string to timedelta object - interval = iso_duration_to_timedelta(logger, interval) - - # Make list of dates from begin to end with interval - dates = [] - date = begin_date - count = 0 - while date <= final_date: - dates.append(date) - date += interval - count += 1 - # Abort if count hits one million - logger.assert_abort(count < 1e6, 'You are planning to read more than one million ' + - 'time steps. This is likely an error. Please check your configuration.') - - # Get the datasets configuration - datasets_config = get(eva_dict, logger, 'datasets') - - # Assert that datasets_config is the same length as dates - logger.assert_abort(len(datasets_config) == len(dates), 'When running in time series mode ' + - 'the number of datasets must be the same as the number of dates.') - - # Loop over datasets reading each one in turn, internally appending the data_collections - for ind, dataset_config in enumerate(datasets_config): - - # Create a temporary collection for this time step - data_collections_tmp = DataCollections() - - # Prepare diagnostic data - logger.info('Running data driver') - timing.start('DataDriverExecute') - data_driver(dataset_config, data_collections_tmp, timing, logger) - timing.stop('DataDriverExecute') - - # Perform any transforms on the fly - if 'transforms' in eva_dict: - logger.info(f'Running transform driver') - timing.start('TransformDriverExecute') - transform_driver(eva_dict, data_collections_tmp, timing, logger) - timing.stop('TransformDriverExecute') - - # Collapse data into time series - collapse_collection_to_time_series(logger, ind, dates, time_series_config, data_collections, - data_collections_tmp) - - if not suppress_collection_display: - logger.info('Computing of Eva time series complete: status of collection:') - data_collections.display_collections() + if not suppress_collection_display: + logger.info('Computing of Eva time series complete: status of collection:') + data_collections.display_collections() # -------------------------------------------------------------------------------------------------- diff --git a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml new file mode 100644 index 0000000..a05de02 --- /dev/null +++ b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml @@ -0,0 +1,119 @@ +suppress_collection_display: false +datasets: +- name: control + type: IodaObsSpace + filenames: + - ${data_input_path}/amsua_n19.20230726T030000Z.nc4 + channels: 3,8 + groups: + - name: ObsValue + variables: + - brightnessTemperature + - name: GsiHofXBc + - name: hofx0 + - name: MetaData + - name: oman +- name: control + type: IodaObsSpace + filenames: + - ${data_input_path}/amsua_n19.20230726T090000Z.nc4 + channels: 3,8 + groups: + - name: ObsValue + variables: + - brightnessTemperature + - name: GsiHofXBc + - name: hofx0 + - name: MetaData + - name: oman +- name: experiment + type: IodaObsSpace + filenames: + - ${data_input_path}/amsua_n19.20230726T030000Z.nc4 + channels: 3,8 + groups: + - name: ObsValue + variables: + - brightnessTemperature + - name: GsiHofXBc + - name: hofx0 + - name: MetaData + - name: oman +- name: experiment + type: IodaObsSpace + filenames: + - ${data_input_path}/amsua_n19.20230726T090000Z.nc4 + channels: 3,8 + groups: + - name: ObsValue + variables: + - brightnessTemperature + - name: GsiHofXBc + - name: hofx0 + - name: MetaData + - name: oman +transforms: +- transform: arithmetic + new name: control::ObsValueMinusHofx::${variable} + equals: control::ObsValue::${variable}-control::hofx0::${variable} + for: + variable: &id001 + - brightnessTemperature +- transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx0::${variable} + for: + variable: *id001 +time_series: +- begin_date: '2023-07-26T03:00:00' + final_date: '2023-07-26T09:00:00' + interval: PT6H + collection: experiment + variables: + - ObsValueMinusHofx::brightnessTemperature + aggregation_methods: + - mean + dimension: Location +- begin_date: '2023-07-26T03:00:00' + final_date: '2023-07-26T09:00:00' + interval: PT6H + collection: control + variables: + - ObsValueMinusHofx::brightnessTemperature + aggregation_methods: + - mean + dimension: Location +graphics: + plotting_backend: Emcpy + figure_list: + - figure: + layout: + - 1 + - 1 + title: Mean OmB | AMSU-A NOAA-19 | Ch 3 | ObsValueMinusHofx::brightnessTemperature + output name: time_series/amsua_n19/brightnessTemperature_mean/3/time_series_compare_omb.png + plots: + - add_xlabel: Datetime + add_ylabel: JEDI h(x) + add_grid: null + add_legend: + loc: upper left + layers: + - type: LinePlot + x: + variable: control_time_series::MetaData::Dates + y: + variable: control_time_series::ObsValueMinusHofx::brightnessTemperature_mean + channel: 3 + markersize: 5 + color: black + label: Observation minus h(x) - ctrl + - type: LinePlot + x: + variable: experiment_time_series::MetaData::Dates + y: + variable: experiment_time_series::ObsValueMinusHofx::brightnessTemperature_mean + channel: 3 + markersize: 5 + color: blue + label: Observation minus h(x) - exp diff --git a/src/eva/tests/data/ctrl_amsua_n19.20230726T030000Z.nc4 b/src/eva/tests/data/ctrl_amsua_n19.20230726T030000Z.nc4 new file mode 100644 index 0000000..a9abbb0 --- /dev/null +++ b/src/eva/tests/data/ctrl_amsua_n19.20230726T030000Z.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203397d73233ef2cb87a683db53cb58180293b0267d11adbcb51ae43b398db13 +size 15913177 diff --git a/src/eva/tests/data/ctrl_amsua_n19.20230726T090000Z.nc4 b/src/eva/tests/data/ctrl_amsua_n19.20230726T090000Z.nc4 new file mode 100644 index 0000000..028ffca --- /dev/null +++ b/src/eva/tests/data/ctrl_amsua_n19.20230726T090000Z.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d758f7d798ee53eff82b7d6cbdacfa36ad7b5ad59378a8bad1885ca22c1d6394 +size 16885881 diff --git a/src/eva/tests/data/exp_amsua_n19.20230726T030000Z.nc4 b/src/eva/tests/data/exp_amsua_n19.20230726T030000Z.nc4 new file mode 100644 index 0000000..54e40ea --- /dev/null +++ b/src/eva/tests/data/exp_amsua_n19.20230726T030000Z.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d977a42b4bd9283d2515bf1cce53a4cb0bb887da66dd03bc783738562d861b45 +size 15913177 diff --git a/src/eva/tests/data/exp_amsua_n19.20230726T090000Z.nc4 b/src/eva/tests/data/exp_amsua_n19.20230726T090000Z.nc4 new file mode 100644 index 0000000..c4de4c4 --- /dev/null +++ b/src/eva/tests/data/exp_amsua_n19.20230726T090000Z.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f21b2df6d766c6decad154345d6872e7a3a4db2235bf002213890b45678e845 +size 16885881 diff --git a/src/eva/time_series/time_series.py b/src/eva/time_series/time_series.py index 399356e..4120653 100644 --- a/src/eva/time_series/time_series.py +++ b/src/eva/time_series/time_series.py @@ -26,7 +26,7 @@ # -------------------------------------------------------------------------------------------------- -def collapse_collection_to_time_series(logger, ind, dates, time_series_config, data_collections, +def collapse_collection_to_time_series(logger, ind, date, time_series_config, data_collections, data_collections_tmp): # Parse the configuration @@ -83,7 +83,7 @@ def collapse_collection_to_time_series(logger, ind, dates, time_series_config, d # ------------------------------------------------------------------------- dims = {dim: dataset_aggregated.sizes[dim] for dim in dataset_aggregated.dims} data_array_shape = tuple(dims[dim] for dim in dims) - dataset_aggregated['MetaData::Dates'] = xr.DataArray(np.full(data_array_shape, dates[ind]), + dataset_aggregated['MetaData::Dates'] = xr.DataArray(np.full(data_array_shape, date), dims=dataset_aggregated.dims) # Add the time index to the aggregated dataset @@ -93,8 +93,8 @@ def collapse_collection_to_time_series(logger, ind, dates, time_series_config, d # Append the dataset with the aggregation concat_dimension = 'TimeIndex' if ind > 0 else None - data_collections.create_or_add_to_collection('time_series', dataset_aggregated, - concat_dimension) + data_collections.create_or_add_to_collection(f'{collection_to_ts}_time_series', + dataset_aggregated, concat_dimension) # -------------------------------------------------------------------------------------------------- From 232f57e05c0c733af4f35a9b6807e9782f13a1b5 Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 12:41:46 -0400 Subject: [PATCH 2/6] test fix --- .../testIodaObsSpaceAmsuaN19_TimeSeries.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml index 477ceb2..f907c7c 100644 --- a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml +++ b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml @@ -30,16 +30,16 @@ transforms: time_series: - begin_date: '2020-12-14T21:00:00' - final_date: '2020-12-15T03:00:00' - interval: 'PT6H' + - begin_date: '2020-12-14T21:00:00' + final_date: '2020-12-15T03:00:00' + interval: 'PT6H' - collection: experiment - variables: - - ObsValueMinusHofx::brightnessTemperature - aggregation_methods: - - mean - dimension: Location + collection: experiment + variables: + - ObsValueMinusHofx::brightnessTemperature + aggregation_methods: + - mean + dimension: Location graphics: From 2d204c97af904d98be24f9056f733d7eada9fb27 Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 13:56:44 -0400 Subject: [PATCH 3/6] test fixes --- .github/workflows/yaml_coding_norms.yml | 4 ++-- requirements-github.txt | 2 +- src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/yaml_coding_norms.yml b/.github/workflows/yaml_coding_norms.yml index d56e1e5..fa115b7 100644 --- a/.github/workflows/yaml_coding_norms.yml +++ b/.github/workflows/yaml_coding_norms.yml @@ -18,7 +18,7 @@ jobs: name: Check YAML Coding Norms runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - id: yaml-lint name: yaml-lint uses: ibiqlik/action-yamllint@v3 @@ -27,7 +27,7 @@ jobs: format: colored config_file: .yamllint.yml - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: yamllint-logfile diff --git a/requirements-github.txt b/requirements-github.txt index 789d706..2de8171 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -8,7 +8,7 @@ xarray>=2022.6.0 seaborn>=0.12.2 hvplot>=0.8.2 nbconvert>=6.5.4 -bokeh<3.5.0,>=3.4.0 +bokeh<3.6.0,>=3.5.0 geopandas>=0.13.2 geoviews>=1.10.0 nbsite diff --git a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml index f907c7c..0d085cf 100644 --- a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml +++ b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_TimeSeries.yaml @@ -63,9 +63,9 @@ graphics: layers: - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::ObsValueMinusHofx::brightnessTemperature_mean + variable: experiment_time_series::ObsValueMinusHofx::brightnessTemperature_mean channel: 3 markersize: 5 color: 'black' From d4adbfca279aa8de60f5232638ed8f882edc59a1 Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 14:06:35 -0400 Subject: [PATCH 4/6] application test fix --- ...JediVariationalBiasCorrectionAmsuaN19.yaml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/eva/tests/config/testJediVariationalBiasCorrectionAmsuaN19.yaml b/src/eva/tests/config/testJediVariationalBiasCorrectionAmsuaN19.yaml index 60cd9bb..c993a32 100644 --- a/src/eva/tests/config/testJediVariationalBiasCorrectionAmsuaN19.yaml +++ b/src/eva/tests/config/testJediVariationalBiasCorrectionAmsuaN19.yaml @@ -13,13 +13,13 @@ datasets: time_series: - begin_date: '2020-12-15T00:00:00' - final_date: '2020-12-15T06:00:00' - interval: 'PT6H' + - begin_date: '2020-12-15T00:00:00' + final_date: '2020-12-15T06:00:00' + interval: 'PT6H' - collection: experiment - variables: - - all + collection: experiment + variables: + - all graphics: @@ -43,45 +43,45 @@ graphics: layers: - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::BiasCoefficients::constant + variable: experiment_time_series::BiasCoefficients::constant channel: 7 markersize: 5 color: 'blue' label: 'BiasCoefficients::constant' - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::BiasCoefficients::sensorScanAngle + variable: experiment_time_series::BiasCoefficients::sensorScanAngle channel: 7 markersize: 5 color: 'orange' label: 'BiasCoefficients::sensorScanAngle' - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::BiasCoefficients::sensorScanAngle_order_2 + variable: experiment_time_series::BiasCoefficients::sensorScanAngle_order_2 channel: 7 markersize: 5 color: 'green' label: 'BiasCoefficients::sensorScanAngle_order_2' - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::BiasCoefficients::sensorScanAngle_order_3 + variable: experiment_time_series::BiasCoefficients::sensorScanAngle_order_3 channel: 7 markersize: 5 color: 'red' label: 'BiasCoefficients::sensorScanAngle_order_3' - type: LinePlot x: - variable: time_series::MetaData::Dates + variable: experiment_time_series::MetaData::Dates y: - variable: time_series::BiasCoefficients::sensorScanAngle_order_4 + variable: experiment_time_series::BiasCoefficients::sensorScanAngle_order_4 channel: 7 markersize: 5 color: 'purple' From c70caed2cdc5d61098f1416c3f07680145c70ee0 Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 14:27:54 -0400 Subject: [PATCH 5/6] testing --- src/eva/eva_driver.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/eva/eva_driver.py b/src/eva/eva_driver.py index 2a9dec9..d092762 100644 --- a/src/eva/eva_driver.py +++ b/src/eva/eva_driver.py @@ -153,11 +153,12 @@ def read_transform_time_series(logger, timing, eva_dict, data_collections): # Save transforms to transform_dict based on collection transform_dict = defaultdict(list) - for transform in get(eva_dict, logger, 'transforms'): - # Get collection name - name = transform['new name'].split('::')[0] - if name == time_series_config['collection']: - transform_dict['transforms'].append(transform) + if 'transforms' in eva_dict: + for transform in get(eva_dict, logger, 'transforms'): + # Get collection name + name = transform['new name'].split('::')[0] + if name == time_series_config['collection']: + transform_dict['transforms'].append(transform) # Assert that datasets_config is the same length as dates logger.assert_abort(len(datasets_config) == len(dates), 'When running in time ' + @@ -177,7 +178,7 @@ def read_transform_time_series(logger, timing, eva_dict, data_collections): timing.stop('DataDriverExecute') # Perform any transforms on the fly - if 'transforms' in transform_dict: + if transform_dict: logger.info(f'Running transform driver') timing.start('TransformDriverExecute') transform_driver(transform_dict, data_collections_tmp, timing, logger) From 02808b58c4c96e53fb3a1a8dcb084869654a6237 Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 8 Oct 2024 15:02:17 -0400 Subject: [PATCH 6/6] fix application test --- .../testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml index a05de02..5f78aa5 100644 --- a/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml +++ b/src/eva/tests/config/testIodaObsSpaceAmsuaN19_Multiple_TimeSeries.yaml @@ -3,7 +3,7 @@ datasets: - name: control type: IodaObsSpace filenames: - - ${data_input_path}/amsua_n19.20230726T030000Z.nc4 + - ${data_input_path}/ctrl_amsua_n19.20230726T030000Z.nc4 channels: 3,8 groups: - name: ObsValue @@ -16,7 +16,7 @@ datasets: - name: control type: IodaObsSpace filenames: - - ${data_input_path}/amsua_n19.20230726T090000Z.nc4 + - ${data_input_path}/ctrl_amsua_n19.20230726T090000Z.nc4 channels: 3,8 groups: - name: ObsValue @@ -29,7 +29,7 @@ datasets: - name: experiment type: IodaObsSpace filenames: - - ${data_input_path}/amsua_n19.20230726T030000Z.nc4 + - ${data_input_path}/exp_amsua_n19.20230726T030000Z.nc4 channels: 3,8 groups: - name: ObsValue @@ -42,7 +42,7 @@ datasets: - name: experiment type: IodaObsSpace filenames: - - ${data_input_path}/amsua_n19.20230726T090000Z.nc4 + - ${data_input_path}/exp_amsua_n19.20230726T090000Z.nc4 channels: 3,8 groups: - name: ObsValue