diff --git a/.gitignore b/.gitignore index b6e47617..4830b678 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ dmypy.json # Pyre type checker .pyre/ + +# Swap files +*.swp +*.swo diff --git a/requirements-github.txt b/requirements-github.txt index 449c5ad0..881fcd31 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -12,3 +12,4 @@ bokeh>=3.1.1 geopandas>=0.13.2 geoviews>=1.10.0 nbsite +git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy diff --git a/requirements-spackstack1.4.txt b/requirements-spackstack1.4.txt new file mode 100644 index 00000000..3cb461b3 --- /dev/null +++ b/requirements-spackstack1.4.txt @@ -0,0 +1,25 @@ +# Packages in spack stack +setuptools==59.4.0 +pyyaml==6.0 +pycodestyle==2.8.0 +netCDF4==1.5.3 +matplotlib==3.7.1 +cartopy==0.21.1 +scipy==1.9.3 +xarray==2022.3.0 +pandas==1.4.0 +numpy==1.22.3 + +# Not explicitly part of eva but dependcies of eva dependencies already in spack-stack +# versions need to be set to avoid other versions being picked +pyproj==3.1.0 +importlib-metadata==4.8.2 + +# Additional packages +git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy +scikit-learn +seaborn +hvplot +nbconvert +bokeh +geopandas diff --git a/requirements.txt b/requirements.txt index 9813251c..e46c37ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,19 @@ +setuptools>=59.4.0 pyyaml>=6.0 pycodestyle>=2.8.0 netCDF4>=1.5.3 -matplotlib>=3.5.2 -cartopy>=0.20.2 -scikit-learn>=1.0.2 -xarray>=0.11.3 +matplotlib>=3.7.1 +cartopy>=0.21.1 +scipy>=1.9.3 +xarray>=2022.3.0 +pandas>=1.4.0 +numpy>=1.22.3 + +# Additional packages +git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy +scikit-learn +seaborn +hvplot +nbconvert +bokeh +geopandas diff --git a/requirements_emc.txt b/requirements_emc.txt new file mode 100644 index 00000000..7b52934d --- /dev/null +++ b/requirements_emc.txt @@ -0,0 +1,13 @@ +setuptools>=59.4.0 +pyyaml>=6.0 +pycodestyle>=2.8.0 +netCDF4>=1.5.3 +matplotlib>=3.7.1 +cartopy>=0.21.1 +scipy>=1.9.3 +xarray>=2022.3.0 +pandas>=1.4.0 +numpy>=1.22.3 + +# Additional packages +git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy diff --git a/setup.py b/setup.py index 8f0f52da..a0fe66f8 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import setuptools + setuptools.setup( name='eva', version='1.5.0', @@ -31,20 +32,6 @@ 'Natural Language :: English', 'Operating System :: OS Independent'], python_requires='>=3.6', - install_requires=[ - 'pyyaml>=6.0', - 'pycodestyle>=2.8.0', - 'netCDF4>=1.5.3', - 'matplotlib>=3.5.2', - 'cartopy>=0.18.0', - 'scikit-learn>=1.0.2', - 'xarray>=0.11.3', - 'emcpy @ git+https://github.com/NOAA-EMC/' + - 'emcpy@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy', - - # Option dependency for making density plots - # 'seaborn>=0.12', - ], package_data={ '': [ 'tests/config/*', diff --git a/src/eva/eva_driver.py b/src/eva/eva_driver.py index 546a1dab..c9e79470 100644 --- a/src/eva/eva_driver.py +++ b/src/eva/eva_driver.py @@ -14,7 +14,7 @@ from eva.utilities.timing import Timing from eva.data.data_driver import data_driver from eva.transforms.transform_driver import transform_driver -from eva.plotting.emcpy.plot_tools.figure_driver import figure_driver +from eva.plotting.batch.base.plot_tools.figure_driver import figure_driver from eva.data.data_collections import DataCollections from eva.utilities.utils import load_yaml_file import argparse diff --git a/src/eva/eva_interactive.py b/src/eva/eva_interactive.py index 37b84111..0112b31c 100644 --- a/src/eva/eva_interactive.py +++ b/src/eva/eva_interactive.py @@ -23,7 +23,7 @@ from eva.transforms.arithmetic import arithmetic, generate_arithmetic_config from eva.transforms.accept_where import accept_where, generate_accept_where_config -import eva.plotting.hvplot.interactive_plot_tools as plot +import eva.plotting.interactive.interactive_plot_tools as plot # -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/__init__.py b/src/eva/plotting/batch/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/base/__init__.py b/src/eva/plotting/batch/base/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/base/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/emcpy/__init__.py b/src/eva/plotting/batch/base/diagnostics/__init__.py similarity index 100% rename from src/eva/plotting/emcpy/__init__.py rename to src/eva/plotting/batch/base/diagnostics/__init__.py diff --git a/src/eva/plotting/emcpy/diagnostics/density.py b/src/eva/plotting/batch/base/diagnostics/density.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/density.py rename to src/eva/plotting/batch/base/diagnostics/density.py diff --git a/src/eva/plotting/emcpy/diagnostics/histogram.py b/src/eva/plotting/batch/base/diagnostics/histogram.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/histogram.py rename to src/eva/plotting/batch/base/diagnostics/histogram.py diff --git a/src/eva/plotting/emcpy/diagnostics/horizontal_line.py b/src/eva/plotting/batch/base/diagnostics/horizontal_line.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/horizontal_line.py rename to src/eva/plotting/batch/base/diagnostics/horizontal_line.py diff --git a/src/eva/plotting/emcpy/diagnostics/line_plot.py b/src/eva/plotting/batch/base/diagnostics/line_plot.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/line_plot.py rename to src/eva/plotting/batch/base/diagnostics/line_plot.py diff --git a/src/eva/plotting/emcpy/diagnostics/map_gridded.py b/src/eva/plotting/batch/base/diagnostics/map_gridded.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/map_gridded.py rename to src/eva/plotting/batch/base/diagnostics/map_gridded.py diff --git a/src/eva/plotting/emcpy/diagnostics/map_scatter.py b/src/eva/plotting/batch/base/diagnostics/map_scatter.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/map_scatter.py rename to src/eva/plotting/batch/base/diagnostics/map_scatter.py diff --git a/src/eva/plotting/batch/base/diagnostics/scatter.py b/src/eva/plotting/batch/base/diagnostics/scatter.py new file mode 100644 index 00000000..f3d743e4 --- /dev/null +++ b/src/eva/plotting/batch/base/diagnostics/scatter.py @@ -0,0 +1,103 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import os +import numpy as np + +from abc import ABC, abstractmethod + +# -------------------------------------------------------------------------------------------------- + + +class Scatter(ABC): + + """Base class for creating scatter plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a scatter plot on a map based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the scatter plot on a map. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a scatter plot on a map based on the provided + configuration. The scatter plot is created using a declarative plotting library from EMCPy + (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "longitude": {"variable": "collection::group::variable"}, + "latitude": {"variable": "collection::group::variable"}, + "data": {"variable": "collection::group::variable", + "channel": "channel_name"}, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + map_scatter_plot = MapScatter(config, logger, None) + """ + self.config = config + self.logger = logger + self.dataobj = dataobj + self.xdata = [] + self.ydata = [] + self.plotobj = None + +# -------------------------------------------------------------------------------------------------- + + def data_prep(self): + + # Get the data to plot from the data_collection + # --------------------------------------------- + var0 = self.config['x']['variable'] + var1 = self.config['y']['variable'] + + var0_cgv = var0.split('::') + var1_cgv = var1.split('::') + + if len(var0_cgv) != 3: + self.logger.abort('Scatter: comparison first var \'var0\' does not appear to ' + + 'be in the required format of collection::group::variable.') + if len(var1_cgv) != 3: + self.logger.abort('Scatter: comparison first var \'var1\' does not appear to ' + + 'be in the required format of collection::group::variable.') + + # Optionally get the channel to plot + channel = None + if 'channel' in self.config: + channel = self.config.get('channel') + + xdata = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel) + xdata1 = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2]) + ydata = self.dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel) + + # see if we need to slice data + xdata = slice_var_from_str(self.config['x'], xdata, self.logger) + ydata = slice_var_from_str(self.config['y'], ydata, self.logger) + + # scatter data should be flattened + xdata = xdata.flatten() + ydata = ydata.flatten() + + # Remove NaN values to enable regression + # -------------------------------------- + mask = ~np.isnan(xdata) + xdata = xdata[mask] + ydata = ydata[mask] + + mask = ~np.isnan(ydata) + self.xdata = xdata[mask] + self.ydata = ydata[mask] + + @abstractmethod + def configure_plot(self): + pass + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/emcpy/diagnostics/vertical_line.py b/src/eva/plotting/batch/base/diagnostics/vertical_line.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/vertical_line.py rename to src/eva/plotting/batch/base/diagnostics/vertical_line.py diff --git a/src/eva/plotting/emcpy/diagnostics/__init__.py b/src/eva/plotting/batch/base/plot_tools/__init__.py similarity index 100% rename from src/eva/plotting/emcpy/diagnostics/__init__.py rename to src/eva/plotting/batch/base/plot_tools/__init__.py diff --git a/src/eva/plotting/emcpy/plot_tools/dynamic_config.py b/src/eva/plotting/batch/base/plot_tools/dynamic_config.py similarity index 100% rename from src/eva/plotting/emcpy/plot_tools/dynamic_config.py rename to src/eva/plotting/batch/base/plot_tools/dynamic_config.py diff --git a/src/eva/plotting/emcpy/plot_tools/figure_driver.py b/src/eva/plotting/batch/base/plot_tools/figure_driver.py similarity index 78% rename from src/eva/plotting/emcpy/plot_tools/figure_driver.py rename to src/eva/plotting/batch/base/plot_tools/figure_driver.py index 706c65e9..28464d38 100644 --- a/src/eva/plotting/emcpy/plot_tools/figure_driver.py +++ b/src/eva/plotting/batch/base/plot_tools/figure_driver.py @@ -14,7 +14,6 @@ from eva.utilities.stats import stats_helper from eva.utilities.utils import get_schema, camelcase_to_underscore, parse_channel_list from eva.utilities.utils import replace_vars_dict -from emcpy.plots.create_plots import CreatePlot, CreateFigure import copy import importlib as im import os @@ -40,7 +39,32 @@ def figure_driver(config, data_collections, timing, logger): # Get list of graphics from configuration # ------------------- - graphics = config.get("graphics") + graphics_section = config.get('graphics') + graphics = graphics_section.get('figure_list') + + # Get plotting backend + # -------------------- + backend = graphics_section.get('plotting_backend') + + if backend not in ['Emcpy', 'Hvplot']: + logger.abort('Backend not found. \ + Available backends: Emcpy, Hvplot') + + if backend == 'Hvplot': + try: + import hvplot + except ImportError: + logger.abort("The hvplot backend is not available since \ + hvplot is not in the environment.") + + # Create handler + # -------------- + handler_class_name = backend + 'FigureHandler' + handler_module_name = camelcase_to_underscore(handler_class_name) + handler_full_module = 'eva.plotting.batch.' + \ + backend.lower() + '.plot_tools.' + handler_module_name + handler_class = getattr(im.import_module(handler_full_module), handler_class_name) + handler = handler_class() # Loop through specified graphics # ------------------- @@ -56,8 +80,8 @@ def figure_driver(config, data_collections, timing, logger): # update figure conf based on schema # ---------------------------------- - fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting', - 'emcpy', 'defaults', 'figure.yaml')) + fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting', 'batch', + backend.lower(), 'defaults', 'figure.yaml')) figure_conf = get_schema(fig_schema, figure_conf, logger) # pass configurations and make graphic(s) @@ -111,19 +135,20 @@ def figure_driver(config, data_collections, timing, logger): **batch_conf_this) # Make plot - make_figure(figure_conf_fill, plots_conf_fill, + make_figure(handler, figure_conf_fill, plots_conf_fill, dynamic_options_conf_fill, data_collections, logger) else: # make just one figure per configuration - make_figure(figure_conf, plots_conf, dynamic_options_conf, data_collections, logger) + make_figure(handler, figure_conf, plots_conf, + dynamic_options_conf, data_collections, logger) timing.stop('Graphics Loop') # -------------------------------------------------------------------------------------------------- -def make_figure(figure_conf, plots, dynamic_options, data_collections, logger): +def make_figure(handler, figure_conf, plots, dynamic_options, data_collections, logger): """ Generates a figure based on the provided configuration and plots. @@ -143,7 +168,8 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger): # Adjust the plots configs if there are dynamic options # ----------------------------------------------------- for dynamic_option in dynamic_options: - dynamic_option_module = im.import_module("eva.plotting.emcpy.plot_tools.dynamic_config") + mod_name = "eva.plotting.batch.base.plot_tools.dynamic_config" + dynamic_option_module = im.import_module(mod_name) dynamic_option_method = getattr(dynamic_option_module, dynamic_option['type']) plots = dynamic_option_method(logger, dynamic_option, plots, data_collections) @@ -158,12 +184,23 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger): for plot in plots: layer_list = [] for layer in plot.get("layers"): - eva_class_name = layer.get("type") - eva_module_name = camelcase_to_underscore(eva_class_name) - full_module = "eva.plotting.emcpy.diagnostics."+eva_module_name - layer_class = getattr(im.import_module(full_module), eva_class_name) - # use the translator class to go from eva to declarative plotting - layer_list.append(layer_class(layer, logger, data_collections).plotobj) + + # Temporary case to handle different diagnostics + if handler.BACKEND_NAME == 'Emcpy': + eva_class_name = layer.get("type") + eva_module_name = camelcase_to_underscore(eva_class_name) + full_module = "eva.plotting.batch.emcpy.diagnostics."+eva_module_name + layer_class = getattr(im.import_module(full_module), eva_class_name) + layer_list.append(layer_class(layer, logger, data_collections).plotobj) + else: + eva_class_name = handler.BACKEND_NAME + layer.get("type") + eva_module_name = camelcase_to_underscore(eva_class_name) + full_module = handler.MODULE_NAME + eva_module_name + layer_class = getattr(im.import_module(full_module), eva_class_name) + layer = layer_class(layer, logger, data_collections) + layer.data_prep() + layer_list.append(layer.configure_plot()) + # get mapping dictionary proj = None domain = None @@ -174,7 +211,7 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger): domain = mapoptions['domain'] # create a subplot based on specified layers - plotobj = CreatePlot(plot_layers=layer_list, projection=proj, domain=domain) + plotobj = handler.create_plot(layer_list, proj, domain) # make changes to subplot based on YAML configuration for key, value in plot.items(): if key not in ['layers', 'mapping', 'statistics']: @@ -191,9 +228,10 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger): plot_list.append(plotobj) # create figure - fig = CreateFigure(nrows=figure_conf['layout'][0], - ncols=figure_conf['layout'][1], - figsize=tuple(figure_conf['figure size'])) + nrows = figure_conf['layout'][0] + ncols = figure_conf['layout'][1] + figsize = tuple(figure_conf['figure size']) + fig = handler.create_figure(nrows, ncols, figsize) fig.plot_list = plot_list fig.create_figure() diff --git a/src/eva/plotting/emcpy/plot_tools/__init__.py b/src/eva/plotting/batch/emcpy/__init__.py similarity index 100% rename from src/eva/plotting/emcpy/plot_tools/__init__.py rename to src/eva/plotting/batch/emcpy/__init__.py diff --git a/src/eva/plotting/emcpy/defaults/density.yaml b/src/eva/plotting/batch/emcpy/defaults/density.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/density.yaml rename to src/eva/plotting/batch/emcpy/defaults/density.yaml diff --git a/src/eva/plotting/emcpy/defaults/figure.yaml b/src/eva/plotting/batch/emcpy/defaults/figure.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/figure.yaml rename to src/eva/plotting/batch/emcpy/defaults/figure.yaml diff --git a/src/eva/plotting/emcpy/defaults/histogram.yaml b/src/eva/plotting/batch/emcpy/defaults/histogram.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/histogram.yaml rename to src/eva/plotting/batch/emcpy/defaults/histogram.yaml diff --git a/src/eva/plotting/emcpy/defaults/horizontal_line.yaml b/src/eva/plotting/batch/emcpy/defaults/horizontal_line.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/horizontal_line.yaml rename to src/eva/plotting/batch/emcpy/defaults/horizontal_line.yaml diff --git a/src/eva/plotting/emcpy/defaults/line_plot.yaml b/src/eva/plotting/batch/emcpy/defaults/line_plot.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/line_plot.yaml rename to src/eva/plotting/batch/emcpy/defaults/line_plot.yaml diff --git a/src/eva/plotting/emcpy/defaults/map_gridded.yaml b/src/eva/plotting/batch/emcpy/defaults/map_gridded.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/map_gridded.yaml rename to src/eva/plotting/batch/emcpy/defaults/map_gridded.yaml diff --git a/src/eva/plotting/emcpy/defaults/map_scatter.yaml b/src/eva/plotting/batch/emcpy/defaults/map_scatter.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/map_scatter.yaml rename to src/eva/plotting/batch/emcpy/defaults/map_scatter.yaml diff --git a/src/eva/plotting/emcpy/defaults/scatter.yaml b/src/eva/plotting/batch/emcpy/defaults/scatter.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/scatter.yaml rename to src/eva/plotting/batch/emcpy/defaults/scatter.yaml diff --git a/src/eva/plotting/emcpy/defaults/scatter_density.yaml b/src/eva/plotting/batch/emcpy/defaults/scatter_density.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/scatter_density.yaml rename to src/eva/plotting/batch/emcpy/defaults/scatter_density.yaml diff --git a/src/eva/plotting/emcpy/defaults/vertical_line.yaml b/src/eva/plotting/batch/emcpy/defaults/vertical_line.yaml similarity index 100% rename from src/eva/plotting/emcpy/defaults/vertical_line.yaml rename to src/eva/plotting/batch/emcpy/defaults/vertical_line.yaml diff --git a/src/eva/plotting/hvplot/__init__.py b/src/eva/plotting/batch/emcpy/diagnostics/__init__.py similarity index 100% rename from src/eva/plotting/hvplot/__init__.py rename to src/eva/plotting/batch/emcpy/diagnostics/__init__.py diff --git a/src/eva/plotting/batch/emcpy/diagnostics/density.py b/src/eva/plotting/batch/emcpy/diagnostics/density.py new file mode 100644 index 00000000..e5b12da3 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/density.py @@ -0,0 +1,91 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import emcpy.plots.plots +import os +import numpy as np + + +# -------------------------------------------------------------------------------------------------- + + +class Density(): + + """Base class for creating density plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a density plot based on the provided configuration and data. + + Args: + config (dict): A dictionary containing the configuration for the density plot. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a density plot based on the provided configuration and + data. The density plot is created using a declarative plotting library from EMCPy + (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "data": { + "variable": "collection::group::variable", + "channel": "channel_name", + "slicing": "slice expression" + }, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + dataobj = DataObject() + density_plot = Density(config, logger, dataobj) + """ + + # Get the data to plot from the data_collection + # --------------------------------------------- + varstr = config['data']['variable'] + var_cgv = varstr.split('::') + + if len(var_cgv) != 3: + logger.abort('In Density the variable \'var_cgv\' does not appear to ' + + 'be in the required format of collection::group::variable.') + + # Optionally get the channel to plot + channel = None + if 'channel' in config['data']: + channel = config['data'].get('channel') + + data = dataobj.get_variable_data(var_cgv[0], var_cgv[1], var_cgv[2], channel) + + # See if we need to slice data + data = slice_var_from_str(config['data'], data, logger) + + # Density data should be flattened + data = data.flatten() + + # Missing data should also be removed + mask = ~np.isnan(data) + data = data[mask] + + # Create declarative plotting density object + # -------------------------------------------- + self.plotobj = emcpy.plots.plots.Density(data) + + # Get defaults from schema + # ------------------------ + layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'batch', 'emcpy', 'defaults', + 'density.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['type', 'schema', 'data'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/emcpy_scatter.py b/src/eva/plotting/batch/emcpy/diagnostics/emcpy_scatter.py new file mode 100644 index 00000000..121d4bff --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/emcpy_scatter.py @@ -0,0 +1,32 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object +import emcpy.plots.plots +import os + +from eva.plotting.batch.base.diagnostics.scatter import Scatter + +# -------------------------------------------------------------------------------------------------- + + +class EmcpyScatter(Scatter): + + def configure_plot(self): + + # Create declarative plotting Scatter object + # ------------------------------------------ + self.plotobj = emcpy.plots.plots.Scatter(self.xdata, self.ydata) + + # Get defaults from schema + # ------------------------ + layer_schema = self.config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'emcpy', 'defaults', 'scatter.yaml')) + new_config = get_schema(layer_schema, self.config, self.logger) + delvars = ['x', 'y', 'type', 'schema'] + for d in delvars: + new_config.pop(d, None) + self.plotobj = update_object(self.plotobj, new_config, self.logger) + + return self.plotobj + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/histogram.py b/src/eva/plotting/batch/emcpy/diagnostics/histogram.py new file mode 100644 index 00000000..bf9f0905 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/histogram.py @@ -0,0 +1,91 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import emcpy.plots.plots +import os +import numpy as np + + +# -------------------------------------------------------------------------------------------------- + + +class Histogram(): + + """Base class for creating histogram plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a histogram plot based on the provided configuration and data. + + Args: + config (dict): A dictionary containing the configuration for the histogram plot. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a histogram plot based on the provided configuration + and data. The histogram plot is created using a declarative plotting library from EMCPy + (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "data": { + "variable": "collection::group::variable", + "channel": "channel_name", + "slicing": "slice expression" + }, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + dataobj = DataObject() + histogram_plot = Histogram(config, logger, dataobj) + """ + + # Get the data to plot from the data_collection + # --------------------------------------------- + varstr = config['data']['variable'] + var_cgv = varstr.split('::') + + if len(var_cgv) != 3: + logger.abort('In Histogram the variable \'var_cgv\' does not appear to ' + + 'be in the required format of collection::group::variable.') + + # Optionally get the channel to plot + channel = None + if 'channel' in config['data']: + channel = config['data'].get('channel') + + data = dataobj.get_variable_data(var_cgv[0], var_cgv[1], var_cgv[2], channel) + + # See if we need to slice data + data = slice_var_from_str(config['data'], data, logger) + + # Histogram data should be flattened + data = data.flatten() + + # Missing data should also be removed + mask = ~np.isnan(data) + data = data[mask] + + # Create declarative plotting histogram object + # -------------------------------------------- + self.plotobj = emcpy.plots.plots.Histogram(data) + + # Get defaults from schema + # ------------------------ + layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'batch', 'emcpy', 'defaults', + 'histogram.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['type', 'schema', 'data'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/horizontal_line.py b/src/eva/plotting/batch/emcpy/diagnostics/horizontal_line.py new file mode 100644 index 00000000..b89ddc71 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/horizontal_line.py @@ -0,0 +1,61 @@ +from eva.eva_path import return_eva_path +from eva.utilities.utils import get_schema, update_object +import emcpy.plots.plots +import os + + +# -------------------------------------------------------------------------------------------------- + + +class HorizontalLine(): + + """Base class for creating horizontal line plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a horizontal line plot based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the horizontal line plot. + logger (Logger): An instance of the logger for logging messages. + + This class initializes and configures a horizontal line plot based on the provided + configuration. The horizontal line plot is created using a declarative plotting library from + EMCPy (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "y": 0.5, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + horizontal_line_plot = HorizontalLine(config, logger) + """ + + # Get the y value to plot + # ----------------------- + yval = config['y'] + + # Create declarative plotting HorizontalLine object + # ------------------------------------------- + self.plotobj = emcpy.plots.plots.HorizontalLine(yval) + + # Get defaults from schema + # ------------------------ + layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'batch', 'emcpy', 'defaults', + 'horizontal_line.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['type', 'schema'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/line_plot.py b/src/eva/plotting/batch/emcpy/diagnostics/line_plot.py new file mode 100644 index 00000000..1738cf8b --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/line_plot.py @@ -0,0 +1,106 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import emcpy.plots.plots +import os +import numpy as np + + +# -------------------------------------------------------------------------------------------------- + + +class LinePlot(): + + """Base class for creating line plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a line plot based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the line plot. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a line plot based on the provided configuration. + The line plot is created using a declarative plotting library from EMCPy + (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "x": {"variable": "collection::group::variable"}, + "y": {"variable": "collection::group::variable"}, + "channel": "channel_name", + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + line_plot = LinePlot(config, logger, None) + """ + + # Get the data to plot from the data_collection + # --------------------------------------------- + var0 = config['x']['variable'] + var1 = config['y']['variable'] + + var0_cgv = var0.split('::') + var1_cgv = var1.split('::') + + if len(var0_cgv) != 3: + logger.abort('In Scatter comparison the first variable \'var0\' does not appear to ' + + 'be in the required format of collection::group::variable.') + if len(var1_cgv) != 3: + logger.abort('In Scatter comparison the first variable \'var1\' does not appear to ' + + 'be in the required format of collection::group::variable.') + + # Optionally get the channel|level to plot + channel = None + if 'channel' in config: + channel = config.get('channel') + level = None + if 'level' in config: + level = config.get('level') + + xdata = dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel, level) + ydata = dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel, level) + + # see if we need to slice data + xdata = slice_var_from_str(config['x'], xdata, logger) + ydata = slice_var_from_str(config['y'], ydata, logger) + + # line plot data should be flattened + xdata = xdata.flatten() + ydata = ydata.flatten() + + # Remove NaN values to enable regression + # -------------------------------------- + mask = ~np.isnan(xdata) + xdata = xdata[mask] + ydata = ydata[mask] + + mask = ~np.isnan(ydata) + xdata = xdata[mask] + ydata = ydata[mask] + + # Create declarative plotting LinePlot object + # ------------------------------------------- + self.plotobj = emcpy.plots.plots.LinePlot(xdata, ydata) + + # Get defaults from schema + # ------------------------ + layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'batch', 'emcpy', 'defaults', + 'line_plot.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['x', 'y', 'type', 'schema', 'channel', 'level'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/map_gridded.py b/src/eva/plotting/batch/emcpy/diagnostics/map_gridded.py new file mode 100644 index 00000000..1e09057c --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/map_gridded.py @@ -0,0 +1,70 @@ +from eva.eva_path import return_eva_path +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import emcpy.plots.map_plots +import os + + +# -------------------------------------------------------------------------------------------------- + + +class MapGridded(): + + """Base class for creating map gridded plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a gridded map plot based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the gridded map plot. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a gridded map plot based on the provided + configuration. The gridded map plot is created using a declarative plotting library from + EMCPy (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "longitude": {"variable": "collection::group::variable"}, + "latitude": {"variable": "collection::group::variable"}, + "data": {"variable": "collection::group::variable"}, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + map_plot = MapGridded(config, logger, None) + """ + + # prepare data based on config + lonvar_cgv = config['longitude']['variable'].split('::') + lonvar = dataobj.get_variable_data(lonvar_cgv[0], lonvar_cgv[1], lonvar_cgv[2], None) + lonvar = slice_var_from_str(config['longitude'], lonvar, logger) + latvar_cgv = config['latitude']['variable'].split('::') + latvar = dataobj.get_variable_data(latvar_cgv[0], latvar_cgv[1], latvar_cgv[2], None) + latvar = slice_var_from_str(config['latitude'], latvar, logger) + datavar_cgv = config['data']['variable'].split('::') + datavar = dataobj.get_variable_data(datavar_cgv[0], datavar_cgv[1], datavar_cgv[2], None) + datavar = slice_var_from_str(config['data'], datavar, logger) + + # create declarative plotting MapGridded object + self.plotobj = emcpy.plots.map_plots.MapGridded(latvar, lonvar, datavar) + # get defaults from schema + layer_schema = config.get("schema", + os.path.join(return_eva_path(), + 'plotting', + 'batch', 'emcpy', 'defaults', + 'map_gridded.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['longitude', 'latitude', 'data', 'type', 'schema'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/diagnostics/map_scatter.py b/src/eva/plotting/batch/emcpy/diagnostics/map_scatter.py new file mode 100644 index 00000000..9544aba1 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/map_scatter.py @@ -0,0 +1,83 @@ +from eva.eva_path import return_eva_path +from eva.utilities.utils import get_schema, update_object, slice_var_from_str +import emcpy.plots.map_plots +import os +import numpy as np + + +# -------------------------------------------------------------------------------------------------- + + +class MapScatter(): + + """Base class for creating map scatter plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a scatter plot on a map based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the scatter plot on a map. + logger (Logger): An instance of the logger for logging messages. + dataobj: An instance of the data object containing input data. + + This class initializes and configures a scatter plot on a map based on the provided + configuration. The scatter plot is created using a declarative plotting library from EMCPy + (https://github.com/NOAA-EMC/emcpy). + + Example: + :: + + config = { + "longitude": {"variable": "collection::group::variable"}, + "latitude": {"variable": "collection::group::variable"}, + "data": {"variable": "collection::group::variable", + "channel": "channel_name"}, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + map_scatter_plot = MapScatter(config, logger, None) + """ + + # prepare data based on config + # Optionally get the channel to plot + channel = None + if 'channel' in config['data']: + channel = config['data'].get('channel') + lonvar_cgv = config['longitude']['variable'].split('::') + lonvar = dataobj.get_variable_data(lonvar_cgv[0], lonvar_cgv[1], lonvar_cgv[2], None) + lonvar = slice_var_from_str(config['longitude'], lonvar, logger) + latvar_cgv = config['latitude']['variable'].split('::') + latvar = dataobj.get_variable_data(latvar_cgv[0], latvar_cgv[1], latvar_cgv[2], None) + latvar = slice_var_from_str(config['latitude'], latvar, logger) + datavar_cgv = config['data']['variable'].split('::') + datavar = dataobj.get_variable_data(datavar_cgv[0], datavar_cgv[1], datavar_cgv[2], channel) + datavar = slice_var_from_str(config['data'], datavar, logger) + # scatter data should be flattened + lonvar = lonvar.flatten() + latvar = latvar.flatten() + datavar = datavar.flatten() + + # If everything is nan plotting will fail so just plot some large values + if np.isnan(datavar).all(): + datavar[np.isnan(datavar)] = 1.0e38 + + # create declarative plotting MapScatter object + self.plotobj = emcpy.plots.map_plots.MapScatter(latvar, lonvar, datavar) + # get defaults from schema + layer_schema = config.get("schema", + os.path.join(return_eva_path(), + 'plotting', + 'batch', 'emcpy', 'defaults', + 'map_scatter.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['longitude', 'latitude', 'data', 'type', 'schema'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/emcpy/diagnostics/scatter.py b/src/eva/plotting/batch/emcpy/diagnostics/scatter.py similarity index 96% rename from src/eva/plotting/emcpy/diagnostics/scatter.py rename to src/eva/plotting/batch/emcpy/diagnostics/scatter.py index 8426172f..f5e084ae 100644 --- a/src/eva/plotting/emcpy/diagnostics/scatter.py +++ b/src/eva/plotting/batch/emcpy/diagnostics/scatter.py @@ -93,7 +93,8 @@ def __init__(self, config, logger, dataobj): # Get defaults from schema # ------------------------ layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', - 'emcpy', 'defaults', 'scatter.yaml')) + 'batch', 'emcpy', 'defaults', + 'scatter.yaml')) config = get_schema(layer_schema, config, logger) delvars = ['x', 'y', 'type', 'schema'] for d in delvars: diff --git a/src/eva/plotting/batch/emcpy/diagnostics/vertical_line.py b/src/eva/plotting/batch/emcpy/diagnostics/vertical_line.py new file mode 100644 index 00000000..a16c67a9 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/diagnostics/vertical_line.py @@ -0,0 +1,62 @@ +from eva.eva_path import return_eva_path +from eva.utilities.utils import get_schema, update_object +import emcpy.plots.plots +import os + + +# -------------------------------------------------------------------------------------------------- + + +class VerticalLine(): + + """Base class for creating vertical line plots.""" + + def __init__(self, config, logger, dataobj): + + """ + Creates a vertical line plot based on the provided configuration. + + Args: + config (dict): A dictionary containing the configuration for the vertical line plot. + logger (Logger): An instance of the logger for logging messages. + dataobj: Not used in this context. + + This class initializes and configures a vertical line plot based on the provided + configuration. The vertical line plot is created using a declarative plotting library from + EMCPy (https://github.com/NOAA-EMC/emcpy). + + Example: + + :: + + config = { + "x": 10, + "plot_property": "property_value", + "plot_option": "option_value", + "schema": "path_to_schema_file.yaml" + } + logger = Logger() + vertical_line_plot = VerticalLine(config, logger, None) + """ + + # Get the x value to plot + # ----------------------- + xval = config['x'] + + # Create declarative plotting HorizontalLine object + # ------------------------------------------- + self.plotobj = emcpy.plots.plots.VerticalLine(xval) + + # Get defaults from schema + # ------------------------ + layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting', + 'batch', 'emcpy', 'defaults', + 'vertical_line.yaml')) + config = get_schema(layer_schema, config, logger) + delvars = ['type', 'schema'] + for d in delvars: + config.pop(d, None) + self.plotobj = update_object(self.plotobj, config, logger) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/emcpy/plot_tools/__init__.py b/src/eva/plotting/batch/emcpy/plot_tools/__init__.py new file mode 100644 index 00000000..ac1c0bc4 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/plot_tools/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/emcpy/plot_tools/emcpy_figure_handler.py b/src/eva/plotting/batch/emcpy/plot_tools/emcpy_figure_handler.py new file mode 100644 index 00000000..2787f340 --- /dev/null +++ b/src/eva/plotting/batch/emcpy/plot_tools/emcpy_figure_handler.py @@ -0,0 +1,17 @@ +from emcpy.plots.create_plots import CreatePlot, CreateFigure +from eva.eva_path import return_eva_path +import os + + +class EmcpyFigureHandler(): + + def __init__(self): + # Define default paths to modules + self.BACKEND_NAME = "Emcpy" + self.MODULE_NAME = "eva.plotting.batch.emcpy.diagnostics." + + def create_plot(self, layer_list, proj, domain): + return CreatePlot(plot_layers=layer_list, projection=proj, domain=domain) + + def create_figure(self, nrows, ncols, figsize): + return CreateFigure(nrows=nrows, ncols=ncols, figsize=figsize) diff --git a/src/eva/plotting/batch/hvplot/__init__.py b/src/eva/plotting/batch/hvplot/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/hvplot/defaults/__init__.py b/src/eva/plotting/batch/hvplot/defaults/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/defaults/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/hvplot/defaults/figure.yaml b/src/eva/plotting/batch/hvplot/defaults/figure.yaml new file mode 100644 index 00000000..62157f54 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/defaults/figure.yaml @@ -0,0 +1,6 @@ +# figure default schema +layout: [1,1] +figure file type: html +output path: ./ +output name: 'myfig' +figure size: [8,8] diff --git a/src/eva/plotting/batch/hvplot/diagnostics/__init__.py b/src/eva/plotting/batch/hvplot/diagnostics/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/diagnostics/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/hvplot/diagnostics/hvplot_scatter.py b/src/eva/plotting/batch/hvplot/diagnostics/hvplot_scatter.py new file mode 100644 index 00000000..71abc662 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/diagnostics/hvplot_scatter.py @@ -0,0 +1,46 @@ +from eva.eva_path import return_eva_path +from eva.utilities.config import get +from eva.utilities.utils import get_schema, update_object +import os +import pandas as pd +import hvplot.pandas +import holoviews as hv +from scipy.stats import linregress + +from eva.plotting.batch.base.diagnostics.scatter import Scatter + +# -------------------------------------------------------------------------------------------------- + + +class HvplotScatter(Scatter): + + def configure_plot(self): + # Save and access name of variables + df = pd.DataFrame() + df['xdata'] = self.xdata + df['ydata'] = self.ydata + color = self.config['color'] + size = self.config['markersize'] + label = self.config['label'] + + # add line statistics to legend label + try: + plot_for_slope = df.hvplot.scatter('xdata', 'ydata') + slope = hv.Slope.from_scatter(plot_for_slope) + slope_attrs = linregress(self.xdata, self.ydata) + slope_expression = 'y='+f'{slope.slope:.3f}'+"x+"+f'{slope.y_intercept:.3f}' + r_sq = ', r^2: ' + f'{slope_attrs.rvalue**2:.3f}' + slope_label = "y="+f'{slope.slope:.3f}'+"x+"+f'{slope.y_intercept:.3f}'+r_sq + plot = df.hvplot.scatter('xdata', 'ydata', + s=size, c=color, label=label+", "+slope_label) + new_plot = hv.Overlay([plot, plot]) + plotobj = new_plot * slope + plotobj.opts(show_legend=False) + except Exception: + plot = df.hvplot.scatter('xdata', 'ydata', s=size, c=color, label=label) + plotobj = hv.Overlay([plot, plot]) + plotobj.opts(show_legend=False) + + return plotobj + +# -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/hvplot/plot_tools/__init__.py b/src/eva/plotting/batch/hvplot/plot_tools/__init__.py new file mode 100644 index 00000000..ebda1763 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/plot_tools/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/batch/hvplot/plot_tools/create_plots.py b/src/eva/plotting/batch/hvplot/plot_tools/create_plots.py new file mode 100644 index 00000000..0c9971e7 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/plot_tools/create_plots.py @@ -0,0 +1,99 @@ +from bokeh.plotting import save, output_file +import hvplot as hv +import os + + +class CreatePlot: + + def __init__(self, plot_layers=[], projection=None, domain=None): + self.plot_layers = plot_layers + if projection: + self.projection = projection + if domain: + self.domain = domain + + def add_grid(self, **kwargs): + self.grid = { + **kwargs + } + + def add_legend(self, **kwargs): + self.legend = { + **kwargs + } + + def add_xlabel(self, xlabel, labelpad=None, + loc='center', **kwargs): + self.xlabel = { + 'xlabel': xlabel, + 'labelpad': labelpad, + 'loc': loc, + **kwargs + } + + def add_ylabel(self, ylabel, labelpad=None, + loc='center', **kwargs): + self.ylabel = { + 'ylabel': ylabel, + 'labelpad': labelpad, + 'loc': loc, + **kwargs + } + + +class CreateFigure: + + def __init__(self, nrows=1, ncols=1, figsize=(8, 6)): + self.nrows = nrows + self.ncols = ncols + self.figsize = figsize + self.plot_list = [] + self.fig = None + + def create_figure(self): + # Needs work, need to combine all the layers + # and figure out how subplots will work + self.fig = self.plot_list[0].plot_layers[0] + plot_obj = self.plot_list[0] + + # Add all features to the figure + for feat in vars(plot_obj).keys(): + self._plot_features(plot_obj, feat) + + def add_suptitle(self, text, **kwargs): + self.fig.opts(title=text) + + def save_figure(self, pathfile, **kwargs): + pathfile_dir = os.path.dirname(pathfile) + if not os.path.exists(pathfile_dir): + os.makedirs(pathfile_dir) + bokeh_fig = hv.render(self.fig, backend='bokeh') + output_file(pathfile) + save(bokeh_fig) + + def close_figure(self): + pass + + def _plot_features(self, plot_obj, feature): + + feature_dict = { + 'xlabel': self._plot_xlabel, + 'ylabel': self._plot_ylabel, + 'legend': self._plot_legend, + 'grid': self._plot_grid, + } + + if feature in feature_dict: + feature_dict[feature](vars(plot_obj)[feature]) + + def _plot_grid(self, grid): + self.fig.opts(show_grid=True) + + def _plot_xlabel(self, xlabel): + self.fig.opts(xlabel=xlabel['xlabel']) + + def _plot_ylabel(self, ylabel): + self.fig.opts(ylabel=ylabel['ylabel']) + + def _plot_legend(self, legend): + self.fig.opts(legend_position='top_left', show_legend=True) diff --git a/src/eva/plotting/batch/hvplot/plot_tools/hvplot_figure_handler.py b/src/eva/plotting/batch/hvplot/plot_tools/hvplot_figure_handler.py new file mode 100644 index 00000000..b9990662 --- /dev/null +++ b/src/eva/plotting/batch/hvplot/plot_tools/hvplot_figure_handler.py @@ -0,0 +1,17 @@ +from eva.eva_path import return_eva_path +from eva.plotting.batch.hvplot.plot_tools.create_plots import CreatePlot, CreateFigure +import os + + +class HvplotFigureHandler(): + + def __init__(self): + # Define default paths to modules + self.BACKEND_NAME = "Hvplot" + self.MODULE_NAME = "eva.plotting.batch.hvplot.diagnostics." + + def create_plot(self, layer_list, proj, domain): + return CreatePlot(plot_layers=layer_list, projection=proj, domain=domain) + + def create_figure(self, nrows, ncols, figsize): + return CreateFigure(nrows=nrows, ncols=ncols, figsize=figsize) diff --git a/src/eva/plotting/interactive/__init__.py b/src/eva/plotting/interactive/__init__.py new file mode 100644 index 00000000..ac1c0bc4 --- /dev/null +++ b/src/eva/plotting/interactive/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/eva/plotting/hvplot/interactive_plot_tools.py b/src/eva/plotting/interactive/interactive_plot_tools.py similarity index 100% rename from src/eva/plotting/hvplot/interactive_plot_tools.py rename to src/eva/plotting/interactive/interactive_plot_tools.py diff --git a/src/eva/tests/config/testCubedSphereRestart.yaml b/src/eva/tests/config/testCubedSphereRestart.yaml index b8e8950e..2f00b47c 100644 --- a/src/eva/tests/config/testCubedSphereRestart.yaml +++ b/src/eva/tests/config/testCubedSphereRestart.yaml @@ -19,6 +19,10 @@ datasets: orography variables: [geolon, geolat] graphics: + + plotting_backend: Emcpy + figure_list: + # Map plots # --------- diff --git a/src/eva/tests/config/testGsiObsSpaceAmsuaMetop-A.yaml b/src/eva/tests/config/testGsiObsSpaceAmsuaMetop-A.yaml index 2740b5f1..9f37ddd7 100644 --- a/src/eva/tests/config/testGsiObsSpaceAmsuaMetop-A.yaml +++ b/src/eva/tests/config/testGsiObsSpaceAmsuaMetop-A.yaml @@ -23,6 +23,10 @@ transforms: variable: *variables graphics: + + plotting_backend: Emcpy + figure_list: + # Histogram plots # --------------- diff --git a/src/eva/tests/config/testGsiObsSpaceConvT.yaml b/src/eva/tests/config/testGsiObsSpaceConvT.yaml index 9cee4552..fa05a98f 100644 --- a/src/eva/tests/config/testGsiObsSpaceConvT.yaml +++ b/src/eva/tests/config/testGsiObsSpaceConvT.yaml @@ -12,6 +12,10 @@ datasets: Longitude] graphics: + + plotting_backend: Emcpy + figure_list: + # Map plots # --------- diff --git a/src/eva/tests/config/testHvplotIodaObsSpaceAmsuaN19.yaml b/src/eva/tests/config/testHvplotIodaObsSpaceAmsuaN19.yaml new file mode 100644 index 00000000..08fb915e --- /dev/null +++ b/src/eva/tests/config/testHvplotIodaObsSpaceAmsuaN19.yaml @@ -0,0 +1,159 @@ +datasets: + - name: experiment + type: IodaObsSpace + filenames: + - ${data_input_path}/ioda_obs_space.amsua_n19.hofx.2020-12-14T210000Z.nc4 + channels: &channels 3,8 + # Note: channelNumber is automatically added to the output and should not + # be listed below + groups: + - name: ObsValue + variables: &variables [brightnessTemperature] + - name: GsiHofXBc + #- name: GsiEffectiveQC + - name: hofx + - name: EffectiveQC + - name: MetaData + +transforms: + + # Generate omb for GSI + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # Generate omb for JEDI + - transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + for: + variable: *variables + + # Generate hofx difference + - transform: arithmetic + new name: experiment::HofxMinusGsiHofXBc::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # Generate hofx that passed QC for JEDI + - transform: accept where + new name: experiment::hofxPassedQc::${variable} + starting field: experiment::hofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate GSI hofx that passed JEDI QC + - transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for GSI + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + +graphics: + plotting_backend: Hvplot + figure_list: + + # Correlation scatter plots + # ------------------------- + + + # GSI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | AMSU-A NOAA-19 | ${variable_title}' + output name: observation_hvplot_scatter_plots/amsua_n19/${variable}/${channel}/gsi_hofx_vs_obs_amsua_n19_${variable}_${channel}.html + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + + # JEDI h(x) vs GSI h(x) + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | AMSU-A NOAA-19 | ${variable_title}' + output name: observation_hvplot_scatter_plots/amsua_n19/${variable}/${channel}/gsi_hofx_vs_jedi_hofx_amsua_n19_${variable}_${channel}.html + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + + # JEDI omb vs GSI omb + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb| AMSU-A NOAA-19 | ${variable_title}' + output name: observation_hvplot_scatter_plots/amsua_n19/${variable}/${channel}/gsi_omb_vs_jedi_omb_amsua_n19_${variable}_${channel}.html + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusHofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI omb vs JEDI omb (all obs)' + diff --git a/src/eva/tests/config/testIodaObsSpaceAircraft.yaml b/src/eva/tests/config/testIodaObsSpaceAircraft.yaml index 8aad999f..ec1ba627 100644 --- a/src/eva/tests/config/testIodaObsSpaceAircraft.yaml +++ b/src/eva/tests/config/testIodaObsSpaceAircraft.yaml @@ -73,6 +73,9 @@ transforms: graphics: + plotting_backend: Emcpy + figure_list: + # Observation correlation scatter plots # ------------------------------------- diff --git a/src/eva/tests/config/testIodaObsSpaceAmsuaN19.yaml b/src/eva/tests/config/testIodaObsSpaceAmsuaN19.yaml index 8c25d2a8..110f65f1 100644 --- a/src/eva/tests/config/testIodaObsSpaceAmsuaN19.yaml +++ b/src/eva/tests/config/testIodaObsSpaceAmsuaN19.yaml @@ -76,6 +76,9 @@ transforms: graphics: + plotting_backend: Emcpy + figure_list: + # Correlation scatter plots # ------------------------- diff --git a/src/eva/tests/config/testIodaObsSpaceIASI_Metop-A.yaml b/src/eva/tests/config/testIodaObsSpaceIASI_Metop-A.yaml index b2f319eb..b6ebd7cd 100644 --- a/src/eva/tests/config/testIodaObsSpaceIASI_Metop-A.yaml +++ b/src/eva/tests/config/testIodaObsSpaceIASI_Metop-A.yaml @@ -52,6 +52,9 @@ transforms: graphics: + plotting_backend: Emcpy + figure_list: + # ---------- Statistical Plot ---------- # JEDI h(x) vs GSI h(x) # ------------------------- diff --git a/src/eva/tests/config/testJediLog.yaml b/src/eva/tests/config/testJediLog.yaml index 2ace22d0..5121397d 100644 --- a/src/eva/tests/config/testJediLog.yaml +++ b/src/eva/tests/config/testJediLog.yaml @@ -13,6 +13,10 @@ datasets: # Make plots graphics: + + plotting_backend: Emcpy + figure_list: + - figure: layout: [3,1] figure size: [12,10] diff --git a/src/eva/tests/config/testLatLon.yaml b/src/eva/tests/config/testLatLon.yaml index 6cc0cf20..dd85923d 100644 --- a/src/eva/tests/config/testLatLon.yaml +++ b/src/eva/tests/config/testLatLon.yaml @@ -6,6 +6,10 @@ datasets: variables: [T_inc, lat, lon] graphics: + + plotting_backend: Emcpy + figure_list: + # Map plots # --------- diff --git a/src/eva/tests/config/testMonDataSpaceHirs4Metop-A.yaml b/src/eva/tests/config/testMonDataSpaceHirs4Metop-A.yaml index 8f235818..692e0c2b 100644 --- a/src/eva/tests/config/testMonDataSpaceHirs4Metop-A.yaml +++ b/src/eva/tests/config/testMonDataSpaceHirs4Metop-A.yaml @@ -19,6 +19,10 @@ datasets: variables: &variables [count, cycle] graphics: + + plotting_backend: Emcpy + figure_list: + # Time series plots # --------------- - figure: diff --git a/src/eva/tests/config/testMonSummary.yaml b/src/eva/tests/config/testMonSummary.yaml index afaa04db..87d1de5c 100644 --- a/src/eva/tests/config/testMonSummary.yaml +++ b/src/eva/tests/config/testMonSummary.yaml @@ -81,6 +81,10 @@ transforms: graphics: + + plotting_backend: Emcpy + figure_list: + # Summary plots # --------------- - figure: diff --git a/src/eva/tests/config/testSocaRestart.yaml b/src/eva/tests/config/testSocaRestart.yaml index ed751eb9..2bb9d57d 100644 --- a/src/eva/tests/config/testSocaRestart.yaml +++ b/src/eva/tests/config/testSocaRestart.yaml @@ -10,6 +10,10 @@ datasets: coordinate variables: [lon, lat] graphics: + + plotting_backend: Emcpy + figure_list: + # Map plots # --------- diff --git a/src/eva/tests/config/testTwoDatasetsOnePlot.yaml b/src/eva/tests/config/testTwoDatasetsOnePlot.yaml index 1d3eca7c..6f04acd6 100644 --- a/src/eva/tests/config/testTwoDatasetsOnePlot.yaml +++ b/src/eva/tests/config/testTwoDatasetsOnePlot.yaml @@ -27,6 +27,10 @@ datasets: orography variables: [geolon, geolat] graphics: + + plotting_backend: Emcpy + figure_list: + # Map plots # ---------