Refactoring for Hvplot batch plotting (JCSDA-internal#158)

* bokeh batch plot

* bokeh components

* first pass diag inherit

* adding figure handler

* bokeh scatter plot

* more hvplot changes

* adding figure_list to tests

* adding in new test for hvplot

* python code style

* enforcing plotting backend in yamls

* requirements

* coding norms

* Sort out requirements files a bit

* emcpy in github requirements

* review changes

* Revert "review changes"

This reverts commit 9214290.

* review changes again

---------

Co-authored-by: danholdaway <[email protected]>

.gitignore

@@ -127,3 +127,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Swap files
+*.swp
+*.swo

requirements-github.txt

@@ -12,3 +12,4 @@ bokeh>=3.1.1
 geopandas>=0.13.2
 geoviews>=1.10.0
 nbsite
+git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy

requirements-spackstack1.4.txt

@@ -0,0 +1,25 @@
+# Packages in spack stack
+setuptools==59.4.0
+pyyaml==6.0
+pycodestyle==2.8.0
+netCDF4==1.5.3
+matplotlib==3.7.1
+cartopy==0.21.1
+scipy==1.9.3
+xarray==2022.3.0
+pandas==1.4.0
+numpy==1.22.3
+
+# Not explicitly part of eva but dependcies of eva dependencies already in spack-stack
+# versions need to be set to avoid other versions being picked
+pyproj==3.1.0
+importlib-metadata==4.8.2
+
+# Additional packages
+git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy
+scikit-learn
+seaborn
+hvplot
+nbconvert
+bokeh
+geopandas

requirements.txt

@@ -1,7 +1,19 @@
+setuptools>=59.4.0
 pyyaml>=6.0
 pycodestyle>=2.8.0
 netCDF4>=1.5.3
-matplotlib>=3.5.2
-cartopy>=0.20.2
-scikit-learn>=1.0.2
-xarray>=0.11.3
+matplotlib>=3.7.1
+cartopy>=0.21.1
+scipy>=1.9.3
+xarray>=2022.3.0
+pandas>=1.4.0
+numpy>=1.22.3
+
+# Additional packages
+git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy
+scikit-learn
+seaborn
+hvplot
+nbconvert
+bokeh
+geopandas

requirements_emc.txt

@@ -0,0 +1,13 @@
+setuptools>=59.4.0
+pyyaml>=6.0
+pycodestyle>=2.8.0
+netCDF4>=1.5.3
+matplotlib>=3.7.1
+cartopy>=0.21.1
+scipy>=1.9.3
+xarray>=2022.3.0
+pandas>=1.4.0
+numpy>=1.22.3
+
+# Additional packages
+git+https://github.com/NOAA-EMC/emcpy.git@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy

setup.py

@@ -14,6 +14,7 @@
 
 import setuptools
 
+
 setuptools.setup(
     name='eva',
     version='1.5.0',
@@ -31,20 +32,6 @@
         'Natural Language :: English',
         'Operating System :: OS Independent'],
     python_requires='>=3.6',
-    install_requires=[
-        'pyyaml>=6.0',
-        'pycodestyle>=2.8.0',
-        'netCDF4>=1.5.3',
-        'matplotlib>=3.5.2',
-        'cartopy>=0.18.0',
-        'scikit-learn>=1.0.2',
-        'xarray>=0.11.3',
-        'emcpy @ git+https://github.com/NOAA-EMC/' +
-        'emcpy@9b6756341e9ae963baa7d3a256b8ada3da688d04#egg=emcpy',
-
-        # Option dependency for making density plots
-        # 'seaborn>=0.12',
-    ],
     package_data={
         '': [
                'tests/config/*',

src/eva/eva_driver.py

@@ -14,7 +14,7 @@
 from eva.utilities.timing import Timing
 from eva.data.data_driver import data_driver
 from eva.transforms.transform_driver import transform_driver
-from eva.plotting.emcpy.plot_tools.figure_driver import figure_driver
+from eva.plotting.batch.base.plot_tools.figure_driver import figure_driver
 from eva.data.data_collections import DataCollections
 from eva.utilities.utils import load_yaml_file
 import argparse

src/eva/eva_interactive.py

@@ -23,7 +23,7 @@
 from eva.transforms.arithmetic import arithmetic, generate_arithmetic_config
 from eva.transforms.accept_where import accept_where, generate_accept_where_config
 
-import eva.plotting.hvplot.interactive_plot_tools as plot
+import eva.plotting.interactive.interactive_plot_tools as plot
 
 
 # --------------------------------------------------------------------------------------------------

src/eva/plotting/batch/__init__.py

@@ -0,0 +1,9 @@
+# (C) Copyright 2023 United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+import os
+
+repo_directory = os.path.dirname(__file__)

src/eva/plotting/batch/base/__init__.py

@@ -0,0 +1,9 @@
+# (C) Copyright 2023 United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+import os
+
+repo_directory = os.path.dirname(__file__)

src/eva/plotting/batch/base/diagnostics/scatter.py

@@ -0,0 +1,103 @@
+from eva.eva_path import return_eva_path
+from eva.utilities.config import get
+from eva.utilities.utils import get_schema, update_object, slice_var_from_str
+import os
+import numpy as np
+
+from abc import ABC, abstractmethod
+
+# --------------------------------------------------------------------------------------------------
+
+
+class Scatter(ABC):
+
+    """Base class for creating scatter plots."""
+
+    def __init__(self, config, logger, dataobj):
+
+        """
+        Creates a scatter plot on a map based on the provided configuration.
+
+        Args:
+            config (dict): A dictionary containing the configuration for the scatter plot on a map.
+            logger (Logger): An instance of the logger for logging messages.
+            dataobj: An instance of the data object containing input data.
+
+        This class initializes and configures a scatter plot on a map based on the provided
+        configuration. The scatter plot is created using a declarative plotting library from EMCPy
+        (https://github.com/NOAA-EMC/emcpy).
+
+        Example:
+
+            ::
+
+                    config = {
+                        "longitude": {"variable": "collection::group::variable"},
+                        "latitude": {"variable": "collection::group::variable"},
+                        "data": {"variable": "collection::group::variable",
+                                 "channel": "channel_name"},
+                        "plot_property": "property_value",
+                        "plot_option": "option_value",
+                        "schema": "path_to_schema_file.yaml"
+                    }
+                    logger = Logger()
+                    map_scatter_plot = MapScatter(config, logger, None)
+        """
+        self.config = config
+        self.logger = logger
+        self.dataobj = dataobj
+        self.xdata = []
+        self.ydata = []
+        self.plotobj = None
+
+# --------------------------------------------------------------------------------------------------
+
+    def data_prep(self):
+
+        # Get the data to plot from the data_collection
+        # ---------------------------------------------
+        var0 = self.config['x']['variable']
+        var1 = self.config['y']['variable']
+
+        var0_cgv = var0.split('::')
+        var1_cgv = var1.split('::')
+
+        if len(var0_cgv) != 3:
+            self.logger.abort('Scatter: comparison first var \'var0\' does not appear to ' +
+                              'be in the required format of collection::group::variable.')
+        if len(var1_cgv) != 3:
+            self.logger.abort('Scatter: comparison first var \'var1\' does not appear to ' +
+                              'be in the required format of collection::group::variable.')
+
+        # Optionally get the channel to plot
+        channel = None
+        if 'channel' in self.config:
+            channel = self.config.get('channel')
+
+        xdata = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel)
+        xdata1 = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2])
+        ydata = self.dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel)
+
+        # see if we need to slice data
+        xdata = slice_var_from_str(self.config['x'], xdata, self.logger)
+        ydata = slice_var_from_str(self.config['y'], ydata, self.logger)
+
+        # scatter data should be flattened
+        xdata = xdata.flatten()
+        ydata = ydata.flatten()
+
+        # Remove NaN values to enable regression
+        # --------------------------------------
+        mask = ~np.isnan(xdata)
+        xdata = xdata[mask]
+        ydata = ydata[mask]
+
+        mask = ~np.isnan(ydata)
+        self.xdata = xdata[mask]
+        self.ydata = ydata[mask]
+
+    @abstractmethod
+    def configure_plot(self):
+        pass
+
+# --------------------------------------------------------------------------------------------------

src/eva/plotting/emcpy/plot_tools/figure_driver.py → src/eva/plotting/batch/base/plot_tools/figure_driver.py

@@ -14,7 +14,6 @@
 from eva.utilities.stats import stats_helper
 from eva.utilities.utils import get_schema, camelcase_to_underscore, parse_channel_list
 from eva.utilities.utils import replace_vars_dict
-from emcpy.plots.create_plots import CreatePlot, CreateFigure
 import copy
 import importlib as im
 import os
@@ -40,7 +39,32 @@ def figure_driver(config, data_collections, timing, logger):
 
     # Get list of graphics from configuration
     # -------------------
-    graphics = config.get("graphics")
+    graphics_section = config.get('graphics')
+    graphics = graphics_section.get('figure_list')
+
+    # Get plotting backend
+    # --------------------
+    backend = graphics_section.get('plotting_backend')
+
+    if backend not in ['Emcpy', 'Hvplot']:
+        logger.abort('Backend not found. \
+                     Available backends: Emcpy, Hvplot')
+
+    if backend == 'Hvplot':
+        try:
+            import hvplot
+        except ImportError:
+            logger.abort("The hvplot backend is not available since \
+                         hvplot is not in the environment.")
+
+    # Create handler
+    # --------------
+    handler_class_name = backend + 'FigureHandler'
+    handler_module_name = camelcase_to_underscore(handler_class_name)
+    handler_full_module = 'eva.plotting.batch.' + \
+                          backend.lower() + '.plot_tools.' + handler_module_name
+    handler_class = getattr(im.import_module(handler_full_module), handler_class_name)
+    handler = handler_class()
 
     # Loop through specified graphics
     # -------------------
@@ -56,8 +80,8 @@ def figure_driver(config, data_collections, timing, logger):
 
         # update figure conf based on schema
         # ----------------------------------
-        fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting',
-                                                            'emcpy', 'defaults', 'figure.yaml'))
+        fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting', 'batch',
+                                     backend.lower(), 'defaults', 'figure.yaml'))
         figure_conf = get_schema(fig_schema, figure_conf, logger)
 
         # pass configurations and make graphic(s)
@@ -111,19 +135,20 @@ def figure_driver(config, data_collections, timing, logger):
                                                                   **batch_conf_this)
 
                     # Make plot
-                    make_figure(figure_conf_fill, plots_conf_fill,
+                    make_figure(handler, figure_conf_fill, plots_conf_fill,
                                 dynamic_options_conf_fill, data_collections, logger)
 
         else:
             # make just one figure per configuration
-            make_figure(figure_conf, plots_conf, dynamic_options_conf, data_collections, logger)
+            make_figure(handler, figure_conf, plots_conf,
+                        dynamic_options_conf, data_collections, logger)
     timing.stop('Graphics Loop')
 
 
 # --------------------------------------------------------------------------------------------------
 
 
-def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
+def make_figure(handler, figure_conf, plots, dynamic_options, data_collections, logger):
     """
     Generates a figure based on the provided configuration and plots.
 
@@ -143,7 +168,8 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
     # Adjust the plots configs if there are dynamic options
     # -----------------------------------------------------
     for dynamic_option in dynamic_options:
-        dynamic_option_module = im.import_module("eva.plotting.emcpy.plot_tools.dynamic_config")
+        mod_name = "eva.plotting.batch.base.plot_tools.dynamic_config"
+        dynamic_option_module = im.import_module(mod_name)
         dynamic_option_method = getattr(dynamic_option_module, dynamic_option['type'])
         plots = dynamic_option_method(logger, dynamic_option, plots, data_collections)
 
@@ -158,12 +184,23 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
     for plot in plots:
         layer_list = []
         for layer in plot.get("layers"):
-            eva_class_name = layer.get("type")
-            eva_module_name = camelcase_to_underscore(eva_class_name)
-            full_module = "eva.plotting.emcpy.diagnostics."+eva_module_name
-            layer_class = getattr(im.import_module(full_module), eva_class_name)
-            # use the translator class to go from eva to declarative plotting
-            layer_list.append(layer_class(layer, logger, data_collections).plotobj)
+
+            # Temporary case to handle different diagnostics
+            if handler.BACKEND_NAME == 'Emcpy':
+                eva_class_name = layer.get("type")
+                eva_module_name = camelcase_to_underscore(eva_class_name)
+                full_module = "eva.plotting.batch.emcpy.diagnostics."+eva_module_name
+                layer_class = getattr(im.import_module(full_module), eva_class_name)
+                layer_list.append(layer_class(layer, logger, data_collections).plotobj)
+            else:
+                eva_class_name = handler.BACKEND_NAME + layer.get("type")
+                eva_module_name = camelcase_to_underscore(eva_class_name)
+                full_module = handler.MODULE_NAME + eva_module_name
+                layer_class = getattr(im.import_module(full_module), eva_class_name)
+                layer = layer_class(layer, logger, data_collections)
+                layer.data_prep()
+                layer_list.append(layer.configure_plot())
+
         # get mapping dictionary
         proj = None
         domain = None
@@ -174,7 +211,7 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
             domain = mapoptions['domain']
 
         # create a subplot based on specified layers
-        plotobj = CreatePlot(plot_layers=layer_list, projection=proj, domain=domain)
+        plotobj = handler.create_plot(layer_list, proj, domain)
         # make changes to subplot based on YAML configuration
         for key, value in plot.items():
             if key not in ['layers', 'mapping', 'statistics']:
@@ -191,9 +228,10 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
         plot_list.append(plotobj)
 
     # create figure
-    fig = CreateFigure(nrows=figure_conf['layout'][0],
-                       ncols=figure_conf['layout'][1],
-                       figsize=tuple(figure_conf['figure size']))
+    nrows = figure_conf['layout'][0]
+    ncols = figure_conf['layout'][1]
+    figsize = tuple(figure_conf['figure size'])
+    fig = handler.create_figure(nrows, ncols, figsize)
     fig.plot_list = plot_list
     fig.create_figure()