Refs #9558 - Moved data annotations to Daviz settings Control Panel

docs/HISTORY.txt

@@ -5,6 +5,8 @@ Changelog
 ----------------------
 * Upgrade step: Within "Plone > Site setup > Add-ons" click on
   upgrade button available for eea.app.visualization
+* Change: Moved data annotations to Daviz settings Control Panel
+  [voineali refs #9558]
 * Change: Cleanup old 'sections' code
   [avoinea]
 * Feature: Added confirm dialog in order to prevent accidentally disable of

eea/app/visualization/config.py

@@ -8,3 +8,11 @@
 
 from zope.i18nmessageid.message import MessageFactory
 EEAMessageFactory = MessageFactory('eea')
+
+DATA_ANNOTATIONS = [
+    '', 'n/a', 'n.a.', 'na', 'n.a',
+    '.', ':', '-', '_', '/', '\\', "[]", "{}", "()", "<>", "*",
+    'empty', "<empty>", 'not set', "<not set>", "notset",
+    "<notset>", 'none', "<none>", "missing", "<missing>",
+    'undefined', "<undefined>", "null", "<null>"
+]

eea/app/visualization/controlpanel/configure.zcml

@@ -1,20 +1,26 @@
-<configure xmlns="http://namespaces.zope.org/zope"
-           xmlns:browser="http://namespaces.zope.org/browser">
+<configure
+  xmlns="http://namespaces.zope.org/zope"
+  xmlns:browser="http://namespaces.zope.org/browser">
 
-    <!-- ZMI Edit view -->
-    <browser:page 
-        name="zmi_edit_html"
-        for="*" 
-        class=".davizsettings.DavizSettingsZMIEditForm" 
-        permission="zope2.ViewManagementScreens"
+  <!-- ZMI Edit view -->
+  <browser:page
+    name="zmi_edit_html"
+    for="*"
+    class=".davizsettings.DavizSettingsZMIEditForm"
+    permission="zope2.ViewManagementScreens"
     />
 
-    <!-- Control panel --> 
-    <browser:page 
-        name="daviz-settings"
-        for="Products.CMFPlone.interfaces.IPloneSiteRoot" 
-        class=".davizsettings.DavizSettingsControlPanelEditForm" 
-        permission="cmf.ManagePortal"
+  <!-- Control panel -->
+  <browser:page
+    name="daviz-settings"
+    for="Products.CMFPlone.interfaces.IPloneSiteRoot"
+    class=".davizsettings.DavizSettingsControlPanelEditForm"
+    permission="cmf.ManagePortal"
+    />
+
+  <utility
+    name="eea.app.visualizations.DataSection"
+    factory=".controlpanel.DataSection"
     />
 
 </configure>

eea/app/visualization/controlpanel/controlpanel.py

@@ -0,0 +1,26 @@
+""" Daviz Settings Section
+"""
+from zope import schema
+from zope.interface import implements
+from zope.formlib.form import FormFields
+from eea.app.visualization.controlpanel.interfaces import IDavizSection
+from eea.app.visualization.config import EEAMessageFactory as _
+
+class DataSection(object):
+    """ Daviz  Settings Section
+    """
+    implements(IDavizSection)
+    prefix = 'data'
+    title = 'Data Settings'
+
+    def __init__(self):
+        self.form_fields = FormFields(
+            schema.Text(
+                __name__='data.annotations',
+                title=_(u"Annotations"),
+                description=_("Data annotations - lowercase - to be "
+                              "ignored/handled as annotations when processing "
+                              "data tables. One per line "
+                              "(e.g. 'n/a', 'n.a.', ':')"),
+                required=False),
+            )

eea/app/visualization/controlpanel/davizsettings.py

@@ -96,9 +96,9 @@ def handle_save_action_daviz(self, saction, data):
             value = self.request.get(field_name, None)
             self.context.settings[field] = value
 
-        self.request.SESSION['messages'] = ["Saved changes. (%s)" 
+        self.request.SESSION['messages'] = ["Saved changes. (%s)"
             % (datetime.now())]
-        self.request.RESPONSE.redirect(self.context.absolute_url() + 
+        self.request.RESPONSE.redirect(self.context.absolute_url() +
             '/manage_workspace')
 
 
@@ -133,6 +133,8 @@ def handle_save_action_daviz(self, saction, data):
             if self.prefix:
                 field_name = self.prefix + "." + field
             value = self.request.get(field_name, None)
+            if isinstance(value, (str, unicode)):
+                value = value.replace('\r\n', '\n')
             self.context.settings[field] = value
         IStatusMessage(self.request).addStatusMessage(u"Settings saved")
         self.request.response.redirect("@@daviz-settings")
@@ -142,4 +144,3 @@ def handle_cancel_action_daviz(self, saction, data):
         """ Cancel action """
         IStatusMessage(self.request).addStatusMessage(u"Edit cancelled")
         self.request.response.redirect("@@overview-controlpanel")
-

eea/app/visualization/converter/types/guess.py

@@ -7,6 +7,8 @@
 from plone.i18n.normalizer.interfaces import IIDNormalizer
 from eea.app.visualization.converter.types.interfaces import IGuessType
 from eea.app.visualization.converter.types.interfaces import IGuessTypes
+from eea.app.visualization.config import DATA_ANNOTATIONS
+from eea.app.visualization.interfaces import IDavizSettings
 
 REGEX = re.compile(r"[\W]+")
 
@@ -35,14 +37,21 @@ class GuessTypes(object):
     """
     implements(IGuessTypes)
 
-    #XXX Move this to ControlPanel
-    missing = (
-        '', 'n/a', 'n.a.', 'na', 'n.a',
-        '.', ':', '-', '_', '/', '\\', "[]", "{}", "()", "<>", "*",
-        'empty', "<empty>", 'not set', "<not set>", "notset", "<notset>",
-        'none', "<none>", "missing", "<missing>", 'undefined', "<undefined>",
-        "null", "<null>"
-    )
+    @property
+    def missing(self):
+        """ Missing annotations
+        """
+        tool = queryUtility(IDavizSettings)
+        if not tool:
+            return set(DATA_ANNOTATIONS)
+
+        anno = tool.settings.get('data.annotations')
+        if anno is None:
+            return set(DATA_ANNOTATIONS)
+
+        anno = set(anno.splitlines())
+        anno.add('')
+        return anno
 
     def column_type(self, column):
         """ Get column and type from column name
@@ -111,10 +120,12 @@ def guessBody(self, table, header, output):
         utilities = getUtilitiesFor(IGuessType)
         utilities = sorted(utilities, cmp=compare)
 
+        missing = self.missing
+
         for row in table:
             for index, cell in enumerate(row):
                 # Skip missing values
-                if cell.lower().strip() in self.missing:
+                if cell.lower().strip() in missing:
                     continue
 
                 label = header[index]

eea/app/visualization/interfaces.py

@@ -39,6 +39,10 @@
     IVisualizationEditResources
 )
 
+# Settings
+from eea.app.visualization.controlpanel.interfaces import IDavizSettings
+from eea.app.visualization.controlpanel.interfaces import IDavizSection
+
 __all__ = (
     IPossibleVisualization.__name__,
     IVisualizationEnabled.__name__,
@@ -61,4 +65,6 @@
     IVisualizationViews.__name__,
     IVisualizationViewResources.__name__,
     IVisualizationEditResources.__name__,
+    IDavizSettings.__name__,
+    IDavizSection.__name__,
 )

eea/app/visualization/upgrades/configure.zcml

@@ -14,6 +14,11 @@
       handler="eea.app.visualization.upgrades.evolve62.fix_column_labels"
       />
 
+    <genericsetup:upgradeStep
+      title="Add daviz data settings to portal_daviz"
+      handler="eea.app.visualization.upgrades.evolve62.update_davizSettings"
+      />
+
     <genericsetup:upgradeDepends
       title="Import profile steps with changes: javascripts"
       import_steps="jsregistry"

eea/app/visualization/upgrades/evolve62.py

@@ -1,10 +1,12 @@
 """ Evolve to version 6.2
 """
 import logging
-from zope.component import queryAdapter
+from zope.component import queryAdapter, queryUtility
 from zope.component.interface import interfaceToName
 from eea.app.visualization.interfaces import IVisualizationConfig
 from eea.app.visualization.interfaces import IVisualizationEnabled
+from eea.app.visualization.interfaces import IDavizSettings
+from eea.app.visualization.config import DATA_ANNOTATIONS
 from Products.CMFCore.utils import getToolByName
 logger = logging.getLogger("eea.app.visualization.upgrades")
 
@@ -36,3 +38,19 @@ def fix_column_labels(context):
             if isinstance(config['label'], str):
                 config['label'] = config['label'].decode('utf-8')
         mutator.json = data
+
+def update_davizSettings(context):
+    """ Add data settings to daviz settings
+    """
+    logger.info('Adding daviz data settings...')
+    tool = queryUtility(IDavizSettings)
+    if not tool:
+        logger.info('Adding daviz data settings... Nothing to do')
+        return
+
+    if tool.settings.get('data.annotations'):
+        logger.info('Adding daviz data settings... Nothing to do')
+        return
+
+    tool.settings['data.annotations'] = u"\n".join(DATA_ANNOTATIONS)
+    logger.info('Adding daviz data settings... DONE')