[QCDP24 30 32 34] added email notifications (#11)

* [QCDP24-30] added comment notification

* [QCDP24-32] added deletion notification

* [QCDP24-34] added follower notification

* Added state column to `datarequest` table
Updated `delete_datarequest` action to soft delete

* [QCDP24-32] implemented state filter

* [QCDP24-32] force active state on update

* Fixed lint issues

---------

Co-authored-by: Mark Calvert <[email protected]>

ckanext/datarequests/actions.py

@@ -20,12 +20,13 @@
 
 import datetime
 import logging
+
 try:
     from html import escape
 except ImportError:
     from cgi import escape
 
-from ckan import authz
+from ckan import authz, model
 from ckan.lib import mailer
 from ckan.lib.redis import connect_to_redis
 from ckan.plugins import toolkit as tk
@@ -44,12 +45,12 @@
 THROTTLE_ERROR = "Too many requests submitted, please wait {} minutes and try again"
 
 
-def _get_user(user_id):
+def _get_user(user_id, keep_email=False):
     try:
         if user_id in USERS_CACHE:
             return USERS_CACHE[user_id]
         else:
-            user = tk.get_action('user_show')({'ignore_auth': True}, {'id': user_id})
+            user = tk.get_action('user_show')({'ignore_auth': True, 'keep_email': keep_email}, {'id': user_id})
             USERS_CACHE[user_id] = user
             return user
     except Exception as e:
@@ -168,21 +169,26 @@ def _get_datarequest_followers(context, datarequest_dict):
     followers = db.DataRequestFollower.get(datarequest_id=datarequest_id)
     for follower in followers:
         if follower.user_id != context['auth_user_obj'].id:
-            follower.user = _get_user(follower.user_id)
-            users.append({
-                'email': follower.user['email'],
-                'name': follower.user['name']
-            })
+            follower.user = _get_user(follower.user_id, True)
+            if follower.user.get('email', None):
+                users.append({
+                    'email': follower.user['email'],
+                    'name': follower.user['name'] or follower.user['email'],
+                })
 
     return users
 
 
-def _send_mail(action_type, datarequest, job_title=None, context=None):
-    user_list = [{
-        'email': config.get('ckanext.datarequests.internal_data_catalogue_support_team_email'),
-        'name': config.get('ckanext.datarequests.internal_data_catalogue_support_team_name')
-    }]
-    if action_type == 'new_datarequest':
+def _send_mail(action_type, datarequest, job_title=None, context=None, comment=None):
+    user_list = []
+
+    def get_catalog_support_team():
+        user_list.append({
+            'email': config.get('ckanext.datarequests.internal_data_catalogue_support_team_email'),
+            'name': config.get('ckanext.datarequests.internal_data_catalogue_support_team_name')
+        })
+
+    def get_dataset_poc():
         dataset = _get_package(datarequest.get('requested_dataset'))
         dataset_poi_email = dataset.get('point_of_contact_email') if dataset else None
         dataset_poi_name = dataset.get('point_of_contact') if dataset else None
@@ -191,7 +197,8 @@ def _send_mail(action_type, datarequest, job_title=None, context=None):
                 'email': dataset_poi_email,
                 'name': dataset_poi_name
             })
-    elif action_type == 'update_datarequest':
+
+    def get_datarequest_creator():
         requester_email = datarequest['user']['email']
         requester_name = datarequest['user']['name']
         if requester_email:
@@ -200,13 +207,48 @@ def _send_mail(action_type, datarequest, job_title=None, context=None):
                 'name': requester_name
             })
 
+    def get_datarequest_followers():
         followers = _get_datarequest_followers(context, datarequest)
         user_list.extend(followers)
 
+    match action_type:
+        case 'new_datarequest':
+            get_catalog_support_team()
+            get_dataset_poc()
+
+        case 'update_datarequest':
+            get_catalog_support_team()
+            get_datarequest_creator()
+
+        case 'comment_datarequest':
+            get_catalog_support_team()
+            get_datarequest_followers()
+
+            if datarequest['user']['id'] == comment['user_id']:
+                # If this comment from datarequest creator, notify the dataset POC.
+                get_dataset_poc()
+            else:
+                get_datarequest_creator()
+
+        case 'delete_datarequest':
+            get_catalog_support_team()
+            get_datarequest_followers()
+
+        case 'update_datarequest_follower':
+            get_datarequest_followers()
+
+    # Load requesting organisation.
+    if datarequest.get('requesting_organisation'):
+        org = _get_organization(datarequest['requesting_organisation'])
+        if org:
+            datarequest['requesting_organisation_dict'] = org
+
+    # Sends the email to users.
     for user in user_list:
         try:
             extra_vars = {
                 'datarequest': datarequest,
+                'comment': comment,
                 'user': user,
                 'site_title': config.get('ckan.site_title'),
                 'site_url': config.get('ckan.site_url')
@@ -404,18 +446,34 @@ def update_datarequest(context, data_dict):
     # Validate data
     validator.validate_datarequest(context, data_dict)
 
+    # Track changes in the data request
+    has_changes = False
+    old_data_json = _dictize_datarequest(data_req)
+    for key, value in data_dict.items():
+        if old_data_json[key] != value:
+            has_changes = True
+            break
+
     # Set the data provided by the user in the data_red
     current_status = data_req.status
     _undictize_datarequest_basic(data_req, data_dict)
     new_status = data_req.status
 
+    # Always force datarequest to active state when updating, some older dataset may be in null state
+    data_req.state = model.State.ACTIVE
+
     session.add(data_req)
     session.commit()
 
     datarequest_dict = _dictize_datarequest(data_req)
 
     if current_status != new_status:
         _send_mail('update_datarequest', datarequest_dict, 'Data Request Status Change Email', context)
+        has_changes = True
+
+    # Send follower and email notifications if there is changes in the data request
+    if has_changes:
+        _send_mail('update_datarequest_follower', datarequest_dict, 'Data Request Updated Email', context)
 
     return datarequest_dict
 
@@ -482,6 +540,9 @@ def list_datarequests(context, data_dict):
     # Filter by status
     status = data_dict.get('status', None)
 
+    #  Filter by state
+    state = data_dict.get('state', None)
+
     # Free text filter
     q = data_dict.get('q', None)
 
@@ -495,7 +556,7 @@ def list_datarequests(context, data_dict):
     # Call the function
     db_datarequests = db.DataRequest.get_ordered_by_date(requesting_organisation=requesting_organisation,
                                                          user_id=user_id, status=status,
-                                                         q=q, desc=desc)
+                                                         q=q, desc=desc, state=state)
 
     # Dictize the results
     datarequests = []
@@ -592,10 +653,14 @@ def delete_datarequest(context, data_dict):
         raise tk.ObjectNotFound(tk._('Data Request %s not found in the data base') % datarequest_id)
 
     data_req = result[0]
-    session.delete(data_req)
+    data_req.delete()
     session.commit()
 
-    return _dictize_datarequest(data_req)
+    # Send emails
+    datarequest_dict = _dictize_datarequest(data_req)
+    _send_mail('delete_datarequest', datarequest_dict, 'Data Request Deletion Email', context)
+
+    return datarequest_dict
 
 
 def close_datarequest(context, data_dict):
@@ -695,7 +760,13 @@ def comment_datarequest(context, data_dict):
     session.add(comment)
     session.commit()
 
-    return _dictize_comment(comment)
+    comment_dict = _dictize_comment(comment)
+
+    # Send emails
+    datarequest_dict = _dictize_datarequest(db.DataRequest.get(id=datarequest_id)[0])
+    _send_mail('comment_datarequest', datarequest_dict, 'Data Request Comment Email', context, comment_dict)
+
+    return comment_dict
 
 
 def show_datarequest_comment(context, data_dict):

ckanext/datarequests/constants.py

@@ -32,7 +32,7 @@
 FOLLOW_DATAREQUEST = 'follow_datarequest'
 UNFOLLOW_DATAREQUEST = 'unfollow_datarequest'
 PURGE_DATAREQUESTS = 'purge_datarequests'
-NAME_MAX_LENGTH = 100
+NAME_MAX_LENGTH = 1000
 DESCRIPTION_MAX_LENGTH = 1000
 COMMENT_MAX_LENGTH = DESCRIPTION_MAX_LENGTH
 DATAREQUESTS_PER_PAGE = 10

ckanext/datarequests/db.py

@@ -38,24 +38,30 @@ def uuid4():
     return str(uuid.uuid4())
 
 
-class DataRequest(model.DomainObject):
+class DataRequest(model.core.StatefulObjectMixin, model.DomainObject):
 
     @classmethod
     def get(cls, **kw):
         '''Finds all the instances required.'''
         query = model.Session.query(cls).autoflush(False)
+        query = query.filter(or_(cls.state == model.core.State.ACTIVE, cls.state is None))
         return query.filter_by(**kw).all()
 
     @classmethod
     def datarequest_exists(cls, title):
         '''Returns true if there is a Data Request with the same title (case insensitive)'''
         query = model.Session.query(cls).autoflush(False)
+        query = query.filter(or_(cls.state == model.core.State.ACTIVE, cls.state is None))
         return query.filter(func.lower(cls.title) == func.lower(title)).first() is not None
 
     @classmethod
-    def get_ordered_by_date(cls, requesting_organisation=None, user_id=None, closed=None, q=None, desc=False, status=None):
+    def get_ordered_by_date(cls, requesting_organisation=None, user_id=None, closed=None, q=None, desc=False, status=None, state=None):
         '''Personalized query'''
         query = model.Session.query(cls).autoflush(False)
+        if state is None:
+            query = query.filter(or_(cls.state == model.core.State.ACTIVE, cls.state is None))
+        else:
+            query = query.filter_by(state=state)
 
         params = {}
 
@@ -121,7 +127,7 @@ def get_ordered_by_date(cls, requesting_organisation=None, user_id=None, closed=
     @classmethod
     def get_open_datarequests_number(cls):
         '''Returns the number of data requests that are open'''
-        return model.Session.query(func.count(cls.id)).filter_by(closed=False).scalar()
+        return model.Session.query(func.count(cls.id)).filter_by(closed=False).filter(or_(cls.state == model.core.State.ACTIVE, cls.state is None)).scalar()
 
 
 class Comment(model.DomainObject):
@@ -186,6 +192,7 @@ def get_datarequest_followers_number(cls, **kw):
                               sa.Column('data_outputs_description', sa.types.Unicode(constants.DESCRIPTION_MAX_LENGTH), primary_key=False, default=u''),
                               sa.Column('status', sa.types.Unicode(constants.MAX_LENGTH_255), primary_key=False, default=u'Assigned'),
                               sa.Column('requested_dataset', sa.types.Unicode(constants.MAX_LENGTH_255), primary_key=False, default=u''),
+                              sa.Column('state', sa.types.UnicodeText, default=model.core.State.ACTIVE),
                               extend_existing=True
                               )
 
@@ -281,3 +288,12 @@ def update_db(deprecated_model=None):
         if 'requested_dataset' not in meta.tables['datarequests'].columns:
             log.info("DataRequests-UpdateDB: 'requested_dataset' field does not exist, adding...")
             DDL('ALTER TABLE "datarequests" ADD COLUMN "requested_dataset" text COLLATE pg_catalog."default";').execute(model.Session.get_bind())
+
+        # change the title field to 1000 characters if it is still 100
+        if 'title' in meta.tables['datarequests'].columns and meta.tables['datarequests'].columns['title'].type.length == 100:
+            log.info("DataRequests-UpdateDB: 'title' field exists and length is 100, changing to 1000 characters...")
+            DDL('ALTER TABLE "datarequests" ALTER COLUMN "title" TYPE varchar(1000)').execute(model.Session.get_bind())
+
+        if 'state' not in meta.tables['datarequests'].columns:
+            log.info("DataRequests-UpdateDB: 'state' field does not exist, adding...")
+            DDL('ALTER TABLE "datarequests" ADD COLUMN "state" text COLLATE pg_catalog."default";').execute(model.Session.get_bind())

ckanext/datarequests/templates/emails/bodies/comment_datarequest.txt

@@ -0,0 +1,8 @@
+A new data access request comment has been added.
+To view or reply to this comment, follow this link: {{ site_url }}/datarequest/comment/{{ datarequest.id }}
+
+Comment: {{ comment.comment }}
+
+User: {{ comment.user.fullname or comment.user.name }}
+
+Do not reply to this email.

ckanext/datarequests/templates/emails/bodies/delete_datarequest.txt

@@ -0,0 +1,14 @@
+A data access request has been deleted.  This is a soft delete and the details will remain on the portal database.  This notification is for record keeping purposes only.
+
+Data Request Creator: {{ datarequest.user.name }}
+Requested data: {{ datarequest.title }} ({{ datarequest.requested_dataset }})
+Data use type: {{ datarequest.data_use_type }}
+Purpose of data use: {{ datarequest.description }}
+Who will access this data: {{ datarequest.who_will_access_this_data }}
+Requesting organisation: {{ datarequest.requesting_organisation_dict.name }} ({{ datarequest.requesting_organisation }})
+Data storage environment: {{ datarequest.data_storage_environment }}
+Data outputs type: {{ datarequest.data_outputs_type }}
+Data outputs description: {{ datarequest.data_outputs_description }}
+Status: {{ datarequest.status }}
+
+Do not reply to this email.

ckanext/datarequests/templates/emails/bodies/update_datarequest_follower.txt

@@ -0,0 +1,9 @@
+An update has been made to a data access request you are following.
+
+Requested data: {{ datarequest.title }}
+
+To view the data access request, follow this link: {{ site_url }}/datarequest/{{ datarequest.id }}
+
+If you require assistance, please contact the Internal Data Catalogue management team at [email protected].
+
+Do not reply to this email.

ckanext/datarequests/templates/emails/subjects/comment_datarequest.txt

@@ -0,0 +1 @@
+Queensland Government Internal Data Catalogue – Data access request comment

ckanext/datarequests/templates/emails/subjects/delete_datarequest.txt

@@ -0,0 +1 @@
+Queensland Government Internal Data Catalogue – Data access request deletion

ckanext/datarequests/templates/emails/subjects/update_datarequest_follower.txt

@@ -0,0 +1 @@
+Queensland Government Internal Data Catalogue – Data access request