Merge pull request #3018 from raft-tech/2473-data-submission-notifica…

…tion

2473 - Upcoming data deadline notification

tdrs-backend/tdpservice/email/email_enums.py

@@ -13,3 +13,4 @@ class EmailType(Enum):
     REQUEST_DENIED = 'request-denied.html'
     DEACTIVATION_WARNING = 'account-deactivation-warning.html'
     ACCOUNT_DEACTIVATED = 'account-deactivated.html'
+    UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html'

tdrs-backend/tdpservice/email/tasks.py

@@ -11,6 +11,11 @@
 import logging
 from tdpservice.email.helpers.account_access_requests import send_num_access_requests_email
 from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email
+from tdpservice.stts.models import STT
+from tdpservice.data_files.models import DataFile
+from tdpservice.email.email import automated_email, log
+from tdpservice.email.email_enums import EmailType
+from tdpservice.parsers.util import calendar_to_fiscal
 
 
 logger = logging.getLogger(__name__)
@@ -75,3 +80,84 @@ def email_admin_num_access_requests():
                                    subject,
                                    email_context,
                                    )
+
+
+@shared_task
+def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter):
+    """Send all Data Analysts a reminder to submit if they have not already."""
+    now = datetime.now()
+    fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter)
+
+    all_locations = STT.objects.all()
+
+    reminder_locations = []
+    year_quarter_files = DataFile.objects.all().filter(
+        year=fiscal_year,
+        quarter=fiscal_quarter,
+    )
+
+    for loc in all_locations:
+        submitted_sections = year_quarter_files.filter(stt=loc).values_list('section', flat=True).distinct()
+        required_sections = loc.filenames.keys()
+
+        submitted_all_sections = True
+        for s in required_sections:
+            if s not in submitted_sections:
+                submitted_all_sections = False
+
+        if not submitted_all_sections:
+            reminder_locations.append(loc)
+
+    template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value
+    text_message = f'Your datafiles are due by {due_date}.'
+
+    all_data_analysts = User.objects.all().filter(
+        account_approval_status=AccountApprovalStatusChoices.APPROVED,
+        groups=Group.objects.get(name='Data Analyst')
+    )
+
+    for loc in reminder_locations:
+        tanf_ssp_label = 'TANF and SSP' if loc.ssp else 'TANF'
+        subject = f'Action Requested: Please submit your {tanf_ssp_label} data files'
+
+        recipients = all_data_analysts.filter(stt=loc)
+
+        for rec in recipients:
+            context = {
+                'first_name': rec.first_name,
+                'fiscal_year': fiscal_year,
+                'fiscal_quarter': fiscal_quarter,
+                'submission_deadline': due_date,
+                'url': settings.FRONTEND_BASE_URL,
+                'subject': subject
+            }
+
+            logger_context = {
+                'user_id': rec.id,
+                'object_id': rec.id,
+                'object_repr': rec.username,
+            }
+
+            automated_email(
+                email_path=template_path,
+                recipient_email=rec.username,
+                subject=subject,
+                email_context=context,
+                text_message=text_message,
+                logger_context=logger_context
+            )
+
+        if len(recipients) == 0:
+            system_user, created = User.objects.get_or_create(username='system')
+            if created:
+                log('Created reserved system user.')
+
+            logger_context = {
+                'user_id': system_user.pk,
+                'object_id': loc.id,
+                'object_repr': loc.name,
+            }
+            log(
+                f"{loc.name} has no recipients for data submission deadline reminder.",
+                logger_context=logger_context if not settings.DEBUG else None
+            )

tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html

@@ -0,0 +1,27 @@
+{% extends 'base.html' %}
+{% block content %}
+
+<!-- Body copy -->
+<p>Hello {{ first_name }},</p>
+<p>This is a friendly reminder that your data files are due in 5 days. </p>
+<p>Please sign in to the <a href="{{ url }}" target="_blank">TANF Data Portal</a> 
+to upload and submit data files for <b>Fiscal Year {{ fiscal_year }} - {{ fiscal_quarter}}</b> by <b>{{ submission_deadline }}.</b></p>
+
+<!-- This link uses descriptive text to inform the user what will happen with the link is tapped. -->
+<!-- It also uses inline styles since some email clients won't render embedded styles from the head. -->
+<!-- <a href="" style="color: #336a90; text-decoration: underline;">Reset your password now</a> -->
+<p><a class="button" href="{{ url }}" style="background-color:#336a90;border-radius:4px;color:#ffffff;display:inline-block;font-family:sans-serif;font-size:18px;font-weight:bold;line-height:60px;text-align:center;text-decoration:none;width: auto; padding-left: 24px; padding-right: 24px;-webkit-text-size-adjust:none;">Submit your data files</a></p>
+
+<b>Need help?</b><br>
+We're here for you! Check out the <a href="http://tdp-project-updates.app.cloud.gov/knowledge-center/" target="_blank">TDP Knowledge Center</a> for specific gudiance on  <a href="https://tdp-project-updates.app.cloud.gov/knowledge-center/uploading-data.html" target="_blank">Submitting Data Files</a> and <a href="https://tdp-project-updates.app.cloud.gov/knowledge-center/faq.html">Frequently Asked Questions</a>.
+<p> 
+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Please reach out to the TDP support team at <a href="mailto:[email protected]" target="_blank">[email protected]</a> if you have any questions or need assistance.
+<p>Thank you,</p>
+<p>The TDP Team</p>
+
+
+<!-- This link uses descriptive text to inform the user what will happen with the link is tapped. -->
+<!-- It also uses inline styles since some email clients won't render embedded styles from the head. -->
+<!-- <a href="" style="color: #336a90; text-decoration: underline;">Reset your password now</a> -->
+
+{% endblock %}

tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py

@@ -0,0 +1,134 @@
+"""Test function for sending upcoming data deadline reminders."""
+import pytest
+from datetime import datetime
+from django.core import mail
+from tdpservice.email.tasks import send_data_submission_reminder
+from django.contrib.auth.models import Group
+from tdpservice.stts.models import STT
+from tdpservice.users.models import User
+from tdpservice.data_files.models import DataFile
+
+
+@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [
+    ('February 14', 'Oct - Dec', 'Q1'),
+    ('May 15th', 'Jan - Mar', 'Q2'),
+    ('August 14th', 'Apr - Jun', 'Q3'),
+    ('November 14th', 'Jul - Sep', 'Q4'),
+])
+@pytest.mark.django_db
+def test_upcoming_deadline_sends_no_sections_submitted(
+        due_date, reporting_period, fiscal_quarter):
+    """Test that the send_deactivation_warning_email function runs when no sections have been submitted."""
+    stt = STT.objects.create(
+        name='Arkansas',
+        filenames={
+            "Active Case Data": "test-filename.txt",
+            "Closed Case Data": "test-filename-closed.txt",
+        }
+    )
+
+    data_analyst = User.objects.create(
+        username='[email protected]',
+        stt=stt,
+        account_approval_status='Approved'
+    )
+    data_analyst.groups.add(Group.objects.get(name='Data Analyst'))
+    data_analyst.save()
+
+    send_data_submission_reminder(due_date, reporting_period, fiscal_quarter)
+
+    assert len(mail.outbox) == 1
+    assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files"
+
+
+@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [
+    ('February 14', 'Oct - Dec', 'Q1'),
+    ('May 15th', 'Jan - Mar', 'Q2'),
+    ('August 14th', 'Apr - Jun', 'Q3'),
+    ('November 14th', 'Jul - Sep', 'Q4'),
+])
+@pytest.mark.django_db
+def test_upcoming_deadline_sends_some_sections_submitted(
+        due_date, reporting_period, fiscal_quarter):
+    """Test that the send_deactivation_warning_email function runs when some sections have been submitted."""
+    stt = STT.objects.create(
+        name='Arkansas',
+        filenames={
+            "Active Case Data": "test-filename.txt",
+            "Closed Case Data": "test-filename-closed.txt",
+        }
+    )
+
+    data_analyst = User.objects.create(
+        username='[email protected]',
+        stt=stt,
+        account_approval_status='Approved'
+    )
+    data_analyst.groups.add(Group.objects.get(name='Data Analyst'))
+    data_analyst.save()
+
+    now = datetime.now()
+    fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year
+
+    _ = DataFile.create_new_version({
+        "section": 'Active Case Data',
+        "quarter": fiscal_quarter,
+        "year": fiscal_year,
+        "stt": stt,
+        "user": data_analyst,
+    })
+
+    send_data_submission_reminder(due_date, reporting_period, fiscal_quarter)
+
+    assert len(mail.outbox) == 1
+    assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files"
+
+
+@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [
+    ('February 14', 'Oct - Dec', 'Q1'),
+    ('May 15th', 'Jan - Mar', 'Q2'),
+    ('August 14th', 'Apr - Jun', 'Q3'),
+    ('November 14th', 'Jul - Sep', 'Q4'),
+])
+@pytest.mark.django_db
+def test_upcoming_deadline_no_send_when_all_sections_complete(
+        due_date, reporting_period, fiscal_quarter):
+    """Test that the send_deactivation_warning_email function does not run when all sections have been submitted."""
+    stt = STT.objects.create(
+        name='Arkansas',
+        filenames={
+            "Active Case Data": "test-filename.txt",
+            "Closed Case Data": "test-filename-closed.txt",
+        }
+    )
+
+    data_analyst = User.objects.create(
+        username='[email protected]',
+        stt=stt,
+        account_approval_status='Approved'
+    )
+    data_analyst.groups.add(Group.objects.get(name='Data Analyst'))
+    data_analyst.save()
+
+    now = datetime.now()
+    fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year
+
+    _ = DataFile.create_new_version({
+        "section": 'Active Case Data',
+        "quarter": fiscal_quarter,
+        "year": fiscal_year,
+        "stt": stt,
+        "user": data_analyst,
+    })
+
+    _ = DataFile.create_new_version({
+        "section": 'Closed Case Data',
+        "quarter": fiscal_quarter,
+        "year": fiscal_year,
+        "stt": stt,
+        "user": data_analyst,
+    })
+
+    send_data_submission_reminder(due_date, reporting_period, fiscal_quarter)
+
+    assert len(mail.outbox) == 0

tdrs-backend/tdpservice/parsers/util.py

@@ -152,6 +152,12 @@ def fiscal_to_calendar(year, fiscal_quarter):
     ind_qtr = array.index(int_qtr)  # get the index so we can easily wrap-around end of array
     return year, "Q{}".format(array[ind_qtr - 1])  # return the previous quarter
 
+
+def calendar_to_fiscal(calendar_year, fiscal_quarter):
+    """Decrement the calendar year if in Q1."""
+    return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year
+
+
 def transform_to_months(quarter):
     """Return a list of months in a quarter depending the quarter's format."""
     match quarter:

tdrs-backend/tdpservice/settings/common.py

@@ -479,6 +479,50 @@ class Common(Configuration):
             'task': 'tdpservice.email.tasks.email_admin_num_access_requests',
             'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST)
         },
+        'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': {
+            'task': 'tdpservice.email.tasks.send_data_submission_reminder',
+            # Feb 9 at 1pm UTC (9am EST)
+            'schedule': crontab(month_of_year='2', day_of_month='9', hour='13', minute='0'),
+            # 'schedule': crontab(minute='*/3'),
+            'kwargs': {
+                'due_date': 'February 14th',
+                'reporting_period': 'Oct - Dec',
+                'fiscal_quarter': 'Q1',
+            }
+        },
+        'Email Data Analyst Q2 Upcoming Submission Deadline Reminder': {
+            'task': 'tdpservice.email.tasks.send_data_submission_reminder',
+            # May 10 at 1pm UTC (9am EST)
+            'schedule': crontab(month_of_year='5', day_of_month='10', hour='13', minute='0'),
+            # 'schedule': crontab(minute='*/3'),
+            'kwargs': {
+                'due_date': 'May 15th',
+                'reporting_period': 'Jan - Mar',
+                'fiscal_quarter': 'Q2',
+            }
+        },
+        'Email Data Analyst Q3 Upcoming Submission Deadline Reminder': {
+            'task': 'tdpservice.email.tasks.send_data_submission_reminder',
+            # Aug 9 at 1pm UTC (9am EST)
+            'schedule': crontab(month_of_year='8', day_of_month='9', hour='13', minute='0'),
+            # 'schedule': crontab(minute='*/3'),
+            'kwargs': {
+                'due_date': 'August 14th',
+                'reporting_period': 'Apr - Jun',
+                'fiscal_quarter': 'Q3',
+            }
+        },
+        'Email Data Analyst Q4 Upcoming Submission Deadline Reminder': {
+            'task': 'tdpservice.email.tasks.send_data_submission_reminder',
+            # Nov 9 at 1pm UTC (9am EST)
+            'schedule': crontab(month_of_year='11', day_of_month='9', hour='13', minute='0'),
+            # 'schedule': crontab(minute='*/3'),
+            'kwargs': {
+                'due_date': 'November 14th',
+                'reporting_period': 'Jul - Sep',
+                'fiscal_quarter': 'Q4',
+            }
+        },
     }
 
     CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None)