Merge branch 'develop' into 3087-datafile-newest-filter

raft-tech · Sep 6, 2024 · 158aeb3 · 158aeb3
2 parents f0aeae7 + db43855
commit 158aeb3
Show file tree

Hide file tree

Showing 12 changed files with 274 additions and 10 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -95,7 +95,7 @@ tasks:
     desc: Restart backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   backend-bash:
     desc: Open a shell in the backend container
@@ -178,7 +178,7 @@ tasks:
     desc: Restart frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   frontend-av:
     desc: Start frontend with optional clamav service

diff --git a/docs/Sprint-Review/sprint-106-summary.md b/docs/Sprint-Review/sprint-106-summary.md
@@ -0,0 +1,97 @@
+# sprint-106-summary
+
+8/14/202 - 8/27/2024
+
+### <mark style="color:orange;">Priority Setting</mark>
+
+* Reparsing &#x20;
+  * Reasoning: OFA has ability to reparse and the data process with latest system logic which will enable STTs to get error reports that are meaningful and relevant and OFA gets the needed data. System, as a whole, will be more flexible with data flow (able to hot fix and hit reparse).  Additionally, we want to avoid missing data and be able to repopulate the database after cleaning the data. &#x20;
+* Data Access Strategy&#x20;
+  * Reasoning: Creating daily blockers &#x20;
+* Admin Console Improvements&#x20;
+  * Reasoning: Trigger has been met to refine tickets (Research Synthesis)
+* Improved Dev Tooling&#x20;
+  * improve test\_parse.py 2641&#x20;
+  * separate celery 2592&#x20;
+  * Reasoning: Developing / refining tickets to support the above priorities – These tickets will enhance capabilities while the above are being flushed out &#x20;
+
+### <mark style="color:orange;">Sprint Goal</mark>
+
+**Dev:**
+
+_**Reparsing, Admin Console Improvements, Application Health Monitoring work, and Improved Dev Tooling**_
+
+* \#2965 — As tech lead, I want a database seed implemented for testing
+* \#3102  — Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export&#x20;
+* \#2561 — As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days
+* \#3110 — Spike - Investigate Custom Filter Integration
+* \#3137 — \[bug] OFA unable to export data to csv by record type and fiscal period
+* \#3074 — TDP Data Files page permissions for DIGIT & Sys Admin user groups
+* \#3076  — Admin Filter Enhancements for Data Files Page&#x20;
+
+**DevOps:**
+
+_**Successful deployments across environments and pipeline stability investments**_
+
+*
+
+**Design:**
+
+_**Support reviews, In-app banner to support parsed data, Continue Error Audit (Cat 4)**_
+
+* \#2968  — \[Design Deliverable] Update Error Audit for Cat 4 / QA
+* \#3114 — \[Design Spike] In-app banner for submission history pages w/ data parsed before May 2024
+* \#3143 — August release notes — Knowledge Center & Email Template
+
+
+
+## Tickets
+
+### Completed/Merged
+
+* [#2985 \[Design Deliverable\] Email spec for Admin Notification for stuck files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2985)
+* [#2996 Add dynamic field name to cat4 error messages](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2996)
+* [#3143 August release notes — Knowledge Center & Email Template](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3143)
+* [#3061 \[a11y fix\] Django multi-select filter ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3061)
+* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954)
+* [#3079 DB Backup Script Fix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3079)
+
+### Submitted (QASP Review, OCIO Review)
+
+* [#3064 Re-parse Meta Model](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3064)
+* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3065)
+* [#2792 \[Error Audit\] Category 3 error messages clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2792)
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883)
+* [#2561 As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2561)
+* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3078)
+
+### Ready to Merge
+
+### Closed (Not Merged)
+
+* [#3147 S3 buckets contain fewer datafiles than DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3147)
+
+### Moved to Next Sprint&#x20;
+
+**In Progress**&#x20;
+
+* [#2965 As tech lead, I want a database seed implemented for testing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2965)
+* [#2458 Integrate Nexus into CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2458)
+* [#3137 \[bug\] OFA unable to export data to csv by record type and fiscal period](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3137)
+* [#2968 \[Design Deliverable\] Update Error Audit for Cat 4 / QA](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2968)
+* [#3060 As a TDP user, I need to stay logged in when I'm actively using the system](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3060)
+* [#3074 TDP Data Files page permissions for DIGIT & Sys Admin user groups](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3074)
+* [#3114 \[Design Spike\] In-app banner for submission history pages w/ data parsed before May 2024](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3114)
+* [#3142 \[Research Spike\] Get more detail about Yun & DIGIT's data workflow and use cases](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3142)
+
+#### Blocked
+
+*
+
+**Raft Review**
+
+* [#3043 Sentry: Local environment for Debugging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3043)
+* [#3110 Spike - Investigate Custom Filter Integration](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3110)
+* [#3102 Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3102)
+* [#3087 Admin By Newest Filter Enhancements for Data Files Page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3087)
+* [#3076 Admin Filter Enhancements for Data Files Page ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3076)
diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py
@@ -13,4 +13,5 @@ class EmailType(Enum):
     REQUEST_DENIED = 'request-denied.html'
     DEACTIVATION_WARNING = 'account-deactivation-warning.html'
     ACCOUNT_DEACTIVATED = 'account-deactivated.html'
+    ACCOUNT_DEACTIVATED_ADMIN = 'account-deactivated-admin.html'
     UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html'
diff --git a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py
@@ -3,12 +3,10 @@
 from tdpservice.email.email import automated_email
 from datetime import datetime, timedelta, timezone
 from django.conf import settings
-
+from tdpservice.users.models import User
 
 def send_deactivation_warning_email(users, days):
     """Send an email to users that are about to be deactivated."""
-    from tdpservice.users.models import User
-
     template_path = EmailType.DEACTIVATION_WARNING.value
     text_message = f'Your account will be deactivated in {days} days.'
     subject = f'Account Deactivation Warning: {days} days remaining'

diff --git a/tdrs-backend/tdpservice/email/helpers/admin_notifications.py b/tdrs-backend/tdpservice/email/helpers/admin_notifications.py
@@ -0,0 +1,35 @@
+"""helper functions to administer user accounts."""
+
+def email_admin_deactivated_user(user):
+    """Send an email to OFA Admins when a user is deactivated."""
+    from tdpservice.users.models import User
+    from tdpservice.email.email_enums import EmailType
+    from tdpservice.email.email import automated_email, log
+    from tdpservice.email.tasks import get_ofa_admin_user_emails
+
+    recipient_emails = get_ofa_admin_user_emails()
+    logger_context = {
+        'user_id': user.id,
+        'object_id': user.id,
+        'object_repr': user.username,
+        'content_type': User,
+    }
+
+    template_path = EmailType.ACCOUNT_DEACTIVATED_ADMIN.value
+    text_message = 'A user account has been deactivated.'
+    subject = ' TDP User Account Deactivated due to Inactivity'
+    context = {
+        'user': user,
+    }
+
+    log(f"Preparing email to OFA Admins for deactivated user {user.username}", logger_context=logger_context)
+
+    for recipient_email in recipient_emails:
+        automated_email(
+            email_path=template_path,
+            recipient_email=recipient_email,
+            subject=subject,
+            email_context=context,
+            text_message=text_message,
+            logger_context=logger_context
+        )
diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py
@@ -16,10 +16,35 @@
 from tdpservice.email.email import automated_email, log
 from tdpservice.email.email_enums import EmailType
 from tdpservice.parsers.util import calendar_to_fiscal
+from tdpservice.email.helpers.admin_notifications import email_admin_deactivated_user
 
 
 logger = logging.getLogger(__name__)
 
+@shared_task
+def deactivate_users():
+    """Deactivate users that have not logged in in the last 180 days."""
+    users_to_deactivate = User.objects.filter(
+        last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=180),
+        account_approval_status=AccountApprovalStatusChoices.APPROVED,
+    )
+
+    for user in users_to_deactivate:
+        user.account_approval_status = AccountApprovalStatusChoices.DEACTIVATED
+        user.groups.clear()
+        user.save()
+
+        logger_context = {
+            'user_id': user.id,
+            'object_id': user.id,
+            'object_repr': user.username,
+        }
+        email_admin_deactivated_user(user)
+        log(
+            f"Deactivated user {user.username} for inactivity.",
+            logger_context=logger_context if not settings.DEBUG else None
+        )
+
 
 @shared_task
 def check_for_accounts_needing_deactivation_warning():

diff --git a/tdrs-backend/tdpservice/email/templates/account-deactivated-admin.html b/tdrs-backend/tdpservice/email/templates/account-deactivated-admin.html
@@ -0,0 +1,18 @@
+{% extends 'base.html' %}
+{% block content %}
+<!-- Body copy -->
+<p style="color: #000000;">
+
+    <p>The following user account for the TANF Data Portal (TDP) has been deactivated.</p>
+
+    <p>Account Information:</p>
+    <ul>
+        <li>Name: {{ user.first_name }}</li>
+        <li>Last name: {{ user.last_name }}</li>
+        <li>Email: {{ user.email }}</li>
+    </ul>
+
+    <p>Thank you,</p>
+    TDP Team
+</p>
+{% endblock %}
diff --git a/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py b/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py
@@ -30,6 +30,22 @@ def test_deactivation_email_10_days(user, mocker):
     assert tdpservice.email.helpers.account_deactivation_warning.send_deactivation_warning_email.called_once_with(
         users=[user], days=10)
 
+@pytest.mark.django_db
+def test_deactivate_users(user, mocker):
+    """Test that the deactivate_users task runs."""
+    mocker.patch(
+        'tdpservice.email.helpers.admin_notifications.email_admin_deactivated_user',
+        return_value=None
+    )
+    user.groups.add()
+    user.last_login = datetime.now(tz=timezone.utc) - timedelta(days=181)
+    user.account_approval_status = AccountApprovalStatusChoices.APPROVED
+    user.save()
+    tdpservice.email.tasks.deactivate_users()
+    assert user.groups.count() == 0
+    assert tdpservice.email.helpers.admin_notifications.email_admin_deactivated_user.called_once_with(user)
+
+
 @pytest.mark.django_db
 def test_deactivation_email_3_days(user, mocker):
     """Test that the check_for_accounts_needing_deactivation_warning task runs."""

diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py
@@ -475,6 +475,13 @@ class Common(Configuration):
                 'expires': 15.0,
             },
         },
+        'Deactivate Users': {
+            'task': 'tdpservice.email.tasks.deactivate_users',
+            'schedule': crontab(day_of_week='*', hour='13', minute='0'), # Every day at 1pm UTC (9am EST)
+            'options': {
+                'expires': 15.0,
+            },
+        },
         'Email Admin Number of Access Requests' : {
             'task': 'tdpservice.email.tasks.email_admin_num_access_requests',
             'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST)

diff --git a/tdrs-backend/tdpservice/users/migrations/0041_users_digit_group_add_datafile_permission.py b/tdrs-backend/tdpservice/users/migrations/0041_users_digit_group_add_datafile_permission.py
@@ -0,0 +1,58 @@
+# Generated by Django 3.2.5 on 2021-08-16 14:10
+from django.contrib.auth.models import Group
+from django.db import migrations
+
+from tdpservice.users.permissions import (
+    add_permissions_q,
+    get_permission_ids_for_model,
+    view_permissions_q
+)
+
+
+def set_digit_team_permissions(apps, schema_editor):
+    """Set relevant Group Permissions for DIGIT Team group."""
+    digit = (
+        apps.get_model('auth', 'Group').objects.get(name='DIGIT Team')
+    )
+
+    stt_permissions = get_permission_ids_for_model(
+        'stts',
+        'stt',
+        filters=[view_permissions_q]
+    )
+
+
+    datafile_permissions = get_permission_ids_for_model(
+        'data_files',
+        'datafile',
+        filters=[view_permissions_q, add_permissions_q]
+    )
+
+    # Assign model permissions
+    digit.permissions.add(*datafile_permissions, *stt_permissions)
+
+def unset_digit_team_permissions(apps, schema_editor):
+    """Remove all Group Permissions added to DIGIT Team."""
+    digit = (
+        apps.get_model('auth', 'Group').objects.get(name='DIGIT Team')
+    )
+    datafile_permissions = get_permission_ids_for_model(
+        'data_files',
+        'datafile',
+        filters=[view_permissions_q, add_permissions_q]
+    )
+    digit.permissions.remove(*datafile_permissions)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('auth', '__latest__'),
+        ('users', '0040_users_digit_group_permissions'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            set_digit_team_permissions,
+            reverse_code=unset_digit_team_permissions
+        )
+    ]
diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py
@@ -180,6 +180,8 @@ def test_digit_team_permissions(digit_team):
     expected_permissions = {'parsers.view_parsererror',
                             'parsers.view_datafilesummary',
                             'data_files.view_datafile',
+                            'data_files.add_datafile',
+                            'stts.view_stt',
                             'search_indexes.view_ssp_m3',
                             'search_indexes.view_tribal_tanf_t5',
                             'search_indexes.view_tribal_tanf_t3',
@@ -200,7 +202,7 @@ def test_digit_team_permissions(digit_team):
                             'search_indexes.view_tanf_t4',
                             'search_indexes.view_ssp_m6',
                             'search_indexes.view_tribal_tanf_t2',
-                            'search_indexes.view_tanf_t6'
+                            'search_indexes.view_tanf_t6',
                             }
     group_permissions = digit_team.get_group_permissions()
     assert group_permissions == expected_permissions
diff --git a/tdrs-frontend/src/components/Reports/Reports.jsx b/tdrs-frontend/src/components/Reports/Reports.jsx
@@ -38,6 +38,9 @@ function Reports() {
   // The logged in user saved in our redux `auth` state object
   const user = useSelector((state) => state.auth.user)
   const isOFAAdmin = useSelector(selectPrimaryUserRole)?.name === 'OFA Admin'
+  const isDIGITTeam = useSelector(selectPrimaryUserRole)?.name === 'DIGIT Team'
+  const isSystemAdmin =
+    useSelector(selectPrimaryUserRole)?.name === 'OFA System Admin'
   const sttList = useSelector((state) => state?.stts?.sttList)
 
   const [errorModalVisible, setErrorModalVisible] = useState(false)
@@ -59,7 +62,8 @@ function Reports() {
     Q4: 'Quarter 4 (July - September)',
   }
 
-  const currentStt = isOFAAdmin ? selectedStt : userProfileStt
+  const currentStt =
+    isOFAAdmin || isDIGITTeam || isSystemAdmin ? selectedStt : userProfileStt
 
   const stt = sttList?.find((stt) => stt?.name === currentStt)
 
@@ -68,7 +72,8 @@ function Reports() {
 
   const errorsCount = formValidation.errors
 
-  const missingStt = !isOFAAdmin && !currentStt
+  const missingStt =
+    !isOFAAdmin && !isDIGITTeam && !isSystemAdmin && !currentStt
 
   const errorsRef = useRef(null)
 
@@ -194,7 +199,7 @@ function Reports() {
       )
       const touchedFields = Object.keys(touched).length
 
-      const expected_fields = isOFAAdmin ? 3 : 2
+      const expected_fields = isOFAAdmin || isDIGITTeam || isSystemAdmin ? 3 : 2
 
       const errors = touchedFields === 3 ? expected_fields - form.length : 0
 
@@ -215,6 +220,8 @@ function Reports() {
     setFormValidationState,
     touched,
     isOFAAdmin,
+    isDIGITTeam,
+    isSystemAdmin,
   ])
 
   return (
@@ -237,7 +244,7 @@ function Reports() {
           </div>
         )}
         <form>
-          {isOFAAdmin && (
+          {(isOFAAdmin || isDIGITTeam || isSystemAdmin) && (
             <div
               className={classNames('usa-form-group maxw-mobile margin-top-4', {
                 'usa-form-group--error': formValidation.stt,