diff --git a/Taskfile.yml b/Taskfile.yml
index 5985f0604..2c67784b9 100644
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -95,7 +95,7 @@ tasks:
desc: Restart backend web server
dir: tdrs-backend
cmds:
- - docker-compose -f docker-compose.yml restart -d
+ - docker-compose -f docker-compose.yml restart
backend-bash:
desc: Open a shell in the backend container
@@ -178,7 +178,7 @@ tasks:
desc: Restart frontend web server
dir: tdrs-frontend
cmds:
- - docker-compose -f docker-compose.yml restart -d
+ - docker-compose -f docker-compose.yml restart
frontend-av:
desc: Start frontend with optional clamav service
diff --git a/docs/Sprint-Review/sprint-106-summary.md b/docs/Sprint-Review/sprint-106-summary.md
new file mode 100644
index 000000000..6d211ed81
--- /dev/null
+++ b/docs/Sprint-Review/sprint-106-summary.md
@@ -0,0 +1,97 @@
+# sprint-106-summary
+
+8/14/202 - 8/27/2024
+
+### Priority Setting
+
+* Reparsing
+ * Reasoning: OFA has ability to reparse and the data process with latest system logic which will enable STTs to get error reports that are meaningful and relevant and OFA gets the needed data. System, as a whole, will be more flexible with data flow (able to hot fix and hit reparse). Additionally, we want to avoid missing data and be able to repopulate the database after cleaning the data.
+* Data Access Strategy
+ * Reasoning: Creating daily blockers
+* Admin Console Improvements
+ * Reasoning: Trigger has been met to refine tickets (Research Synthesis)
+* Improved Dev Tooling
+ * improve test\_parse.py 2641
+ * separate celery 2592
+ * Reasoning: Developing / refining tickets to support the above priorities – These tickets will enhance capabilities while the above are being flushed out
+
+### Sprint Goal
+
+**Dev:**
+
+_**Reparsing, Admin Console Improvements, Application Health Monitoring work, and Improved Dev Tooling**_
+
+* \#2965 — As tech lead, I want a database seed implemented for testing
+* \#3102 — Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export
+* \#2561 — As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days
+* \#3110 — Spike - Investigate Custom Filter Integration
+* \#3137 — \[bug] OFA unable to export data to csv by record type and fiscal period
+* \#3074 — TDP Data Files page permissions for DIGIT & Sys Admin user groups
+* \#3076 — Admin Filter Enhancements for Data Files Page
+
+**DevOps:**
+
+_**Successful deployments across environments and pipeline stability investments**_
+
+*
+
+**Design:**
+
+_**Support reviews, In-app banner to support parsed data, Continue Error Audit (Cat 4)**_
+
+* \#2968 — \[Design Deliverable] Update Error Audit for Cat 4 / QA
+* \#3114 — \[Design Spike] In-app banner for submission history pages w/ data parsed before May 2024
+* \#3143 — August release notes — Knowledge Center & Email Template
+
+
+
+## Tickets
+
+### Completed/Merged
+
+* [#2985 \[Design Deliverable\] Email spec for Admin Notification for stuck files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2985)
+* [#2996 Add dynamic field name to cat4 error messages](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2996)
+* [#3143 August release notes — Knowledge Center & Email Template](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3143)
+* [#3061 \[a11y fix\] Django multi-select filter ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3061)
+* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954)
+* [#3079 DB Backup Script Fix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3079)
+
+### Submitted (QASP Review, OCIO Review)
+
+* [#3064 Re-parse Meta Model](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3064)
+* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3065)
+* [#2792 \[Error Audit\] Category 3 error messages clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2792)
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883)
+* [#2561 As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2561)
+* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3078)
+
+### Ready to Merge
+
+### Closed (Not Merged)
+
+* [#3147 S3 buckets contain fewer datafiles than DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3147)
+
+### Moved to Next Sprint
+
+**In Progress**
+
+* [#2965 As tech lead, I want a database seed implemented for testing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2965)
+* [#2458 Integrate Nexus into CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2458)
+* [#3137 \[bug\] OFA unable to export data to csv by record type and fiscal period](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3137)
+* [#2968 \[Design Deliverable\] Update Error Audit for Cat 4 / QA](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2968)
+* [#3060 As a TDP user, I need to stay logged in when I'm actively using the system](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3060)
+* [#3074 TDP Data Files page permissions for DIGIT & Sys Admin user groups](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3074)
+* [#3114 \[Design Spike\] In-app banner for submission history pages w/ data parsed before May 2024](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3114)
+* [#3142 \[Research Spike\] Get more detail about Yun & DIGIT's data workflow and use cases](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3142)
+
+#### Blocked
+
+*
+
+**Raft Review**
+
+* [#3043 Sentry: Local environment for Debugging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3043)
+* [#3110 Spike - Investigate Custom Filter Integration](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3110)
+* [#3102 Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3102)
+* [#3087 Admin By Newest Filter Enhancements for Data Files Page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3087)
+* [#3076 Admin Filter Enhancements for Data Files Page ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3076)
diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py
index 3b7028681..4527b6016 100644
--- a/tdrs-backend/tdpservice/email/email_enums.py
+++ b/tdrs-backend/tdpservice/email/email_enums.py
@@ -13,4 +13,5 @@ class EmailType(Enum):
REQUEST_DENIED = 'request-denied.html'
DEACTIVATION_WARNING = 'account-deactivation-warning.html'
ACCOUNT_DEACTIVATED = 'account-deactivated.html'
+ ACCOUNT_DEACTIVATED_ADMIN = 'account-deactivated-admin.html'
UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html'
diff --git a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py
index 75af0a4ef..9851c155c 100644
--- a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py
+++ b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py
@@ -3,12 +3,10 @@
from tdpservice.email.email import automated_email
from datetime import datetime, timedelta, timezone
from django.conf import settings
-
+from tdpservice.users.models import User
def send_deactivation_warning_email(users, days):
"""Send an email to users that are about to be deactivated."""
- from tdpservice.users.models import User
-
template_path = EmailType.DEACTIVATION_WARNING.value
text_message = f'Your account will be deactivated in {days} days.'
subject = f'Account Deactivation Warning: {days} days remaining'
diff --git a/tdrs-backend/tdpservice/email/helpers/admin_notifications.py b/tdrs-backend/tdpservice/email/helpers/admin_notifications.py
new file mode 100644
index 000000000..594e48710
--- /dev/null
+++ b/tdrs-backend/tdpservice/email/helpers/admin_notifications.py
@@ -0,0 +1,35 @@
+"""helper functions to administer user accounts."""
+
+def email_admin_deactivated_user(user):
+ """Send an email to OFA Admins when a user is deactivated."""
+ from tdpservice.users.models import User
+ from tdpservice.email.email_enums import EmailType
+ from tdpservice.email.email import automated_email, log
+ from tdpservice.email.tasks import get_ofa_admin_user_emails
+
+ recipient_emails = get_ofa_admin_user_emails()
+ logger_context = {
+ 'user_id': user.id,
+ 'object_id': user.id,
+ 'object_repr': user.username,
+ 'content_type': User,
+ }
+
+ template_path = EmailType.ACCOUNT_DEACTIVATED_ADMIN.value
+ text_message = 'A user account has been deactivated.'
+ subject = ' TDP User Account Deactivated due to Inactivity'
+ context = {
+ 'user': user,
+ }
+
+ log(f"Preparing email to OFA Admins for deactivated user {user.username}", logger_context=logger_context)
+
+ for recipient_email in recipient_emails:
+ automated_email(
+ email_path=template_path,
+ recipient_email=recipient_email,
+ subject=subject,
+ email_context=context,
+ text_message=text_message,
+ logger_context=logger_context
+ )
diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py
index 089ea4e8d..179eeed86 100644
--- a/tdrs-backend/tdpservice/email/tasks.py
+++ b/tdrs-backend/tdpservice/email/tasks.py
@@ -16,10 +16,35 @@
from tdpservice.email.email import automated_email, log
from tdpservice.email.email_enums import EmailType
from tdpservice.parsers.util import calendar_to_fiscal
+from tdpservice.email.helpers.admin_notifications import email_admin_deactivated_user
logger = logging.getLogger(__name__)
+@shared_task
+def deactivate_users():
+ """Deactivate users that have not logged in in the last 180 days."""
+ users_to_deactivate = User.objects.filter(
+ last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=180),
+ account_approval_status=AccountApprovalStatusChoices.APPROVED,
+ )
+
+ for user in users_to_deactivate:
+ user.account_approval_status = AccountApprovalStatusChoices.DEACTIVATED
+ user.groups.clear()
+ user.save()
+
+ logger_context = {
+ 'user_id': user.id,
+ 'object_id': user.id,
+ 'object_repr': user.username,
+ }
+ email_admin_deactivated_user(user)
+ log(
+ f"Deactivated user {user.username} for inactivity.",
+ logger_context=logger_context if not settings.DEBUG else None
+ )
+
@shared_task
def check_for_accounts_needing_deactivation_warning():
diff --git a/tdrs-backend/tdpservice/email/templates/account-deactivated-admin.html b/tdrs-backend/tdpservice/email/templates/account-deactivated-admin.html
new file mode 100644
index 000000000..d8b5130fa
--- /dev/null
+++ b/tdrs-backend/tdpservice/email/templates/account-deactivated-admin.html
@@ -0,0 +1,18 @@
+{% extends 'base.html' %}
+{% block content %}
+
+
+
+
The following user account for the TANF Data Portal (TDP) has been deactivated.
+
+ Account Information:
+
+ - Name: {{ user.first_name }}
+ - Last name: {{ user.last_name }}
+ - Email: {{ user.email }}
+
+
+ Thank you,
+ TDP Team
+
+{% endblock %}
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py b/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py
index 369688d1a..1f4c2caf7 100644
--- a/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py
+++ b/tdrs-backend/tdpservice/scheduling/test/test_user_deactivation_warning.py
@@ -30,6 +30,22 @@ def test_deactivation_email_10_days(user, mocker):
assert tdpservice.email.helpers.account_deactivation_warning.send_deactivation_warning_email.called_once_with(
users=[user], days=10)
+@pytest.mark.django_db
+def test_deactivate_users(user, mocker):
+ """Test that the deactivate_users task runs."""
+ mocker.patch(
+ 'tdpservice.email.helpers.admin_notifications.email_admin_deactivated_user',
+ return_value=None
+ )
+ user.groups.add()
+ user.last_login = datetime.now(tz=timezone.utc) - timedelta(days=181)
+ user.account_approval_status = AccountApprovalStatusChoices.APPROVED
+ user.save()
+ tdpservice.email.tasks.deactivate_users()
+ assert user.groups.count() == 0
+ assert tdpservice.email.helpers.admin_notifications.email_admin_deactivated_user.called_once_with(user)
+
+
@pytest.mark.django_db
def test_deactivation_email_3_days(user, mocker):
"""Test that the check_for_accounts_needing_deactivation_warning task runs."""
diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py
index 7a7baad72..89acbe10c 100644
--- a/tdrs-backend/tdpservice/settings/common.py
+++ b/tdrs-backend/tdpservice/settings/common.py
@@ -475,6 +475,13 @@ class Common(Configuration):
'expires': 15.0,
},
},
+ 'Deactivate Users': {
+ 'task': 'tdpservice.email.tasks.deactivate_users',
+ 'schedule': crontab(day_of_week='*', hour='13', minute='0'), # Every day at 1pm UTC (9am EST)
+ 'options': {
+ 'expires': 15.0,
+ },
+ },
'Email Admin Number of Access Requests' : {
'task': 'tdpservice.email.tasks.email_admin_num_access_requests',
'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST)
diff --git a/tdrs-backend/tdpservice/users/migrations/0041_users_digit_group_add_datafile_permission.py b/tdrs-backend/tdpservice/users/migrations/0041_users_digit_group_add_datafile_permission.py
new file mode 100644
index 000000000..67dbd70ab
--- /dev/null
+++ b/tdrs-backend/tdpservice/users/migrations/0041_users_digit_group_add_datafile_permission.py
@@ -0,0 +1,58 @@
+# Generated by Django 3.2.5 on 2021-08-16 14:10
+from django.contrib.auth.models import Group
+from django.db import migrations
+
+from tdpservice.users.permissions import (
+ add_permissions_q,
+ get_permission_ids_for_model,
+ view_permissions_q
+)
+
+
+def set_digit_team_permissions(apps, schema_editor):
+ """Set relevant Group Permissions for DIGIT Team group."""
+ digit = (
+ apps.get_model('auth', 'Group').objects.get(name='DIGIT Team')
+ )
+
+ stt_permissions = get_permission_ids_for_model(
+ 'stts',
+ 'stt',
+ filters=[view_permissions_q]
+ )
+
+
+ datafile_permissions = get_permission_ids_for_model(
+ 'data_files',
+ 'datafile',
+ filters=[view_permissions_q, add_permissions_q]
+ )
+
+ # Assign model permissions
+ digit.permissions.add(*datafile_permissions, *stt_permissions)
+
+def unset_digit_team_permissions(apps, schema_editor):
+ """Remove all Group Permissions added to DIGIT Team."""
+ digit = (
+ apps.get_model('auth', 'Group').objects.get(name='DIGIT Team')
+ )
+ datafile_permissions = get_permission_ids_for_model(
+ 'data_files',
+ 'datafile',
+ filters=[view_permissions_q, add_permissions_q]
+ )
+ digit.permissions.remove(*datafile_permissions)
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('auth', '__latest__'),
+ ('users', '0040_users_digit_group_permissions'),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ set_digit_team_permissions,
+ reverse_code=unset_digit_team_permissions
+ )
+ ]
diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py
index 608305131..ae53b3cda 100644
--- a/tdrs-backend/tdpservice/users/test/test_permissions.py
+++ b/tdrs-backend/tdpservice/users/test/test_permissions.py
@@ -180,6 +180,8 @@ def test_digit_team_permissions(digit_team):
expected_permissions = {'parsers.view_parsererror',
'parsers.view_datafilesummary',
'data_files.view_datafile',
+ 'data_files.add_datafile',
+ 'stts.view_stt',
'search_indexes.view_ssp_m3',
'search_indexes.view_tribal_tanf_t5',
'search_indexes.view_tribal_tanf_t3',
@@ -200,7 +202,7 @@ def test_digit_team_permissions(digit_team):
'search_indexes.view_tanf_t4',
'search_indexes.view_ssp_m6',
'search_indexes.view_tribal_tanf_t2',
- 'search_indexes.view_tanf_t6'
+ 'search_indexes.view_tanf_t6',
}
group_permissions = digit_team.get_group_permissions()
assert group_permissions == expected_permissions
diff --git a/tdrs-frontend/src/components/Reports/Reports.jsx b/tdrs-frontend/src/components/Reports/Reports.jsx
index 1f00df2c2..82e161e37 100644
--- a/tdrs-frontend/src/components/Reports/Reports.jsx
+++ b/tdrs-frontend/src/components/Reports/Reports.jsx
@@ -38,6 +38,9 @@ function Reports() {
// The logged in user saved in our redux `auth` state object
const user = useSelector((state) => state.auth.user)
const isOFAAdmin = useSelector(selectPrimaryUserRole)?.name === 'OFA Admin'
+ const isDIGITTeam = useSelector(selectPrimaryUserRole)?.name === 'DIGIT Team'
+ const isSystemAdmin =
+ useSelector(selectPrimaryUserRole)?.name === 'OFA System Admin'
const sttList = useSelector((state) => state?.stts?.sttList)
const [errorModalVisible, setErrorModalVisible] = useState(false)
@@ -59,7 +62,8 @@ function Reports() {
Q4: 'Quarter 4 (July - September)',
}
- const currentStt = isOFAAdmin ? selectedStt : userProfileStt
+ const currentStt =
+ isOFAAdmin || isDIGITTeam || isSystemAdmin ? selectedStt : userProfileStt
const stt = sttList?.find((stt) => stt?.name === currentStt)
@@ -68,7 +72,8 @@ function Reports() {
const errorsCount = formValidation.errors
- const missingStt = !isOFAAdmin && !currentStt
+ const missingStt =
+ !isOFAAdmin && !isDIGITTeam && !isSystemAdmin && !currentStt
const errorsRef = useRef(null)
@@ -194,7 +199,7 @@ function Reports() {
)
const touchedFields = Object.keys(touched).length
- const expected_fields = isOFAAdmin ? 3 : 2
+ const expected_fields = isOFAAdmin || isDIGITTeam || isSystemAdmin ? 3 : 2
const errors = touchedFields === 3 ? expected_fields - form.length : 0
@@ -215,6 +220,8 @@ function Reports() {
setFormValidationState,
touched,
isOFAAdmin,
+ isDIGITTeam,
+ isSystemAdmin,
])
return (
@@ -237,7 +244,7 @@ function Reports() {
)}