Merge branch 'develop' into 2429/single-clamav

docs/Sprint-Review/sprint-82-summary.md

@@ -0,0 +1,49 @@
+
+# Sprint 82 Summary
+
+09/13/23 - 09/26/23
+
+Velocity: Dev (13)
+
+## Sprint Goal
+* Continue parsing engine development for TANF Section (04) and SSP (01), close out subsmission history and metadata workflows (1613/12/10).
+* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users
+* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline.
+
+## Tickets
+### Completed/Merged
+* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board)
+* [#2700 Deployment/migration issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2700)
+
+### Ready to Merge
+* N/A
+
+### Submitted (QASP Review, OCIO Review)
+* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612)
+
+### Closed (not merged)
+* N/A
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+
+* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429)
+* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695)
+* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411)
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610)
+* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111)
+* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664)
+
+### Demo
+* Internal:
+    * 1613
+
+
+

tdrs-backend/tdpservice/data_files/admin.py

@@ -3,19 +3,73 @@
 
 from ..core.utils import ReadOnlyAdminMixin
 from .models import DataFile, LegacyFileTransfer
+from tdpservice.parsers.models import DataFileSummary, ParserError
+from django.conf import settings
+from django.utils.html import format_html
 
+DOMAIN = settings.FRONTEND_BASE_URL
+
+class DataFileSummaryPrgTypeFilter(admin.SimpleListFilter):
+    """Admin class filter for Program Type on datafile model."""
+
+    title = 'Program Type'
+    parameter_name = 'program_type'
+
+    def lookups(self, request, model_admin):
+        """Return a list of tuples."""
+        return [
+            ('TAN', 'TAN'),
+            ('SSP', 'SSP'),
+        ]
+
+    def queryset(self, request, queryset):
+        """Return a queryset."""
+        if self.value():
+            query_set_ids = [df.id for df in queryset if df.prog_type == self.value()]
+            return queryset.filter(id__in=query_set_ids)
+        else:
+            return queryset
 
 @admin.register(DataFile)
 class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
     """Admin class for DataFile models."""
 
+    def status(self, obj):
+        """Return the status of the data file summary."""
+        return DataFileSummary.objects.get(datafile=obj).status
+
+    def case_totals(self, obj):
+        """Return the case totals."""
+        return DataFileSummary.objects.get(datafile=obj).case_aggregates
+
+    def error_report_link(self, obj):
+        """Return the link to the error report."""
+        pe_len = ParserError.objects.filter(file=obj).count()
+
+        filtered_parserror_list_url = f'{DOMAIN}/admin/parsers/parsererror/?file=' + str(obj.id)
+        # have to find the error id from obj
+        return format_html("<a href='{url}'>{field}</a>",
+                           field="Parser Errors: " + str(pe_len),
+                           url=filtered_parserror_list_url)
+
+    error_report_link.allow_tags = True
+
+    def data_file_summary(self, obj):
+        """Return the data file summary."""
+        df = DataFileSummary.objects.get(datafile=obj)
+        return format_html("<a href='{url}'>{field}</a>",
+                           field=f'{df.id}' + ":" + df.get_status(),
+                           url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/")
+
     list_display = [
         'id',
         'stt',
         'year',
         'quarter',
         'section',
         'version',
+        'data_file_summary',
+        'error_report_link',
     ]
 
     list_filter = [
@@ -25,6 +79,8 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
         'user',
         'year',
         'version',
+        'summary__status',
+        DataFileSummaryPrgTypeFilter
     ]
 
 @admin.register(LegacyFileTransfer)

tdrs-backend/tdpservice/data_files/models.py

@@ -152,6 +152,20 @@ class Meta:
                                         null=True
                                         )
 
+    @property
+    def prog_type(self):
+        """Return the program type for a given section."""
+        # e.g., 'SSP Closed Case Data'
+        if self.section.startswith('SSP'):
+            return 'SSP'
+        elif self.section.startswith('Tribal'):
+            return 'TAN'  # problematic, do we need to infer tribal entirely from tribe/fips code?
+        else:
+            return 'TAN'
+
+        # TODO: if given a datafile (section), we can reverse back to the program b/c the
+        # section string has "tribal/ssp" in it, then process of elimination we have tanf
+
     @property
     def filename(self):
         """Return the correct filename for this data file."""

tdrs-backend/tdpservice/data_files/test/test_admin.py

@@ -0,0 +1,22 @@
+"""Test DataFileAdmin methods."""
+import pytest
+from django.contrib.admin.sites import AdminSite
+
+from tdpservice.data_files.admin import DataFileAdmin
+from tdpservice.data_files.models import DataFile
+from tdpservice.data_files.test.factories import DataFileFactory
+from tdpservice.parsers.test.factories import DataFileSummaryFactory
+from django.conf import settings
+
+@pytest.mark.django_db
+def test_DataFileAdmin_status():
+    """Test DataFileAdmin status method."""
+    data_file = DataFileFactory()
+    data_file_summary = DataFileSummaryFactory(datafile=data_file)
+    data_file_admin = DataFileAdmin(DataFile, AdminSite())
+
+    assert data_file_admin.status(data_file) == data_file_summary.status
+    assert data_file_admin.case_totals(data_file) == data_file_summary.case_aggregates
+    DOMAIN = settings.FRONTEND_BASE_URL
+    assert data_file_admin.error_report_link(data_file) == \
+        f"<a href='{DOMAIN}/admin/parsers/parsererror/?file={data_file.id}'>Parser Errors: 0</a>"

tdrs-frontend/cypress/e2e/accounts/accounts.js

@@ -8,7 +8,9 @@ Then('{string} sees a Request Access form', (username) => {
 
 Then('{string} can see the hompage', (username) => {
   cy.visit('/home')
-  cy.contains('You have been approved for access to TDP.').should('exist')
+  cy.wait(2000).then(() => {
+    cy.contains('You have been approved for access to TDP.').should('exist')
+  })
 })
 
 When('{string} is in begin state', (username) => {