Merge branch 'develop' into 2411-metadata-parsed-datafiles-3

tdrs-backend/tdpservice/data_files/serializers.py

@@ -12,6 +12,7 @@
 from tdpservice.security.models import ClamAVFileScan
 from tdpservice.stts.models import STT
 from tdpservice.users.models import User
+from tdpservice.parsers.serializers import DataFileSummarySerializer
 logger = logging.getLogger(__name__)
 
 class DataFileSerializer(serializers.ModelSerializer):
@@ -22,6 +23,7 @@ class DataFileSerializer(serializers.ModelSerializer):
     user = serializers.PrimaryKeyRelatedField(queryset=User.objects.all())
     ssp = serializers.BooleanField(write_only=True)
     has_error = serializers.SerializerMethodField()
+    summary = DataFileSummarySerializer(many=False, read_only=True)
 
     class Meta:
         """Metadata."""
@@ -45,6 +47,7 @@ class Meta:
             's3_location',
             's3_versioning_id',
             'has_error',
+            'summary'
         ]
 
         read_only_fields = ("version",)

tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py

@@ -14,5 +14,5 @@ class Migration(migrations.Migration):
             model_name='parsererror',
             name='error_type',
             field=models.TextField(choices=[('1', 'File pre-check'), ('2', 'Record value invalid'), ('3', 'Record value consistency'), ('4', 'Case consistency'), ('5', 'Section consistency'), ('6', 'Historical consistency')], max_length=128),
-        )
+        ),
     ]

tdrs-backend/tdpservice/parsers/migrations/0008_alter_datafilesummary_datafile.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.15 on 2023-07-20 20:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0012_datafile_s3_versioning_id'),
+        ('parsers', '0007_datafilesummary'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafilesummary',
+            name='datafile',
+            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='data_files.datafile'),
+        ),
+    ]

tdrs-backend/tdpservice/parsers/migrations/0009_alter_datafilesummary_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2023-08-23 12:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parsers', '0008_alter_datafilesummary_datafile'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafilesummary',
+            name='status',
+            field=models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Partially Accepted with Errors', 'Partially Accepted'), ('Rejected', 'Rejected')], default='Pending', max_length=50),
+        ),
+    ]

tdrs-backend/tdpservice/parsers/models.py

@@ -77,6 +77,7 @@ class Status(models.TextChoices):
         PENDING = "Pending"  # file has been uploaded, but not validated
         ACCEPTED = "Accepted"
         ACCEPTED_WITH_ERRORS = "Accepted with Errors"
+        PARTIALLY_ACCEPTED = "Partially Accepted with Errors"
         REJECTED = "Rejected"
 
     status = models.CharField(
@@ -85,7 +86,7 @@ class Status(models.TextChoices):
         default=Status.PENDING,
     )
 
-    datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE)
+    datafile = models.OneToOneField(DataFile, on_delete=models.CASCADE, related_name="summary")
 
     case_aggregates = models.JSONField(null=True, blank=False)
 
@@ -100,11 +101,17 @@ def get_status(self):
                                 .exclude(error_message__icontains="trailer")\
                                 .exclude(error_message__icontains="Unknown Record_Type was found.")
 
+        row_precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\
+                                    .filter(field_name="Record_Type")\
+                                    .exclude(error_message__icontains="trailer")
+
         if errors is None:
             return DataFileSummary.Status.PENDING
         elif errors.count() == 0:
             return DataFileSummary.Status.ACCEPTED
         elif precheck_errors.count() > 0:
             return DataFileSummary.Status.REJECTED
+        elif row_precheck_errors.count() > 0:
+            return DataFileSummary.Status.PARTIALLY_ACCEPTED
         else:
             return DataFileSummary.Status.ACCEPTED_WITH_ERRORS

tdrs-backend/tdpservice/parsers/parse.py

@@ -20,7 +20,7 @@ def parse_datafile(datafile):
     header_line = rawfile.readline().decode().strip()
     header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(
         header_line,
-        util.make_generate_parser_error(datafile, 1)
+        util.make_generate_file_precheck_parser_error(datafile, 1)
     )
     if not header_is_valid:
         logger.info(f"Preparser Error: {len(header_errors)} header errors encountered.")

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -608,7 +608,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field
 
     parse.parse_datafile(bad_tanf_s1__row_missing_required_field)
 
-    assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS
+    assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED
 
     parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field)
     assert parser_errors.count() == 4

tdrs-backend/tdpservice/parsers/util.py

@@ -35,7 +35,7 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m
         row_number=line_number,
         column_number=getattr(field, 'item', None),
         item_number=getattr(field, 'item', None),
-        field_name=getattr(field, 'name', None),
+        field_name=getattr(field, 'name', None) if hasattr(field, 'name') else field,
         rpt_month_year=getattr(record, 'RPT_MONTH_YEAR', None),
         case_number=getattr(record, 'CASE_NUMBER', None),
         error_message=error_message,
@@ -64,6 +64,22 @@ def generate(schema, error_category, error_message, record=None, field=None):
     return generate
 
 
+def make_generate_file_precheck_parser_error(datafile, line_number):
+    """Configure a generate_parser_error that acts as a file pre-check error."""
+    def generate(schema, error_category, error_message, record=None, field=None):
+        return generate_parser_error(
+            datafile=datafile,
+            line_number=line_number,
+            schema=schema,
+            error_category=error_category,
+            error_message=error_message,
+            record=record,
+            field=None,  # purposely overridden to force a "Rejected" status for certain file precheck errors
+        )
+
+    return generate
+
+
 class SchemaManager:
     """Manages one or more RowSchema's and runs all parsers and validators."""
 

tdrs-backend/tdpservice/scheduling/parser_task.py

@@ -19,7 +19,8 @@ def parse(data_file_id):
     # for undetermined amount of time.
     data_file = DataFile.objects.get(id=data_file_id)
 
-    logger.info(f"DataFile parsing started for file -> {repr(data_file)}")
+    logger.info(f"DataFile parsing started for file {data_file.filename}")
+
     dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING)
     errors = parse_datafile(data_file)
     dfs.status = dfs.get_status()

tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx

@@ -3,6 +3,13 @@ import axios from 'axios'
 import PropTypes from 'prop-types'
 import { useDispatch, useSelector } from 'react-redux'
 import { fileUploadSections } from '../../reducers/reports'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
+import {
+  faCheckCircle,
+  faExclamationCircle,
+  faXmarkCircle,
+  faClock,
+} from '@fortawesome/free-solid-svg-icons'
 import Paginator from '../Paginator'
 import { getAvailableFileList, download } from '../../actions/reports'
 import { useEffect } from 'react'
@@ -11,6 +18,39 @@ import { getParseErrors } from '../../actions/createXLSReport'
 
 const formatDate = (dateStr) => new Date(dateStr).toLocaleString()
 
+const SubmissionSummaryStatusIcon = ({ status }) => {
+  let icon = null
+  let color = null
+
+  switch (status) {
+    case 'Pending':
+      icon = faClock
+      color = '#005EA2'
+      break
+    case 'Accepted':
+      icon = faCheckCircle
+      color = '#40bb45'
+      break
+    case 'Partially Accepted with Errors':
+      icon = faExclamationCircle
+      color = '#ec4e11'
+      break
+    case 'Accepted with Errors':
+      icon = faExclamationCircle
+      color = '#ec4e11'
+      break
+    case 'Rejected':
+      icon = faXmarkCircle
+      color = '#bb0000'
+      break
+    default:
+      break
+  }
+  return (
+    <FontAwesomeIcon className="margin-right-1" icon={icon} color={color} />
+  )
+}
+
 const SubmissionHistoryRow = ({ file }) => {
   const dispatch = useDispatch()
 
@@ -41,14 +81,24 @@ const SubmissionHistoryRow = ({ file }) => {
           {file.fileName}
         </button>
       </td>
+      <td>
+        {file.summary ? (
+          <>
+            <SubmissionSummaryStatusIcon status={file.summary.status} />
+            {file.summary && file.summary.status
+              ? file.summary.status
+              : 'Pending'}
+          </>
+        ) : (
+          'N/A'
+        )}
+      </td>
       <td>
         {file.hasError > 0 ? (
           <button className="section-download" onClick={returned_errors}>
             {file.year}-{file.quarter}-{file.section}.xlsx
           </button>
-        ) : (
-          'Currently Unavailable'
-        )}
+        ) : null}
       </td>
     </tr>
   )
@@ -78,6 +128,7 @@ const SectionSubmissionHistory = ({ section, label, files }) => {
                 <th>Submitted On</th>
                 <th>Submitted By</th>
                 <th>File Name</th>
+                <th>Acceptance Status</th>
                 <th>Error Reports (In development)</th>
               </tr>
             </thead>

tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js

@@ -244,8 +244,6 @@ describe('SubmissionHistory', () => {
     expect(
       screen.queryByText('Error Reports (In development)')
     ).toBeInTheDocument()
-
-    expect(screen.queryByText('Currently Unavailable')).toBeInTheDocument()
   })
 
   it('Shows SSP results when SSP-MOE file type selected', () => {
@@ -339,4 +337,63 @@ describe('SubmissionHistory', () => {
     expect(screen.queryByText('test5.txt')).toBeInTheDocument()
     expect(screen.queryByText('test6.txt')).not.toBeInTheDocument()
   })
+
+  it.each([
+    'Pending',
+    'Accepted',
+    'Accepted with Errors',
+    'Partially Accepted with Errors',
+    'Rejected',
+    null,
+  ])('Shows the submission acceptance status', (status) => {
+    const state = {
+      reports: {
+        files: [
+          {
+            id: '123',
+            fileName: 'test1.txt',
+            fileType: 'TANF',
+            quarter: 'Q1',
+            section: 'Active Case Data',
+            uuid: '123-4-4-321',
+            year: '2023',
+            s3_version_id: '321-0-0-123',
+            createdAt: '12/12/2012 12:12',
+            submittedBy: '[email protected]',
+            summary: {
+              datafile: '123',
+              status: status,
+              case_aggregates: {
+                Oct: {
+                  total: 0,
+                  accepted: 0,
+                  rejected: 0,
+                },
+                Nov: {
+                  total: 0,
+                  accepted: 0,
+                  rejected: 0,
+                },
+                Dec: {
+                  total: 0,
+                  accepted: 0,
+                  rejected: 0,
+                },
+              },
+            },
+          },
+        ],
+      },
+    }
+
+    const store = appConfigureStore(state)
+    const dispatch = jest.fn(store.dispatch)
+    store.dispatch = dispatch
+
+    setup(store)
+
+    expect(screen.queryByText('Acceptance Status')).toBeInTheDocument()
+    expect(screen.queryByText('test1.txt')).toBeInTheDocument()
+    expect(screen.queryByText(status || 'Pending')).toBeInTheDocument()
+  })
 })

tdrs-frontend/src/reducers/reports.js

@@ -66,6 +66,7 @@ export const serializeApiDataFile = (dataFile) => ({
   createdAt: dataFile.created_at,
   submittedBy: dataFile.submitted_by,
   hasError: dataFile.has_error,
+  summary: dataFile.summary,
 })
 
 const initialState = {
@@ -131,6 +132,7 @@ const reports = (state = initialState, action) => {
             created_at: null,
             submitted_by: null,
             has_error: null,
+            summary: null,
           })
         }),
       }