Merge branch '2599-error-report-corrections-impact-wpr-metrics' of ht…

…tps://github.com/raft-tech/TANF-app into 1114-tribal-s1

tdrs-backend/Pipfile

@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]

tdrs-backend/tdpservice/parsers/fields.py

@@ -4,12 +4,13 @@
 
 logger = logging.getLogger(__name__)
 
+
 def value_is_empty(value, length):
     """Handle 'empty' values as field inputs."""
     empty_values = [
-        ' '*length,  # '     '
-        '#'*length,  # '#####'
-        '_'*length,  # '_____'
+        " " * length,  # '     '
+        "#" * length,  # '#####'
+        "_" * length,  # '_____'
     ]
 
     return value is None or value in empty_values
@@ -18,9 +19,20 @@ def value_is_empty(value, length):
 class Field:
     """Provides a mapping between a field name and its position."""
 
-    def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
+    def __init__(
+        self,
+        item,
+        name,
+        friendly_name,
+        type,
+        startIndex,
+        endIndex,
+        required=True,
+        validators=[],
+    ):
         self.item = item
         self.name = name
+        self.friendly_name = friendly_name
         self.type = type
         self.startIndex = startIndex
         self.endIndex = endIndex
@@ -37,32 +49,43 @@ def __repr__(self):
 
     def parse_value(self, line):
         """Parse the value for a field given a line, startIndex, endIndex, and field type."""
-        value = line[self.startIndex:self.endIndex]
+        value = line[self.startIndex: self.endIndex]
 
-        if value_is_empty(value, self.endIndex-self.startIndex):
-            logger.debug(f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty.")
+        if value_is_empty(value, self.endIndex - self.startIndex):
+            logger.debug(
+                f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty."
+            )
             return None
 
         match self.type:
-            case 'number':
+            case "number":
                 try:
                     value = int(value)
                     return value
                 except ValueError:
                     logger.error(f"Error parsing field value: {value} to integer.")
                     return None
-            case 'string':
+            case "string":
                 return value
             case _:
                 logger.warn(f"Unknown field type: {self.type}.")
                 return None
 
+
 class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
-    def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True,
+    def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
                  validators=[], **kwargs):
-        super().__init__(item, name, type, startIndex, endIndex, required, validators)
+        super().__init__(
+            item=item,
+            name=name,
+            type=type,
+            friendly_name=friendly_name,
+            startIndex=startIndex,
+            endIndex=endIndex,
+            required=required,
+            validators=validators)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -150,16 +150,18 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
+                # get field from field name
+                fields = [self.get_field_by_name(name) for name in field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
                         error_message=validator_error,
                         record=instance,
-                        field=None
+                        field=fields,
                     )
                 )
 

tdrs-backend/tdpservice/parsers/schema_defs/header.py

@@ -20,6 +20,7 @@
         Field(
             item="2",
             name="title",
+            friendly_name="title",
             type="string",
             startIndex=0,
             endIndex=6,
@@ -31,6 +32,7 @@
         Field(
             item="4",
             name="year",
+            friendly_name="year",
             type="number",
             startIndex=6,
             endIndex=10,
@@ -40,6 +42,7 @@
         Field(
             item="5",
             name="quarter",
+            friendly_name="quarter",
             type="string",
             startIndex=10,
             endIndex=11,
@@ -49,6 +52,7 @@
         Field(
             item="6",
             name="type",
+            friendly_name="type",
             type="string",
             startIndex=11,
             endIndex=12,
@@ -58,6 +62,7 @@
         Field(
             item="1",
             name="state_fips",
+            friendly_name="state fips",
             type="string",
             startIndex=12,
             endIndex=14,
@@ -77,6 +82,7 @@
         Field(
             item="3",
             name="tribe_code",
+            friendly_name="tribe code",
             type="string",
             startIndex=14,
             endIndex=17,
@@ -86,6 +92,7 @@
         Field(
             item="7",
             name="program_type",
+            friendly_name="program type",
             type="string",
             startIndex=17,
             endIndex=20,
@@ -95,6 +102,7 @@
         Field(
             item="8",
             name="edit",
+            friendly_name="edit",
             type="string",
             startIndex=20,
             endIndex=21,
@@ -104,6 +112,7 @@
         Field(
             item="9",
             name="encryption",
+            friendly_name="encryption",
             type="string",
             startIndex=21,
             endIndex=22,
@@ -113,6 +122,7 @@
         Field(
             item="10",
             name="update",
+            friendly_name="update",
             type="string",
             startIndex=22,
             endIndex=23,