2599-error-report-excel-format (#2734)

* Added formating for header and autofit columns * Formatted the headers * added year/month to the columns * Added contants - translation column * added friendly names to T1 and T2 * added friendly name to m1 and m2 * added friendly name to m3 * added friendly_name to t3 * added friendly_name to t4 and t5 * added friendly_name to t7 * correct missing friendly_name * correction on failing tests * addedfriendly name to excel report * linting * linting * linting * delete contants.py * added test for json field in error model * linting * linting * linting * 2599-added friendly name to postparsing validators * refining the validator tests * added returning fields names to validators * added friendly_name to error field * linting * corrections on views/tests * corrections for fields * failing test corrected * failing test corrected * correcting test failures * linting * corrected the excel fiel generator * removed excessive space in validator * linting * listing * added m6 * lint * corrected new line break * refactored validator logic * linting and correction on t1 * friendly_name correction from comments * friendly_name correction * corrected failing test for m5 * refactor the field_json creation DRY * friendly_name corrections * linting and cleaning errors * linting * correction on friendly_names * corrected friendly_name for test_util * correction child care - number of months * fixed a few more typos and some spacing. (#2767) * fixed a few more typos and some spacing. * fixed linting issues * missed a spot. --------- Co-authored-by: George Hudson <[email protected]> * fields_json Backward compatibility * remove PATCH from black list * backward compatibility * linting * add PATCH to allowed * readded internal_name * added a function * remove NoneType from fields_json * linting * Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py Co-authored-by: Alex P. <[email protected]> * Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py Co-authored-by: Alex P. <[email protected]> * corrections on friendly names * Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py Co-authored-by: Alex P. <[email protected]> * Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py Co-authored-by: Alex P. <[email protected]> * correction on friendly names --------- Co-authored-by: George Hudson <[email protected]> Co-authored-by: George Hudson <[email protected]> Co-authored-by: Alex P. <[email protected]>
raft-tech · Dec 18, 2023 · ae1abca · ae1abca
1 parent 8028ad0
commit ae1abca
Show file tree

Hide file tree

Showing 29 changed files with 3,927 additions and 1,698 deletions.
diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile
@@ -58,7 +58,8 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
+openpyxl = "==3.1.2"
 sendgrid = "==6.10.0"
 
 [requires]

diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock
diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
@@ -9,9 +9,20 @@
 class Field:
     """Provides a mapping between a field name and its position."""
 
-    def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
+    def __init__(
+        self,
+        item,
+        name,
+        friendly_name,
+        type,
+        startIndex,
+        endIndex,
+        required=True,
+        validators=[],
+    ):
         self.item = item
         self.name = name
+        self.friendly_name = friendly_name
         self.type = type
         self.startIndex = startIndex
         self.endIndex = endIndex
@@ -36,25 +47,34 @@ def parse_value(self, line):
             return None
 
         match self.type:
-            case 'number':
+            case "number":
                 try:
                     value = int(value)
                     return value
                 except ValueError:
                     logger.error(f"Error parsing field value: {value} to integer.")
                     return None
-            case 'string':
+            case "string":
                 return value
             case _:
                 logger.warn(f"Unknown field type: {self.type}.")
                 return None
 
+
 class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
-    def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True,
+    def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
                  validators=[], **kwargs):
-        super().__init__(item, name, type, startIndex, endIndex, required, validators)
+        super().__init__(
+            item=item,
+            name=name,
+            type=type,
+            friendly_name=friendly_name,
+            startIndex=startIndex,
+            endIndex=endIndex,
+            required=required,
+            validators=validators)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py
@@ -151,16 +151,18 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
+                # get field from field name
+                fields = [self.get_field_by_name(name) for name in field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
                         error_message=validator_error,
                         record=instance,
-                        field=None
+                        field=fields,
                     )
                 )
 

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py
@@ -20,6 +20,7 @@
         Field(
             item="2",
             name="title",
+            friendly_name="title",
             type="string",
             startIndex=0,
             endIndex=6,
@@ -31,6 +32,7 @@
         Field(
             item="4",
             name="year",
+            friendly_name="year",
             type="number",
             startIndex=6,
             endIndex=10,
@@ -40,6 +42,7 @@
         Field(
             item="5",
             name="quarter",
+            friendly_name="quarter",
             type="string",
             startIndex=10,
             endIndex=11,
@@ -49,6 +52,7 @@
         Field(
             item="6",
             name="type",
+            friendly_name="type",
             type="string",
             startIndex=11,
             endIndex=12,
@@ -58,74 +62,23 @@
         Field(
             item="1",
             name="state_fips",
+            friendly_name="state fips",
             type="string",
             startIndex=12,
             endIndex=14,
             required=True,
             validators=[
-                validators.oneOf(
-                    (
-                        "01",
-                        "02",
-                        "04",
-                        "05",
-                        "06",
-                        "08",
-                        "09",
-                        "10",
-                        "11",
-                        "12",
-                        "13",
-                        "15",
-                        "16",
-                        "17",
-                        "18",
-                        "19",
-                        "20",
-                        "21",
-                        "22",
-                        "23",
-                        "24",
-                        "25",
-                        "26",
-                        "27",
-                        "28",
-                        "29",
-                        "30",
-                        "31",
-                        "32",
-                        "33",
-                        "34",
-                        "35",
-                        "36",
-                        "37",
-                        "38",
-                        "39",
-                        "40",
-                        "41",
-                        "42",
-                        "44",
-                        "45",
-                        "46",
-                        "47",
-                        "48",
-                        "49",
-                        "50",
-                        "51",
-                        "53",
-                        "54",
-                        "55",
-                        "56",
-                        "66",
-                        "72",
-                        "78",
-                    )
-                )
+                validators.oneOf(("01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15",
+                                  "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27",
+                                  "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
+                                  "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53",
+                                  "54", "55", "56", "66", "72", "78"))
             ],
         ),
         Field(
             item="3",
             name="tribe_code",
+            friendly_name="tribe code",
             type="string",
             startIndex=14,
             endIndex=17,
@@ -135,6 +88,7 @@
         Field(
             item="7",
             name="program_type",
+            friendly_name="program type",
             type="string",
             startIndex=17,
             endIndex=20,
@@ -144,6 +98,7 @@
         Field(
             item="8",
             name="edit",
+            friendly_name="edit",
             type="string",
             startIndex=20,
             endIndex=21,
@@ -153,6 +108,7 @@
         Field(
             item="9",
             name="encryption",
+            friendly_name="encryption",
             type="string",
             startIndex=21,
             endIndex=22,
@@ -162,6 +118,7 @@
         Field(
             item="10",
             name="update",
+            friendly_name="update",
             type="string",
             startIndex=22,
             endIndex=23,