3113 added validation skip error

raft-tech · Aug 6, 2024 · 6b8c434 · 6b8c434
1 parent cfe6e8a
commit 6b8c434
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 18 deletions.
diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
@@ -18,6 +18,7 @@ def __init__(
         endIndex,
         required=True,
         validators=[],
+        ignore_errors=False,
     ):
         self.item = item
         self.name = name
@@ -27,6 +28,7 @@ def __init__(
         self.endIndex = endIndex
         self.required = required
         self.validators = validators
+        self.ignore_errors = ignore_errors
 
     def create(self, item, name, length, start, end, type):
         """Create a new field."""
@@ -64,7 +66,7 @@ class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
     def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
-                 validators=[], **kwargs):
+                 validators=[], ignore_errors=False, **kwargs):
         super().__init__(
             item=item,
             name=name,
@@ -73,7 +75,8 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex,
             startIndex=startIndex,
             endIndex=endIndex,
             required=required,
-            validators=validators)
+            validators=validators,
+            ignore_errors=ignore_errors)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
@@ -34,6 +34,10 @@ def parse_datafile(datafile, dfs):
         errors['header'] = header_errors
         bulk_create_errors({1: header_errors}, 1, flush=True)
         return errors
+    elif header_is_valid and len(header_errors) > 0:
+        logger.info(f"Preparser Warning: {len(header_errors)} header warnings encountered.")
+        errors['header'] = header_errors
+        bulk_create_errors({1: header_errors}, 1, flush=True)
 
     field_values = schema_defs.header.get_field_values_by_names(header_line,
                                                                 {"encryption", "tribe_code", "state_fips"})

diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py
@@ -141,7 +141,7 @@ def run_field_validators(self, instance, generate_error):
             if (field.required and not is_empty) or should_validate:
                 for validator in field.validators:
                     validator_is_valid, validator_error = validator(value, self, field.friendly_name, field.item)
-                    is_valid = False if not validator_is_valid else is_valid
+                    is_valid = False if (not validator_is_valid and not field.ignore_errors) else is_valid
                     if validator_error:
                         errors.append(
                             generate_error(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py
@@ -122,11 +122,17 @@
             startIndex=22,
             endIndex=23,
             required=True,
-            validators=[validators.matches("D",
-                                           error_func=lambda eargs: ("HEADER Update Indicator must be set to D "
-                                                                     f"instead of {eargs.value}. Please review "
-                                                                     "Exporting Complete Data Using FTANF in the "
-                                                                     "Knowledge Center."))],
+            validators=[
+                validators.matches(
+                    "D",
+                    error_func=lambda eargs: (
+                        "HEADER Update Indicator must be set to D "
+                        f"instead of {eargs.value}. Please review "
+                        "Exporting Complete Data Using FTANF in the "
+                        "Knowledge Center."
+                    ),
+                )],
+            ignore_errors=True,
         ),
     ],
 )
diff --git a/tdrs-backend/tdpservice/parsers/test/test_header.py b/tdrs-backend/tdpservice/parsers/test/test_header.py
@@ -59,8 +59,8 @@ def test_header_cleanup(test_datafile):
     # Encryption error
     ("HEADER20204A06   TAN1AD", False, "HEADER Item 9 (encryption): A is not in [ , E]."),
     # Update error
-    ("HEADER20204A06   TAN1EA", False, ("HEADER Update Indicator must be set to D instead of A. Please review "
-                                        "Exporting Complete Data Using FTANF in the Knowledge Center.")),
+    ("HEADER20204A06   TAN1EA", True, ("HEADER Update Indicator must be set to D instead of A. Please review "
+                                       "Exporting Complete Data Using FTANF in the Knowledge Center.")),
 ])
 @pytest.mark.django_db
 def test_header_fields(test_datafile, header_line, is_valid, error):

diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1610,16 +1610,12 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file
 
     parser_errors = ParserError.objects.filter(file=tanf_section_1_file_with_bad_update_indicator)
 
-    assert parser_errors.count() == 1
-
-    error = parser_errors.first()
+    assert parser_errors.count() == 5
 
-    assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE
-    assert error.error_message == ("HEADER Update Indicator must be set to D "
-                                   "instead of U. Please review "
-                                   "Exporting Complete Data Using FTANF in the "
-                                   "Knowledge Center.")
+    error_messages = [error.error_message for error in parser_errors]
 
+    assert "HEADER Update Indicator must be set to D instead of U. Please review" + \
+        " Exporting Complete Data Using FTANF in the Knowledge Center." in error_messages
 
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):