diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 076743096..2431148d0 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -18,6 +18,7 @@ def __init__( endIndex, required=True, validators=[], + ignore_errors=False, ): self.item = item self.name = name @@ -27,6 +28,7 @@ def __init__( self.endIndex = endIndex self.required = required self.validators = validators + self.ignore_errors = ignore_errors def create(self, item, name, length, start, end, type): """Create a new field.""" @@ -64,7 +66,7 @@ class TransformField(Field): """Represents a field that requires some transformation before serializing.""" def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True, - validators=[], **kwargs): + validators=[], ignore_errors=False, **kwargs): super().__init__( item=item, name=name, @@ -73,7 +75,8 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex, startIndex=startIndex, endIndex=endIndex, required=required, - validators=validators) + validators=validators, + ignore_errors=ignore_errors) self.transform_func = transform_func self.kwargs = kwargs diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 6b1f1a338..f5c8418c1 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -34,6 +34,10 @@ def parse_datafile(datafile, dfs): errors['header'] = header_errors bulk_create_errors({1: header_errors}, 1, flush=True) return errors + elif header_is_valid and len(header_errors) > 0: + logger.info(f"Preparser Warning: {len(header_errors)} header warnings encountered.") + errors['header'] = header_errors + bulk_create_errors({1: header_errors}, 1, flush=True) field_values = schema_defs.header.get_field_values_by_names(header_line, {"encryption", "tribe_code", "state_fips"}) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 7dd01556f..970f83cbe 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -141,7 +141,7 @@ def run_field_validators(self, instance, generate_error): if (field.required and not is_empty) or should_validate: for validator in field.validators: validator_is_valid, validator_error = validator(value, self, field.friendly_name, field.item) - is_valid = False if not validator_is_valid else is_valid + is_valid = False if (not validator_is_valid and not field.ignore_errors) else is_valid if validator_error: errors.append( generate_error( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index 67475fd5f..2ef98f56e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -122,11 +122,17 @@ startIndex=22, endIndex=23, required=True, - validators=[validators.matches("D", - error_func=lambda eargs: ("HEADER Update Indicator must be set to D " - f"instead of {eargs.value}. Please review " - "Exporting Complete Data Using FTANF in the " - "Knowledge Center."))], + validators=[ + validators.matches( + "D", + error_func=lambda eargs: ( + "HEADER Update Indicator must be set to D " + f"instead of {eargs.value}. Please review " + "Exporting Complete Data Using FTANF in the " + "Knowledge Center." + ), + )], + ignore_errors=True, ), ], ) diff --git a/tdrs-backend/tdpservice/parsers/test/test_header.py b/tdrs-backend/tdpservice/parsers/test/test_header.py index 18079bc68..ceca0a35e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_header.py +++ b/tdrs-backend/tdpservice/parsers/test/test_header.py @@ -59,8 +59,8 @@ def test_header_cleanup(test_datafile): # Encryption error ("HEADER20204A06 TAN1AD", False, "HEADER Item 9 (encryption): A is not in [ , E]."), # Update error - ("HEADER20204A06 TAN1EA", False, ("HEADER Update Indicator must be set to D instead of A. Please review " - "Exporting Complete Data Using FTANF in the Knowledge Center.")), + ("HEADER20204A06 TAN1EA", True, ("HEADER Update Indicator must be set to D instead of A. Please review " + "Exporting Complete Data Using FTANF in the Knowledge Center.")), ]) @pytest.mark.django_db def test_header_fields(test_datafile, header_line, is_valid, error): diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index fd24bb8fe..cda1b3663 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1610,16 +1610,12 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file parser_errors = ParserError.objects.filter(file=tanf_section_1_file_with_bad_update_indicator) - assert parser_errors.count() == 1 - - error = parser_errors.first() + assert parser_errors.count() == 5 - assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_message == ("HEADER Update Indicator must be set to D " - "instead of U. Please review " - "Exporting Complete Data Using FTANF in the " - "Knowledge Center.") + error_messages = [error.error_message for error in parser_errors] + assert "HEADER Update Indicator must be set to D instead of U. Please review" + \ + " Exporting Complete Data Using FTANF in the Knowledge Center." in error_messages @pytest.mark.django_db() def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):