Deprecate Validators

tdrs-backend/tdpservice/data_files/views.py

@@ -146,7 +146,7 @@ def download(self, request, pk=None):
     def download_error_report(self, request, pk=None):
         """Generate and return the parsing error report xlsx."""
         datafile = self.get_object()
-        all_errors = ParserError.objects.filter(file=datafile)
+        all_errors = ParserError.objects.filter(file=datafile, deprecated=False)
         filtered_errors = get_prioritized_queryset(all_errors)
 
         return Response(get_xls_serialized_file(all_errors, filtered_errors))

tdrs-backend/tdpservice/parsers/aggregates.py

@@ -21,6 +21,7 @@ def case_aggregates_by_month(df, dfs_status):
     schema_models = [model for model in schema_models_dict.values()]
 
     aggregate_data = {"months": [], "rejected": 0}
+    all_errors = ParserError.objects.filter(file=df, deprecated=False)
     for month in month_list:
         total = 0
         cases_with_errors = 0
@@ -46,8 +47,7 @@ def case_aggregates_by_month(df, dfs_status):
             case_numbers = case_numbers.union(curr_case_numbers)
 
         total += len(case_numbers)
-        cases_with_errors += ParserError.objects.filter(file=df, case_number__in=case_numbers)\
-            .distinct('case_number').count()
+        cases_with_errors += all_errors.filter(case_number__in=case_numbers).distinct('case_number').count()
         accepted = total - cases_with_errors
 
         aggregate_data['months'].append({"month": month,
@@ -57,8 +57,8 @@ def case_aggregates_by_month(df, dfs_status):
     error_type_query = Query(error_type=ParserErrorCategoryChoices.PRE_CHECK) | \
         Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY)
 
-    aggregate_data['rejected'] = ParserError.objects.filter(error_type_query, file=df)\
-        .distinct("row_number").exclude(row_number=0).count()
+    aggregate_data['rejected'] = all_errors.filter(error_type_query).distinct("row_number")\
+        .exclude(row_number=0).count()
 
     return aggregate_data
 
@@ -70,7 +70,7 @@ def total_errors_by_month(df, dfs_status):
 
     total_errors_data = {"months": []}
 
-    errors = ParserError.objects.all().filter(file=df)
+    errors = ParserError.objects.all().filter(file=df, deprecated=False)
 
     for month in month_list:
         if dfs_status == "Rejected":

tdrs-backend/tdpservice/parsers/case_consistency_validator.py

@@ -9,6 +9,7 @@
 from tdpservice.parsers.validators.util import ValidationErrorArgs
 from tdpservice.parsers.validators.category3 import format_error_context
 import logging
+import warnings
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +55,7 @@ def __get_error_context(self, field_name, schema):
                                          )
         return format_error_context(error_args)
 
-    def __generate_and_add_error(self, schema, record, field, line_num, msg):
+    def __generate_and_add_error(self, schema, record, field, line_num, msg, deprecated=False):
         """Generate a ParserError and add it to the `generated_errors` list."""
         err = self.generate_error(
             error_category=ParserErrorCategoryChoices.CASE_CONSISTENCY,
@@ -63,6 +64,7 @@ def __generate_and_add_error(self, schema, record, field, line_num, msg):
             record=record,
             field=field,
             error_message=msg,
+            deprecated=deprecated
         )
         self.generated_errors.append(err)
 
@@ -340,11 +342,14 @@ def __validate_s1_records_are_related(self):
 
     def __validate_case_closure_employment(self, t4, t5s, error_msg):
         """
-        Validate case closure.
+        Validate case closure. (DEPRECATED, always returns zero.).
 
         If case closure reason = 01:employment, then at least one person on
         the case must have employment status = 1:Yes in the same month.
         """
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4_record, t4_schema, line_num = t4
 
@@ -362,19 +367,23 @@ def __validate_case_closure_employment(self, t4, t5s, error_msg):
                 t4_record,
                 "EMPLOYMENT_STATUS",
                 line_num,
-                error_msg
+                error_msg,
+                deprecated=True
             )
             num_errors += 1
 
-        return num_errors
+        return 0
 
     def __validate_case_closure_ftl(self, t4, t5s, error_msg):
         """
-        Validate case closure.
+        Validate case closure. (DEPRECATED, always returns zero.).
 
         If closure reason = FTL, then at least one person who is HoH
         or spouse of HoH on case must have FTL months >=60.
         """
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4_record, t4_schema, line_num = t4
 
@@ -393,11 +402,12 @@ def __validate_case_closure_ftl(self, t4, t5s, error_msg):
                 t4_record,
                 "COUNTABLE_MONTH_FED_TIME",
                 line_num,
-                error_msg
+                error_msg,
+                deprecated=True
             )
             num_errors += 1
 
-        return num_errors
+        return 0
 
     def __validate_s2_records_are_related(self):
         """
@@ -466,6 +476,10 @@ def __validate_s2_records_are_related(self):
         return num_errors
 
     def __validate_t5_atd_and_ssi(self):
+        """Validate aid totally disabled and SSI. (DEPRECATED, always returns zero.)."""
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4s, t4_model_name, t5s, t5_model_name = self.__get_s2_triplets_and_names()
 
@@ -492,7 +506,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} Adults in territories must have a valid "
                         f"value for {self.__get_error_context('REC_AID_TOTALLY_DISABLED', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
             elif is_state and rec_atd == 1:
@@ -504,7 +519,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in states should not have a value "
                         f"of 1 for {self.__get_error_context('REC_AID_TOTALLY_DISABLED', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
 
@@ -517,7 +533,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in territories must have value = 2:No for "
                         f"{self.__get_error_context('REC_SSI', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
             elif is_state and family_affiliation == 1 and rec_ssi not in {1, 2}:
@@ -529,8 +546,9 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in states must have a valid value for "
                         f"{self.__get_error_context('REC_SSI', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
 
-        return num_errors
+        return 0

tdrs-backend/tdpservice/parsers/migrations/0012_parsererror_deprecated.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2024-12-18 16:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parsers', '0011_parsererror_row_number_optional'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='parsererror',
+            name='deprecated',
+            field=models.BooleanField(default=False),
+        ),
+    ]

tdrs-backend/tdpservice/parsers/models.py

@@ -44,6 +44,8 @@ class Meta:
     created_at = models.DateTimeField(auto_now_add=True)
     fields_json = models.JSONField(null=True)
 
+    deprecated = models.BooleanField(default=False)
+
     @property
     def rpt_month_name(self):
         """Return the month name."""

tdrs-backend/tdpservice/parsers/parse.py

@@ -61,7 +61,7 @@ def parse_datafile(datafile, dfs):
 
     # Validate tribe code in submission across program type and fips code
     generate_error = util.make_generate_parser_error(datafile, 1)
-    tribe_is_valid, tribe_error = category1.validate_tribe_fips_program_agree(
+    tribe_is_valid, tribe_error, _ = category1.validate_tribe_fips_program_agree(
         header['program_type'],
         field_values["tribe_code"],
         field_values["state_fips"],
@@ -76,7 +76,7 @@ def parse_datafile(datafile, dfs):
         return errors
 
     # Ensure file section matches upload section
-    section_is_valid, section_error = category1.validate_header_section_matches_submission(
+    section_is_valid, section_error, _ = category1.validate_header_section_matches_submission(
         datafile,
         get_section_reference(program_type, section),
         util.make_generate_parser_error(datafile, 1)
@@ -89,7 +89,7 @@ def parse_datafile(datafile, dfs):
         bulk_create_errors(unsaved_parser_errors, 1, flush=True)
         return errors
 
-    rpt_month_year_is_valid, rpt_month_year_error = category1.validate_header_rpt_month_year(
+    rpt_month_year_is_valid, rpt_month_year_error, _ = category1.validate_header_rpt_month_year(
         datafile,
         header,
         util.make_generate_parser_error(datafile, 1)

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -98,7 +98,7 @@ def run_preparsing_validators(self, line, generate_error):
                 friendly_name=field.friendly_name if field else 'record type',
                 item_num=field.item if field else '0',
             )
-            validator_is_valid, validator_error = validator(line, eargs)
+            validator_is_valid, validator_error, is_deprecated = validator(line, eargs)
             is_valid = False if not validator_is_valid else is_valid
 
             is_quiet_preparser_errors = (
@@ -113,7 +113,8 @@ def run_preparsing_validators(self, line, generate_error):
                         error_category=ParserErrorCategoryChoices.PRE_CHECK,
                         error_message=validator_error,
                         record=None,
-                        field="Record_Type"
+                        field="Record_Type",
+                        deprecated=is_deprecated,
                     )
                 )
 
@@ -152,7 +153,7 @@ def run_field_validators(self, instance, generate_error):
             should_validate = not field.required and not is_empty
             if (field.required and not is_empty) or should_validate:
                 for validator in field.validators:
-                    validator_is_valid, validator_error = validator(value, eargs)
+                    validator_is_valid, validator_error, is_deprecated = validator(value, eargs)
                     is_valid = False if (not validator_is_valid and not field.ignore_errors) else is_valid
                     if validator_error:
                         errors.append(
@@ -161,7 +162,8 @@ def run_field_validators(self, instance, generate_error):
                                 error_category=ParserErrorCategoryChoices.FIELD_VALUE,
                                 error_message=validator_error,
                                 record=instance,
-                                field=field
+                                field=field,
+                                deprecated=is_deprecated
                             )
                         )
             elif field.required:
@@ -187,7 +189,7 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error, field_names = validator(instance, self)
+            validator_is_valid, validator_error, field_names, is_deprecated = validator(instance, self)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
                 # get field from field name
@@ -199,6 +201,7 @@ def run_postparsing_validators(self, instance, generate_error):
                         error_message=validator_error,
                         record=instance,
                         field=fields,
+                        deprecated=is_deprecated
                     )
                 )
 

tdrs-backend/tdpservice/parsers/test/test_case_consistency.py

@@ -612,7 +612,7 @@ def test_section2_validator_fail_case_closure_employment(
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 1
-        assert num_errors == 1
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         assert errors[0].error_message == (
             f"At least one person on the case must have Item {emp_status_item_num} (Employment Status) = 1:Yes in the "
@@ -688,7 +688,7 @@ def test_section2_validator_fail_case_closure_ftl(self, small_correct_file, head
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 1
-        assert num_errors == 1
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         is_tribal = "Tribal" in header["program_type"]
         tribe_or_fed = "Tribal" if is_tribal else "Federal"
@@ -974,7 +974,7 @@ def test_section2_aabd_ssi_validator_fail_territory_adult_aabd(self, small_corre
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 2
-        assert num_errors == 2
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         assert errors[0].error_message == (
             f"{t5_model_name} Adults in territories must have a valid value for Item {ratd_item_num} "
@@ -1124,7 +1124,7 @@ def test_section2_atd_ssi_validator_fail_state_atdd(self, small_correct_file, he
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 2
-        assert num_errors == 2
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         assert errors[0].error_message == (
             f"{t5_model_name} People in states should not have a value of 1 for Item {item_no} ("
@@ -1204,7 +1204,7 @@ def test_section2_aabd_ssi_validator_fail_territory_ssi(self, small_correct_file
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 2
-        assert num_errors == 2
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         assert errors[0].error_message == (
             f"{t5_model_name} People in territories must have value = 2:No for Item {rec_ssi_item_num} "
@@ -1284,7 +1284,7 @@ def test_section2_atd_ssi_validator_fail_state_ssi(self, small_correct_file, hea
         errors = case_consistency_validator.get_generated_errors()
 
         assert len(errors) == 1
-        assert num_errors == 1
+        assert num_errors == 0
         assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY
         assert errors[0].error_message == (
             f"{t5_model_name} People in states must have a valid value for Item {rec_ssi_item_num} "

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -878,8 +878,8 @@ def test_parse_tanf_section2_file(tanf_section2_file, dfs):
 
     parse.parse_datafile(tanf_section2_file, dfs)
 
-    assert TANF_T4.objects.all().count() == 206
-    assert TANF_T5.objects.all().count() == 548
+    assert TANF_T4.objects.all().count() == 223
+    assert TANF_T5.objects.all().count() == 605
 
     parser_errors = ParserError.objects.filter(file=tanf_section2_file)