2795 s1 duplicates

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -100,8 +100,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response):
 
         assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
             + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \
-            + " then number of months T1: 0 is not larger than 0."
+        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \
+            "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER."
 
     @staticmethod
     def assert_error_report_ssp_file_content_matches_with_friendly_names(response):
@@ -132,8 +132,9 @@ def assert_error_report_file_content_matches_without_friendly_names(response):
 
         assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
             + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then "
-                                                                  "NBR_MONTHS T1: 0 is not larger than 0.")
+        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ("Every T1 record should have at least one "
+                                                                  "corresponding T2 or T3 record with the same "
+                                                                  "RPT_MONTH_YEAR and CASE_NUMBER.")
 
     @staticmethod
     def assert_data_file_exists(data_file_data, version, user):

tdrs-backend/tdpservice/parsers/aggregates.py

@@ -1,9 +1,10 @@
 """Aggregate methods for the parsers."""
 from .row_schema import SchemaManager
-from .models import ParserError
+from .models import ParserError, ParserErrorCategoryChoices
 from .util import month_to_int, \
     transform_to_months, fiscal_to_calendar, get_prog_from_section
 from .schema_defs.utils import get_program_models, get_text_from_df
+from django.db.models import Q
 
 
 def case_aggregates_by_month(df, dfs_status):
@@ -39,22 +40,25 @@ def case_aggregates_by_month(df, dfs_status):
             if isinstance(schema_model, SchemaManager):
                 schema_model = schema_model.schemas[0]
 
-            curr_case_numbers = set(schema_model.document.Django.model.objects.filter(datafile=df)
-                                    .filter(RPT_MONTH_YEAR=rpt_month_year)
+            curr_case_numbers = set(schema_model.document.Django.model.objects.filter(datafile=df,
+                                                                                      RPT_MONTH_YEAR=rpt_month_year)
                                     .distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True))
             case_numbers = case_numbers.union(curr_case_numbers)
 
         total += len(case_numbers)
-        cases_with_errors += ParserError.objects.filter(file=df).filter(
-            case_number__in=case_numbers).distinct('case_number').count()
+        cases_with_errors += ParserError.objects.filter(file=df, case_number__in=case_numbers)\
+            .distinct('case_number').count()
         accepted = total - cases_with_errors
 
         aggregate_data['months'].append({"month": month,
                                          "accepted_without_errors": accepted,
                                          "accepted_with_errors": cases_with_errors})
 
-    aggregate_data['rejected'] = ParserError.objects.filter(file=df).filter(case_number=None).distinct("row_number")\
-        .exclude(row_number=0).count()
+    error_type_query = Q(error_type=ParserErrorCategoryChoices.PRE_CHECK) | \
+        Q(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY)
+
+    aggregate_data['rejected'] = ParserError.objects.filter(error_type_query, file=df)\
+        .distinct("row_number").exclude(row_number=0).count()
 
     return aggregate_data