diff --git a/tdrs-backend/tdpservice/parsers/case_consistency_validator.py b/tdrs-backend/tdpservice/parsers/case_consistency_validator.py index 6bdc81a68..db90a99e8 100644 --- a/tdrs-backend/tdpservice/parsers/case_consistency_validator.py +++ b/tdrs-backend/tdpservice/parsers/case_consistency_validator.py @@ -1,6 +1,9 @@ """Class definition for Category Four validator.""" +from datetime import datetime from .models import ParserErrorCategoryChoices +from .util import get_years_apart +from tdpservice.stts.models import STT from tdpservice.parsers.schema_defs.utils import get_program_model import logging @@ -43,7 +46,7 @@ def __add_record_to_sorted_object(self, record_schema_pair): class CaseConsistencyValidator: """Caches records of the same case and month to perform category four validation while actively parsing.""" - def __init__(self, header, program_type, generate_error): + def __init__(self, header, program_type, stt_type, generate_error): self.header = header self.record_schema_pairs = SortedRecordSchemaPairs() self.current_case = None @@ -56,6 +59,7 @@ def __init__(self, header, program_type, generate_error): self.generated_errors = [] self.total_cases_cached = 0 self.total_cases_validated = 0 + self.stt_type = stt_type def __get_model(self, model_str): """Return a model for the current program type/section given the model's string name.""" @@ -137,10 +141,12 @@ def __validate_section1(self, num_errors): def __validate_section2(self, num_errors): """Perform TANF Section 2 category four validation on all cached records.""" num_errors += self.__validate_s2_records_are_related() + num_errors += self.__validate_t5_aabd_and_ssi() return num_errors def __validate_family_affiliation(self, num_errors, t1s, t2s, t3s, error_msg): """Validate at least one record in t2s+t3s has FAMILY_AFFILIATION == 1.""" + num_errors = 0 passed = False for record, schema in t2s + t3s: family_affiliation = getattr(record, 'FAMILY_AFFILIATION') @@ -185,6 +191,19 @@ def __validate_s1_records_are_related(self): t3s = reporting_year_cases.get(t3_model, []) if len(t1s) > 0: + if len(t1s) > 1: # likely to be captured by "no duplicates" validator + for record, schema in t1s[1:]: + self.__generate_and_add_error( + schema, + record, + field='RPT_MONTH_YEAR', + msg=( + f'There should only be one {t1_model_name} record ' + f'per RPT_MONTH_YEAR and CASE_NUMBER.' + ) + ) + num_errors += 1 + if len(t2s) == 0 and len(t3s) == 0: for record, schema in t1s: self.__generate_and_add_error( @@ -237,6 +256,65 @@ def __validate_s1_records_are_related(self): return num_errors + def __validate_case_closure_employment(self, t4, t5s, error_msg): + """ + Validate case closure. + + If case closure reason = 01:employment, then at least one person on + the case must have employment status = 1:Yes in the same month. + """ + num_errors = 0 + t4_record, t4_schema = t4 + + passed = False + for record, schema in t5s: + employment_status = getattr(record, 'EMPLOYMENT_STATUS') + + if employment_status == 1: + passed = True + break + + if not passed: + self.__generate_and_add_error( + t4_schema, + t4_record, + 'EMPLOYMENT_STATUS', + error_msg + ) + num_errors += 1 + + return num_errors + + def __validate_case_closure_ftl(self, t4, t5s, error_msg): + """ + Validate case closure. + + If closure reason = FTL, then at least one person who is HoH + or spouse of HoH on case must have FTL months >=60. + """ + num_errors = 0 + t4_record, t4_schema = t4 + + passed = False + for record, schema in t5s: + relationship_hoh = getattr(record, 'RELATIONSHIP_HOH') + ftl_months = getattr(record, 'COUNTABLE_MONTH_FED_TIME') + + if (relationship_hoh == '01' or relationship_hoh == '02') and int(ftl_months) >= 60: + passed = True + break + + if not passed: + self.__generate_and_add_error( + t4_schema, + t4_record, + 'COUNTABLE_MONTH_FED_TIME', + error_msg + ) + num_errors += 1 + + return num_errors + def __validate_s2_records_are_related(self): """ Validate section 2 records are related. @@ -259,6 +337,34 @@ def __validate_s2_records_are_related(self): t5s = reporting_year_cases.get(t5_model, []) if len(t4s) > 0: + if len(t4s) > 1: + for record, schema in t4s[1:]: + self.__generate_and_add_error( + schema, + record, + field='RPT_MONTH_YEAR', + msg=( + f'There should only be one {t4_model_name} record ' + f'per RPT_MONTH_YEAR and CASE_NUMBER.' + ) + ) + num_errors += 1 + else: + t4 = t4s[0] + t4_record, t4_schema = t4 + closure_reason = getattr(t4_record, 'CLOSURE_REASON') + + if closure_reason == '01': + num_errors += self.__validate_case_closure_employment(t4, t5s, ( + 'At least one person on the case must have employment status = 1:Yes in the ' + 'same RPT_MONTH_YEAR since CLOSURE_REASON = 1:Employment/excess earnings.' + )) + elif closure_reason == '03' and not is_ssp: + num_errors += self.__validate_case_closure_ftl(t4, t5s, ( + 'At least one person who is head-of-household or spouse of head-of-household ' + 'on case must have countable months toward time limit >= 60 since ' + 'CLOSURE_REASON = 03: federal 5 year time limit.' + )) if len(t5s) == 0: for record, schema in t4s: self.__generate_and_add_error( @@ -288,3 +394,74 @@ def __validate_s2_records_are_related(self): num_errors += 1 return num_errors + + def __validate_t5_aabd_and_ssi(self): + print('validate t5') + num_errors = 0 + is_ssp = self.program_type == 'SSP' + + t5_model_name = 'M5' if is_ssp else 'T5' + t5_model = self.__get_model(t5_model_name) + + is_state = self.stt_type == STT.EntityType.STATE + is_territory = self.stt_type == STT.EntityType.TERRITORY + + for rpt_month_year, reporting_year_cases in self.record_schema_pairs.sorted_cases.items(): + t5s = reporting_year_cases.get(t5_model, []) + + for record, schema in t5s: + rec_aabd = getattr(record, 'REC_AID_TOTALLY_DISABLED') + rec_ssi = getattr(record, 'REC_SSI') + family_affiliation = getattr(record, 'FAMILY_AFFILIATION') + dob = getattr(record, 'DATE_OF_BIRTH') + + rpt_month_year_dd = f'{rpt_month_year}01' + rpt_date = datetime.strptime(rpt_month_year_dd, '%Y%m%d') + dob_date = datetime.strptime(dob, '%Y%m%d') + is_adult = get_years_apart(rpt_date, dob_date) >= 19 + + if is_territory and is_adult and (rec_aabd != 1 and rec_aabd != 2): + self.__generate_and_add_error( + schema, + record, + field='REC_AID_TOTALLY_DISABLED', + msg=( + f'{t5_model_name} Adults in territories must have a valid ' + 'value for REC_AID_TOTALLY_DISABLED.' + ) + ) + num_errors += 1 + elif is_state and rec_aabd != 2: + self.__generate_and_add_error( + schema, + record, + field='REC_AID_TOTALLY_DISABLED', + msg=( + f'{t5_model_name} People in states should not have a value ' + 'of 1 for REC_AID_TOTALLY_DISABLED.' + ) + ) + num_errors += 1 + + if is_territory and rec_ssi != 2: + self.__generate_and_add_error( + schema, + record, + field='REC_SSI', + msg=( + f'{t5_model_name} People in territories must have value = 2:No for REC_SSI.' + ) + ) + num_errors += 1 + elif is_state and family_affiliation == 1: + self.__generate_and_add_error( + schema, + record, + field='REC_SSI', + msg=( + f'{t5_model_name} People in states must have a valid value for REC_SSI.' + ) + ) + num_errors += 1 + + return num_errors diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 54149bd7a..1a2f339c4 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -73,6 +73,7 @@ def parse_datafile(datafile, dfs): case_consistency_validator = CaseConsistencyValidator( header, program_type, + datafile.stt.type, util.make_generate_parser_error(datafile, None) ) diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index ae04dfe77..861a524c5 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -269,7 +269,7 @@ class Meta: STRATUM = 1 ZIP_CODE = "11111" DISPOSITION = 1 - CLOSURE_REASON = '01' + CLOSURE_REASON = '02' REC_SUB_HOUSING = 1 REC_MED_ASSIST = 1 REC_FOOD_STAMPS = 1 @@ -287,7 +287,7 @@ class Meta: RPT_MONTH_YEAR = 202301 CASE_NUMBER = "1" FAMILY_AFFILIATION = 1 - DATE_OF_BIRTH = "02091997" + DATE_OF_BIRTH = "19970209" SSN = "123456789" RACE_HISPANIC = 1 RACE_AMER_INDIAN = 1 @@ -515,7 +515,7 @@ class Meta: STRATUM = 1 ZIP_CODE = "11111" DISPOSITION = 1 - CLOSURE_REASON = '01' + CLOSURE_REASON = '02' REC_SUB_HOUSING = 1 REC_MED_ASSIST = 1 REC_FOOD_STAMPS = 1 @@ -533,7 +533,7 @@ class Meta: RPT_MONTH_YEAR = 202301 CASE_NUMBER = "1" FAMILY_AFFILIATION = 1 - DATE_OF_BIRTH = "02091997" + DATE_OF_BIRTH = "19970209" SSN = "123456789" RACE_HISPANIC = 1 RACE_AMER_INDIAN = 1 @@ -773,7 +773,7 @@ class Meta: STRATUM = 1 ZIP_CODE = "11111" DISPOSITION = 1 - CLOSURE_REASON = '01' + CLOSURE_REASON = '02' REC_SUB_HOUSING = 1 REC_MED_ASSIST = 1 REC_FOOD_STAMPS = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_case_consistency.py b/tdrs-backend/tdpservice/parsers/test/test_case_consistency.py index 764967f49..f18eb7e3e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_case_consistency.py +++ b/tdrs-backend/tdpservice/parsers/test/test_case_consistency.py @@ -6,6 +6,7 @@ from .. import schema_defs, util from ..case_consistency_validator import CaseConsistencyValidator from tdpservice.parsers.models import ParserErrorCategoryChoices +from tdpservice.stts.models import STT logger = logging.getLogger(__name__) @@ -64,6 +65,7 @@ def test_add_record(self, small_correct_file_header, small_correct_file, tanf_s1 case_consistency_validator = CaseConsistencyValidator( small_correct_file_header, small_correct_file_header['program_type'], + STT.EntityType.STATE, util.make_generate_parser_error(small_correct_file, None) ) @@ -103,28 +105,32 @@ def test_add_record(self, small_correct_file_header, small_correct_file, tanf_s1 assert case_consistency_validator.total_cases_cached == 2 assert case_consistency_validator.total_cases_validated == 2 - @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff", [ + @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff,stt_type", [ ( {"type": "A", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT1Factory, schema_defs.tanf.t1.schemas[0], 'T1'), (factories.TanfT2Factory, schema_defs.tanf.t2.schemas[0], 'T2'), (factories.TanfT3Factory, schema_defs.tanf.t3.schemas[0], 'T3'), + STT.EntityType.STATE, ), ( {"type": "A", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT1Factory, schema_defs.tribal_tanf.t1.schemas[0], 'T1'), (factories.TribalTanfT2Factory, schema_defs.tribal_tanf.t2.schemas[0], 'T2'), (factories.TribalTanfT3Factory, schema_defs.tribal_tanf.t3.schemas[0], 'T3'), + STT.EntityType.TRIBE, ), ( {"type": "A", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM1Factory, schema_defs.ssp.m1.schemas[0], 'M1'), (factories.SSPM2Factory, schema_defs.ssp.m2.schemas[0], 'M2'), (factories.SSPM3Factory, schema_defs.ssp.m3.schemas[0], 'M3'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_section1_validator_pass(self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff): + def test_section1_records_are_related_validator_pass( + self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff, stt_type): """Test records are related validator success case.""" (T1Factory, t1_schema, t1_model_name) = T1Stuff (T2Factory, t2_schema, t2_model_name) = T2Stuff @@ -133,6 +139,7 @@ def test_records_are_related_section1_validator_pass(self, small_correct_file, h case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -141,10 +148,6 @@ def test_records_are_related_section1_validator_pass(self, small_correct_file, h RPT_MONTH_YEAR=202010, CASE_NUMBER='123', ), - T1Factory.create( - RPT_MONTH_YEAR=202010, - CASE_NUMBER='123' - ), ] for t1 in t1s: case_consistency_validator.add_record(t1, t1_schema, False) @@ -186,29 +189,32 @@ def test_records_are_related_section1_validator_pass(self, small_correct_file, h assert len(errors) == 0 assert num_errors == 0 - @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff", [ + @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff,stt_type", [ ( {"type": "A", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT1Factory, schema_defs.tanf.t1.schemas[0], 'T1'), (factories.TanfT2Factory, schema_defs.tanf.t2.schemas[0], 'T2'), (factories.TanfT3Factory, schema_defs.tanf.t3.schemas[0], 'T3'), + STT.EntityType.STATE, ), ( {"type": "A", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT1Factory, schema_defs.tribal_tanf.t1.schemas[0], 'T1'), (factories.TribalTanfT2Factory, schema_defs.tribal_tanf.t2.schemas[0], 'T2'), (factories.TribalTanfT3Factory, schema_defs.tribal_tanf.t3.schemas[0], 'T3'), + STT.EntityType.TRIBE, ), ( {"type": "A", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM1Factory, schema_defs.ssp.m1.schemas[0], 'M1'), (factories.SSPM2Factory, schema_defs.ssp.m2.schemas[0], 'M2'), (factories.SSPM3Factory, schema_defs.ssp.m3.schemas[0], 'M3'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_s1_validator_fail_no_t2_or_t3( - self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff): + def test_section1_records_are_related_validator_fail_no_t2_or_t3( + self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff, stt_type): """Test records are related validator fails with no t2s or t3s.""" (T1Factory, t1_schema, t1_model_name) = T1Stuff (T2Factory, t2_schema, t2_model_name) = T2Stuff @@ -217,6 +223,7 @@ def test_records_are_related_s1_validator_fail_no_t2_or_t3( case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -225,10 +232,6 @@ def test_records_are_related_s1_validator_fail_no_t2_or_t3( RPT_MONTH_YEAR=202010, CASE_NUMBER='123' ), - T1Factory.create( - RPT_MONTH_YEAR=202010, - CASE_NUMBER='123' - ), ] for t1 in t1s: case_consistency_validator.add_record(t1, t1_schema, False) @@ -237,41 +240,40 @@ def test_records_are_related_s1_validator_fail_no_t2_or_t3( errors = case_consistency_validator.get_generated_errors() - assert len(errors) == 2 - assert num_errors == 2 + assert len(errors) == 1 + assert num_errors == 1 assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY assert errors[0].error_message == ( f'Every {t1_model_name} record should have at least one corresponding ' f'{t2_model_name} or {t3_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' ) - assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert errors[1].error_message == ( - f'Every {t1_model_name} record should have at least one corresponding ' - f'{t2_model_name} or {t3_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' - ) - @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff", [ + @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff,stt_type", [ ( {"type": "A", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT1Factory, schema_defs.tanf.t1.schemas[0], 'T1'), (factories.TanfT2Factory, schema_defs.tanf.t2.schemas[0], 'T2'), (factories.TanfT3Factory, schema_defs.tanf.t3.schemas[0], 'T3'), + STT.EntityType.STATE, ), ( {"type": "A", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT1Factory, schema_defs.tribal_tanf.t1.schemas[0], 'T1'), (factories.TribalTanfT2Factory, schema_defs.tribal_tanf.t2.schemas[0], 'T2'), (factories.TribalTanfT3Factory, schema_defs.tribal_tanf.t3.schemas[0], 'T3'), + STT.EntityType.TRIBE, ), ( {"type": "A", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM1Factory, schema_defs.ssp.m1.schemas[0], 'M1'), (factories.SSPM2Factory, schema_defs.ssp.m2.schemas[0], 'M2'), (factories.SSPM3Factory, schema_defs.ssp.m3.schemas[0], 'M3'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_s1_validator_fail_no_t1(self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff): + def test_section1_records_are_related_validator_fail_no_t1( + self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff, stt_type): """Test records are related validator fails with no t1s.""" (T1Factory, t1_schema, t1_model_name) = T1Stuff (T2Factory, t2_schema, t2_model_name) = T2Stuff @@ -280,6 +282,7 @@ def test_records_are_related_s1_validator_fail_no_t1(self, small_correct_file, h case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -340,30 +343,33 @@ def test_records_are_related_s1_validator_fail_no_t1(self, small_correct_file, h f'{t1_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' ) - @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff", [ + @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff,stt_type", [ ( {"type": "A", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT1Factory, schema_defs.tanf.t1.schemas[0], 'T1'), (factories.TanfT2Factory, schema_defs.tanf.t2.schemas[0], 'T2'), (factories.TanfT3Factory, schema_defs.tanf.t3.schemas[0], 'T3'), + STT.EntityType.STATE, ), ( {"type": "A", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT1Factory, schema_defs.tribal_tanf.t1.schemas[0], 'T1'), (factories.TribalTanfT2Factory, schema_defs.tribal_tanf.t2.schemas[0], 'T2'), (factories.TribalTanfT3Factory, schema_defs.tribal_tanf.t3.schemas[0], 'T3'), + STT.EntityType.TRIBE, ), ( {"type": "A", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM1Factory, schema_defs.ssp.m1.schemas[0], 'M1'), (factories.SSPM2Factory, schema_defs.ssp.m2.schemas[0], 'M2'), (factories.SSPM3Factory, schema_defs.ssp.m3.schemas[0], 'M3'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_validator_s1_fail_no_family_affiliation( - self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff): - """Test records are related validator fails when no t2 or t3 has family_affiliation == 1.""" + def test_section1_records_are_related_validator_fail_multiple_t1s( + self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff, stt_type): + """Test records are related validator fails when there are multiple t1s.""" (T1Factory, t1_schema, t1_model_name) = T1Stuff (T2Factory, t2_schema, t2_model_name) = T2Stuff (T3Factory, t3_schema, t3_model_name) = T3Stuff @@ -371,6 +377,7 @@ def test_records_are_related_validator_s1_fail_no_family_affiliation( case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -387,6 +394,95 @@ def test_records_are_related_validator_s1_fail_no_family_affiliation( for t1 in t1s: case_consistency_validator.add_record(t1, t1_schema, False) + t2s = [ + T2Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=1, + ), + T2Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + ), + ] + for t2 in t2s: + case_consistency_validator.add_record(t2, t2_schema, False) + + t3s = [ + T3Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=1, + ), + T3Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + ), + ] + for t3 in t3s: + case_consistency_validator.add_record(t3, t3_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 1 + assert num_errors == 1 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'There should only be one {t1_model_name} record ' + f'per RPT_MONTH_YEAR and CASE_NUMBER.' + ) + + @pytest.mark.parametrize("header,T1Stuff,T2Stuff,T3Stuff,stt_type", [ + ( + {"type": "A", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT1Factory, schema_defs.tanf.t1.schemas[0], 'T1'), + (factories.TanfT2Factory, schema_defs.tanf.t2.schemas[0], 'T2'), + (factories.TanfT3Factory, schema_defs.tanf.t3.schemas[0], 'T3'), + STT.EntityType.STATE, + ), + ( + {"type": "A", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT1Factory, schema_defs.tribal_tanf.t1.schemas[0], 'T1'), + (factories.TribalTanfT2Factory, schema_defs.tribal_tanf.t2.schemas[0], 'T2'), + (factories.TribalTanfT3Factory, schema_defs.tribal_tanf.t3.schemas[0], 'T3'), + STT.EntityType.TRIBE, + ), + ( + {"type": "A", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM1Factory, schema_defs.ssp.m1.schemas[0], 'M1'), + (factories.SSPM2Factory, schema_defs.ssp.m2.schemas[0], 'M2'), + (factories.SSPM3Factory, schema_defs.ssp.m3.schemas[0], 'M3'), + STT.EntityType.STATE, + ), + ]) + @pytest.mark.django_db + def test_section1_records_are_related_validator_fail_no_family_affiliation( + self, small_correct_file, header, T1Stuff, T2Stuff, T3Stuff, stt_type): + """Test records are related validator fails when no t2 or t3 has family_affiliation == 1.""" + (T1Factory, t1_schema, t1_model_name) = T1Stuff + (T2Factory, t2_schema, t2_model_name) = T2Stuff + (T3Factory, t3_schema, t3_model_name) = T3Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + stt_type, + util.make_generate_parser_error(small_correct_file, None) + ) + + t1s = [ + T1Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123' + ), + ] + for t1 in t1s: + case_consistency_validator.add_record(t1, t1_schema, False) + t2s = [ T2Factory.create( RPT_MONTH_YEAR=202010, @@ -421,40 +517,37 @@ def test_records_are_related_validator_s1_fail_no_family_affiliation( errors = case_consistency_validator.get_generated_errors() - assert len(errors) == 2 - assert num_errors == 2 + assert len(errors) == 1 + assert num_errors == 1 assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY assert errors[0].error_message == ( f'Every {t1_model_name} record should have at least one corresponding ' f'{t2_model_name} or {t3_model_name} record with the same RPT_MONTH_YEAR and ' f'CASE_NUMBER, where FAMILY_AFFILIATION==1' ) - assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert errors[1].error_message == ( - f'Every {t1_model_name} record should have at least one corresponding ' - f'{t2_model_name} or {t3_model_name} record with the same RPT_MONTH_YEAR and ' - f'CASE_NUMBER, where FAMILY_AFFILIATION==1' - ) - @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ ( {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, ), ( {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, ), ( {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_section2_validator_pass(self, small_correct_file, header, T4Stuff, T5Stuff): + def test_section2_validator_pass(self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): """Test records are related validator section 2 success case.""" (T4Factory, t4_schema, t4_model_name) = T4Stuff (T5Factory, t5_schema, t5_model_name) = T5Stuff @@ -462,6 +555,7 @@ def test_records_are_related_section2_validator_pass(self, small_correct_file, h case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -470,10 +564,6 @@ def test_records_are_related_section2_validator_pass(self, small_correct_file, h RPT_MONTH_YEAR=202010, CASE_NUMBER='123', ), - T4Factory.create( - RPT_MONTH_YEAR=202010, - CASE_NUMBER='123' - ), ] for t4 in t4s: case_consistency_validator.add_record(t4, t4_schema, False) @@ -482,12 +572,16 @@ def test_records_are_related_section2_validator_pass(self, small_correct_file, h T5Factory.create( RPT_MONTH_YEAR=202010, CASE_NUMBER='123', - FAMILY_AFFILIATION=1, + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 ), T5Factory.create( RPT_MONTH_YEAR=202010, CASE_NUMBER='123', FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 ), ] for t5 in t5s: @@ -500,32 +594,36 @@ def test_records_are_related_section2_validator_pass(self, small_correct_file, h assert len(errors) == 0 assert num_errors == 0 - @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ ( {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, ), ( {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, ), ( {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_section2_validator_fail_no_t5s(self, small_correct_file, header, T4Stuff, T5Stuff): - """Test records are related validator fails with no t5s.""" + def test_section2_validator_fail_multiple_t4s(self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): + """Test records are related validator section 2 success case.""" (T4Factory, t4_schema, t4_model_name) = T4Stuff (T5Factory, t5_schema, t5_model_name) = T5Stuff case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) @@ -542,62 +640,97 @@ def test_records_are_related_section2_validator_fail_no_t5s(self, small_correct_ for t4 in t4s: case_consistency_validator.add_record(t4, t4_schema, False) + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + num_errors = case_consistency_validator.validate() errors = case_consistency_validator.get_generated_errors() - assert len(errors) == 2 - assert num_errors == 2 + assert len(errors) == 1 + assert num_errors == 1 assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY assert errors[0].error_message == ( - f'Every {t4_model_name} record should have at least one corresponding ' - f'{t5_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' - ) - assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert errors[1].error_message == ( - f'Every {t4_model_name} record should have at least one corresponding ' - f'{t5_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + f'There should only be one {t4_model_name} record ' + f'per RPT_MONTH_YEAR and CASE_NUMBER.' ) - @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ ( {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, ), ( {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, ), ( {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + STT.EntityType.STATE, ), ]) @pytest.mark.django_db - def test_records_are_related_section2_validator_fail_no_t4s(self, small_correct_file, header, T4Stuff, T5Stuff): - """Test records are related validator fails with no t4s.""" + def test_section2_validator_fail_case_closure_employment( + self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): + """Test records are related validator section 2 success case.""" (T4Factory, t4_schema, t4_model_name) = T4Stuff (T5Factory, t5_schema, t5_model_name) = T5Stuff case_consistency_validator = CaseConsistencyValidator( header, header['program_type'], + stt_type, util.make_generate_parser_error(small_correct_file, None) ) + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + CLOSURE_REASON='01' + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + t5s = [ T5Factory.create( RPT_MONTH_YEAR=202010, CASE_NUMBER='123', - FAMILY_AFFILIATION=1, + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2, + EMPLOYMENT_STATUS=3, ), T5Factory.create( RPT_MONTH_YEAR=202010, CASE_NUMBER='123', FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2, + EMPLOYMENT_STATUS=2, ), ] for t5 in t5s: @@ -607,15 +740,638 @@ def test_records_are_related_section2_validator_fail_no_t4s(self, small_correct_ errors = case_consistency_validator.get_generated_errors() - assert len(errors) == 2 - assert num_errors == 2 + assert len(errors) == 1 + assert num_errors == 1 assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY assert errors[0].error_message == ( - f'Every {t5_model_name} record should have at least one corresponding ' - f'{t4_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + 'At least one person on the case must have employment status = 1:Yes' + ' in the same RPT_MONTH_YEAR since CLOSURE_REASON = 1:Employment/excess earnings.' ) - assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert errors[1].error_message == ( - f'Every {t5_model_name} record should have at least one corresponding ' - f'{t4_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, + ), + ]) + @pytest.mark.django_db + def test_section2_validator_fail_case_closure_ftl(self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + stt_type, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + CLOSURE_REASON='03' + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2, + RELATIONSHIP_HOH='10', + COUNTABLE_MONTH_FED_TIME='059', + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2, + RELATIONSHIP_HOH='03', + COUNTABLE_MONTH_FED_TIME='001', + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 1 + assert num_errors == 1 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + 'At least one person who is head-of-household or spouse of head-of-household ' + 'on case must have countable months toward time limit >= 60 since ' + 'CLOSURE_REASON = 03: federal 5 year time limit.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + STT.EntityType.STATE, + ), + ]) + @pytest.mark.django_db + def test_section2_records_are_related_validator_fail_no_t5s( + self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): + """Test records are related validator fails with no t5s.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + stt_type, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 1 + assert num_errors == 1 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'Every {t4_model_name} record should have at least one corresponding ' + f'{t5_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff,stt_type", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + STT.EntityType.STATE, + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + STT.EntityType.TRIBE, + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + STT.EntityType.STATE, + ), + ]) + @pytest.mark.django_db + def test_section2_records_are_related_validator_fail_no_t4s( + self, small_correct_file, header, T4Stuff, T5Stuff, stt_type): + """Test records are related validator fails with no t4s.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + stt_type, + util.make_generate_parser_error(small_correct_file, None) + ) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 2 + assert num_errors == 2 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'Every {t5_model_name} record should have at least one corresponding ' + f'{t4_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + ) + assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[1].error_message == ( + f'Every {t5_model_name} record should have at least one corresponding ' + f'{t4_model_name} record with the same RPT_MONTH_YEAR and CASE_NUMBER.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_pass_territory_adult_aadb(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.TERRITORY, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=1, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 0 + assert num_errors == 0 + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_fail_territory_adult_aabd(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.TERRITORY, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=1, + REC_AID_TOTALLY_DISABLED=0, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=0, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 2 + assert num_errors == 2 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'{t5_model_name} Adults in territories must have a valid value for REC_AID_TOTALLY_DISABLED.' + ) + assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[1].error_message == ( + f'{t5_model_name} Adults in territories must have a valid value for REC_AID_TOTALLY_DISABLED.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_pass_territory_child_aabd(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.TERRITORY, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="20170209", + FAMILY_AFFILIATION=1, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="20170209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 0 + assert num_errors == 0 + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_fail_state_aabd(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.STATE, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="20170209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 2 + assert num_errors == 2 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'{t5_model_name} People in states should not have a value of 1 for REC_AID_TOTALLY_DISABLED.' + ) + assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[1].error_message == ( + f'{t5_model_name} People in states should not have a value of 1 for REC_AID_TOTALLY_DISABLED.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_fail_territory_ssi(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.TERRITORY, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=1, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=1 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=2, + REC_AID_TOTALLY_DISABLED=1, + REC_SSI=1 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 2 + assert num_errors == 2 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'{t5_model_name} People in territories must have value = 2:No for REC_SSI.' + ) + assert errors[1].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[1].error_message == ( + f'{t5_model_name} People in territories must have value = 2:No for REC_SSI.' + ) + + @pytest.mark.parametrize("header,T4Stuff,T5Stuff", [ + ( + {"type": "C", "program_type": "TAN", "year": 2020, "quarter": "4"}, + (factories.TanfT4Factory, schema_defs.tanf.t4.schemas[0], 'T4'), + (factories.TanfT5Factory, schema_defs.tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "Tribal TAN", "year": 2020, "quarter": "4"}, + (factories.TribalTanfT4Factory, schema_defs.tribal_tanf.t4.schemas[0], 'T4'), + (factories.TribalTanfT5Factory, schema_defs.tribal_tanf.t5.schemas[0], 'T5'), + ), + ( + {"type": "C", "program_type": "SSP", "year": 2020, "quarter": "4"}, + (factories.SSPM4Factory, schema_defs.ssp.m4.schemas[0], 'M4'), + (factories.SSPM5Factory, schema_defs.ssp.m5.schemas[0], 'M5'), + ), + ]) + @pytest.mark.django_db + def test_section2_aabd_ssi_validator_fail_state_ssi(self, small_correct_file, header, T4Stuff, T5Stuff): + """Test records are related validator section 2 success case.""" + (T4Factory, t4_schema, t4_model_name) = T4Stuff + (T5Factory, t5_schema, t5_model_name) = T5Stuff + + case_consistency_validator = CaseConsistencyValidator( + header, + header['program_type'], + STT.EntityType.STATE, + util.make_generate_parser_error(small_correct_file, None) + ) + + t4s = [ + T4Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + ), + ] + for t4 in t4s: + case_consistency_validator.add_record(t4, t4_schema, False) + + t5s = [ + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=1, + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + T5Factory.create( + RPT_MONTH_YEAR=202010, + CASE_NUMBER='123', + DATE_OF_BIRTH="19970209", + FAMILY_AFFILIATION=2, # validator only applies to fam_affil = 1; won't generate error + REC_AID_TOTALLY_DISABLED=2, + REC_SSI=2 + ), + ] + for t5 in t5s: + case_consistency_validator.add_record(t5, t5_schema, False) + + num_errors = case_consistency_validator.validate() + + errors = case_consistency_validator.get_generated_errors() + + assert len(errors) == 1 + assert num_errors == 1 + assert errors[0].error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert errors[0].error_message == ( + f'{t5_model_name} People in states must have a valid value for REC_SSI.' ) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b617829cb..da6ec2455 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1011,10 +1011,7 @@ def test_parse_tanf_section2_file(tanf_section2_file, dfs): parser_errors = ParserError.objects.filter(file=tanf_section2_file) err = parser_errors.first() - assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == "REC_OASDI_INSURANCE is required but a value was not provided." - assert err.content_type.model == "tanf_t5" - assert err.object_id is not None + assert err.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY @pytest.fixture diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index cd50f9c4a..84797958e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -1,9 +1,10 @@ """Test the methods of RowSchema to ensure parsing and validation work in all individual cases.""" import pytest +from datetime import datetime from ..fields import Field from ..row_schema import RowSchema, SchemaManager -from ..util import make_generate_parser_error, create_test_datafile +from ..util import make_generate_parser_error, create_test_datafile, get_years_apart def passing_validator(): @@ -525,3 +526,16 @@ def postparse_validator(): assert is_valid is False assert errors[0].fields_json == {'friendly_name': {'FIRST': 'first', 'SECOND': 'second'}} assert errors[0].error_message == "an Error" + + +@pytest.mark.parametrize("rpt_date_str,date_str,expected", [ + ('20200102', '20100101', 10), + ('20200102', '20100106', 9), + ('20200101', '20200102', 0), + ('20200101', '20210102', -1), +]) +def test_get_years_apart(rpt_date_str, date_str, expected): + """Test the get_years_apart util function.""" + rpt_date = datetime.strptime(rpt_date_str, '%Y%m%d') + date = datetime.strptime(date_str, '%Y%m%d') + assert int(get_years_apart(rpt_date, date)) == expected diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 3a5528391..287c58cff 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -167,3 +167,10 @@ def get_quarter_from_month(month): month = year_month[4:] quarter = get_quarter_from_month(month) return year, quarter + + +def get_years_apart(rpt_month_year_date, date): + """Return the number of years (double) between rpt_month_year_date and the target date - both `datetime`s.""" + delta = rpt_month_year_date - date + age = delta.days/365.25 + return age