From 6b5065862c4eb310ed842b7eec2a3db7a26d5d86 Mon Sep 17 00:00:00 2001 From: jtimpe <111305129+jtimpe@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:10:05 -0500 Subject: [PATCH] Feature/2709 ssp section 1 validation (#2723) * cat 2 validation * ssp s1 cat 3 validators * debug ssp section errors * change string validators to number validators * add df to case agg query * update ssp s1 test * update ssp tests * fix lint * update test * rm prints * update validator * Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * fix the validator i broke and update tests * udpate validator (again) * lint * Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * lint * rm unneeded cat3 validator * update docstring * rm unused validator * update m2 validator * fix education level validation for m3 * add required=False to allow blanks * fix other education level m3 validator * rm schema comments * fix tests --------- Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 223 ++++++++--- .../tdpservice/parsers/schema_defs/ssp/m2.py | 350 ++++++++++++++---- .../tdpservice/parsers/schema_defs/ssp/m3.py | 272 +++++++++++--- .../parsers/test/data/small_ssp_section1.txt | 38 +- .../tdpservice/parsers/test/test_parse.py | 71 ++-- tdrs-backend/tdpservice/parsers/util.py | 3 +- tdrs-backend/tdpservice/parsers/validators.py | 7 +- 7 files changed, 746 insertions(+), 218 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 7d907dfd4..479a243cf 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -14,92 +14,231 @@ preparsing_validators=[ validators.hasLength(150), ], - postparsing_validators=[], + postparsing_validators=[ + validators.if_then_validator( + condition_field='CASH_AMOUNT', condition_function=validators.isLargerThan(0), + result_field='NBR_MONTHS', result_function=validators.isLargerThan(0), + ), + validators.if_then_validator( + condition_field='CC_AMOUNT', condition_function=validators.isLargerThan(0), + result_field='CHILDREN_COVERED', result_function=validators.isLargerThan(0), + ), + validators.if_then_validator( + condition_field='CC_AMOUNT', condition_function=validators.isLargerThan(0), + result_field='CC_NBR_MONTHS', result_function=validators.isLargerThan(0), + ), + validators.if_then_validator( + condition_field='TRANSP_AMOUNT', condition_function=validators.isLargerThan(0), + result_field='TRANSP_NBR_MONTHS', result_function=validators.isLargerThan(0), + ), + validators.if_then_validator( + condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0), + result_field='WORK_REQ_SANCTION', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0), + result_field='SANC_TEEN_PARENT', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0), + result_field='NON_COOPERATION_CSE', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0), + result_field='FAILURE_TO_COMPLY', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0), + result_field='OTHER_SANCTION', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0), + result_field='FAMILY_CAP', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0), + result_field='REDUCTIONS_ON_RECEIPTS', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0), + result_field='OTHER_NON_SANCTION', result_function=validators.oneOf((1, 2)), + ), + validators.sumIsLarger([ + "AMT_FOOD_STAMP_ASSISTANCE", + "AMT_SUB_CC", + "CASH_AMOUNT", + "CC_AMOUNT", + "CC_NBR_MONTHS" + ], 0) + ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, - required=True, validators=[]), + required=True, validators=[ + validators.isAlphaNumeric() + ]), Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, - required=True, validators=[]), + required=True, validators=[ + validators.isNumber(), + ]), Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[]), + required=False, validators=[ + validators.isInStringRange(0, 99), + ]), Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, - required=True, validators=[]), + required=True, validators=[ + validators.isNumber(), + ]), Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(1, 99), + ]), Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(1, 3), + ]), Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(1, 3), + ]), Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(1, 2), + ]), Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(1, 2), + ]), Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2), + ]), Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2), + ]), Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, - required=True, validators=[]), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, - required=True, validators=[]), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, - required=True, validators=[]), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, - required=True, validators=[]), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 9), + ]), Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0), + ]), Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]), + ]), Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 9), + ]), Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), ] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 0b1b06379..58c99ccac 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -16,140 +16,352 @@ preparsing_validators=[ validators.hasLength(150), ], - postparsing_validators=[], + postparsing_validators=[ + validators.validate__FAM_AFF__SSN(), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='SSN', result_function=validators.validateSSN(), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_HISPANIC', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_AMER_INDIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_ASIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_BLACK', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_HAWAIIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='RACE_WHITE', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='MARITAL_STATUS', result_function=validators.isInLimits(1, 5), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 2), + result_field='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='EDUCATION_LEVEL', result_function=validators.or_validators( + validators.isInStringRange(1, 16), + validators.isInStringRange(98, 99) + ), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='COOPERATION_CHILD_SUPPORT', result_function=validators.oneOf((1, 2, 9)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), + result_field='EMPLOYMENT_STATUS', result_function=validators.isInLimits(1, 3), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='WORK_ELIGIBLE_INDICATOR', result_function=validators.or_validators( + validators.isInLimits(1, 9), + validators.oneOf((11, 12)) + ), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='WORK_PART_STATUS', result_function=validators.oneOf([ + 1, 2, 5, 7, 9, + 15, 16, 17, 18, 99 + ]), + ), + validators.if_then_validator( + condition_field='WORK_ELIGIBLE_INDICATOR', condition_function=validators.isInLimits(1, 5), + result_field='WORK_PART_STATUS', result_function=validators.notMatches(99), + ), + ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, - required=True, validators=[]), + required=True, validators=[ + validators.isAlphaNumeric() + ]), Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2, 3, 5]) + ]), Field(item="27", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, - required=True, validators=[]), - Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), + Field(item="28", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, + required=True, validators=[ + validators.isLargerThan(0) + ]), TransformField(transform_func=ssp_ssn_decryption_func, item="29", name='SSN', type='string', - startIndex=29, endIndex=38, required=True, validators=[], is_encrypted=False), - Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, - validators=[]), + startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], + is_encrypted=False), + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, + validators=[ + validators.isInLimits(0, 2) + ]), Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, - required=True, validators=[]), + required=True, validators=[ + validators.isLargerThanOrEqualTo(0) + ]), Field(item="32A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="32B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="32C", name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, - required=True, validators=[]), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0) + ]), Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, - required=True, validators=[]), - Field(item="34", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 5) + ]), + Field(item="34", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, + required=True, validators=[ + validators.isInStringRange(1, 10) + ]), Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 3) + ]), Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 9) + ]), Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, - required=True, validators=[]), + required=False, validators=[ + validators.or_validators( + validators.isInLimits(0, 16), + validators.isInLimits(98, 99) + ) + ]), Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 1, 2, 3, 9]) + ]), Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 1, 2, 9]) + ]), Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 3) + ]), Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, - required=True, validators=[]), + required=True, validators=[ + validators.or_validators( + validators.isInLimits(1, 4), + validators.isInLimits(6, 9), + validators.isInLimits(11, 12) + ) + ]), Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([1, 2, 5, 7, 9, 15, 16, 17, 18, 19, 99]) + ]), Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 99) + ]), Field(item="58", name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="59B", name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="59C", name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="59D", name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="59E", name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), ], ) ] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 301e1b03e..38e66ee35 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,50 +13,144 @@ preparsing_validators=[ validators.notEmpty(start=19, end=60), ], - postparsing_validators=[], + postparsing_validators=[ + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='SSN', result_function=validators.validateSSN(), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_HISPANIC', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_AMER_INDIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_ASIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_BLACK', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_HAWAIIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_WHITE', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RELATIONSHIP_HOH', result_function=validators.isInLimits(4, 9), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='PARENT_MINOR_CHILD', result_function=validators.oneOf((1, 2, 3)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='EDUCATION_LEVEL', result_function=validators.notMatches(99), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(2), + result_field='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2, 3, 9)), + ), + ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, - required=True, validators=[]), + required=True, validators=[ + validators.isAlphaNumeric() + ]), Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2, 4]) + ]), Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), TransformField(transform_func=ssp_ssn_decryption_func, item="62", name='SSN', type='string', startIndex=28, - endIndex=37, required=True, validators=[], is_encrypted=False), + endIndex=37, required=True, is_encrypted=False, validators=[ + validators.validateSSN() + ]), Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9) + ]), Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, - required=True, validators=[]), + required=False, validators=[ + validators.isInStringRange(0, 10) + ]), Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 2, 3]) + ]), Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, - required=True, validators=[]), + required=True, validators=[ + validators.or_validators( + validators.isInStringRange(0, 16), + validators.isInStringRange(98, 99) + ) + ]), Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 1, 2, 3, 9]) + ]), Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, - required=True, validators=[]) + required=True, validators=[ + validators.isInLimits(0, 9999) + ]) ] ) @@ -66,50 +160,144 @@ preparsing_validators=[ validators.notEmpty(start=60, end=101), ], - postparsing_validators=[], + postparsing_validators=[ + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='SSN', result_function=validators.validateSSN(), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_HISPANIC', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_AMER_INDIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_ASIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_BLACK', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_HAWAIIAN', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RACE_WHITE', result_function=validators.isInLimits(1, 2), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='RELATIONSHIP_HOH', result_function=validators.isInStringRange(4, 9), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), + result_field='PARENT_MINOR_CHILD', result_function=validators.oneOf((1, 2, 3)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='EDUCATION_LEVEL', result_function=validators.notMatches(99), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1), + result_field='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), + ), + validators.if_then_validator( + condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(2), + result_field='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2, 3, 9)), + ), + ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, - required=True, validators=[]), + required=True, validators=[ + validators.isAlphaNumeric() + ]), Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2, 4]) + ]), Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, - required=True, validators=[]), + required=True, validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ]), TransformField(transform_func=ssp_ssn_decryption_func, item="62", name='SSN', type='string', startIndex=69, - endIndex=78, required=True, validators=[], is_encrypted=False), + endIndex=78, required=True, is_encrypted=False, validators=[ + validators.validateSSN() + ]), Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 2) + ]), Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9) + ]), Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, - required=True, validators=[]), + required=True, validators=[ + validators.oneOf([1, 2]) + ]), Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, - required=True, validators=[]), + required=False, validators=[ + validators.isInLimits(0, 10) + ]), Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 2, 3]) + ]), Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, - required=True, validators=[]), + required=True, validators=[ + validators.or_validators( + validators.isInStringRange(0, 16), + validators.isInStringRange(98, 99) + ) + ]), Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, - required=True, validators=[]), + required=False, validators=[ + validators.oneOf([0, 1, 2, 3, 9]) + ]), Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, - required=True, validators=[]), + required=True, validators=[ + validators.isInLimits(0, 9999) + ]), Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, - required=True, validators=[]) + required=True, validators=[ + validators.isInLimits(0, 9999) + ]) ] ) diff --git a/tdrs-backend/tdpservice/parsers/test/data/small_ssp_section1.txt b/tdrs-backend/tdpservice/parsers/test/data/small_ssp_section1.txt index 3feb5b513..2b63873a7 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/small_ssp_section1.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/small_ssp_section1.txt @@ -1,20 +1,20 @@ -HEADER20184A24 SSP1EU -M1201810111111111272140140035102133110027300000000000000010540000000000000000000000000000000000222222000000002229 -M2201810111111111271219811103WTTT#PW@W2221222222225012211111011935000000000000000000000000000000000000000000000000000000000000225300000000000000000000 -M320181011111111127120110615WTTTP99B#22212222204301100000000 -M1201810111111111982000340017102133110221300000000000000010180000000000000000000000000000000000222222000000002229 -M2201810111111111981219840123WTTT#@@PT2222212222221012211111011728000000000000000000000000000000000000000000000000000000000000175100000000000000000000 -M320181011111111198120130122WTTT##0WP22222112204398100000000 -M1201810111111112212090340591102121110288300000000000000010060000000000000000000000000000000000222222000000002229 -M2201810111111112211219840523WTTT#TBZW1222212222221012211112011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000 -M320181011111111221120160615WTTTYY#WB12222112204398100000000 -M1201810111111113562000340427102131110015300000404000000010080000000000000000000000000000000000222222000000002229 -M2201810111111113561219820116WTTTP@#YY1222222222221012211111011936000000000000000000000000000000000000000000000000000000000000205300000000000000000000 -M320181011111111356120030726WTTT0@#ZT12222222204309100000000 -M1201810111111114312120140429105233110741300000000000000010010000000000000000000000000000000000222222000000002229 -M2201810111111114311219850523WTTT0ZB9#2222212222223011212111011955000000000000000000000000000000000000000000000000000000000000482100000000000000000000 -M2201810111111114311219870621WTTTYPYZ@2222211222221101211112011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000 -M320181011111111431120180615WTTT#YW0022222122204398100000001120180615WTTT#YW002222212220439810000000 -M320181011111111431120170701WTTT0B0WB22222122204398100000000 -M320181011111111431120140515WTTTPY@YW22222112204398100000000 +HEADER20234A24 SSP1EU +M1202310111111111272140140035102133110027300000000000000010540000000000000000000000000000000000222222000000002229 +M2202310111111111271219811103WTTT#PW@W2221222222225012211111011935000000000000000000000000000000000000000000000000000000000000225300000000000000000000 +M320231011111111127120110615WTTTP99B#22212222204301100000000 +M1202310111111111982000340017102133110221300000000000000010180000000000000000000000000000000000222222000000002229 +M2202310111111111981219840123WTTT#@@PT2222212222221012211111011728000000000000000000000000000000000000000000000000000000000000175100000000000000000000 +M320231011111111198120130122WTTT##0WP22222112204398100000000 +M1202310111111112212090340591102121110288300000000000000010060000000000000000000000000000000000222222000000002229 +M2202310111111112211219840523WTTT#TBZW1222212222221012211112011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +M320231011111111221120160615WTTTYY#WB12222112204398100000000 +M1202310111111113562000340427102131110015300000404000000010080000000000000000000000000000000000222222000000002229 +M2202310111111113561219820116WTTTP@#YY1222222222221012211111011936000000000000000000000000000000000000000000000000000000000000205300000000000000000000 +M320231011111111356120030726WTTT0@#ZT12222222204309100000000 +M1202310111111114312120140429105233110741300000000000000010010000000000000000000000000000000000222222000000002229 +M2202310111111114311219850523WTTT0ZB9#2222212222223011212111011955000000000000000000000000000000000000000000000000000000000000482100000000000000000000 +M2202310111111114311219870621WTTTYPYZ@2222211222221101211112011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +M320231011111111431120180615WTTT#YW0022222122204398100000001120180615WTTT#YW002222212220439810000000 +M320231011111111431120170701WTTT0B0WB22222122204398100000000 +M320231011111111431120140515WTTTPY@YW22222112204398100000000 TRAILER0033991 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index d5b126dcc..ce41eed48 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -411,38 +411,27 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): expected_m2_record_count = 6 expected_m3_record_count = 8 - small_ssp_section1_datafile.year = 2019 + small_ssp_section1_datafile.year = 2024 small_ssp_section1_datafile.quarter = 'Q1' small_ssp_section1_datafile.save() dfs.datafile = small_ssp_section1_datafile dfs.save() - errors = parse.parse_datafile(small_ssp_section1_datafile) + parse.parse_datafile(small_ssp_section1_datafile) dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 1, 'months': [ - {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 5, 'month': 'Oct'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 20 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Trailer length is 15 but must be 23 characters.' - assert err.content_type is None - assert err.object_id is None - assert errors == { - 'trailer': [err] - } + assert parser_errors.count() == 16 assert SSP_M1.objects.count() == expected_m1_record_count assert SSP_M2.objects.count() == expected_m2_record_count assert SSP_M3.objects.count() == expected_m3_record_count @@ -464,15 +453,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile): parse.parse_datafile(ssp_section1_datafile) parser_errors = ParserError.objects.filter(file=ssp_section1_datafile) - assert parser_errors.count() == 10 - - err = parser_errors.first() - - assert err.row_number == 10339 - assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == 'EARNED_INCOME is required but a value was not provided.' - assert err.content_type is not None - assert err.object_id is not None + assert parser_errors.count() == 19846 assert SSP_M1.objects.count() == expected_m1_record_count assert SSP_M2.objects.count() == expected_m2_record_count @@ -650,49 +631,51 @@ def bad_ssp_s1__row_missing_required_field(stt_user, stt): @pytest.mark.django_db() def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_field): """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" - errors = parse.parse_datafile(bad_ssp_s1__row_missing_required_field) + parse.parse_datafile(bad_ssp_s1__row_missing_required_field) parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) - assert parser_errors.count() == 5 + assert parser_errors.count() == 9 - row_2_error = parser_errors.get(row_number=2) + row_2_error = parser_errors.get( + row_number=2, + error_message='RPT_MONTH_YEAR is required but a value was not provided.' + ) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == 'RPT_MONTH_YEAR is required but a value was not provided.' assert row_2_error.content_type.model == 'ssp_m1' assert row_2_error.object_id is not None - row_3_error = parser_errors.get(row_number=3) + row_3_error = parser_errors.get( + row_number=3, + error_message='RPT_MONTH_YEAR is required but a value was not provided.' + ) assert row_3_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_3_error.error_message == 'RPT_MONTH_YEAR is required but a value was not provided.' assert row_3_error.content_type.model == 'ssp_m2' assert row_3_error.object_id is not None - row_4_error = parser_errors.get(row_number=4) + row_4_error = parser_errors.get( + row_number=4, + error_message='RPT_MONTH_YEAR is required but a value was not provided.' + ) assert row_4_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_4_error.error_message == 'RPT_MONTH_YEAR is required but a value was not provided.' assert row_4_error.content_type.model == 'ssp_m3' assert row_4_error.object_id is not None - row_5_error = parser_errors.get(row_number=5) + row_5_error = parser_errors.get( + row_number=5, + error_message='Unknown Record_Type was found.' + ) assert row_5_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_5_error.error_message == 'Unknown Record_Type was found.' assert row_5_error.content_type is None assert row_5_error.object_id is None - trailer_error = parser_errors.get(row_number=6) + trailer_error = parser_errors.get( + row_number=6, + error_message='Trailer length is 15 but must be 23 characters.' + ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'Trailer length is 15 but must be 23 characters.' assert trailer_error.content_type is None assert trailer_error.object_id is None - assert errors == { - "2_0": [row_2_error], - "3_0": [row_3_error], - "4_0": [row_4_error], - "5_0": [row_5_error], - 'trailer': [trailer_error], - } - @pytest.mark.django_db def test_dfs_set_case_aggregates(test_datafile, dfs): """Test that the case aggregates are set correctly.""" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 6ed48e44e..3417ecfb2 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -320,7 +320,8 @@ def case_aggregates_by_month(df, dfs_status): case_numbers = case_numbers.union(curr_case_numbers) total += len(case_numbers) - cases_with_errors += ParserError.objects.filter(case_number__in=case_numbers).distinct('case_number').count() + cases_with_errors += ParserError.objects.filter(file=df).filter( + case_number__in=case_numbers).distinct('case_number').count() accepted = total - cases_with_errors aggregate_data['months'].append({"month": month, diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d5637f620..e516a6530 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -298,7 +298,12 @@ def validateRace(): # outlier validators def validate__FAM_AFF__SSN(): - """If item 30 ==2 and item 42 ==1 or 2, then item 33 != 000000000 -- 999999999.""" + """ + Validate social security number provided. + + If item FAMILY_AFFILIATION ==2 and item CITIZENSHIP_STATUS ==1 or 2, + then item SSN != 000000000 -- 999999999. + """ # value is instance def validate(instance): FAMILY_AFFILIATION = instance['FAMILY_AFFILIATION'] if type(instance) is dict else \