From f94c1fff79f4d14127d499371a7767864cae1d84 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 13 Sep 2023 11:41:57 -0600 Subject: [PATCH 01/20] - Added support to fields that can allow blanks as valid fields - Added new tests - Updated make validator to handle type errors --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 35 ++-- .../tdpservice/parsers/schema_defs/tanf/t2.py | 154 ++++++++++++------ .../tdpservice/parsers/schema_defs/tanf/t3.py | 54 ++++-- .../parsers/test/test_validators.py | 48 ++++++ tdrs-backend/tdpservice/parsers/validators.py | 20 ++- 5 files changed, 229 insertions(+), 82 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 08e171c22..0ba3d002d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -93,7 +93,8 @@ ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, required=True, validators=[ - validators.isInStringRange(0, 99), + validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()), ]), Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ @@ -129,7 +130,8 @@ ]), Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ - validators.isInLimits(0, 2), + validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()), ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ @@ -137,7 +139,8 @@ ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ - validators.isInLimits(0, 3), + validators.or_validators(validators.isInLimits(0, 3), + validators.numIsBlank()), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ @@ -181,19 +184,23 @@ ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ - validators.isLargerThanOrEqualTo(0), + validators.or_validators(validators.isLargerThanOrEqualTo(0), + validators.numIsBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ - validators.isLargerThanOrEqualTo(0), + validators.or_validators(validators.isLargerThanOrEqualTo(0), + validators.numIsBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ - validators.isLargerThanOrEqualTo(0), + validators.or_validators(validators.isLargerThanOrEqualTo(0), + validators.numIsBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ - validators.isLargerThanOrEqualTo(0), + validators.or_validators(validators.isLargerThanOrEqualTo(0), + validators.numIsBlank()), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ @@ -205,7 +212,8 @@ ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ - validators.oneOf([0, 1, 2]), + validators.or_validators(validators.oneOf([0, 1, 2]), + validators.numIsBlank()), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ @@ -245,8 +253,12 @@ ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, required=True, validators=[ - validators.or_validators(validators.matches('9'), validators.isEmpty()), - validators.isAlphaNumeric(), + validators.or_validators( + validators.and_validators( + validators.or_validators(validators.matches('9'), validators.isEmpty()), + validators.isAlphaNumeric() + ), + validators.isBlank()), ]), Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ @@ -255,7 +267,8 @@ ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ - validators.oneOf([1, 2]), + validators.or_validators(validators.oneOf([1, 2]), + validators.numIsBlank()), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 74f0b9d52..dc2886e00 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -110,17 +110,23 @@ endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, - validators=[validators.isInLimits(0, 2)]), + validators=[validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, @@ -130,106 +136,150 @@ Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, - validators=[validators.isLargerThanOrEqualTo(0),]), + validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), + validators.numIsBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, - validators=[validators.isInLimits(0, 5),]), + validators=[validators.or_validators(validators.isInLimits(0, 5), + validators.numIsBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, - validators=[validators.isInLimits(0, 3),]), + validators=[validators.or_validators(validators.isInLimits(0, 3), + validators.numIsBlank()),]), Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, - validators=[validators.isInLimits(0, 9),]), + validators=[validators.or_validators(validators.isInLimits(0, 9), + validators.numIsBlank()),]), Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), - validators.isInStringRange(98, 99) - )]), + validators.isInStringRange(98, 99), + validators.isBlank())]), Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, - validators=[validators.oneOf([0, 1, 2, 9])]), + validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, - validators=[validators.oneOf([0, 1, 2, 9]),]), + validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()),]), Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, - validators=[validators.isInStringRange(0, 999),]), + validators=[validators.or_validators(validators.isInStringRange(0, 999), + validators.isBlank()),]), Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=True, validators=[validators.isInLimits(0, 9),]), + required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), + validators.numIsBlank()),]), Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, - validators=[validators.isInLimits(0, 3),]), + validators=[validators.or_validators(validators.isInLimits(0, 3), + validators.numIsBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, - validators=[validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99'])]), + validators=[validators.or_validators( + validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), + validators.isBlank()),]), Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ - validators.isInStringRange(0, 99), - ]), + validators.or_validators( + validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, - validators=[validators.isInStringRange(0, 99),]), + validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - required=False, validators=[validators.isInStringRange(0, 99),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isBlank()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=False, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, - required=False, validators=[validators.isInStringRange(0, 9999),]), + required=False, validators=[validators.or_validators(validators.isInStringRange(0, 9999), + validators.isBlank()),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=False, validators=[validators.isInStringRange(0, 9999),]), Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index d22d848df..351f98628 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -84,17 +84,23 @@ endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -109,11 +115,13 @@ ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, required=True, validators=[ - validators.isInStringRange(0, 10) + validators.or_validators(validators.isInStringRange(0, 10), + validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ - validators.oneOf([0, 2, 3]) + validators.or_validators(validators.oneOf([0, 2, 3]), + validators.numIsBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[ @@ -124,7 +132,8 @@ ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ - validators.oneOf([0, 1, 2, 9]) + validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=False, validators=[ @@ -213,17 +222,23 @@ endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=True, validators=[validators.validateRace()]), + required=True, validators=[validators.or_validators(validators.validateRace(), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -238,11 +253,13 @@ ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, required=True, validators=[ - validators.isInStringRange(0, 10) + validators.or_validators(validators.isInStringRange(0, 10), + validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ - validators.oneOf([0, 2, 3]) + validators.or_validators(validators.oneOf([0, 2, 3]), + validators.numIsBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[ @@ -253,7 +270,8 @@ ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ - validators.oneOf([0, 1, 2, 9]) + validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=False, validators=[ diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index ecca06561..197467543 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -1024,3 +1024,51 @@ def test_sum_of_recipients(self, record): assert result == (False, "The sum of ['NUM_ADULT_RECIPIENTS', 'NUM_CHILD_RECIPIENTS'] does not equal " + "NUM_RECIPIENTS.") + +@pytest.mark.django_db() +def test_t1_blank_fields(): + """Test blank validators on expected fields for T1 record.""" + t1 = TanfT1Factory.create() + t1.TRANSITION_SERVICES_AMOUNT = None + t1.STRATUM = " " + + stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) + + tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()) + + assert stratum_val(t1.STRATUM) == (True, None) + assert tsa_val(t1.TRANSITION_SERVICES_AMOUNT) == (True, None) + +@pytest.mark.django_db() +def test_t2_blank_fields(): + """Test blank validators on expected fields for T2 record.""" + t2 = TanfT2Factory.create() + t2.RACE_HISPANIC = None + t2.EDUCATION_LEVEL = " " + t2.WORK_EXPERIENCE_HOL = " " + + race_val = validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()) + ed_level_val = validators.or_validators(validators.isInStringRange(0, 16), + validators.isInStringRange(98, 99), + validators.isBlank()) + hol_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) + + assert race_val(t2.RACE_HISPANIC) == (True, None) + assert ed_level_val(t2.EDUCATION_LEVEL) == (True, None) + assert hol_val(t2.WORK_EXPERIENCE_HOL) == (True, None) + +@pytest.mark.django_db() +def test_t3_blank_fields(): + """Test blank validators on expected fields for T3 record.""" + t3 = TanfT3Factory.create() + t3.RACE_HISPANIC = None + t3.RELATIONSHIP_HOH = " " + t3.CITIZENSHIP_STATUS = None + + race_val = validators.or_validators(validators.validateRace(), validators.numIsBlank()) + hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) + cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) + + assert race_val(t3.RACE_HISPANIC) == (True, None) + assert hoh_val(t3.RELATIONSHIP_HOH) == (True, None) + assert cit_status_val(t3.CITIZENSHIP_STATUS) == (True, None) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index c811a6ef1..6bd50a85a 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -4,12 +4,23 @@ from .models import ParserErrorCategoryChoices from tdpservice.data_files.models import DataFile from datetime import date +import logging + +logger = logging.getLogger(__name__) # higher order validator func def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - return lambda value: (True, None) if value is not None and validator_func(value) else (False, error_func(value)) + def validator(value): + try: + if validator_func(value): + return (True, None) + return (False, error_func(value)) + except Exception as e: + logger.debug(f"Caught exception in make_validator. Exception: {e}") + return (False, error_func(value)) + return validator def or_validators(*args, **kwargs): @@ -166,6 +177,13 @@ def isBlank(): lambda value: f'{value} is not blank.' ) +def numIsBlank(): + """Validate that a number field was all blanks at parse time.""" + return make_validator( + lambda value: value == None, + lambda value: f'{value} is not a blank integer.' + ) + def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( From 8a12d794a891603ba7d0457b87527263cdd2133d Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Sep 2023 12:09:53 -0600 Subject: [PATCH 02/20] - Updating blank fields to not be required --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 20 +++--- .../tdpservice/parsers/schema_defs/tanf/t2.py | 64 +++++++++---------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 36 +++++------ 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 0ba3d002d..f74a70f05 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,7 +92,7 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()), ]), @@ -129,7 +129,7 @@ validators.isInLimits(1, 2), ]), Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()), ]), @@ -138,7 +138,7 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()), ]), @@ -183,22 +183,22 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), @@ -211,7 +211,7 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), validators.numIsBlank()), ]), @@ -252,7 +252,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - required=True, validators=[ + required=False, validators=[ validators.or_validators( validators.and_validators( validators.or_validators(validators.matches('9'), validators.isEmpty()), @@ -266,7 +266,7 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([1, 2]), validators.numIsBlank()), ]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index dc2886e00..fe6d64cfd 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,22 +109,22 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=False, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, @@ -133,92 +133,92 @@ validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=False, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=False, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=False, validators=[validators.or_validators(validators.isInLimits(0, 5), validators.numIsBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=False, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=False, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), - Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=False, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), + required=False, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=False, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=False, validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), validators.isBlank()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=False, validators=[ validators.or_validators( validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 351f98628..e3cdab0ad 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -84,22 +84,22 @@ endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ @@ -114,12 +114,12 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) ]), @@ -131,7 +131,7 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) ]), @@ -222,22 +222,22 @@ endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=True, validators=[validators.or_validators(validators.validateRace(), + required=False, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ @@ -252,12 +252,12 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) ]), @@ -269,7 +269,7 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) ]), From 5ede638d75ec99df778556c15a67ae79b4390135 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Sep 2023 13:30:46 -0600 Subject: [PATCH 03/20] - Fix lint errors - Add datafile --- .../tdpservice/parsers/schema_defs/tanf/t2.py | 11 +++++---- .../tdpservice/parsers/schema_defs/tanf/t3.py | 24 +++++++++---------- .../test/data/tanf_section1_blanks.txt | 5 ++++ tdrs-backend/tdpservice/parsers/validators.py | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index fe6d64cfd..8c226f52f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -158,9 +158,9 @@ Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), - Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=False, - validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()),]), + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + required=False, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()),]), Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), @@ -169,7 +169,7 @@ validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, required=False, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=False, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), @@ -179,7 +179,8 @@ )]), Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=False, validators=[validators.or_validators( - validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), + validators.oneOf(['01', '02', '05', '07', '09', '15', + '16', '17', '18', '19', '99']), validators.isBlank()),]), Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index e3cdab0ad..340d32d59 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -85,22 +85,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -223,22 +223,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=False, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt new file mode 100644 index 000000000..8a03c2945 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt @@ -0,0 +1,5 @@ +HEADER20204A06 TAN1EN +T120201011111111112230 4033611102131 0000300000000000008730010000000000000000 00000000000000222222000000002229012 +T2202010111111111121219740114WTTTTTY@W222122222222101221211001472 01 00000000000000000000000000000000000000000000000000000000000000000000000000000000000291 +T320201011111111112120190127WTTTT90W0 2212222204 98100000000 +TRAILER0002643 diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 6bd50a85a..eafa5a7fb 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -180,7 +180,7 @@ def isBlank(): def numIsBlank(): """Validate that a number field was all blanks at parse time.""" return make_validator( - lambda value: value == None, + lambda value: value is None, lambda value: f'{value} is not a blank integer.' ) From 780a6c7eb99b66c6e674b11e0c6bdb81c7539b44 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Sep 2023 11:05:23 -0600 Subject: [PATCH 04/20] - updating fields to be required until guidance comes out on what is not reqruired --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 20 ++-- .../tdpservice/parsers/schema_defs/tanf/t2.py | 112 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 44 +++---- 3 files changed, 88 insertions(+), 88 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index f74a70f05..0ba3d002d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,7 +92,7 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()), ]), @@ -129,7 +129,7 @@ validators.isInLimits(1, 2), ]), Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()), ]), @@ -138,7 +138,7 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()), ]), @@ -183,22 +183,22 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), @@ -211,7 +211,7 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), validators.numIsBlank()), ]), @@ -252,7 +252,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - required=False, validators=[ + required=True, validators=[ validators.or_validators( validators.and_validators( validators.or_validators(validators.matches('9'), validators.isEmpty()), @@ -266,7 +266,7 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([1, 2]), validators.numIsBlank()), ]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 8c226f52f..5145f69b5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,22 +109,22 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=False, + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=False, + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=False, + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=False, + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=False, + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, @@ -133,161 +133,161 @@ validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=False, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=False, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=False, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[validators.or_validators(validators.isInLimits(0, 5), validators.numIsBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=False, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=False, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=False, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - required=False, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=False, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=False, validators=[validators.or_validators(validators.isInLimits(0, 9), + required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=False, + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=False, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), validators.isBlank()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=False, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=False, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=False, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=False, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=False, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=False, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=False, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ validators.or_validators( validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=False, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=False, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=False, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=False, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=False, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=False, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, - required=False, validators=[validators.isInStringRange(0, 9999),]), + required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, - required=False, validators=[validators.or_validators(validators.isInStringRange(0, 9999), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9999), validators.isBlank()),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, - required=False, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, + required=True, validators=[validators.isInStringRange(0, 9999),]), + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=True, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=True, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=True, validators=[validators.isInStringRange(0, 9999),]), ], )] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 340d32d59..3315cf49a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -84,22 +84,22 @@ endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ @@ -114,12 +114,12 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) ]), @@ -131,16 +131,16 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, - required=False, validators=[ + required=True, validators=[ validators.isInStringRange(0, 9999) ]), Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, - required=False, validators=[ + required=True, validators=[ validators.isInStringRange(0, 9999) ]), ], @@ -222,22 +222,22 @@ endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=False, validators=[validators.or_validators(validators.validateRace(), + required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ @@ -252,12 +252,12 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) ]), @@ -269,16 +269,16 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, - required=False, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, - required=False, validators=[ + required=True, validators=[ validators.isInStringRange(0, 9999) ]), Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, - required=False, validators=[ + required=True, validators=[ validators.isInStringRange(0, 9999) ]), ], From 07506776668ad9a63759ee433c6cdbb1bb0a9c70 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Sep 2023 11:44:56 -0600 Subject: [PATCH 05/20] - ADding None check --- tdrs-backend/tdpservice/parsers/validators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index eafa5a7fb..93f316922 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -84,7 +84,8 @@ def sumIsLarger(fields, val): def sumIsLargerFunc(value): sum = 0 for field in fields: - sum += value[field] if type(value) is dict else getattr(value, field) + val = value[field] if type(value) is dict else getattr(value, field) + sum += 0 if val is None else val return (True, None) if sum > val else (False, f"The sum of {fields} is not larger than {val}.") From c63ba74c8f023889311b05e217171b0cde2686e4 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Sep 2023 12:21:43 -0600 Subject: [PATCH 06/20] - Error checking --- .../tdpservice/parsers/test/test_parse.py | 16 ++++++++++++++++ tdrs-backend/tdpservice/parsers/validators.py | 9 +++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 4a538ae8a..c3d0526b4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -656,3 +656,19 @@ def test_parse_tanf_section3_file(tanf_section3_file): assert first.NUM_CLOSED_CASES == 3884 assert second.NUM_CLOSED_CASES == 3881 assert third.NUM_CLOSED_CASES == 5453 + +@pytest.fixture +def tanf_section1_file_with_blanks(stt_user, stt): + """Fixture for ADS.E2J.FTP3.TS06.""" + return create_test_datafile('tanf_section1_blanks.txt', stt_user, stt) + +@pytest.mark.django_db() +def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): + parse.parse_datafile(tanf_section1_file_with_blanks) + + parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) + + for e in parser_errors: + print(e.error_message) + + assert False diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 93f316922..3354c49c0 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -18,7 +18,7 @@ def validator(value): return (True, None) return (False, error_func(value)) except Exception as e: - logger.debug(f"Caught exception in make_validator. Exception: {e}") + logger.debug(f"Caught exception in validator. Exception: {e}") return (False, error_func(value)) return validator @@ -71,7 +71,8 @@ def sumIsEqual(condition_field, sum_fields=[]): def sumIsEqualFunc(value): sum = 0 for field in sum_fields: - sum += value[field] if type(value) is dict else getattr(value, field) + val = value[field] if type(value) is dict else getattr(value, field) + sum += 0 if val is None else val condition_val = value[condition_field] if type(value) is dict else getattr(value, condition_field) return (True, None) if sum == condition_val else (False, @@ -84,8 +85,8 @@ def sumIsLarger(fields, val): def sumIsLargerFunc(value): sum = 0 for field in fields: - val = value[field] if type(value) is dict else getattr(value, field) - sum += 0 if val is None else val + temp_val = value[field] if type(value) is dict else getattr(value, field) + sum += 0 if temp_val is None else temp_val return (True, None) if sum > val else (False, f"The sum of {fields} is not larger than {val}.") From 6f32a02b41279523f35b88972c96187ca1572b70 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 11:10:32 -0600 Subject: [PATCH 07/20] - Updated to run field validators even if field is empty but is allowed to be - removed unnecessary validator - updated schemas - --- tdrs-backend/tdpservice/parsers/fields.py | 8 +- tdrs-backend/tdpservice/parsers/row_schema.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 70 ++------- .../tdpservice/parsers/schema_defs/tanf/t2.py | 147 ++++++++++-------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 68 ++++---- .../parsers/test/test_validators.py | 8 +- tdrs-backend/tdpservice/parsers/validators.py | 11 +- 7 files changed, 146 insertions(+), 168 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index fa4c73101..7d5ad05df 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -18,7 +18,7 @@ def value_is_empty(value, length): class Field: """Provides a mapping between a field name and its position.""" - def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]): + def __init__(self, item, name, type, startIndex, endIndex, can_be_empty=False, required=True, validators=[]): self.item = item self.name = name self.type = type @@ -26,6 +26,7 @@ def __init__(self, item, name, type, startIndex, endIndex, required=True, valida self.endIndex = endIndex self.required = required self.validators = validators + self.can_be_empty = can_be_empty def create(self, item, name, length, start, end, type): """Create a new field.""" @@ -60,8 +61,9 @@ def parse_value(self, line): class TransformField(Field): """Represents a field that requires some transformation before serializing.""" - def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True, validators=[], **kwargs): - super().__init__(item, name, type, startIndex, endIndex, required, validators) + def __init__(self, transform_func, item, name, type, startIndex, endIndex, can_be_empty=False, required=True, + validators=[], **kwargs): + super().__init__(item, name, type, startIndex, endIndex, can_be_empty, required, validators) self.transform_func = transform_func self.kwargs = kwargs diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index a4faecdf3..c8ca6358d 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -114,7 +114,7 @@ def run_field_validators(self, instance, generate_error): else: value = getattr(instance, field.name, None) - if field.required and not value_is_empty(value, field.endIndex-field.startIndex): + if (field.required and not value_is_empty(value, field.endIndex-field.startIndex)) or field.can_be_empty: for validator in field.validators: validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 0ba3d002d..d1ce9f6c4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,55 +92,19 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()), ]), - Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, - required=True, validators=[ - validators.isNumber(), - ]), - Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, - required=True, validators=[ - validators.oneOf([1, 2]), - ]), - Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, - required=True, validators=[ - validators.oneOf([1, 2]), - ]), - Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, - required=True, validators=[ - validators.isLargerThan(0), - ]), - Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, - required=True, validators=[ - validators.isInLimits(1, 2), - ]), - Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - required=True, validators=[ - validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()), - ]), + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ @@ -183,24 +147,24 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ @@ -211,9 +175,9 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ @@ -252,7 +216,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators( validators.and_validators( validators.or_validators(validators.matches('9'), validators.isEmpty()), @@ -266,9 +230,9 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([1, 2]), - validators.numIsBlank()), + validators.isBlank()), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 5145f69b5..c1150a902 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,172 +109,191 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, + validators.isBlank()),]), + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, + validators.isBlank()),]), + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, + validators.isBlank()),]), + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, + validators.isBlank()),]), + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, + validators.isBlank()),]), + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 5), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, + validators.isBlank()),]), + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.numIsBlank()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, + validators.isBlank()),]), + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.isBlank()),]), + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.numIsBlank()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), + validators.isBlank()),]), + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, can_be_empty=True, + required=True, validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), validators.isBlank()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, validators=[ validators.or_validators( validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 3315cf49a..1c5d04016 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -83,24 +83,24 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + validators.isBlank()),]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + validators.isBlank()),]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + validators.isBlank()),]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + validators.isBlank()),]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + validators.isBlank()),]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -113,15 +113,15 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.numIsBlank()) + validators.isBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[ @@ -130,10 +130,10 @@ validators.isInStringRange(98, 99) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()) + validators.isBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=True, validators=[ @@ -221,24 +221,24 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + validators.isBlank()),]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + validators.isBlank()),]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + validators.isBlank()),]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + validators.isBlank()),]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + validators.isBlank()),]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -251,15 +251,15 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.numIsBlank()) + validators.isBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[ @@ -268,10 +268,10 @@ validators.oneOf(['98', '99']) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()) + validators.isBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=True, validators=[ diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 197467543..ae63a2604 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -1034,7 +1034,7 @@ def test_t1_blank_fields(): stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) - tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()) + tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.isBlank()) assert stratum_val(t1.STRATUM) == (True, None) assert tsa_val(t1.TRANSITION_SERVICES_AMOUNT) == (True, None) @@ -1047,7 +1047,7 @@ def test_t2_blank_fields(): t2.EDUCATION_LEVEL = " " t2.WORK_EXPERIENCE_HOL = " " - race_val = validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()) + race_val = validators.or_validators(validators.isInLimits(0, 2), validators.isBlank()) ed_level_val = validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank()) @@ -1065,9 +1065,9 @@ def test_t3_blank_fields(): t3.RELATIONSHIP_HOH = " " t3.CITIZENSHIP_STATUS = None - race_val = validators.or_validators(validators.validateRace(), validators.numIsBlank()) + race_val = validators.or_validators(validators.validateRace(), validators.isBlank()) hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) - cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) + cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.isBlank()) assert race_val(t3.RACE_HISPANIC) == (True, None) assert hoh_val(t3.RELATIONSHIP_HOH) == (True, None) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 3354c49c0..2c81ef1b7 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -173,19 +173,12 @@ def isAlphaNumeric(): ) def isBlank(): - """Validate that string value is blank.""" + """Validate that value is blank.""" return make_validator( - lambda value: value.isspace(), + lambda value: value.isspace() or value is None, lambda value: f'{value} is not blank.' ) -def numIsBlank(): - """Validate that a number field was all blanks at parse time.""" - return make_validator( - lambda value: value is None, - lambda value: f'{value} is not a blank integer.' - ) - def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( From 870c01d6c84e15b5a440675c69b7f0f3a4dda00a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 13:44:28 -0600 Subject: [PATCH 08/20] - Switching back to numIsBlank --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 70 +++++++-- .../tdpservice/parsers/schema_defs/tanf/t2.py | 147 ++++++++---------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 68 ++++---- .../parsers/test/test_validators.py | 8 +- tdrs-backend/tdpservice/parsers/validators.py | 11 +- 5 files changed, 164 insertions(+), 140 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index d1ce9f6c4..0ba3d002d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,19 +92,55 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()), ]), - + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + required=True, validators=[ + validators.isNumber(), + ]), + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, + required=True, validators=[ + validators.oneOf([1, 2]), + ]), + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + required=True, validators=[ + validators.oneOf([1, 2]), + ]), + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + required=True, validators=[ + validators.isLargerThan(0), + ]), + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + required=True, validators=[ + validators.isInLimits(1, 2), + ]), + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + required=True, validators=[ + validators.or_validators(validators.isInLimits(0, 2), + validators.numIsBlank()), + ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isInLimits(0, 3), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ @@ -147,24 +183,24 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ @@ -175,9 +211,9 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ @@ -216,7 +252,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators( validators.and_validators( validators.or_validators(validators.matches('9'), validators.isEmpty()), @@ -230,9 +266,9 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - can_be_empty=True, required=True, validators=[ + required=True, validators=[ validators.or_validators(validators.oneOf([1, 2]), - validators.isBlank()), + validators.numIsBlank()), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index c1150a902..5145f69b5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,191 +109,172 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, can_be_empty=True, - required=True, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, - can_be_empty=True, required=True, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, - can_be_empty=True, required=True, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, can_be_empty=True, - required=True, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[validators.or_validators(validators.isInLimits(0, 5), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, can_be_empty=True, - required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.isBlank()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.isBlank()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, can_be_empty=True, - required=True, + validators.numIsBlank()),]), + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, can_be_empty=True, - required=True, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isBlank()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, - required=True, + required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + validators.numIsBlank()),]), + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, - can_be_empty=True, required=True, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.isBlank()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, - required=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), + validators.numIsBlank()),]), + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, can_be_empty=True, - required=True, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), validators.isBlank()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, can_be_empty=True, - required=True, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, can_be_empty=True, - required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, required=True, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, required=True, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, required=True, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, required=True, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ validators.or_validators( validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, required=True, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, required=True, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, required=True, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, required=True, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, required=True, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, required=True, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 1c5d04016..3315cf49a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -83,24 +83,24 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -113,15 +113,15 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, can_be_empty=True, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, can_be_empty=True, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.isBlank()) + validators.numIsBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[ @@ -130,10 +130,10 @@ validators.isInStringRange(98, 99) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, can_be_empty=True, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isBlank()) + validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=True, validators=[ @@ -221,24 +221,24 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, + validators.numIsBlank()),]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isBlank()),]), + validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -251,15 +251,15 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, can_be_empty=True, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, can_be_empty=True, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.isBlank()) + validators.numIsBlank()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[ @@ -268,10 +268,10 @@ validators.oneOf(['98', '99']) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, can_be_empty=True, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isBlank()) + validators.numIsBlank()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=True, validators=[ diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index ae63a2604..197467543 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -1034,7 +1034,7 @@ def test_t1_blank_fields(): stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) - tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.isBlank()) + tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()) assert stratum_val(t1.STRATUM) == (True, None) assert tsa_val(t1.TRANSITION_SERVICES_AMOUNT) == (True, None) @@ -1047,7 +1047,7 @@ def test_t2_blank_fields(): t2.EDUCATION_LEVEL = " " t2.WORK_EXPERIENCE_HOL = " " - race_val = validators.or_validators(validators.isInLimits(0, 2), validators.isBlank()) + race_val = validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()) ed_level_val = validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank()) @@ -1065,9 +1065,9 @@ def test_t3_blank_fields(): t3.RELATIONSHIP_HOH = " " t3.CITIZENSHIP_STATUS = None - race_val = validators.or_validators(validators.validateRace(), validators.isBlank()) + race_val = validators.or_validators(validators.validateRace(), validators.numIsBlank()) hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) - cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.isBlank()) + cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) assert race_val(t3.RACE_HISPANIC) == (True, None) assert hoh_val(t3.RELATIONSHIP_HOH) == (True, None) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 2c81ef1b7..3354c49c0 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -173,12 +173,19 @@ def isAlphaNumeric(): ) def isBlank(): - """Validate that value is blank.""" + """Validate that string value is blank.""" return make_validator( - lambda value: value.isspace() or value is None, + lambda value: value.isspace(), lambda value: f'{value} is not blank.' ) +def numIsBlank(): + """Validate that a number field was all blanks at parse time.""" + return make_validator( + lambda value: value is None, + lambda value: f'{value} is not a blank integer.' + ) + def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( From c66fd8adde0866113d85cfc18c1b001a6f7f2eb1 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 13:53:08 -0600 Subject: [PATCH 09/20] -Adding back numIsBlank --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 56 ++------- .../tdpservice/parsers/schema_defs/tanf/t2.py | 119 ++++++++++-------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 36 +++--- .../tdpservice/parsers/test/test_parse.py | 2 +- 4 files changed, 98 insertions(+), 115 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 0ba3d002d..4d3820c1c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,53 +92,17 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()), ]), - Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, - required=True, validators=[ - validators.isNumber(), - ]), - Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, - required=True, validators=[ - validators.oneOf([1, 2]), - ]), - Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, - required=True, validators=[ - validators.oneOf([1, 2]), - ]), - Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, - required=True, validators=[ - validators.isLargerThan(0), - ]), - Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, - required=True, validators=[ - validators.isInLimits(1, 3), - ]), - Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, - required=True, validators=[ - validators.isInLimits(1, 2), - ]), - Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - required=True, validators=[ - validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()), - ]), + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()), ]), @@ -183,22 +147,22 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()), ]), @@ -211,7 +175,7 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), validators.numIsBlank()), ]), @@ -252,7 +216,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators( validators.and_validators( validators.or_validators(validators.matches('9'), validators.isEmpty()), @@ -266,7 +230,7 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - required=True, validators=[ + can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([1, 2]), validators.numIsBlank()), ]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 5145f69b5..5d7946abe 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,22 +109,28 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, @@ -133,148 +139,161 @@ validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 5), validators.numIsBlank()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), validators.isBlank())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), validators.isBlank()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), validators.numIsBlank()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.numIsBlank()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, can_be_empty=True, + required=True, validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), validators.isBlank()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, validators=[ validators.or_validators( validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 3315cf49a..2ae3d022a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -83,22 +83,22 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, @@ -113,12 +113,12 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) @@ -130,7 +130,7 @@ validators.isInStringRange(98, 99) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) @@ -221,22 +221,22 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), validators.numIsBlank()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, @@ -251,12 +251,12 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), validators.numIsBlank()) @@ -268,7 +268,7 @@ validators.oneOf(['98', '99']) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index c3d0526b4..76643e5af 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -669,6 +669,6 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) for e in parser_errors: - print(e.error_message) + print(e.field_name, e.error_message) assert False From ca0b22ced2b5dbf4bcaa973e0bf2a4b50b29be5d Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 14:00:04 -0600 Subject: [PATCH 10/20] - adding isNone since empty string fields are also None --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 18 ++-- .../tdpservice/parsers/schema_defs/tanf/t2.py | 102 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 36 +++---- tdrs-backend/tdpservice/parsers/validators.py | 6 +- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 4d3820c1c..af39fcb5d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -94,7 +94,7 @@ Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()), + validators.isNone()), ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, @@ -104,7 +104,7 @@ Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ @@ -149,22 +149,22 @@ Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ @@ -177,7 +177,7 @@ Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2]), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ @@ -222,7 +222,7 @@ validators.or_validators(validators.matches('9'), validators.isEmpty()), validators.isAlphaNumeric() ), - validators.isBlank()), + validators.isNone()), ]), Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ @@ -232,7 +232,7 @@ Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([1, 2]), - validators.numIsBlank()), + validators.isNone()), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 5d7946abe..aef798a2f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -112,27 +112,27 @@ Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, @@ -141,54 +141,54 @@ validators=[validators.oneOf([1, 2])]), Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), + validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isNone())]), Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 5), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - validators.isBlank())]), + validators.isNone())]), Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), - validators.isBlank()),]), + validators.isNone()),]), Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) @@ -198,108 +198,108 @@ validators=[validators.or_validators( validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']), - validators.isBlank()),]), + validators.isNone()),]), Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, validators=[ validators.or_validators( validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isBlank()),]), + validators.isNone()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9999), - validators.isBlank()),]), + validators.isNone()),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 2ae3d022a..9dc4a1422 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -85,22 +85,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -116,12 +116,12 @@ Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), - validators.isBlank()) + validators.isNone()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.numIsBlank()) + validators.isNone()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[ @@ -133,7 +133,7 @@ Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()) + validators.isNone()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=True, validators=[ @@ -223,22 +223,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.numIsBlank()),]), + validators.isNone()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -254,12 +254,12 @@ Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.isInStringRange(0, 10), - validators.isBlank()) + validators.isNone()) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 2, 3]), - validators.numIsBlank()) + validators.isNone()) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[ @@ -271,7 +271,7 @@ Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, can_be_empty=True, required=True, validators=[ validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.numIsBlank()) + validators.isNone()) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=True, validators=[ diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 3354c49c0..d5edae128 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -179,11 +179,11 @@ def isBlank(): lambda value: f'{value} is not blank.' ) -def numIsBlank(): - """Validate that a number field was all blanks at parse time.""" +def isNone(): + """Validate that a value is None/blank.""" return make_validator( lambda value: value is None, - lambda value: f'{value} is not a blank integer.' + lambda value: f'{value} is not blank.' ) def isInStringRange(lower, upper): From 45644e662b02a1adfee6d8334d724cf92f838662 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 14:18:03 -0600 Subject: [PATCH 11/20] - Added all missing fields to test file - Updated incorrect fields and catch validator error --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 5 +++-- .../tdpservice/parsers/test/data/tanf_section1_blanks.txt | 6 +++--- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index aef798a2f..e93e84d4c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -298,8 +298,9 @@ Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9999), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[validators.or_validators( + validators.isInStringRange(0, 9999), + validators.isNone()),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=True, diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt index 8a03c2945..499f4bdc1 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt @@ -1,5 +1,5 @@ HEADER20204A06 TAN1EN -T120201011111111112230 4033611102131 0000300000000000008730010000000000000000 00000000000000222222000000002229012 -T2202010111111111121219740114WTTTTTY@W222122222222101221211001472 01 00000000000000000000000000000000000000000000000000000000000000000000000000000000000291 -T320201011111111112120190127WTTTT90W0 2212222204 98100000000 +T120201011111111112230 4033611102131 0000 00000000000008730010000000000000000 00002 222200000000222901 +T2202010111111111121219740114WTTTTTY@W 222 1 01 01 0000 0000000000000291 +T320201011111111112120190127WTTTT90W0 222 98 00000000 TRAILER0002643 diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d5edae128..8efeb9d0d 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -337,7 +337,7 @@ def validate(instance): MONTHS_FED_TIME_LIMIT = instance['MONTHS_FED_TIME_LIMIT'] if type(instance) is dict else \ getattr(instance, 'MONTHS_FED_TIME_LIMIT') if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if int(MONTHS_FED_TIME_LIMIT) < 1: + if MONTHS_FED_TIME_LIMIT is None or int(MONTHS_FED_TIME_LIMIT) < 1: return (False, 'If FAMILY_AFFILIATION == 2 and MONTHS_FED_TIME_LIMIT== 1 or 2, then MONTHS_FED_TIME_LIMIT > 1.' ) From ccb80fc3f986d6a6182cd802e9484e6fbe140d9c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 14:23:38 -0600 Subject: [PATCH 12/20] - Updating test --- .../tdpservice/parsers/test/test_parse.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 76643e5af..5643b13eb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -664,11 +664,22 @@ def tanf_section1_file_with_blanks(stt_user, stt): @pytest.mark.django_db() def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): + """Test section 1 fields that are allowed to have blanks.""" parse.parse_datafile(tanf_section1_file_with_blanks) parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) - for e in parser_errors: - print(e.field_name, e.error_message) + assert parser_errors.count() == 23 - assert False + # Should only be cat3 validator errors + for error in parser_errors: + assert error.error_type == ParserErrorCategoryChoices.VALUE_CONSISTENCY + + t1 = TANF_T1.objects.first() + t2 = TANF_T2.objects.first() + t3 = TANF_T3.objects.first() + + assert t1.FAMILY_SANC_ADULT is None + assert t2.MARITAL_STATUS is None + assert t3.CITIZENSHIP_STATUS is None + From 265d498a078dfba6392a903c0e27bc2fa9b7fc0e Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 21 Sep 2023 15:13:26 -0600 Subject: [PATCH 13/20] - Fix lint errors --- .../tdpservice/parsers/schema_defs/tanf/t2.py | 152 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 24 +-- .../tdpservice/parsers/test/test_parse.py | 3 +- 3 files changed, 88 insertions(+), 91 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index e93e84d4c..8eff30e7e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -154,8 +154,8 @@ validators.isNone()),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, can_be_empty=True, - required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), validators.isNone()),]), Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, @@ -173,7 +173,7 @@ validators.isNone()),]), Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isNone()),]), + validators.isNone()),]), Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 999), @@ -184,7 +184,7 @@ validators.isNone()),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.isNone()),]), + validators.isNone()),]), Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), @@ -203,103 +203,101 @@ required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, validators=[ - validators.or_validators( - validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, + can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, + validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), + validators.isNone()),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, can_be_empty=True, required=True, validators=[validators.or_validators( - validators.isInStringRange(0, 9999), + validators.isInStringRange(0, 9999), validators.isNone()),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=True, validators=[validators.isInStringRange(0, 9999),]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 9dc4a1422..b3771230a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -85,22 +85,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -223,22 +223,22 @@ is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + validators.isNone()),]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 5643b13eb..3798f328a 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -674,7 +674,7 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): # Should only be cat3 validator errors for error in parser_errors: assert error.error_type == ParserErrorCategoryChoices.VALUE_CONSISTENCY - + t1 = TANF_T1.objects.first() t2 = TANF_T2.objects.first() t3 = TANF_T3.objects.first() @@ -682,4 +682,3 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): assert t1.FAMILY_SANC_ADULT is None assert t2.MARITAL_STATUS is None assert t3.CITIZENSHIP_STATUS is None - From 78f108b4ce921e189e09016293d104ed2125b12f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 25 Sep 2023 10:15:20 -0600 Subject: [PATCH 14/20] - Re-added missing fields - Updating tests --- .../tdpservice/parsers/schema_defs/tanf/t1.py | 38 ++++++++++++++++++- .../tdpservice/parsers/test/test_parse.py | 17 +++++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index af39fcb5d..633cd3857 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -96,7 +96,43 @@ validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()), ]), - + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + required=True, validators=[ + validators.isNumber(), + ]), + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, + required=True, validators=[ + validators.oneOf([1, 2]), + ]), + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + required=True, validators=[ + validators.oneOf([1, 2]), + ]), + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + required=True, validators=[ + validators.isLargerThan(0), + ]), + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + required=True, validators=[ + validators.isInLimits(1, 3), + ]), + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + required=True, validators=[ + validators.isInLimits(1, 2) + ]), + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + can_be_empty=True, required=True, validators=[ + validators.or_validators(validators.isInLimits(0, 2), + validators.isNone()), + ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ validators.isLargerThanOrEqualTo(0), diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 3798f328a..2a071f686 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -104,12 +104,13 @@ def test_parse_big_file(test_big_file): parse.parse_datafile(test_big_file) parser_errors = ParserError.objects.filter(file=test_big_file) - error_message = 'MONTHS_FED_TIME_LIMIT is required but a value was not provided.' - row_18_error = parser_errors.get(row_number=18, error_message=error_message) - assert row_18_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_18_error.error_message == error_message - assert row_18_error.content_type.model == 'tanf_t2' - assert row_18_error.object_id is not None + error_message = "14 is not in ['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']. " + \ + "or 14 is not blank." + row_118_error = parser_errors.get(row_number=118, error_message=error_message) + assert row_118_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE + assert row_118_error.error_message == error_message + assert row_118_error.content_type.model == 'tanf_t2' + assert row_118_error.object_id is not None assert TANF_T1.objects.count() == expected_t1_record_count assert TANF_T2.objects.count() == expected_t2_record_count @@ -153,7 +154,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header): """Test parsing of bad_missing_header.""" errors = parse.parse_datafile(bad_file_missing_header) - parser_errors = ParserError.objects.filter(file=bad_file_missing_header) + parser_errors = ParserError.objects.filter(file=bad_file_missing_header).order_by('created_at') assert parser_errors.count() == 2 @@ -165,7 +166,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header): assert err.content_type is None assert err.object_id is None assert errors == { - 'header': [parser_errors[1], parser_errors[0]] + 'header': list(parser_errors) } From 2d96c9bfdcadd25b2093d111d9fdfe27b4214e24 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 25 Sep 2023 10:39:08 -0600 Subject: [PATCH 15/20] - Updated tests --- .../parsers/test/test_validators.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 197467543..b0fc905b0 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -1030,11 +1030,11 @@ def test_t1_blank_fields(): """Test blank validators on expected fields for T1 record.""" t1 = TanfT1Factory.create() t1.TRANSITION_SERVICES_AMOUNT = None - t1.STRATUM = " " + t1.STRATUM = None - stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) + stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()) - tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.numIsBlank()) + tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.isNone()) assert stratum_val(t1.STRATUM) == (True, None) assert tsa_val(t1.TRANSITION_SERVICES_AMOUNT) == (True, None) @@ -1044,14 +1044,14 @@ def test_t2_blank_fields(): """Test blank validators on expected fields for T2 record.""" t2 = TanfT2Factory.create() t2.RACE_HISPANIC = None - t2.EDUCATION_LEVEL = " " - t2.WORK_EXPERIENCE_HOL = " " + t2.EDUCATION_LEVEL = None + t2.WORK_EXPERIENCE_HOL = None - race_val = validators.or_validators(validators.isInLimits(0, 2), validators.numIsBlank()) + race_val = validators.or_validators(validators.isInLimits(0, 2), validators.isNone()) ed_level_val = validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - validators.isBlank()) - hol_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isBlank()) + validators.isNone()) + hol_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()) assert race_val(t2.RACE_HISPANIC) == (True, None) assert ed_level_val(t2.EDUCATION_LEVEL) == (True, None) @@ -1062,12 +1062,12 @@ def test_t3_blank_fields(): """Test blank validators on expected fields for T3 record.""" t3 = TanfT3Factory.create() t3.RACE_HISPANIC = None - t3.RELATIONSHIP_HOH = " " + t3.RELATIONSHIP_HOH = None t3.CITIZENSHIP_STATUS = None - race_val = validators.or_validators(validators.validateRace(), validators.numIsBlank()) - hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isBlank()) - cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.numIsBlank()) + race_val = validators.or_validators(validators.validateRace(), validators.isNone()) + hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isNone()) + cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.isNone()) assert race_val(t3.RACE_HISPANIC) == (True, None) assert hoh_val(t3.RELATIONSHIP_HOH) == (True, None) From 681751939cab90708e4b76786ef689561bfc8afe Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 2 Oct 2023 10:01:07 -0600 Subject: [PATCH 16/20] - Fixing errors from merge --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 016b28daf..83f91e684 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -838,7 +838,7 @@ def test_parse_tanf_section3_file(tanf_section3_file): @pytest.fixture def tanf_section1_file_with_blanks(stt_user, stt): """Fixture for ADS.E2J.FTP3.TS06.""" - return create_test_datafile('tanf_section1_blanks.txt', stt_user, stt) + return util.create_test_datafile('tanf_section1_blanks.txt', stt_user, stt) @pytest.mark.django_db() def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): @@ -861,6 +861,7 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): assert t2.MARITAL_STATUS is None assert t3.CITIZENSHIP_STATUS is None +@pytest.fixture def tanf_section4_file(stt_user, stt): """Fixture for ADS.E2J.FTP4.TS06.""" return util.create_test_datafile('ADS.E2J.FTP4.TS06', stt_user, stt, "Stratum Data") From 18c987a4c627a02029dc4a9517b66e2613603f89 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 3 Oct 2023 15:18:10 -0600 Subject: [PATCH 17/20] - Removed extra flag - Updated logic on when to run field validators - Updated schemas --- tdrs-backend/tdpservice/parsers/fields.py | 7 +- tdrs-backend/tdpservice/parsers/row_schema.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 59 ++--- .../tdpservice/parsers/schema_defs/tanf/t2.py | 239 +++++++----------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 112 ++++---- .../test/data/tanf_section1_blanks.txt | 2 +- .../parsers/test/test_validators.py | 48 ---- 7 files changed, 162 insertions(+), 309 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 7d5ad05df..1487cf498 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -18,7 +18,7 @@ def value_is_empty(value, length): class Field: """Provides a mapping between a field name and its position.""" - def __init__(self, item, name, type, startIndex, endIndex, can_be_empty=False, required=True, validators=[]): + def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]): self.item = item self.name = name self.type = type @@ -26,7 +26,6 @@ def __init__(self, item, name, type, startIndex, endIndex, can_be_empty=False, r self.endIndex = endIndex self.required = required self.validators = validators - self.can_be_empty = can_be_empty def create(self, item, name, length, start, end, type): """Create a new field.""" @@ -61,9 +60,9 @@ def parse_value(self, line): class TransformField(Field): """Represents a field that requires some transformation before serializing.""" - def __init__(self, transform_func, item, name, type, startIndex, endIndex, can_be_empty=False, required=True, + def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True, validators=[], **kwargs): - super().__init__(item, name, type, startIndex, endIndex, can_be_empty, required, validators) + super().__init__(item, name, type, startIndex, endIndex, required, validators) self.transform_func = transform_func self.kwargs = kwargs diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index fdda99c4a..83885042c 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -114,7 +114,9 @@ def run_field_validators(self, instance, generate_error): else: value = getattr(instance, field.name, None) - if (field.required and not value_is_empty(value, field.endIndex-field.startIndex)) or field.can_be_empty: + is_empty = value_is_empty(value, field.endIndex-field.startIndex) + should_validate = not field.required and not is_empty + if (field.required and not is_empty) or should_validate: for validator in field.validators: validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 7406fb1cf..a35de7c04 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,9 +92,8 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()), + required=False, validators=[ + validators.isInStringRange(0, 99), ]), Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ @@ -126,21 +125,19 @@ ]), Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ - validators.isInLimits(1, 2) + validators.isInLimits(1, 2), ]), Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()), + required=False, validators=[ + validators.isInLimits(0, 2), ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInLimits(0, 3), - validators.isNone()), + required=False, validators=[ + validators.isInLimits(0, 3), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ @@ -183,24 +180,20 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isNone()), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isNone()), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isNone()), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isNone()), + required=False, validators=[ + validators.isLargerThanOrEqualTo(0), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ @@ -211,9 +204,8 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.oneOf([0, 1, 2]), - validators.isNone()), + required=False, validators=[ + validators.oneOf([0, 1, 2]), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ @@ -252,13 +244,9 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - can_be_empty=True, required=True, validators=[ - validators.or_validators( - validators.and_validators( - validators.or_validators(validators.matches('9'), validators.isEmpty()), - validators.isAlphaNumeric() - ), - validators.isNone()), + required=False, validators=[ + validators.or_validators(validators.matches('9'), validators.isEmpty()), + validators.isAlphaNumeric(), ]), Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ @@ -266,12 +254,11 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.oneOf([1, 2]), - validators.isNone()), + required=False, validators=[ + validators.oneOf([1, 2]), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], )] -) +) \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 8eff30e7e..e599b8fa8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,196 +109,127 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 2), - validators.isNone()),]), + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, + validators=[validators.isInLimits(0, 2)]), + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=False, + validators=[validators.isInLimits(0, 2)]), + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=False, + validators=[validators.isInLimits(0, 2)]), + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=False, + validators=[validators.isInLimits(0, 2)]), + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=False, + validators=[validators.isInLimits(0, 2)]), + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=False, + validators=[validators.isInLimits(0, 2)]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[validators.oneOf([1, 2])]), Field(item="36B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[validators.oneOf([1, 2])]), - Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, - can_be_empty=True, required=True, - validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isNone())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, - can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isLargerThanOrEqualTo(0), - validators.isNone()),]), + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, + validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=False, + validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 5), - validators.isNone()),]), + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=False, + validators=[validators.isInLimits(0, 5),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, - can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.isNone()),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.isNone()),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, can_be_empty=True, - required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=False, + validators=[validators.isInLimits(0, 3),]), + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=False, + validators=[validators.isInLimits(0, 9),]), + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 16), - validators.isInStringRange(98, 99), - validators.isNone())]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isNone()),]), - Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isNone()),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 999), - validators.isNone()),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, - can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + validators.isInStringRange(98, 99) + )]), + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, + validators=[validators.oneOf([0, 1, 2, 9])]), + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=False, + validators=[validators.oneOf([0, 1, 2, 9]),]), + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, + validators=[validators.isInStringRange(0, 999),]), + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=False, + validators=[validators.isInStringRange(0, 99),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - can_be_empty=True, required=True, validators=[validators.or_validators(validators.isInLimits(0, 9), - validators.isNone()),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInLimits(0, 3), - validators.isNone()),]), + required=False, validators=[validators.isInLimits(0, 9),]), + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=False, + validators=[validators.isInLimits(0, 3),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, can_be_empty=True, - required=True, - validators=[validators.or_validators( - validators.oneOf(['01', '02', '05', '07', '09', '15', - '16', '17', '18', '19', '99']), - validators.isNone()),]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, can_be_empty=True, - required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, - can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, can_be_empty=True, required=True, - validators=[validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.isInStringRange(0, 99), - validators.isNone()),]), + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=False, + validators=[validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99'])]), + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=False, validators=[ + validators.isInStringRange(0, 99), + ]), + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=False, + validators=[validators.isInStringRange(0, 99),]), + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=False, + validators=[validators.isInStringRange(0, 99),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, - can_be_empty=True, required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, - can_be_empty=True, required=True, validators=[validators.or_validators( - validators.isInStringRange(0, 9999), - validators.isNone()),]), + required=False, validators=[validators.isInStringRange(0, 9999),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=True, @@ -309,4 +240,4 @@ validators=[validators.isInStringRange(0, 9999),]), ], )] -) +) \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index b3771230a..bef2ff090 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -83,24 +83,18 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + required=False, validators=[validators.validateRace()]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + required=False, validators=[validators.validateRace()]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + required=False, validators=[validators.validateRace()]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + required=False, validators=[validators.validateRace()]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + required=False, validators=[validators.validateRace()]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + required=False, validators=[validators.validateRace()]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -113,15 +107,13 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 10), - validators.isNone()) + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, + required=False, validators=[ + validators.isInStringRange(0, 10) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.oneOf([0, 2, 3]), - validators.isNone()) + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + required=False, validators=[ + validators.oneOf([0, 2, 3]) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[ @@ -130,17 +122,16 @@ validators.isInStringRange(98, 99) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isNone()) + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + required=False, validators=[ + validators.oneOf([0, 1, 2, 9]) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 9999) ]), Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 9999) ]), ], @@ -221,24 +212,18 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), - Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, can_be_empty=True, - required=True, validators=[validators.or_validators(validators.validateRace(), - validators.isNone()),]), + Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + required=False, validators=[validators.validateRace()]), + Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + required=False, validators=[validators.validateRace()]), + Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + required=False, validators=[validators.validateRace()]), + Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + required=False, validators=[validators.validateRace()]), + Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + required=False, validators=[validators.validateRace()]), + Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + required=False, validators=[validators.validateRace()]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -251,15 +236,13 @@ required=True, validators=[ validators.oneOf([1, 2]) ]), - Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.isInStringRange(0, 10), - validators.isNone()) + Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, + required=False, validators=[ + validators.isInStringRange(0, 10) ]), - Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.oneOf([0, 2, 3]), - validators.isNone()) + Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + required=False, validators=[ + validators.oneOf([0, 2, 3]) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[ @@ -268,17 +251,16 @@ validators.oneOf(['98', '99']) ) ]), - Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, can_be_empty=True, - required=True, validators=[ - validators.or_validators(validators.oneOf([0, 1, 2, 9]), - validators.isNone()) + Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + required=False, validators=[ + validators.oneOf([0, 1, 2, 9]) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 9999) ]), Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 9999) ]), ], @@ -289,4 +271,4 @@ child_one, child_two ] -) +) \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt index 499f4bdc1..c9f313404 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt @@ -1,5 +1,5 @@ HEADER20204A06 TAN1EN T120201011111111112230 4033611102131 0000 00000000000008730010000000000000000 00002 222200000000222901 -T2202010111111111121219740114WTTTTTY@W 222 1 01 01 0000 0000000000000291 +T2202010111111111121219740114WTTTTTY@W 2221 1 01 01 0000 0000000000000291 T320201011111111112120190127WTTTT90W0 222 98 00000000 TRAILER0002643 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index b0fc905b0..ecca06561 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -1024,51 +1024,3 @@ def test_sum_of_recipients(self, record): assert result == (False, "The sum of ['NUM_ADULT_RECIPIENTS', 'NUM_CHILD_RECIPIENTS'] does not equal " + "NUM_RECIPIENTS.") - -@pytest.mark.django_db() -def test_t1_blank_fields(): - """Test blank validators on expected fields for T1 record.""" - t1 = TanfT1Factory.create() - t1.TRANSITION_SERVICES_AMOUNT = None - t1.STRATUM = None - - stratum_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()) - - tsa_val = validators.or_validators(validators.isLargerThanOrEqualTo(0), validators.isNone()) - - assert stratum_val(t1.STRATUM) == (True, None) - assert tsa_val(t1.TRANSITION_SERVICES_AMOUNT) == (True, None) - -@pytest.mark.django_db() -def test_t2_blank_fields(): - """Test blank validators on expected fields for T2 record.""" - t2 = TanfT2Factory.create() - t2.RACE_HISPANIC = None - t2.EDUCATION_LEVEL = None - t2.WORK_EXPERIENCE_HOL = None - - race_val = validators.or_validators(validators.isInLimits(0, 2), validators.isNone()) - ed_level_val = validators.or_validators(validators.isInStringRange(0, 16), - validators.isInStringRange(98, 99), - validators.isNone()) - hol_val = validators.or_validators(validators.isInStringRange(0, 99), validators.isNone()) - - assert race_val(t2.RACE_HISPANIC) == (True, None) - assert ed_level_val(t2.EDUCATION_LEVEL) == (True, None) - assert hol_val(t2.WORK_EXPERIENCE_HOL) == (True, None) - -@pytest.mark.django_db() -def test_t3_blank_fields(): - """Test blank validators on expected fields for T3 record.""" - t3 = TanfT3Factory.create() - t3.RACE_HISPANIC = None - t3.RELATIONSHIP_HOH = None - t3.CITIZENSHIP_STATUS = None - - race_val = validators.or_validators(validators.validateRace(), validators.isNone()) - hoh_val = validators.or_validators(validators.isInStringRange(0, 10), validators.isNone()) - cit_status_val = validators.or_validators(validators.oneOf([0, 1, 2, 9]), validators.isNone()) - - assert race_val(t3.RACE_HISPANIC) == (True, None) - assert hoh_val(t3.RELATIONSHIP_HOH) == (True, None) - assert cit_status_val(t3.CITIZENSHIP_STATUS) == (True, None) From 88cc777b2457d8fa985911b845c80e7986172d13 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 4 Oct 2023 08:51:13 -0600 Subject: [PATCH 18/20] - Fixed failing tests --- tdrs-backend/tdpservice/parsers/test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index b2817174e..03997597a 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -267,7 +267,7 @@ def test_field_validators_blank_and_not_required_returns_valid(first): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=False, validators=[ passing_validator(), failing_validator() ]), From f444601013ff7609b1f5488c62b6b4c4ce781134 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 4 Oct 2023 09:10:10 -0600 Subject: [PATCH 19/20] - Fix lint --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 6 +++--- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index a35de7c04..f31bc892c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -261,4 +261,4 @@ validators=[]), ], )] -) \ No newline at end of file +) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index e599b8fa8..1f2088b38 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -147,8 +147,8 @@ )]), Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, validators=[validators.oneOf([0, 1, 2, 9])]), - Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=False, - validators=[validators.oneOf([0, 1, 2, 9]),]), + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + required=False, validators=[validators.oneOf([0, 1, 2, 9]),]), Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, validators=[validators.isInStringRange(0, 999),]), Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=False, @@ -240,4 +240,4 @@ validators=[validators.isInStringRange(0, 9999),]), ], )] -) \ No newline at end of file +) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index bef2ff090..66daa1e59 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -271,4 +271,4 @@ child_one, child_two ] -) \ No newline at end of file +) From c6aba847f5c22b603aab24d870b1f8f8e258feee Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 10 Oct 2023 09:31:12 -0600 Subject: [PATCH 20/20] - Removing unnused validator --- tdrs-backend/tdpservice/parsers/validators.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 80ea500bd..d5637f620 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -177,13 +177,6 @@ def isBlank(): lambda value: f'{value} is not blank.' ) -def isNone(): - """Validate that a value is None/blank.""" - return make_validator( - lambda value: value is None, - lambda value: f'{value} is not blank.' - ) - def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator(