From b0a26929126615c755dd5dfe0261c41bb44c7cf9 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 26 Oct 2023 08:39:58 -0400 Subject: [PATCH] add required=False to allow blanks --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 18 ++-- .../tdpservice/parsers/schema_defs/ssp/m2.py | 94 +++++++++---------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 36 +++---- .../tdpservice/parsers/test/test_parse.py | 6 +- 4 files changed, 77 insertions(+), 77 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 8847ae169..bd882e69f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -96,7 +96,7 @@ validators.isNumber(), ]), Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 99), ]), Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, @@ -128,7 +128,7 @@ validators.isInLimits(1, 2), ]), Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2), ]), Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, @@ -136,7 +136,7 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2), ]), Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, @@ -180,19 +180,19 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, @@ -204,7 +204,7 @@ validators.oneOf([1, 2]), ]), Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 9), # diff than t1 (0-2) ]), Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, @@ -244,7 +244,7 @@ validators.oneOf([1, 2]), ]), Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 9), # validators.isAlphaNumeric(), # diff than t1 (blank or 9, alpha) ]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index a6a3b78d9..92d316912 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -119,28 +119,28 @@ TransformField(transform_func=ssp_ssn_decryption_func, item="29", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, @@ -160,7 +160,7 @@ validators.oneOf([1, 2]) # string field in t2 ]), Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0) ]), Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, @@ -168,7 +168,7 @@ validators.oneOf([1, 2]) ]), Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 5) ]), Field(item="34", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, @@ -178,15 +178,15 @@ # naming difference between t2? Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 3) ]), Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 9) ]), Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, - required=True, validators=[ + required=False, validators=[ validators.or_validators( # validators.isInStringRange(0, 16), validators.isInLimits(0, 16), @@ -195,15 +195,15 @@ ) ]), Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 3, 9]) # option 3 missing from t2 ]), Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 9]) ]), Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 3) ]), Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, @@ -215,163 +215,163 @@ ) ]), Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, - required=True, validators=[ + required=False, validators=[ # validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']) # 15 repeated, no 16 in m2 (assumed typo in doc) validators.oneOf([1, 2, 5, 7, 9, 15, 16, 17, 18, 19, 99]) ]), Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 99) # string in t2 validators.isInLimits(0, 99) ]), @@ -381,7 +381,7 @@ validators.isInLimits(0, 9999) ]), Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 9999) # string in t2 validators.isInLimits(0, 9999) ]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 1f13c781b..f78267990 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -91,27 +91,27 @@ validators.validateSSN() ]), Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) # validateRace? ]), Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, @@ -127,11 +127,11 @@ validators.oneOf([1, 2]) ]), Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, # string in t3 - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 10) ]), Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 2, 3]) ]), Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, # string in t3 @@ -142,7 +142,7 @@ ) ]), Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 3, 9]) # 3 not in t3 ]), Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, # string in t3 @@ -240,27 +240,27 @@ validators.validateSSN() ]), Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2) ]), Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, @@ -276,12 +276,12 @@ validators.oneOf([1, 2]) ]), Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, - required=True, validators=[ + required=False, validators=[ # validators.isInStringRange(0, 10) # string in t3 validators.isInLimits(0, 10) ]), Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 2, 3]) ]), Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, @@ -292,7 +292,7 @@ ) ]), Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 3, 9]) # 3 not in t3 ]), Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 9d5547bd1..1da6a45f9 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -430,7 +430,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): ]} parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) - assert parser_errors.count() == 16 + assert parser_errors.count() == 9 assert SSP_M1.objects.count() == expected_m1_record_count assert SSP_M2.objects.count() == expected_m2_record_count assert SSP_M3.objects.count() == expected_m3_record_count @@ -452,7 +452,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile): parse.parse_datafile(ssp_section1_datafile) parser_errors = ParserError.objects.filter(file=ssp_section1_datafile) - assert parser_errors.count() == 19848 + assert parser_errors.count() == 12779 assert SSP_M1.objects.count() == expected_m1_record_count assert SSP_M2.objects.count() == expected_m2_record_count @@ -633,7 +633,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel parse.parse_datafile(bad_ssp_s1__row_missing_required_field) parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) - assert parser_errors.count() == 9 + assert parser_errors.count() == 7 row_2_error = parser_errors.get( row_number=2,