From 5cae4fb86510ed9dd37e9d78dacdd3ec3c58b939 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Thu, 4 Apr 2024 22:33:20 -0700 Subject: [PATCH 001/105] Final mile procedures and updates additional updates to schemas and friendly names --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 88 +++++----- .../tdpservice/parsers/schema_defs/ssp/m2.py | 143 ++++++++-------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 110 ++++++------- .../tdpservice/parsers/schema_defs/ssp/m4.py | 28 ++-- .../tdpservice/parsers/schema_defs/ssp/m5.py | 67 ++++---- .../tdpservice/parsers/schema_defs/ssp/m6.py | 83 +++++----- .../tdpservice/parsers/schema_defs/ssp/m7.py | 13 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 100 ++++++----- .../tdpservice/parsers/schema_defs/tanf/t2.py | 155 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 114 ++++++------- .../tdpservice/parsers/schema_defs/tanf/t4.py | 28 ++-- .../tdpservice/parsers/schema_defs/tanf/t5.py | 73 ++++----- .../tdpservice/parsers/schema_defs/tanf/t6.py | 113 ++++++------- .../tdpservice/parsers/schema_defs/tanf/t7.py | 13 +- .../parsers/schema_defs/tribal_tanf/t1.py | 93 +++++------ .../parsers/schema_defs/tribal_tanf/t2.py | 120 +++++++------- .../parsers/schema_defs/tribal_tanf/t3.py | 84 +++++----- .../parsers/schema_defs/tribal_tanf/t4.py | 30 ++-- .../parsers/schema_defs/tribal_tanf/t5.py | 73 ++++----- .../parsers/schema_defs/tribal_tanf/t6.py | 113 ++++++------- .../parsers/schema_defs/tribal_tanf/t7.py | 13 +- 21 files changed, 781 insertions(+), 873 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index ad4991cfe..e6465ae27 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -12,10 +12,6 @@ document=SSP_M1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(150), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -102,7 +98,7 @@ Field( item="0", name='RecordType', - friendly_name="record type", + friendly_name="Record Type", type='string', startIndex=0, endIndex=2, @@ -112,7 +108,7 @@ Field( item="3", name='RPT_MONTH_YEAR', - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type='number', startIndex=2, endIndex=8, @@ -125,7 +121,7 @@ Field( item="5", name='CASE_NUMBER', - friendly_name="case number", + friendly_name="Case Number", type='string', startIndex=8, endIndex=19, @@ -135,7 +131,7 @@ Field( item="2", name='COUNTY_FIPS_CODE', - friendly_name="county fips code", + friendly_name="County FIPS Code", type='string', startIndex=19, endIndex=22, @@ -145,7 +141,7 @@ Field( item="4", name='STRATUM', - friendly_name="stratum", + friendly_name="Stratum", type='string', startIndex=22, endIndex=24, @@ -155,7 +151,7 @@ Field( item="6", name='ZIP_CODE', - friendly_name="zip code", + friendly_name="ZIP Code", type='string', startIndex=24, endIndex=29, @@ -165,7 +161,7 @@ Field( item="7", name='DISPOSITION', - friendly_name="disposition", + friendly_name="Disposition", type='number', startIndex=29, endIndex=30, @@ -175,7 +171,7 @@ Field( item="8", name='NBR_FAMILY_MEMBERS', - friendly_name="number of family members", + friendly_name="Number of Family Members", type='number', startIndex=30, endIndex=32, @@ -185,7 +181,7 @@ Field( item="9", name='FAMILY_TYPE', - friendly_name="family type", + friendly_name="Type of Family for Work Participation", type='number', startIndex=32, endIndex=33, @@ -195,7 +191,7 @@ Field( item="10", name='TANF_ASST_IN_6MONTHS', - friendly_name="tanf assistance in 6 months", + friendly_name="Received Assistance Under a State (Tribal) TANF Program Within the Past Six Months", type='number', startIndex=33, endIndex=34, @@ -205,7 +201,7 @@ Field( item="11", name='RECEIVES_SUB_HOUSING', - friendly_name="receives subsidized housing", + friendly_name="Receives Subsidized Housing", type='number', startIndex=34, endIndex=35, @@ -215,7 +211,7 @@ Field( item="12", name='RECEIVES_MED_ASSISTANCE', - friendly_name="receives medical assistance", + friendly_name="Received Medical Assistance", type='number', startIndex=35, endIndex=36, @@ -225,7 +221,7 @@ Field( item="13", name='RECEIVES_FOOD_STAMPS', - friendly_name="receives food assistance", + friendly_name="Receives Assistance from the Supplemental Nutrition Assistance Program (SNAP)", type='number', startIndex=36, endIndex=37, @@ -235,7 +231,7 @@ Field( item="14", name='AMT_FOOD_STAMP_ASSISTANCE', - friendly_name="amount of food stamp assistance/stamps", + friendly_name="Amount of Supplemental Nutrition Assistance Program (SNAP) Benefits", type='number', startIndex=37, endIndex=41, @@ -245,7 +241,7 @@ Field( item="15", name='RECEIVES_SUB_CC', - friendly_name="receives subsidized child care", + friendly_name="Receives Subsidized Child Care:", type='number', startIndex=41, endIndex=42, @@ -255,7 +251,7 @@ Field( item="16", name='AMT_SUB_CC', - friendly_name="amount of subsidized child care", + friendly_name="Amount of Subsidized Child Care", type='number', startIndex=42, endIndex=46, @@ -265,7 +261,7 @@ Field( item="17", name='CHILD_SUPPORT_AMT', - friendly_name="child support amount", + friendly_name="Amount of Child Support", type='number', startIndex=46, endIndex=50, @@ -275,7 +271,7 @@ Field( item="18", name='FAMILY_CASH_RESOURCES', - friendly_name="family cash resources", + friendly_name="Amount of the Family's Cash Resources", type='number', startIndex=50, endIndex=54, @@ -285,7 +281,7 @@ Field( item="19A", name='CASH_AMOUNT', - friendly_name="cash amount", + friendly_name="Cash and Cash Equivalents: Amount", type='number', startIndex=54, endIndex=58, @@ -295,7 +291,7 @@ Field( item="19B", name='NBR_MONTHS', - friendly_name="number of months", + friendly_name="Cash and Cash Equivalents: Number of Months", type='number', startIndex=58, endIndex=61, @@ -305,7 +301,7 @@ Field( item="20A", name='CC_AMOUNT', - friendly_name="child care amount", + friendly_name="SSP-MOE Child Care: Amount", type='number', startIndex=61, endIndex=65, @@ -315,7 +311,7 @@ Field( item="20B", name='CHILDREN_COVERED', - friendly_name="children covered", + friendly_name="SSP-MOE Child Care: Number of Children Covered", type='number', startIndex=65, endIndex=67, @@ -325,7 +321,7 @@ Field( item="20C", name='CC_NBR_MONTHS', - friendly_name="child care - number of months", + friendly_name="SSP-MOE Child Care: Number of Months", type='number', startIndex=67, endIndex=70, @@ -335,7 +331,7 @@ Field( item="21A", name='TRANSP_AMOUNT', - friendly_name="transportation amount", + friendly_name="Transportation and Other Supportive Services: Amount", type='number', startIndex=70, endIndex=74, @@ -345,7 +341,7 @@ Field( item="21B", name='TRANSP_NBR_MONTHS', - friendly_name="transportation - number of months", + friendly_name="Transportation and Other Supportive Services: Number of Months", type='number', startIndex=74, endIndex=77, @@ -355,7 +351,7 @@ Field( item="22A", name='TRANSITION_SERVICES_AMOUNT', - friendly_name="transition services amount", + friendly_name="Transitional Services: Amount", type='number', startIndex=77, endIndex=81, @@ -365,7 +361,7 @@ Field( item="22B", name='TRANSITION_NBR_MONTHS', - friendly_name="transition services - number of months", + friendly_name="Transitional Services: Number of Months", type='number', startIndex=81, endIndex=84, @@ -375,7 +371,7 @@ Field( item="23A", name='OTHER_AMOUNT', - friendly_name="other amount", + friendly_name="Other: Amount", type='number', startIndex=84, endIndex=88, @@ -385,7 +381,7 @@ Field( item="23B", name='OTHER_NBR_MONTHS', - friendly_name="other - number of months", + friendly_name="Other: Number of Months", type='number', startIndex=88, endIndex=91, @@ -395,7 +391,7 @@ Field( item="24AI", name='SANC_REDUCTION_AMT', - friendly_name="sanction reduction amount", + friendly_name="Sanctions: Total Dollar Amount of Reductions due to Sanctions", type='number', startIndex=91, endIndex=95, @@ -405,7 +401,7 @@ Field( item="24AII", name='WORK_REQ_SANCTION', - friendly_name="work requirements sanction", + friendly_name="Sanctions: Work Requirements", type='number', startIndex=95, endIndex=96, @@ -415,7 +411,7 @@ Field( item="24AIII", name='FAMILY_SANC_ADULT', - friendly_name="family sanction for adult", + friendly_name="Sanctions: Code no longer in use", type='number', startIndex=96, endIndex=97, @@ -425,7 +421,7 @@ Field( item="24AIV", name='SANC_TEEN_PARENT', - friendly_name="sanction for teen parent", + friendly_name="Sanction: Teen Parent not Attending School", type='number', startIndex=97, endIndex=98, @@ -435,7 +431,7 @@ Field( item="24AV", name='NON_COOPERATION_CSE', - friendly_name="non-cooperation with child support", + friendly_name="Sanction: Non-Cooperation with Child Support", type='number', startIndex=98, endIndex=99, @@ -445,7 +441,7 @@ Field( item="24AVI", name='FAILURE_TO_COMPLY', - friendly_name="failure to comply", + friendly_name="Sanction: Failure to Comply with an Individual Responsibility Plan ", type='number', startIndex=99, endIndex=100, @@ -455,7 +451,7 @@ Field( item="24AVII", name='OTHER_SANCTION', - friendly_name="other sanction", + friendly_name="Sanction: Other", type='number', startIndex=100, endIndex=101, @@ -465,7 +461,7 @@ Field( item="24B", name='RECOUPMENT_PRIOR_OVRPMT', - friendly_name="recoupment prior overpayment", + friendly_name="Recoupment of Prior Overpayment", type='number', startIndex=101, endIndex=105, @@ -475,7 +471,7 @@ Field( item="24CI", name='OTHER_TOTAL_REDUCTIONS', - friendly_name="other total reductions", + friendly_name="Other: Total Dollar Amount of Reductions for Other Reasons", type='number', startIndex=105, endIndex=109, @@ -485,7 +481,7 @@ Field( item="24CII", name='FAMILY_CAP', - friendly_name="family cap", + friendly_name="Other: Family Cap", type='number', startIndex=109, endIndex=110, @@ -495,7 +491,7 @@ Field( item="24CIII", name='REDUCTIONS_ON_RECEIPTS', - friendly_name="reductions on receipts", + friendly_name="Other: Reduction Based on Time Limit", type='number', startIndex=110, endIndex=111, @@ -505,7 +501,7 @@ Field( item="24CIV", name='OTHER_NON_SANCTION', - friendly_name="other non-sanction", + friendly_name="Other: Non-Sanction, Non-Recoupment ", type='number', startIndex=111, endIndex=112, @@ -515,7 +511,7 @@ Field( item="25", name='WAIVER_EVAL_CONTROL_GRPS', - friendly_name="waiver evaluation experimental and control groups", + friendly_name="Waiver Evaluation Experimental and Control Groups", type='number', startIndex=112, endIndex=113, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index a24db0576..352b6d3fe 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -14,10 +14,6 @@ document=SSP_M2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(150), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -129,7 +125,7 @@ Field( item="0", name='RecordType', - friendly_name="record type", + friendly_name="Record Type", type='string', startIndex=0, endIndex=2, @@ -139,7 +135,7 @@ Field( item="3", name='RPT_MONTH_YEAR', - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type='number', startIndex=2, endIndex=8, @@ -152,7 +148,7 @@ Field( item="5", name='CASE_NUMBER', - friendly_name="case number", + friendly_name="Case Number", type='string', startIndex=8, endIndex=19, @@ -162,7 +158,7 @@ Field( item="26", name='FAMILY_AFFILIATION', - friendly_name="family affiliation", + friendly_name="Family Affiliation", type='number', startIndex=19, endIndex=20, @@ -172,7 +168,7 @@ Field( item="27", name='NONCUSTODIAL_PARENT', - friendly_name="noncustodial parent", + friendly_name="Noncustodial Parent Indicator", type='number', startIndex=20, endIndex=21, @@ -182,21 +178,18 @@ Field( item="28", name='DATE_OF_BIRTH', - friendly_name="date of birth", - type='string', + friendly_name="Date of Birth", + type='number', startIndex=21, endIndex=29, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid()] + validators=[validators.isLargerThan(0)] ), TransformField( transform_func=ssp_ssn_decryption_func, item="29", name='SSN', - friendly_name="social security number", + friendly_name="Social Security Number", type='string', startIndex=29, endIndex=38, @@ -208,7 +201,7 @@ item="30A", name='RACE_HISPANIC', type='number', - friendly_name="race hispanic", + friendly_name="Race/Ethnicity: Hispanic or Latino", startIndex=38, endIndex=39, required=False, @@ -217,7 +210,7 @@ Field( item="30B", name='RACE_AMER_INDIAN', - friendly_name="race american-indian", + friendly_name="Race/Ethnicity: American Indian or Alaska Native", type='number', startIndex=39, endIndex=40, @@ -227,7 +220,7 @@ Field( item="30C", name='RACE_ASIAN', - friendly_name="race asian", + friendly_name="Race/Ethnicity: Asian", type='number', startIndex=40, endIndex=41, @@ -237,7 +230,7 @@ Field( item="30D", name='RACE_BLACK', - friendly_name="race black", + friendly_name="Race/Ethnicity: Black or African American", type='number', startIndex=41, endIndex=42, @@ -247,7 +240,7 @@ Field( item="30E", name='RACE_HAWAIIAN', - friendly_name="race hawaiian", + friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", type='number', startIndex=42, endIndex=43, @@ -257,7 +250,7 @@ Field( item="30F", name='RACE_WHITE', - friendly_name="race white", + friendly_name="Race/Ethnicity: White", type='number', startIndex=43, endIndex=44, @@ -267,7 +260,7 @@ Field( item="31", name='GENDER', - friendly_name="gender", + friendly_name="Gender", type='number', startIndex=44, endIndex=45, @@ -277,7 +270,7 @@ Field( item="32A", name='FED_OASDI_PROGRAM', - friendly_name="federal old-age survivors and disability insurance program", + friendly_name="Receives Disability Benefits: OASDI Program", type='number', startIndex=45, endIndex=46, @@ -287,7 +280,7 @@ Field( item="32B", name='FED_DISABILITY_STATUS', - friendly_name="federal disability status", + friendly_name="Receives Disability Benefits: Federal Disability Status", type='number', startIndex=46, endIndex=47, @@ -297,7 +290,7 @@ Field( item="32C", name='DISABLED_TITLE_XIVAPDT', - friendly_name="received aid under Title XIV-APDT", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type='number', startIndex=47, endIndex=48, @@ -307,7 +300,7 @@ Field( item="32D", name='AID_AGED_BLIND', - friendly_name="receives from aid to the aged, blind, and disabled program", + friendly_name="Receives Disability Benefits: Code no longer in use.", type='number', startIndex=48, endIndex=49, @@ -317,7 +310,7 @@ Field( item="32E", name='RECEIVE_SSI', - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI ", type='number', startIndex=49, endIndex=50, @@ -327,7 +320,7 @@ Field( item="33", name='MARITAL_STATUS', - friendly_name="marital status", + friendly_name="Marital Status", type='number', startIndex=50, endIndex=51, @@ -337,7 +330,7 @@ Field( item="34", name='RELATIONSHIP_HOH', - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type='string', startIndex=51, endIndex=53, @@ -347,7 +340,7 @@ Field( item="35", name='PARENT_MINOR_CHILD', - friendly_name="parent of minor child", + friendly_name="Parent with Minor Child in the Family", type='number', startIndex=53, endIndex=54, @@ -357,7 +350,7 @@ Field( item="36", name='NEEDS_PREGNANT_WOMAN', - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type='number', startIndex=54, endIndex=55, @@ -367,7 +360,7 @@ Field( item="37", name='EDUCATION_LEVEL', - friendly_name="education level", + friendly_name="Educational Level", type='number', startIndex=55, endIndex=57, @@ -381,7 +374,7 @@ Field( item="38", name='CITIZENSHIP_STATUS', - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type='number', startIndex=57, endIndex=58, @@ -391,7 +384,7 @@ Field( item="39", name='COOPERATION_CHILD_SUPPORT', - friendly_name="cooperation with child support", + friendly_name="Cooperated with Child Support", type='number', startIndex=58, endIndex=59, @@ -401,7 +394,7 @@ Field( item="40", name='EMPLOYMENT_STATUS', - friendly_name="employment status", + friendly_name="Employment Status", type='number', startIndex=59, endIndex=60, @@ -411,7 +404,7 @@ Field( item="41", name='WORK_ELIGIBLE_INDICATOR', - friendly_name="work eligible indicator", + friendly_name="Work-Eligible Individual Indicator", type='number', startIndex=60, endIndex=62, @@ -427,7 +420,7 @@ Field( item="42", name='WORK_PART_STATUS', - friendly_name="work participation status", + friendly_name="Work Participation Status", type='number', startIndex=62, endIndex=64, @@ -437,7 +430,7 @@ Field( item="43", name='UNSUB_EMPLOYMENT', - friendly_name="unsubsidized employment", + friendly_name="Unsubsidized Employment", type='number', startIndex=64, endIndex=66, @@ -447,7 +440,7 @@ Field( item="44", name='SUB_PRIVATE_EMPLOYMENT', - friendly_name="subsidized private employment", + friendly_name="Subsidized Private-Sector Employment", type='number', startIndex=66, endIndex=68, @@ -457,7 +450,7 @@ Field( item="45", name='SUB_PUBLIC_EMPLOYMENT', - friendly_name="subsidized public employment", + friendly_name="Subsidized Public-Sector Employment", type='number', startIndex=68, endIndex=70, @@ -467,7 +460,7 @@ Field( item="46A", name='WORK_EXPERIENCE_HOP', - friendly_name="work experience - hours of participation", + friendly_name="Work Experience: Hours of Participation", type='number', startIndex=70, endIndex=72, @@ -477,7 +470,7 @@ Field( item="46B", name='WORK_EXPERIENCE_EA', - friendly_name="work experience - excused absence", + friendly_name="Work Experience: Hours of Excused Absences", type='number', startIndex=72, endIndex=74, @@ -487,7 +480,7 @@ Field( item="46C", name='WORK_EXPERIENCE_HOL', - friendly_name="work experience hours - holiday", + friendly_name="Work Experience: Hours of Holidays", type='number', startIndex=74, endIndex=76, @@ -497,7 +490,7 @@ Field( item="47", name='OJT', - friendly_name="OJT", + friendly_name="On-the-job Training ", type='number', startIndex=76, endIndex=78, @@ -507,7 +500,7 @@ Field( item="48A", name='JOB_SEARCH_HOP', - friendly_name="job search - hours of participation", + friendly_name="Job Search and Job Readiness Assistance: Hours of Participation", type='number', startIndex=78, endIndex=80, @@ -517,7 +510,7 @@ Field( item="48B", name='JOB_SEARCH_EA', - friendly_name="job search - excused absence", + friendly_name="Job Search and Job Readiness Assistance: Hours of Excused Absences", type='number', startIndex=80, endIndex=82, @@ -527,7 +520,7 @@ Field( item="48C", name='JOB_SEARCH_HOL', - friendly_name="job search - holiday", + friendly_name="Job Search and Job Readiness Assistance: Hours of Holidays", type='number', startIndex=82, endIndex=84, @@ -537,7 +530,7 @@ Field( item="49A", name='COMM_SERVICES_HOP', - friendly_name="community services - hours of participation", + friendly_name="Community Service Program: Hours of Participation", type='number', startIndex=84, endIndex=86, @@ -547,7 +540,7 @@ Field( item="49B", name='COMM_SERVICES_EA', - friendly_name="community services - excused absence", + friendly_name="Community Service Program: Hours of Excused Absences", type='number', startIndex=86, endIndex=88, @@ -557,7 +550,7 @@ Field( item="49C", name='COMM_SERVICES_HOL', - friendly_name="community services - holiday", + friendly_name="Community Service Program: Hours of Holidays", type='number', startIndex=88, endIndex=90, @@ -567,7 +560,7 @@ Field( item="50A", name='VOCATIONAL_ED_TRAINING_HOP', - friendly_name="vocational education training - hours of participation", + friendly_name="Vocational Educational Training: Hours of Participation", type='number', startIndex=90, endIndex=92, @@ -577,7 +570,7 @@ Field( item="50B", name='VOCATIONAL_ED_TRAINING_EA', - friendly_name="vocational education training - excused absence", + friendly_name="Vocational Educational Training: Hours of Excused Absences", type='number', startIndex=92, endIndex=94, @@ -587,7 +580,7 @@ Field( item="50C", name='VOCATIONAL_ED_TRAINING_HOL', - friendly_name="vocational education training - holiday", + friendly_name="Vocational Educational Training: Hours of Holidays", type='number', startIndex=94, endIndex=96, @@ -597,7 +590,7 @@ Field( item="51A", name='JOB_SKILLS_TRAINING_HOP', - friendly_name="job skills training - hours of participation", + friendly_name="Job Skills Training: Hours of Participation", type='number', startIndex=96, endIndex=98, @@ -607,7 +600,7 @@ Field( item="51B", name='JOB_SKILLS_TRAINING_EA', - friendly_name="job skills training - excused absence", + friendly_name="Job Skills Training: Hours of Excused Absences", type='number', startIndex=98, endIndex=100, @@ -617,7 +610,7 @@ Field( item="51C", name='JOB_SKILLS_TRAINING_HOL', - friendly_name="job skills training - holiday", + friendly_name="Job Skills Training: Hours of Holidays", type='number', startIndex=100, endIndex=102, @@ -627,7 +620,7 @@ Field( item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', - friendly_name="education no high school diploma - hours of participation", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=102, endIndex=104, @@ -637,7 +630,7 @@ Field( item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', - friendly_name="education no high school diploma - excused absence", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=104, endIndex=106, @@ -647,7 +640,7 @@ Field( item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', - friendly_name="education no high school diploma - holiday", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Holidays", type='number', startIndex=106, endIndex=108, @@ -657,7 +650,7 @@ Field( item="53A", name='SCHOOL_ATTENDENCE_HOP', - friendly_name="school attendance - hours of participation", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=108, endIndex=110, @@ -667,7 +660,7 @@ Field( item="53B", name='SCHOOL_ATTENDENCE_EA', - friendly_name="school attendance - excused absence", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=110, endIndex=112, @@ -677,7 +670,7 @@ Field( item="53C", name='SCHOOL_ATTENDENCE_HOL', - friendly_name="school attendance - holiday", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Holidays", type='number', startIndex=112, endIndex=114, @@ -687,7 +680,7 @@ Field( item="54A", name='PROVIDE_CC_HOP', - friendly_name="provide child care - hours of participation", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Participation", type='number', startIndex=114, endIndex=116, @@ -697,7 +690,7 @@ Field( item="54B", name='PROVIDE_CC_EA', - friendly_name="provide child care - excused absence", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Excused Absences", type='number', startIndex=116, endIndex=118, @@ -707,7 +700,7 @@ Field( item="54C", name='PROVIDE_CC_HOL', - friendly_name="provide child care - holiday", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Holidays", type='number', startIndex=118, endIndex=120, @@ -717,7 +710,7 @@ Field( item="55", name='OTHER_WORK_ACTIVITIES', - friendly_name="other work activities", + friendly_name="Hours of Other Work Activities", type='number', startIndex=120, endIndex=122, @@ -727,7 +720,7 @@ Field( item="56", name='DEEMED_HOURS_FOR_OVERALL', - friendly_name="deemed hours for overall", + friendly_name="Number of Deemed Core Hours for Overall Rate", type='number', startIndex=122, endIndex=124, @@ -737,7 +730,7 @@ Field( item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', - friendly_name="deemed hours for two parents", + friendly_name="Number of Deemed Core Hours for the Two-Parent Rate", type='number', startIndex=124, endIndex=126, @@ -747,7 +740,7 @@ Field( item="58", name='EARNED_INCOME', - friendly_name="earned income", + friendly_name="Amount of Earned Income", type='number', startIndex=126, endIndex=130, @@ -757,7 +750,7 @@ Field( item="59A", name='UNEARNED_INCOME_TAX_CREDIT', - friendly_name="unearned income tax credit", + friendly_name="Amount of Unearned Income: Tax Credit", type='number', startIndex=130, endIndex=134, @@ -767,7 +760,7 @@ Field( item="59B", name='UNEARNED_SOCIAL_SECURITY', - friendly_name="unearned social security", + friendly_name="Amount of Unearned Income: Social Security", type='number', startIndex=134, endIndex=138, @@ -777,7 +770,7 @@ Field( item="59C", name='UNEARNED_SSI', - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: Social Security: SSI Benefit", type='number', startIndex=138, endIndex=142, @@ -787,7 +780,7 @@ Field( item="59D", name='UNEARNED_WORKERS_COMP', - friendly_name="unearned workers compensation", + friendly_name="Amount of Unearned Income: Worker's Compensation", type='number', startIndex=142, endIndex=146, @@ -797,7 +790,7 @@ Field( item="59E", name='OTHER_UNEARNED_INCOME', - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type='number', startIndex=146, endIndex=150, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 2f8971c68..039b7b938 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -11,10 +11,6 @@ document=SSP_M3DataSubmissionDocument(), preparsing_validators=[ validators.notEmpty(start=19, end=60), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -95,7 +91,7 @@ Field( item="0", name='RecordType', - friendly_name="record type", + friendly_name="Record Type", type='string', startIndex=0, endIndex=2, @@ -105,7 +101,7 @@ Field( item="3", name='RPT_MONTH_YEAR', - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type='number', startIndex=2, endIndex=8, @@ -118,7 +114,7 @@ Field( item="5", name='CASE_NUMBER', - friendly_name="case number", + friendly_name="Case Number", type='string', startIndex=8, endIndex=19, @@ -128,7 +124,7 @@ Field( item="60", name='FAMILY_AFFILIATION', - friendly_name="family affiliation", + friendly_name="Family Affiliation", type='number', startIndex=19, endIndex=20, @@ -138,22 +134,21 @@ Field( item="61", name='DATE_OF_BIRTH', - friendly_name="date of birth", + friendly_name="Date of Birth", type='string', startIndex=20, endIndex=28, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ] ), TransformField( transform_func=ssp_ssn_decryption_func, item="62", name='SSN', - friendly_name="social security number", + friendly_name="Social Security Number", type='string', startIndex=28, endIndex=37, @@ -164,7 +159,7 @@ Field( item="63A", name='RACE_HISPANIC', - friendly_name="race hispanic", + friendly_name="Race/Ethnicity: Hispanic or Latino", type='number', startIndex=37, endIndex=38, @@ -174,7 +169,7 @@ Field( item="63B", name='RACE_AMER_INDIAN', - friendly_name="race american-indian", + friendly_name="Race/Ethnicity: American Indian or Alaska Native", type='number', startIndex=38, endIndex=39, @@ -184,7 +179,7 @@ Field( item="63C", name='RACE_ASIAN', - friendly_name="race asian", + friendly_name="Race/Ethnicity: Asian", type='number', startIndex=39, endIndex=40, @@ -194,7 +189,7 @@ Field( item="63D", name='RACE_BLACK', - friendly_name="race black", + friendly_name="Race/Ethnicity: Black or African American", type='number', startIndex=40, endIndex=41, @@ -204,7 +199,7 @@ Field( item="63E", name='RACE_HAWAIIAN', - friendly_name="race hawaiian", + friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", type='number', startIndex=41, endIndex=42, @@ -214,7 +209,7 @@ Field( item="63F", name='RACE_WHITE', - friendly_name="race white", + friendly_name="Race/Ethnicity: White", type='number', startIndex=42, endIndex=43, @@ -224,7 +219,7 @@ Field( item="64", name='GENDER', - friendly_name="gender", + friendly_name="Gender", type='number', startIndex=43, endIndex=44, @@ -234,7 +229,7 @@ Field( item="65A", name='RECEIVE_NONSSI_BENEFITS', - friendly_name="receive non-SSI benefits", + friendly_name="Receives Disability Benefits: Federal Disability Status", type='number', startIndex=44, endIndex=45, @@ -244,7 +239,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or Aged, Blind, and Disabled Under Title XVI-AABD", type='number', startIndex=45, endIndex=46, @@ -254,7 +249,7 @@ Field( item="66", name='RELATIONSHIP_HOH', - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type='number', startIndex=46, endIndex=48, @@ -264,7 +259,7 @@ Field( item="67", name='PARENT_MINOR_CHILD', - friendly_name="parent of minor child", + friendly_name="Parental Status of Minor", type='number', startIndex=48, endIndex=49, @@ -274,7 +269,7 @@ Field( item="68", name='EDUCATION_LEVEL', - friendly_name="education level", + friendly_name="Educational Level", type='number', startIndex=49, endIndex=51, @@ -289,7 +284,7 @@ Field( item="69", name='CITIZENSHIP_STATUS', - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type='number', startIndex=51, endIndex=52, @@ -299,7 +294,7 @@ Field( item="70A", name='UNEARNED_SSI', - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: SSI", type='number', startIndex=52, endIndex=56, @@ -309,7 +304,7 @@ Field( item="70B", name='OTHER_UNEARNED_INCOME', - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type='number', startIndex=56, endIndex=60, @@ -324,10 +319,6 @@ quiet_preparser_errors=True, preparsing_validators=[ validators.notEmpty(start=60, end=101), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -408,7 +399,7 @@ Field( item="0", name='RecordType', - friendly_name="record type", + friendly_name="Record Type", type='string', startIndex=0, endIndex=2, @@ -418,7 +409,7 @@ Field( item="3", name='RPT_MONTH_YEAR', - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type='number', startIndex=2, endIndex=8, @@ -431,7 +422,7 @@ Field( item="5", name='CASE_NUMBER', - friendly_name="case number", + friendly_name="Case Number", type='string', startIndex=8, endIndex=19, @@ -441,7 +432,7 @@ Field( item="60", name='FAMILY_AFFILIATION', - friendly_name="family affiliation", + friendly_name="Family Affiliation", type='number', startIndex=60, endIndex=61, @@ -451,22 +442,21 @@ Field( item="61", name='DATE_OF_BIRTH', - friendly_name="date of birth", + friendly_name="Date of Birth", type='string', startIndex=61, endIndex=69, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(), + ] ), TransformField( transform_func=ssp_ssn_decryption_func, item="62", name='SSN', - friendly_name="social security number", + friendly_name="Social Security Number", type='string', startIndex=69, endIndex=78, @@ -477,7 +467,7 @@ Field( item="63A", name='RACE_HISPANIC', - friendly_name="race hispanic", + friendly_name="Race/Ethnicity: Hispanic or Latino", type='number', startIndex=78, endIndex=79, @@ -487,7 +477,7 @@ Field( item="63B", name='RACE_AMER_INDIAN', - friendly_name="race american-indian", + friendly_name="Race/Ethnicity: American Indian or Alaska Native", type='number', startIndex=79, endIndex=80, @@ -497,7 +487,7 @@ Field( item="63C", name='RACE_ASIAN', - friendly_name="race asian", + friendly_name="Race/Ethnicity: Asian", type='number', startIndex=80, endIndex=81, @@ -507,7 +497,7 @@ Field( item="63D", name='RACE_BLACK', - friendly_name="race black", + friendly_name="Race/Ethnicity: Black or African American", type='number', startIndex=81, endIndex=82, @@ -517,7 +507,7 @@ Field( item="63E", name='RACE_HAWAIIAN', - friendly_name="race hawaiian", + friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", type='number', startIndex=82, endIndex=83, @@ -527,7 +517,7 @@ Field( item="63F", name='RACE_WHITE', - friendly_name="race white", + friendly_name="Race/Ethnicity: White", type='number', startIndex=83, endIndex=84, @@ -537,7 +527,7 @@ Field( item="64", name='GENDER', - friendly_name="gender", + friendly_name="Gender", type='number', startIndex=84, endIndex=85, @@ -547,7 +537,7 @@ Field( item="65A", name='RECEIVE_NONSSI_BENEFITS', - friendly_name="receives non-SSI benefit", + friendly_name="Receives Disability Benefits: Federal Disability Status", type='number', startIndex=85, endIndex=86, @@ -557,7 +547,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="receives ssi", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or Aged, Blind, and Disabled Under Title XVI-AABD", type='number', startIndex=86, endIndex=87, @@ -567,7 +557,7 @@ Field( item="66", name='RELATIONSHIP_HOH', - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type='number', startIndex=87, endIndex=89, @@ -577,7 +567,7 @@ Field( item="67", name='PARENT_MINOR_CHILD', - friendly_name="parent of minor child", + friendly_name="Parental Status of Minor", type='number', startIndex=89, endIndex=90, @@ -587,7 +577,7 @@ Field( item="68", name='EDUCATION_LEVEL', - friendly_name="education level", + friendly_name="Educational Level", type='number', startIndex=90, endIndex=92, @@ -602,7 +592,7 @@ Field( item="69", name='CITIZENSHIP_STATUS', - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type='number', startIndex=92, endIndex=93, @@ -612,7 +602,7 @@ Field( item="70A", name='UNEARNED_SSI', - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: SSI", type='number', startIndex=93, endIndex=97, @@ -622,7 +612,7 @@ Field( item="70B", name='OTHER_UNEARNED_INCOME', - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type='number', startIndex=97, endIndex=101, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 5dabf960d..89f916984 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -12,10 +12,6 @@ document=SSP_M4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(66), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[], @@ -23,7 +19,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -33,7 +29,7 @@ Field( item="3", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -46,7 +42,7 @@ Field( item="5", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -56,7 +52,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="county fips code", + friendly_name="County FIPS Code", type="string", startIndex=19, endIndex=22, @@ -66,7 +62,7 @@ Field( item="4", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=22, endIndex=24, @@ -76,7 +72,7 @@ Field( item="6", name="ZIP_CODE", - friendly_name="zip code", + friendly_name="ZIP Code", type="string", startIndex=24, endIndex=29, @@ -86,7 +82,7 @@ Field( item="7", name="DISPOSITION", - friendly_name="disposition", + friendly_name="Disposition", type="number", startIndex=29, endIndex=30, @@ -96,7 +92,7 @@ Field( item="8", name="CLOSURE_REASON", - friendly_name="closure reason", + friendly_name="Reason for Closure", type="string", startIndex=30, endIndex=32, @@ -111,7 +107,7 @@ Field( item="9", name="REC_SUB_HOUSING", - friendly_name="receives subsidized housing", + friendly_name="Received Subsidized Housing", type="number", startIndex=32, endIndex=33, @@ -121,7 +117,7 @@ Field( item="10`", name="REC_MED_ASSIST", - friendly_name="receives medical assistance", + friendly_name="Received Medical Assistance", type="number", startIndex=33, endIndex=34, @@ -131,7 +127,7 @@ Field( item="11", name="REC_FOOD_STAMPS", - friendly_name="receives food stamps", + friendly_name="Received Assistance from the Supplemental Nutrition Assistance Program (SNAP)", type="number", startIndex=34, endIndex=35, @@ -141,7 +137,7 @@ Field( item="12", name="REC_SUB_CC", - friendly_name="receives subsidized child care", + friendly_name="Received Subsidized Child Care", type="number", startIndex=35, endIndex=36, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 578eebefe..35f5f0121 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -14,10 +14,6 @@ document=SSP_M5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(66), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -108,7 +104,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -118,7 +114,7 @@ Field( item="3", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -131,7 +127,7 @@ Field( item="5", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -141,7 +137,7 @@ Field( item="13", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -151,22 +147,21 @@ Field( item="14", name="DATE_OF_BIRTH", - friendly_name="date of birth", + friendly_name="Date of Birth", type="string", startIndex=20, endIndex=28, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ], + validators=[ + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=ssp_ssn_decryption_func, item="15", name="SSN", - friendly_name="social security number", + friendly_name="Social Security Number", type="string", startIndex=28, endIndex=37, @@ -177,7 +172,7 @@ Field( item="16A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Race/Ethnicity: Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -187,7 +182,7 @@ Field( item="16B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Race/Ethnicity: American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -197,7 +192,7 @@ Field( item="16C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Race/Ethnicity: Asian", type="number", startIndex=39, endIndex=40, @@ -207,7 +202,7 @@ Field( item="16D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Race/Ethnicity: Black or African American", type="number", startIndex=40, endIndex=41, @@ -217,7 +212,7 @@ Field( item="16E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -227,7 +222,7 @@ Field( item="16F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Race/Ethnicity: White", type="number", startIndex=42, endIndex=43, @@ -237,7 +232,7 @@ Field( item="17", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=43, endIndex=44, @@ -247,7 +242,7 @@ Field( item="18A", name="REC_OASDI_INSURANCE", - friendly_name="receives old-age survivors and disability insurance", + friendly_name="Received Disability Benefits: OASDI Program", type="number", startIndex=44, endIndex=45, @@ -257,7 +252,7 @@ Field( item="18B", name="REC_FEDERAL_DISABILITY", - friendly_name="receives federal disability", + friendly_name="Received Disability Benefits: Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -267,7 +262,7 @@ Field( item="18C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="receives aid for totally disabled", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type="number", startIndex=46, endIndex=47, @@ -277,7 +272,7 @@ Field( item="18D", name="REC_AID_AGED_BLIND", - friendly_name="receives from aid to the aged, blind, and disabled program", + friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", type="number", startIndex=47, endIndex=48, @@ -287,7 +282,7 @@ Field( item="18E", name="REC_SSI", - friendly_name="receives SSI", + friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=48, endIndex=49, @@ -297,7 +292,7 @@ Field( item="19", name="MARITAL_STATUS", - friendly_name="marital status", + friendly_name="Marital Status", type="number", startIndex=49, endIndex=50, @@ -307,7 +302,7 @@ Field( item="20", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=50, endIndex=52, @@ -317,7 +312,7 @@ Field( item="21", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent with Minor Child in the Family", type="number", startIndex=52, endIndex=53, @@ -327,7 +322,7 @@ Field( item="22", name="NEEDS_OF_PREGNANT_WOMAN", - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type="number", startIndex=53, endIndex=54, @@ -337,7 +332,7 @@ Field( item="23", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=54, endIndex=56, @@ -352,7 +347,7 @@ Field( item="24", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type="number", startIndex=56, endIndex=57, @@ -367,7 +362,7 @@ Field( item="25", name="EMPLOYMENT_STATUS", - friendly_name="employment status", + friendly_name="Employment Status", type="number", startIndex=57, endIndex=58, @@ -377,7 +372,7 @@ Field( item="26", name="AMOUNT_EARNED_INCOME", - friendly_name="amount of earned income", + friendly_name="Amount of Earned Income", type="string", startIndex=58, endIndex=62, @@ -387,7 +382,7 @@ Field( item="27", name="AMOUNT_UNEARNED_INCOME", - friendly_name="amount of unearned income", + friendly_name="Amount of Earned Income", type="string", startIndex=62, endIndex=66, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index ceea6735b..f7b686caa 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -11,7 +11,6 @@ document=SSP_M6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(259), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -32,7 +31,7 @@ Field( item="0", name='RecordType', - friendly_name='record type', + friendly_name='Record Type', type='string', startIndex=0, endIndex=2, @@ -42,7 +41,7 @@ Field( item="2", name='CALENDAR_QUARTER', - friendly_name='calendar quarter', + friendly_name='Calendar Quarter', type='number', startIndex=2, endIndex=7, @@ -56,7 +55,7 @@ calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', - friendly_name='reporting month and year', + friendly_name='Reporting Year and Month', type='number', startIndex=2, endIndex=7, @@ -69,7 +68,7 @@ Field( item="3A", name='SSPMOE_FAMILIES', - friendly_name='ssp/moe families', + friendly_name='SSP-MOE Families', type='number', startIndex=7, endIndex=15, @@ -79,7 +78,7 @@ Field( item="4A", name='NUM_2_PARENTS', - friendly_name='number of two-parent families', + friendly_name='SSP-MOE Two-Parent Families', type='number', startIndex=31, endIndex=39, @@ -89,7 +88,7 @@ Field( item="5A", name='NUM_1_PARENTS', - friendly_name='number of one-parent families', + friendly_name='SSP-MOE One-Parent Families', type='number', startIndex=55, endIndex=63, @@ -99,7 +98,7 @@ Field( item="6A", name='NUM_NO_PARENTS', - friendly_name='number of no-parent families', + friendly_name='SSP-MOE No-Parent Families', type='number', startIndex=79, endIndex=87, @@ -109,7 +108,7 @@ Field( item="7A", name='NUM_RECIPIENTS', - friendly_name='number of recipients', + friendly_name='SSP-MOE Recipient', type='number', startIndex=103, endIndex=111, @@ -119,7 +118,7 @@ Field( item="8A", name='ADULT_RECIPIENTS', - friendly_name='number of adult recipients', + friendly_name='SSP-MOE Adult Recipients', type='number', startIndex=127, endIndex=135, @@ -129,7 +128,7 @@ Field( item="9A", name='CHILD_RECIPIENTS', - friendly_name='number of child recipients', + friendly_name='SSP-MOE Child Recipients', type='number', startIndex=151, endIndex=159, @@ -139,7 +138,7 @@ Field( item="10A", name='NONCUSTODIALS', - friendly_name='number of noncustodial parents', + friendly_name='Total Number of Noncustodial Parents Participating in Work Activities', type='number', startIndex=175, endIndex=183, @@ -149,7 +148,7 @@ Field( item="11A", name='AMT_ASSISTANCE', - friendly_name='amount of assistance', + friendly_name='SSP-MOE Amount of Assistance', type='number', startIndex=199, endIndex=211, @@ -159,7 +158,7 @@ Field( item="12A", name='CLOSED_CASES', - friendly_name='number of closed cases', + friendly_name='SSP-MOE Number of Closed Cases', type='number', startIndex=235, endIndex=243, @@ -171,10 +170,8 @@ s2 = RowSchema( document=SSP_M6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(259), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -195,7 +192,7 @@ Field( item="0", name='RecordType', - friendly_name='record type', + friendly_name='Record Type', type='string', startIndex=0, endIndex=2, @@ -205,7 +202,7 @@ Field( item="2", name='CALENDAR_QUARTER', - friendly_name='calendar quarter', + friendly_name='Calendar Quarter', type='number', startIndex=2, endIndex=7, @@ -219,7 +216,7 @@ calendar_quarter_to_rpt_month_year(1), item="2B", name='RPT_MONTH_YEAR', - friendly_name='reporting month and year', + friendly_name='Reporting Year and Month', type='number', startIndex=2, endIndex=7, @@ -232,7 +229,7 @@ Field( item="3B", name='SSPMOE_FAMILIES', - friendly_name='ssp/moe families', + friendly_name='SSP-MOE Families', type='number', startIndex=15, endIndex=23, @@ -242,7 +239,7 @@ Field( item="4B", name='NUM_2_PARENTS', - friendly_name='number of two-parent families', + friendly_name='SSP-MOE Two-Parent Families', type='number', startIndex=39, endIndex=47, @@ -252,7 +249,7 @@ Field( item="5B", name='NUM_1_PARENTS', - friendly_name='number of one-parent families', + friendly_name='SSP-MOE One-Parent Families', type='number', startIndex=63, endIndex=71, @@ -262,7 +259,7 @@ Field( item="6B", name='NUM_NO_PARENTS', - friendly_name='number of no-parent families', + friendly_name='SSP-MOE No-Parent Families', type='number', startIndex=87, endIndex=95, @@ -272,7 +269,7 @@ Field( item="7B", name='NUM_RECIPIENTS', - friendly_name='number of recipients', + friendly_name='SSP-MOERecipients', type='number', startIndex=111, endIndex=119, @@ -282,7 +279,7 @@ Field( item="8B", name='ADULT_RECIPIENTS', - friendly_name='number of adult recipients', + friendly_name='SSP-MOE Adult Recipients', type='number', startIndex=135, endIndex=143, @@ -292,7 +289,7 @@ Field( item="9B", name='CHILD_RECIPIENTS', - friendly_name='number of child recipients', + friendly_name='SSP-MOE Child Recipients', type='number', startIndex=159, endIndex=167, @@ -302,7 +299,7 @@ Field( item="10B", name='NONCUSTODIALS', - friendly_name='number of noncustodial parents', + friendly_name='SSP-MOE Noncustodial Parents Participating in Work Activities', type='number', startIndex=183, endIndex=191, @@ -312,7 +309,7 @@ Field( item="11B", name='AMT_ASSISTANCE', - friendly_name='amount of assistance', + friendly_name='SSP-MOE Amount of Assistance', type='number', startIndex=211, endIndex=223, @@ -322,7 +319,7 @@ Field( item="12B", name='CLOSED_CASES', - friendly_name='number of closed cases', + friendly_name='SSP-MOE Number of Closed Cases', type='number', startIndex=243, endIndex=251, @@ -334,10 +331,8 @@ s3 = RowSchema( document=SSP_M6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(259), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -358,7 +353,7 @@ Field( item="0", name='RecordType', - friendly_name='record type', + friendly_name='Record Type', type='string', startIndex=0, endIndex=2, @@ -368,7 +363,7 @@ Field( item="2", name='CALENDAR_QUARTER', - friendly_name='calendar quarter', + friendly_name='Calendar Quarter', type='number', startIndex=2, endIndex=7, @@ -382,7 +377,7 @@ calendar_quarter_to_rpt_month_year(2), item="2B", name='RPT_MONTH_YEAR', - friendly_name='reporting month and year', + friendly_name='Reporting Year and Month', type='number', startIndex=2, endIndex=7, @@ -395,7 +390,7 @@ Field( item="3C", name='SSPMOE_FAMILIES', - friendly_name='ssp/moe families', + friendly_name='SSP-MOE Families', type='number', startIndex=23, endIndex=31, @@ -405,7 +400,7 @@ Field( item="4C", name='NUM_2_PARENTS', - friendly_name='number of two-parent families', + friendly_name='SSP-MOE Two-Parent Families', type='number', startIndex=47, endIndex=55, @@ -415,7 +410,7 @@ Field( item="5C", name='NUM_1_PARENTS', - friendly_name='number of one-parent families', + friendly_name='SSP-MOE One-Parent Families', type='number', startIndex=71, endIndex=79, @@ -425,7 +420,7 @@ Field( item="6C", name='NUM_NO_PARENTS', - friendly_name='number of no-parent families', + friendly_name='SSP-MOE No-Parent Families', type='number', startIndex=95, endIndex=103, @@ -435,7 +430,7 @@ Field( item="7C", name='NUM_RECIPIENTS', - friendly_name='number of recipients', + friendly_name='SSP-MOE Recipients', type='number', startIndex=119, endIndex=127, @@ -445,7 +440,7 @@ Field( item="8C", name='ADULT_RECIPIENTS', - friendly_name='number of adult recipients', + friendly_name='SSP-MOE Adult Recipients', type='number', startIndex=143, endIndex=151, @@ -455,7 +450,7 @@ Field( item="9C", name='CHILD_RECIPIENTS', - friendly_name='number of child recipients', + friendly_name='SSP-MOE Child Recipients', type='number', startIndex=167, endIndex=175, @@ -465,7 +460,7 @@ Field( item="10C", name='NONCUSTODIALS', - friendly_name='number of noncustodial parents', + friendly_name='SSP-MOE Noncustodial Parents Participating in Work Activities', type='number', startIndex=191, endIndex=199, @@ -475,7 +470,7 @@ Field( item="11C", name='AMT_ASSISTANCE', - friendly_name='amount of assistance', + friendly_name='SSP-MOE Amount of Assistance', type='number', startIndex=223, endIndex=235, @@ -485,7 +480,7 @@ Field( item="12C", name='CLOSED_CASES', - friendly_name='number of closed cases', + friendly_name='SSP-MOE Number of Closed Cases', type='number', startIndex=251, endIndex=259, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 2edb161b5..b2d31e994 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -25,14 +25,13 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -42,7 +41,7 @@ Field( item="2", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -56,7 +55,7 @@ transform_func=calendar_quarter_to_rpt_month_year((i - 1) % 3), item="2A", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -69,7 +68,7 @@ Field( item="3", name="TDRS_SECTION_IND", - friendly_name="tdrs section indicator", + friendly_name="TDRS Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -79,7 +78,7 @@ Field( item="4", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=stratum_index, endIndex=stratum_index + 2, @@ -89,7 +88,7 @@ Field( item=families_item_numbers[i - 1], name="FAMILIES_MONTH", - friendly_name="families month", + friendly_name="Families Month", type="number", startIndex=families_index, endIndex=families_index + 7, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 8662b19e6..946e3e692 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -12,10 +12,6 @@ document=TANF_T1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(156), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -116,6 +112,8 @@ "CASH_AMOUNT", "CC_AMOUNT", "TRANSP_AMOUNT", + "TRANSITION_SERVICES_AMOUNT", + "OTHER_AMOUNT", ), 0, ), @@ -134,7 +132,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -147,7 +145,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -157,7 +155,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="county fips code", + friendly_name="County FIPS Code", type="string", startIndex=19, endIndex=22, @@ -169,7 +167,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=22, endIndex=24, @@ -181,7 +179,7 @@ Field( item="7", name="ZIP_CODE", - friendly_name="zip code", + friendly_name="ZIP Code", type="string", startIndex=24, endIndex=29, @@ -193,31 +191,31 @@ Field( item="8", name="FUNDING_STREAM", - friendly_name="funding stream", + friendly_name="Funding Stream", type="number", startIndex=29, endIndex=30, required=True, validators=[ - validators.isInLimits(1, 2), + validators.isInLimits(1, 3), ], ), Field( item="9", name="DISPOSITION", - friendly_name="disposition", + friendly_name="Disposition", type="number", startIndex=30, endIndex=31, required=True, validators=[ - validators.matches(1) + validators.oneOf([1, 2]), ], ), Field( item="10", name="NEW_APPLICANT", - friendly_name="new applicant", + friendly_name="Newly-Approved Applicant", type="number", startIndex=31, endIndex=32, @@ -229,7 +227,7 @@ Field( item="11", name="NBR_FAMILY_MEMBERS", - friendly_name="number of family members", + friendly_name="Number of Family Members", type="number", startIndex=32, endIndex=34, @@ -241,7 +239,7 @@ Field( item="12", name="FAMILY_TYPE", - friendly_name="family type", + friendly_name="Type of Family for Work Participation", type="number", startIndex=34, endIndex=35, @@ -253,19 +251,19 @@ Field( item="13", name="RECEIVES_SUB_HOUSING", - friendly_name="receives subsidized housing", + friendly_name="Receives Subsidized Housing", type="number", startIndex=35, endIndex=36, required=True, validators=[ - validators.isInLimits(1, 2), + validators.isInLimits(1, 3), ], ), Field( item="14", name="RECEIVES_MED_ASSISTANCE", - friendly_name="receives medical assistance", + friendly_name="Receives Medical Assistance", type="number", startIndex=36, endIndex=37, @@ -277,7 +275,7 @@ Field( item="15", name="RECEIVES_FOOD_STAMPS", - friendly_name="receives food stamps", + friendly_name="Receives Assistance from the Supplemental Nutrition Assistance Program (SNAP)", type="number", startIndex=37, endIndex=38, @@ -289,7 +287,7 @@ Field( item="16", name="AMT_FOOD_STAMP_ASSISTANCE", - friendly_name="amount of food stamp assistance", + friendly_name="Amount of Supplemental Nutrition Assistance Program (SNAP) Benefits", type="number", startIndex=38, endIndex=42, @@ -301,7 +299,7 @@ Field( item="17", name="RECEIVES_SUB_CC", - friendly_name="receives subsidized child care", + friendly_name="Received Subsidized Child Care", type="number", startIndex=42, endIndex=43, @@ -313,7 +311,7 @@ Field( item="18", name="AMT_SUB_CC", - friendly_name="amount of subsidized child care", + friendly_name="Amount of Subsidized Child Care", type="number", startIndex=43, endIndex=47, @@ -325,7 +323,7 @@ Field( item="19", name="CHILD_SUPPORT_AMT", - friendly_name="child support amount", + friendly_name="Amount of Child Support", type="number", startIndex=47, endIndex=51, @@ -337,7 +335,7 @@ Field( item="20", name="FAMILY_CASH_RESOURCES", - friendly_name="family cash resources", + friendly_name="Amount of the Family's Cash Resources", type="number", startIndex=51, endIndex=55, @@ -349,7 +347,7 @@ Field( item="21A", name="CASH_AMOUNT", - friendly_name="cash amount", + friendly_name="Cash and Cash Equivalents: Amount", type="number", startIndex=55, endIndex=59, @@ -361,7 +359,7 @@ Field( item="21B", name="NBR_MONTHS", - friendly_name="number of months", + friendly_name="Cash and Cash Equivalents: Number of Months", type="number", startIndex=59, endIndex=62, @@ -373,7 +371,7 @@ Field( item="22A", name="CC_AMOUNT", - friendly_name="child care amount", + friendly_name="TANF Child Care: Amount", type="number", startIndex=62, endIndex=66, @@ -385,7 +383,7 @@ Field( item="22B", name="CHILDREN_COVERED", - friendly_name="children covered", + friendly_name="TANF Child Care: Number of Children Covered", type="number", startIndex=66, endIndex=68, @@ -397,7 +395,7 @@ Field( item="22C", name="CC_NBR_MONTHS", - friendly_name="child care - number of months", + friendly_name="TANF Child Care: Number of Months", type="number", startIndex=68, endIndex=71, @@ -409,7 +407,7 @@ Field( item="23A", name="TRANSP_AMOUNT", - friendly_name="transportation amount", + friendly_name="Transportation and Other Supportive Services: Amount", type="number", startIndex=71, endIndex=75, @@ -421,7 +419,7 @@ Field( item="23B", name="TRANSP_NBR_MONTHS", - friendly_name="transportation - number of months", + friendly_name="Transportation and Other Supportive Services: Number of Months", type="number", startIndex=75, endIndex=78, @@ -433,7 +431,7 @@ Field( item="24A", name="TRANSITION_SERVICES_AMOUNT", - friendly_name="transition services amount", + friendly_name="Transitional Services: Amount", type="number", startIndex=78, endIndex=82, @@ -445,7 +443,7 @@ Field( item="24B", name="TRANSITION_NBR_MONTHS", - friendly_name="transition services - number of months", + friendly_name="Transitional Services: Number of Months", type="number", startIndex=82, endIndex=85, @@ -457,7 +455,7 @@ Field( item="25A", name="OTHER_AMOUNT", - friendly_name="other amount", + friendly_name="Other: Amount", type="number", startIndex=85, endIndex=89, @@ -469,7 +467,7 @@ Field( item="25B", name="OTHER_NBR_MONTHS", - friendly_name="other - number of months", + friendly_name="Other: Number of Months", type="number", startIndex=89, endIndex=92, @@ -481,7 +479,7 @@ Field( item="26AI", name="SANC_REDUCTION_AMT", - friendly_name="sanction reduction amount", + friendly_name="Total Dollar Amount of Reductions due to Sanctions", type="number", startIndex=92, endIndex=96, @@ -493,7 +491,7 @@ Field( item="26AII", name="WORK_REQ_SANCTION", - friendly_name="work requirement sanction", + friendly_name="Work Requirements Sanction", type="number", startIndex=96, endIndex=97, @@ -505,7 +503,7 @@ Field( item="26AIII", name="FAMILY_SANC_ADULT", - friendly_name="family sanction adult", + friendly_name="Sanctions: Code no longer in use", type="number", startIndex=97, endIndex=98, @@ -517,7 +515,7 @@ Field( item="26AIV", name="SANC_TEEN_PARENT", - friendly_name="sanctioned teen parent", + friendly_name="Sanction: Teen Parent not Attending School (or Alternative Training)", type="number", startIndex=98, endIndex=99, @@ -529,7 +527,7 @@ Field( item="26AV", name="NON_COOPERATION_CSE", - friendly_name="non-cooperation with child support", + friendly_name="Sanction: Non-Cooperation with Child Support", type="number", startIndex=99, endIndex=100, @@ -541,7 +539,7 @@ Field( item="26AVI", name="FAILURE_TO_COMPLY", - friendly_name="failure to comply", + friendly_name="Failure to Comply with an Individual Responsibility Plan", type="number", startIndex=100, endIndex=101, @@ -553,7 +551,7 @@ Field( item="26AVII", name="OTHER_SANCTION", - friendly_name="other sanction", + friendly_name="Other Sanction", type="number", startIndex=101, endIndex=102, @@ -565,7 +563,7 @@ Field( item="26B", name="RECOUPMENT_PRIOR_OVRPMT", - friendly_name="recoupment prior overpayment", + friendly_name="Recoupment of Prior Overpayment", type="number", startIndex=102, endIndex=106, @@ -577,7 +575,7 @@ Field( item="26CI", name="OTHER_TOTAL_REDUCTIONS", - friendly_name="other total reductions", + friendly_name="Other: Total Dollar Amount of Reductions", type="number", startIndex=106, endIndex=110, @@ -589,7 +587,7 @@ Field( item="26CII", name="FAMILY_CAP", - friendly_name="family cap", + friendly_name="Other: Family Cap", type="number", startIndex=110, endIndex=111, @@ -601,7 +599,7 @@ Field( item="26CIII", name="REDUCTIONS_ON_RECEIPTS", - friendly_name="reductions on receipts", + friendly_name="Other: Reductions Based on Time Limit", type="number", startIndex=111, endIndex=112, @@ -613,7 +611,7 @@ Field( item="26CIV", name="OTHER_NON_SANCTION", - friendly_name="other non-sanction", + friendly_name="Other: Non-Sanction, Non-Recoupment", type="number", startIndex=112, endIndex=113, @@ -625,7 +623,7 @@ Field( item="27", name="WAIVER_EVAL_CONTROL_GRPS", - friendly_name="waiver evaluation control groups", + friendly_name="Waiver Evaluation Experimental and Control Groups", type="string", startIndex=113, endIndex=114, @@ -641,7 +639,7 @@ Field( item="28", name="FAMILY_EXEMPT_TIME_LIMITS", - friendly_name="family exempt time limits", + friendly_name="Federal Time Limit Exemptions", type="number", startIndex=114, endIndex=116, @@ -651,7 +649,7 @@ Field( item="29", name="FAMILY_NEW_CHILD", - friendly_name="family new child", + friendly_name="TANF Family a New Child-Only Family", type="number", startIndex=116, endIndex=117, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index a20ff3a2b..36fba6621 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -14,10 +14,6 @@ document=TANF_T2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(156), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -126,7 +122,6 @@ result_field="WORK_PART_STATUS", result_function=validators.notMatches("99"), ), - validators.validate__WORK_ELIGIBLE_INDICATOR__HOH__AGE(), ], fields=[ Field( @@ -142,7 +137,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -155,7 +150,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -165,7 +160,7 @@ Field( item="30", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -175,7 +170,7 @@ Field( item="31", name="NONCUSTODIAL_PARENT", - friendly_name="noncustodial parent", + friendly_name="Noncustodial Parent Indicator", type="number", startIndex=20, endIndex=21, @@ -185,22 +180,21 @@ Field( item="32", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=21, endIndex=29, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="33", name="SSN", - friendly_name="social security number", + friendly_name="Social Security Number", type="string", startIndex=29, endIndex=38, @@ -211,7 +205,7 @@ Field( item="34A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Race/Ethnicity: Hispanic or Latino", type="number", startIndex=38, endIndex=39, @@ -221,7 +215,7 @@ Field( item="34B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Race/Ethnicity: American Indian or Alaska Native", type="number", startIndex=39, endIndex=40, @@ -231,7 +225,7 @@ Field( item="34C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Race/Ethnicity: Asian", type="number", startIndex=40, endIndex=41, @@ -241,7 +235,7 @@ Field( item="34D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Race/Ethnicity: Black or African American", type="number", startIndex=41, endIndex=42, @@ -251,7 +245,7 @@ Field( item="34E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", type="number", startIndex=42, endIndex=43, @@ -261,7 +255,7 @@ Field( item="34F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Race/Ethnicity: White", type="number", startIndex=43, endIndex=44, @@ -271,7 +265,7 @@ Field( item="35", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=44, endIndex=45, @@ -283,7 +277,7 @@ Field( item="36A", name="FED_OASDI_PROGRAM", - friendly_name="federal old age survivors and disability insurance program", + friendly_name="Receives Disability Benefits: OASDI Program", type="number", startIndex=45, endIndex=46, @@ -293,7 +287,7 @@ Field( item="36B", name="FED_DISABILITY_STATUS", - friendly_name="federal disability status", + friendly_name="Receives Disability Benefits: Federal Disability Status", type="number", startIndex=46, endIndex=47, @@ -303,7 +297,7 @@ Field( item="36C", name="DISABLED_TITLE_XIVAPDT", - friendly_name="received aid under Title XIV-APDT", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type="string", startIndex=47, endIndex=48, @@ -318,7 +312,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="receives from the aid to the aged, blind, and disabled program", + friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under Title XVI-AABD ", type="number", startIndex=48, endIndex=49, @@ -330,7 +324,7 @@ Field( item="36E", name="RECEIVE_SSI", - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=49, endIndex=50, @@ -342,7 +336,7 @@ Field( item="37", name="MARITAL_STATUS", - friendly_name="marital status", + friendly_name="Marital Status", type="number", startIndex=50, endIndex=51, @@ -354,7 +348,7 @@ Field( item="38", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=51, endIndex=53, @@ -366,7 +360,7 @@ Field( item="39", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent with Minor Child in the Family", type="number", startIndex=53, endIndex=54, @@ -378,7 +372,7 @@ Field( item="40", name="NEEDS_PREGNANT_WOMAN", - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type="number", startIndex=54, endIndex=55, @@ -390,7 +384,7 @@ Field( item="41", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=55, endIndex=57, @@ -405,7 +399,7 @@ Field( item="42", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type="number", startIndex=57, endIndex=58, @@ -415,7 +409,7 @@ Field( item="43", name="COOPERATION_CHILD_SUPPORT", - friendly_name="cooperation with child support", + friendly_name="Cooperated with Child Support", type="number", startIndex=58, endIndex=59, @@ -427,7 +421,7 @@ Field( item="44", name="MONTHS_FED_TIME_LIMIT", - friendly_name="countable months toward federal time limit", + friendly_name="Number of Months Countable toward Federal Time Limit", type="string", startIndex=59, endIndex=62, @@ -439,7 +433,7 @@ Field( item="45", name="MONTHS_STATE_TIME_LIMIT", - friendly_name="months of state time limit", + friendly_name="Number of Countable Months Remaining Under State's Time Limit", type="string", startIndex=62, endIndex=64, @@ -451,7 +445,7 @@ Field( item="46", name="CURRENT_MONTH_STATE_EXEMPT", - friendly_name="current month state exempt", + friendly_name="Current Month Exempt from the State's Time Limit", type="number", startIndex=64, endIndex=65, @@ -463,7 +457,7 @@ Field( item="47", name="EMPLOYMENT_STATUS", - friendly_name="employment status", + friendly_name="Employment Status", type="number", startIndex=65, endIndex=66, @@ -475,7 +469,7 @@ Field( item="48", name="WORK_ELIGIBLE_INDICATOR", - friendly_name="work eligible indicator", + friendly_name="Work-Eligible Individual Indicator", type="string", startIndex=66, endIndex=68, @@ -490,11 +484,11 @@ Field( item="49", name="WORK_PART_STATUS", - friendly_name="work participation status", + friendly_name="Work Participation Status", type="string", startIndex=68, endIndex=70, - required=True, + required=False, validators=[ validators.oneOf( [ @@ -507,6 +501,7 @@ "16", "17", "18", + "19", "99", ] ) @@ -515,7 +510,7 @@ Field( item="50", name="UNSUB_EMPLOYMENT", - friendly_name="unsubsidized employment", + friendly_name="Unsubsidized Employment", type="string", startIndex=70, endIndex=72, @@ -527,7 +522,7 @@ Field( item="51", name="SUB_PRIVATE_EMPLOYMENT", - friendly_name="subsidized private employment", + friendly_name="Subsidized Private-Sector Employment", type="string", startIndex=72, endIndex=74, @@ -539,7 +534,7 @@ Field( item="52", name="SUB_PUBLIC_EMPLOYMENT", - friendly_name="subsidized public employment", + friendly_name="Subsidized Public-Sector Employment ", type="string", startIndex=74, endIndex=76, @@ -551,7 +546,7 @@ Field( item="53A", name="WORK_EXPERIENCE_HOP", - friendly_name="work experience - hours of participation", + friendly_name="Work Experience: Hours of Participation", type="string", startIndex=76, endIndex=78, @@ -563,7 +558,7 @@ Field( item="53B", name="WORK_EXPERIENCE_EA", - friendly_name="work experience - excused absence", + friendly_name="Work Experience: Hours of Excused Absences", type="string", startIndex=78, endIndex=80, @@ -575,7 +570,7 @@ Field( item="53C", name="WORK_EXPERIENCE_HOL", - friendly_name="work experience - holiday", + friendly_name="Work Experience: Hours of Holidays", type="string", startIndex=80, endIndex=82, @@ -587,7 +582,7 @@ Field( item="54", name="OJT", - friendly_name="on the job training", + friendly_name="On-the-job Training", type="string", startIndex=82, endIndex=84, @@ -599,7 +594,7 @@ Field( item="55A", name="JOB_SEARCH_HOP", - friendly_name="job search - hours of participation", + friendly_name="Job Search and Job Readiness Assistance: Hours of Participation", type="string", startIndex=84, endIndex=86, @@ -611,7 +606,7 @@ Field( item="55B", name="JOB_SEARCH_EA", - friendly_name="job search - excused absence", + friendly_name="Job Search and Job Readiness Assistance: Hours of Excused Absences", type="string", startIndex=86, endIndex=88, @@ -623,7 +618,7 @@ Field( item="55C", name="JOB_SEARCH_HOL", - friendly_name="job search - holidays", + friendly_name="Job Search and Job Readiness Assistance: Hours of Holidays", type="string", startIndex=88, endIndex=90, @@ -635,7 +630,7 @@ Field( item="56A", name="COMM_SERVICES_HOP", - friendly_name="community service - hours of participation", + friendly_name="Community Service Programs: Hours of Participation", type="string", startIndex=90, endIndex=92, @@ -647,7 +642,7 @@ Field( item="56B", name="COMM_SERVICES_EA", - friendly_name="community service - excused absence", + friendly_name="Community Service Programs: Hours of Excused Absences", type="string", startIndex=92, endIndex=94, @@ -659,7 +654,7 @@ Field( item="56C", name="COMM_SERVICES_HOL", - friendly_name="community service - hours of leave", + friendly_name="Community Service Programs: Hours of Holidays", type="string", startIndex=94, endIndex=96, @@ -671,7 +666,7 @@ Field( item="57A", name="VOCATIONAL_ED_TRAINING_HOP", - friendly_name="vocational education training - hours of participation", + friendly_name="Vocational Educational Training: Hours of Participation", type="string", startIndex=96, endIndex=98, @@ -683,7 +678,7 @@ Field( item="57B", name="VOCATIONAL_ED_TRAINING_EA", - friendly_name="vocational education training - excused absence", + friendly_name="Vocational Educational Training: Hours of Excused Absences", type="string", startIndex=98, endIndex=100, @@ -695,7 +690,7 @@ Field( item="57C", name="VOCATIONAL_ED_TRAINING_HOL", - friendly_name="vocational education training - hours of leave", + friendly_name="Vocational Educational Training: Hours of Holidays", type="string", startIndex=100, endIndex=102, @@ -707,7 +702,7 @@ Field( item="58A", name="JOB_SKILLS_TRAINING_HOP", - friendly_name="job skills training - hours of participation", + friendly_name="Job Skills Training Directly Related to Employment: Hours of Participation", type="string", startIndex=102, endIndex=104, @@ -719,7 +714,7 @@ Field( item="58B", name="JOB_SKILLS_TRAINING_EA", - friendly_name="job skills training - excused absence", + friendly_name="Job Skills Training Directly Related to Employment: Hours of Excused Absences", type="string", startIndex=104, endIndex=106, @@ -731,7 +726,7 @@ Field( item="58C", name="JOB_SKILLS_TRAINING_HOL", - friendly_name="job skills training - hours of leave", + friendly_name="Job Skills Training Directly Related to Employment: Hours of Holidays", type="string", startIndex=106, endIndex=108, @@ -743,7 +738,7 @@ Field( item="59A", name="ED_NO_HIGH_SCHOOL_DIPL_HOP", - friendly_name="education no high school diploma - hours of participation", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Participation", type="string", startIndex=108, endIndex=110, @@ -755,7 +750,7 @@ Field( item="59B", name="ED_NO_HIGH_SCHOOL_DIPL_EA", - friendly_name="education no high school diploma - excused absence", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate: Hours of Excused Absences", type="string", startIndex=110, endIndex=112, @@ -767,7 +762,7 @@ Field( item="59C", name="ED_NO_HIGH_SCHOOL_DIPL_HOL", - friendly_name="education no high school diploma - holiday", + friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate: Hours of Holidays", type="string", startIndex=112, endIndex=114, @@ -779,7 +774,7 @@ Field( item="60A", name="SCHOOL_ATTENDENCE_HOP", - friendly_name="school attendance - hours of participation", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Participation", type="string", startIndex=114, endIndex=116, @@ -791,7 +786,7 @@ Field( item="60B", name="SCHOOL_ATTENDENCE_EA", - friendly_name="school attendance - excused absence", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Ecused Absences", type="string", startIndex=116, endIndex=118, @@ -803,7 +798,7 @@ Field( item="60C", name="SCHOOL_ATTENDENCE_HOL", - friendly_name="school attendance - holiday", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Holidays", type="string", startIndex=118, endIndex=120, @@ -815,7 +810,7 @@ Field( item="61A", name="PROVIDE_CC_HOP", - friendly_name="provide child care - hours of participation", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Participation", type="string", startIndex=120, endIndex=122, @@ -827,7 +822,7 @@ Field( item="61B", name="PROVIDE_CC_EA", - friendly_name="provide child care - excused absence", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Excused Absences", type="string", startIndex=122, endIndex=124, @@ -839,7 +834,7 @@ Field( item="61C", name="PROVIDE_CC_HOL", - friendly_name="provide child care - holiday", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Holidays", type="string", startIndex=124, endIndex=126, @@ -851,7 +846,7 @@ Field( item="62", name="OTHER_WORK_ACTIVITIES", - friendly_name="other work activities", + friendly_name="Hours of Other Work Activities ", type="string", startIndex=126, endIndex=128, @@ -863,7 +858,7 @@ Field( item="63", name="DEEMED_HOURS_FOR_OVERALL", - friendly_name="deemed hours for overall", + friendly_name="Number of Deemed Core Hours for Overall Rate", type="string", startIndex=128, endIndex=130, @@ -875,7 +870,7 @@ Field( item="64", name="DEEMED_HOURS_FOR_TWO_PARENT", - friendly_name="deemed hours for two parent", + friendly_name="Number of Deemed Core Hours for the Two-Parent Rate", type="string", startIndex=130, endIndex=132, @@ -887,7 +882,7 @@ Field( item="65", name="EARNED_INCOME", - friendly_name="earned income", + friendly_name="Amount of Earned Income", type="string", startIndex=132, endIndex=136, @@ -899,7 +894,7 @@ Field( item="66A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="unearned income tax credit", + friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", type="string", startIndex=136, endIndex=140, @@ -911,7 +906,7 @@ Field( item="66B", name="UNEARNED_SOCIAL_SECURITY", - friendly_name="unearned social security", + friendly_name="Amount of Unearned Income: Social Security", type="string", startIndex=140, endIndex=144, @@ -923,7 +918,7 @@ Field( item="66C", name="UNEARNED_SSI", - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=144, endIndex=148, @@ -935,7 +930,7 @@ Field( item="66D", name="UNEARNED_WORKERS_COMP", - friendly_name="unearned workers compensation", + friendly_name="Ammount of Unearned Income: Worker's Compensation", type="string", startIndex=148, endIndex=152, @@ -947,7 +942,7 @@ Field( item="66E", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=152, endIndex=156, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 8e22974f8..b2212d5d3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -12,10 +12,6 @@ document=TANF_T3DataSubmissionDocument(), preparsing_validators=[ validators.notEmpty(start=19, end=60), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -96,7 +92,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -106,7 +102,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -116,7 +112,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -126,7 +122,7 @@ Field( item="67", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -136,22 +132,21 @@ Field( item="68", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=20, endIndex=28, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1950), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="69", name="SSN", - friendly_name="social security number", + friendly_name="Social Security Number", type="string", startIndex=28, endIndex=37, @@ -162,7 +157,7 @@ Field( item="70A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -172,7 +167,7 @@ Field( item="70B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Native ", type="number", startIndex=38, endIndex=39, @@ -182,7 +177,7 @@ Field( item="70C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=39, endIndex=40, @@ -192,7 +187,7 @@ Field( item="70D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=40, endIndex=41, @@ -202,7 +197,7 @@ Field( item="70E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Ethnicity/Race: Native Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -212,7 +207,7 @@ Field( item="70F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Ethnicity/Race: White", type="number", startIndex=42, endIndex=43, @@ -222,7 +217,7 @@ Field( item="71", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=43, endIndex=44, @@ -232,7 +227,7 @@ Field( item="72A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="receives non-social security act benefits", + friendly_name="Receives Disability Benefits: Under Non-Social Securty Act", type="number", startIndex=44, endIndex=45, @@ -242,7 +237,7 @@ Field( item="72B", name="RECEIVE_SSI", - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: SSI or AABD", type="number", startIndex=45, endIndex=46, @@ -252,7 +247,7 @@ Field( item="73", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=46, endIndex=48, @@ -262,7 +257,7 @@ Field( item="74", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parental status of minor who is not a head-of-household or spouse of the head-of-household", type="number", startIndex=48, endIndex=49, @@ -272,7 +267,7 @@ Field( item="75", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=49, endIndex=51, @@ -287,7 +282,7 @@ Field( item="76", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type="number", startIndex=51, endIndex=52, @@ -297,7 +292,7 @@ Field( item="77A", name="UNEARNED_SSI", - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=52, endIndex=56, @@ -307,7 +302,7 @@ Field( item="77B", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=56, endIndex=60, @@ -322,10 +317,6 @@ quiet_preparser_errors=True, preparsing_validators=[ validators.notEmpty(start=60, end=101), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -406,7 +397,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Recod Type", type="string", startIndex=0, endIndex=2, @@ -416,7 +407,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -426,7 +417,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -436,7 +427,7 @@ Field( item="67", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=60, endIndex=61, @@ -446,22 +437,21 @@ Field( item="68", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=61, endIndex=69, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1950), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="69", name="SSN", - friendly_name="social security number", + friendly_name="Social Security Number", type="string", startIndex=69, endIndex=78, @@ -472,7 +462,7 @@ Field( item="70A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity: Hispanic or Latino", type="number", startIndex=78, endIndex=79, @@ -482,7 +472,7 @@ Field( item="70B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Race: American Indian or Alaska Native", type="number", startIndex=79, endIndex=80, @@ -492,7 +482,7 @@ Field( item="70C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Race: Asian", type="number", startIndex=80, endIndex=81, @@ -502,7 +492,7 @@ Field( item="70D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Race: Black or African American", type="number", startIndex=81, endIndex=82, @@ -512,7 +502,7 @@ Field( item="70E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Race: Native Hawaiian or Other Pacific Islander", type="number", startIndex=82, endIndex=83, @@ -522,7 +512,7 @@ Field( item="70F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Race: White", type="number", startIndex=83, endIndex=84, @@ -532,7 +522,7 @@ Field( item="71", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=84, endIndex=85, @@ -542,7 +532,7 @@ Field( item="72A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="receives non-ssa benefits", + friendly_name="Receives Disability Benefits: Federal Disability Status", type="number", startIndex=85, endIndex=86, @@ -552,7 +542,7 @@ Field( item="72B", name="RECEIVE_SSI", - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: SSI Under Title XVI-SSI or AABD Under Title XVI-AABD", type="number", startIndex=86, endIndex=87, @@ -562,7 +552,7 @@ Field( item="73", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=87, endIndex=89, @@ -572,7 +562,7 @@ Field( item="74", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parental status of minor who is not a head-of-household or spouse of the head-of-household", type="number", startIndex=89, endIndex=90, @@ -582,7 +572,7 @@ Field( item="75", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=90, endIndex=92, @@ -597,7 +587,7 @@ Field( item="76", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type="number", startIndex=92, endIndex=93, @@ -607,7 +597,7 @@ Field( item="77A", name="UNEARNED_SSI", - friendly_name="unearned SSI benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=93, endIndex=97, @@ -617,7 +607,7 @@ Field( item="77B", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=97, endIndex=101, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 6d9304c0c..88185ab84 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -13,10 +13,6 @@ document=TANF_T4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[], @@ -24,7 +20,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -34,7 +30,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -47,7 +43,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -57,7 +53,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="county fips code", + friendly_name="County FIPS Code", type="string", startIndex=19, endIndex=22, @@ -67,7 +63,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=22, endIndex=24, @@ -77,7 +73,7 @@ Field( item="7", name="ZIP_CODE", - friendly_name="zip code", + friendly_name="ZIP Code", type="string", startIndex=24, endIndex=29, @@ -87,7 +83,7 @@ Field( item="8", name="DISPOSITION", - friendly_name="disposition", + friendly_name="Disposition", type="number", startIndex=29, endIndex=30, @@ -97,7 +93,7 @@ Field( item="9", name="CLOSURE_REASON", - friendly_name="closure reason", + friendly_name="Reason for Closure", type="string", startIndex=30, endIndex=32, @@ -112,7 +108,7 @@ Field( item="10", name="REC_SUB_HOUSING", - friendly_name="receives subsidized housing", + friendly_name="Received Subsidized Housing", type="number", startIndex=32, endIndex=33, @@ -122,7 +118,7 @@ Field( item="11", name="REC_MED_ASSIST", - friendly_name="receives medical assistance", + friendly_name="Received Medical Assistance", type="number", startIndex=33, endIndex=34, @@ -132,7 +128,7 @@ Field( item="12", name="REC_FOOD_STAMPS", - friendly_name="receives food stamps", + friendly_name="Received Assistance from the Supplemental Nutrition Assistance Program (SNAP)", type="number", startIndex=34, endIndex=35, @@ -142,7 +138,7 @@ Field( item="13", name="REC_SUB_CC", - friendly_name="receives subsidized child care", + friendly_name="Received Subsidized Child Care", type="number", startIndex=35, endIndex=36, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index b3d456dd3..b87a3af6a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -14,10 +14,6 @@ document=TANF_T5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -109,7 +105,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -119,7 +115,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -132,7 +128,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number", type="string", startIndex=8, endIndex=19, @@ -142,7 +138,7 @@ Field( item="14", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -152,22 +148,21 @@ Field( item="15", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=20, endIndex=28, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="16", name="SSN", - friendly_name="social security number", + friendly_name="Social Security Number", type="string", startIndex=28, endIndex=37, @@ -178,7 +173,7 @@ Field( item="17A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hisapic or Latino", type="number", startIndex=37, endIndex=38, @@ -188,7 +183,7 @@ Field( item="17B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Native ", type="number", startIndex=38, endIndex=39, @@ -198,7 +193,7 @@ Field( item="17C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=39, endIndex=40, @@ -208,7 +203,7 @@ Field( item="17D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=40, endIndex=41, @@ -218,7 +213,7 @@ Field( item="17E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -228,7 +223,7 @@ Field( item="17F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Ethnicity/Race: White", type="number", startIndex=42, endIndex=43, @@ -238,7 +233,7 @@ Field( item="18", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=43, endIndex=44, @@ -248,7 +243,7 @@ Field( item="19A", name="REC_OASDI_INSURANCE", - friendly_name="receives old-age survivors and disability insurance", + friendly_name="Received Disability Benefits: OASDI Program", type="number", startIndex=44, endIndex=45, @@ -258,7 +253,7 @@ Field( item="19B", name="REC_FEDERAL_DISABILITY", - friendly_name="receives federal disability", + friendly_name="Received Disability Benefits: Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -268,7 +263,7 @@ Field( item="19C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="receives aid for totally disabled", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type="number", startIndex=46, endIndex=47, @@ -278,7 +273,7 @@ Field( item="19D", name="REC_AID_AGED_BLIND", - friendly_name="receives from the aid to the aged, blind, and disabled program", + friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", type="number", startIndex=47, endIndex=48, @@ -288,7 +283,7 @@ Field( item="19E", name="REC_SSI", - friendly_name="receives SSI", + friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=48, endIndex=49, @@ -298,7 +293,7 @@ Field( item="20", name="MARITAL_STATUS", - friendly_name="marital status", + friendly_name="Marital Status", type="number", startIndex=49, endIndex=50, @@ -308,7 +303,7 @@ Field( item="21", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=50, endIndex=52, @@ -318,7 +313,7 @@ Field( item="22", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent with Minor Child in the Family", type="number", startIndex=52, endIndex=53, @@ -328,7 +323,7 @@ Field( item="23", name="NEEDS_OF_PREGNANT_WOMAN", - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type="number", startIndex=53, endIndex=54, @@ -338,7 +333,7 @@ Field( item="24", name="EDUCATION_LEVEL", - friendly_name="educational level", + friendly_name="Educational Level", type="string", startIndex=54, endIndex=56, @@ -353,7 +348,7 @@ Field( item="25", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Immigration Status", type="number", startIndex=56, endIndex=57, @@ -368,7 +363,7 @@ Field( item="26", name="COUNTABLE_MONTH_FED_TIME", - friendly_name="countable months toward federal time", + friendly_name="Number of Months Countable Toward Federal Time Limit", type="string", startIndex=57, endIndex=60, @@ -378,7 +373,7 @@ Field( item="27", name="COUNTABLE_MONTHS_STATE_TRIBE", - friendly_name="countable months remaining under state tribe", + friendly_name="Number of Countable Months Remaining Under State's Time Limit", type="string", startIndex=60, endIndex=62, @@ -388,7 +383,7 @@ Field( item="28", name="EMPLOYMENT_STATUS", - friendly_name="employment status", + friendly_name="Employment Status", type="number", startIndex=62, endIndex=63, @@ -398,7 +393,7 @@ Field( item="29", name="AMOUNT_EARNED_INCOME", - friendly_name="amount of earned income", + friendly_name="Amount of Earned Income", type="string", startIndex=63, endIndex=67, @@ -408,7 +403,7 @@ Field( item="30", name="AMOUNT_UNEARNED_INCOME", - friendly_name="amount of unearned income", + friendly_name="Amount of Unearned Income", type="string", startIndex=67, endIndex=71, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index cc0f97d3a..a03ea79de 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -12,7 +12,6 @@ document=TANF_T6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -39,7 +38,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -49,7 +48,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -63,7 +62,7 @@ calendar_quarter_to_rpt_month_year(0), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -76,7 +75,7 @@ Field( item="4A", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applications", type="number", startIndex=7, endIndex=15, @@ -86,7 +85,7 @@ Field( item="5A", name="NUM_APPROVED", - friendly_name="total number of approved applications", + friendly_name="Total Number of Approved Applications", type="number", startIndex=31, endIndex=39, @@ -96,7 +95,7 @@ Field( item="6A", name="NUM_DENIED", - friendly_name="total number of denied applications", + friendly_name="Total Number of Denied Applications", type="number", startIndex=55, endIndex=63, @@ -106,7 +105,7 @@ Field( item="7A", name="ASSISTANCE", - friendly_name="total amount of assistance", + friendly_name="Total Number of Cash Assistance", type="number", startIndex=79, endIndex=91, @@ -116,7 +115,7 @@ Field( item="8A", name="NUM_FAMILIES", - friendly_name="number of families", + friendly_name="Total Number of Families", type="number", startIndex=115, endIndex=123, @@ -126,7 +125,7 @@ Field( item="9A", name="NUM_2_PARENTS", - friendly_name="number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=139, endIndex=147, @@ -136,7 +135,7 @@ Field( item="10A", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of One-Parent Families", type="number", startIndex=163, endIndex=171, @@ -146,7 +145,7 @@ Field( item="11A", name="NUM_NO_PARENTS", - friendly_name="total number of no-parent families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=187, endIndex=195, @@ -156,7 +155,7 @@ Field( item="12A", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=211, endIndex=219, @@ -166,7 +165,7 @@ Field( item="13A", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=235, endIndex=243, @@ -176,7 +175,7 @@ Field( item="14A", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=259, endIndex=267, @@ -186,7 +185,7 @@ Field( item="15A", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents participating in work activities", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=283, endIndex=291, @@ -196,7 +195,7 @@ Field( item="16A", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=307, endIndex=315, @@ -206,7 +205,7 @@ Field( item="17A", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out-of-wedlock births", + friendly_name="Total Number of Non-Marital Births", type="number", startIndex=331, endIndex=339, @@ -216,7 +215,7 @@ Field( item="18A", name="NUM_CLOSED_CASES", - friendly_name="total number of closed cases", + friendly_name="Total Number of Closed Cases", type="number", startIndex=355, endIndex=363, @@ -228,10 +227,8 @@ s2 = RowSchema( document=TANF_T6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -258,7 +255,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -268,7 +265,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -279,7 +276,7 @@ calendar_quarter_to_rpt_month_year(1), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -289,7 +286,7 @@ Field( item="4B", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applications", type="number", startIndex=15, endIndex=23, @@ -299,7 +296,7 @@ Field( item="5B", name="NUM_APPROVED", - friendly_name="total number of approved cases", + friendly_name="Total Number of Approved Applications", type="number", startIndex=39, endIndex=47, @@ -309,7 +306,7 @@ Field( item="6B", name="NUM_DENIED", - friendly_name="total number of denied", + friendly_name="Total Number of Denied Applications", type="number", startIndex=63, endIndex=71, @@ -319,7 +316,7 @@ Field( item="7B", name="ASSISTANCE", - friendly_name="assistance", + friendly_name="Total Amount of Cash Assistance", type="number", startIndex=91, endIndex=103, @@ -329,7 +326,7 @@ Field( item="8B", name="NUM_FAMILIES", - friendly_name="total of number of families", + friendly_name="Total Number of Families", type="number", startIndex=123, endIndex=131, @@ -339,7 +336,7 @@ Field( item="9B", name="NUM_2_PARENTS", - friendly_name="number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=147, endIndex=155, @@ -349,7 +346,7 @@ Field( item="10B", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of One-Parent Families", type="number", startIndex=171, endIndex=179, @@ -359,7 +356,7 @@ Field( item="11B", name="NUM_NO_PARENTS", - friendly_name="total number of no-parent families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=195, endIndex=203, @@ -369,7 +366,7 @@ Field( item="12B", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=219, endIndex=227, @@ -379,7 +376,7 @@ Field( item="13B", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=243, endIndex=251, @@ -389,7 +386,7 @@ Field( item="14B", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=267, endIndex=275, @@ -399,7 +396,7 @@ Field( item="15B", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=291, endIndex=299, @@ -409,7 +406,7 @@ Field( item="16B", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=315, endIndex=323, @@ -419,7 +416,7 @@ Field( item="17B", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out-of-wedlock births", + friendly_name="Total Number of Non-Marital Births", type="number", startIndex=339, endIndex=347, @@ -429,7 +426,7 @@ Field( item="18B", name="NUM_CLOSED_CASES", - friendly_name="total number of closed assets", + friendly_name="Total Number of Closed Cases", type="number", startIndex=363, endIndex=371, @@ -441,10 +438,8 @@ s3 = RowSchema( document=TANF_T6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -471,7 +466,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -481,7 +476,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -492,7 +487,7 @@ calendar_quarter_to_rpt_month_year(2), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -502,7 +497,7 @@ Field( item="4C", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applications", type="number", startIndex=23, endIndex=31, @@ -512,7 +507,7 @@ Field( item="5C", name="NUM_APPROVED", - friendly_name="total number of approved applications", + friendly_name="Total Number of Applications", type="number", startIndex=47, endIndex=55, @@ -522,7 +517,7 @@ Field( item="6C", name="NUM_DENIED", - friendly_name="total number of denied applications", + friendly_name="Total Number of Denied Applications", type="number", startIndex=71, endIndex=79, @@ -532,7 +527,7 @@ Field( item="7C", name="ASSISTANCE", - friendly_name="total amount of assistance", + friendly_name="Total Number of Cash Assistance", type="number", startIndex=103, endIndex=115, @@ -542,7 +537,7 @@ Field( item="8C", name="NUM_FAMILIES", - friendly_name="total number of families", + friendly_name="Total Number of Families", type="number", startIndex=131, endIndex=139, @@ -552,7 +547,7 @@ Field( item="9C", name="NUM_2_PARENTS", - friendly_name="number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=155, endIndex=163, @@ -562,7 +557,7 @@ Field( item="10C", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of One-Parent Families", type="number", startIndex=179, endIndex=187, @@ -572,7 +567,7 @@ Field( item="11C", name="NUM_NO_PARENTS", - friendly_name="total number of no-parent families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=203, endIndex=211, @@ -582,7 +577,7 @@ Field( item="12C", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=227, endIndex=235, @@ -592,7 +587,7 @@ Field( item="13C", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=251, endIndex=259, @@ -602,7 +597,7 @@ Field( item="14C", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=275, endIndex=283, @@ -612,7 +607,7 @@ Field( item="15C", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=299, endIndex=307, @@ -622,7 +617,7 @@ Field( item="16C", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=323, endIndex=331, @@ -632,7 +627,7 @@ Field( item="17C", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out-of-wedlock births", + friendly_name="Total Number of Non-Marital Births", type="number", startIndex=347, endIndex=355, @@ -642,7 +637,7 @@ Field( item="18C", name="NUM_CLOSED_CASES", - friendly_name="total number of closed cases", + friendly_name="Total Number of Closed Cases", type="number", startIndex=371, endIndex=379, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index c247c6427..edf7299f2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -25,14 +25,13 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -42,7 +41,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -56,7 +55,7 @@ transform_func=calendar_quarter_to_rpt_month_year(month_index), item="3A", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -69,7 +68,7 @@ Field( item="4", name="TDRS_SECTION_IND", - friendly_name="tdrs section indicator", + friendly_name="TDRS Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -79,7 +78,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=stratum_index, endIndex=stratum_index + 2, @@ -89,7 +88,7 @@ Field( item=families_value_item_number, name="FAMILIES_MONTH", - friendly_name="families month", + friendly_name="Families Month", type="number", startIndex=families_index, endIndex=families_index + 7, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 0f337df7e..f36411a72 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -5,16 +5,13 @@ from ... import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument + t1 = SchemaManager( schemas=[ RowSchema( document=Tribal_TANF_T1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(122), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -125,7 +122,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -135,7 +132,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -148,7 +145,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -158,7 +155,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="county fips code", + friendly_name="County FIPS Code", type="string", startIndex=19, endIndex=22, @@ -170,7 +167,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=22, endIndex=24, @@ -182,7 +179,7 @@ Field( item="7", name="ZIP_CODE", - friendly_name="zip code", + friendly_name="ZIP Code", type="string", startIndex=24, endIndex=29, @@ -194,7 +191,7 @@ Field( item="8", name="FUNDING_STREAM", - friendly_name="funding stream", + friendly_name="Funding Stream", type="number", startIndex=29, endIndex=30, @@ -206,7 +203,7 @@ Field( item="9", name="DISPOSITION", - friendly_name="disposition", + friendly_name="Disposition", type="number", startIndex=30, endIndex=31, @@ -218,7 +215,7 @@ Field( item="10", name="NEW_APPLICANT", - friendly_name="new applicant", + friendly_name="New Applicant", type="number", startIndex=31, endIndex=32, @@ -230,7 +227,7 @@ Field( item="11", name="NBR_FAMILY_MEMBERS", - friendly_name="number of family members", + friendly_name="Number of Family Members", type="number", startIndex=32, endIndex=34, @@ -242,7 +239,7 @@ Field( item="12", name="FAMILY_TYPE", - friendly_name="family type", + friendly_name="Type of Family for Work Participation", type="number", startIndex=34, endIndex=35, @@ -254,7 +251,7 @@ Field( item="13", name="RECEIVES_SUB_HOUSING", - friendly_name="receives subsidized housing", + friendly_name="Receives Subsidized Housing", type="number", startIndex=35, endIndex=36, @@ -266,7 +263,7 @@ Field( item="14", name="RECEIVES_MED_ASSISTANCE", - friendly_name="receives medical assistance", + friendly_name="Receives Medical Assistance", type="number", startIndex=36, endIndex=37, @@ -278,7 +275,7 @@ Field( item="15", name="RECEIVES_FOOD_STAMPS", - friendly_name="receives food stamps", + friendly_name="Receives Food Stamps", type="number", startIndex=37, endIndex=38, @@ -290,7 +287,7 @@ Field( item="16", name="AMT_FOOD_STAMP_ASSISTANCE", - friendly_name="amount of food stamp assistance", + friendly_name="Amount of Food Stamp Assistance", type="number", startIndex=38, endIndex=42, @@ -302,7 +299,7 @@ Field( item="17", name="RECEIVES_SUB_CC", - friendly_name="receives subsidized child care", + friendly_name="Receives Subsidized Child Care", type="number", startIndex=42, endIndex=43, @@ -314,7 +311,7 @@ Field( item="18", name="AMT_SUB_CC", - friendly_name="amount of subsidized child care", + friendly_name="Amount of Subsidized Child Care", type="number", startIndex=43, endIndex=47, @@ -326,7 +323,7 @@ Field( item="19", name="CHILD_SUPPORT_AMT", - friendly_name="child support amount", + friendly_name="Amount of Child Support", type="number", startIndex=47, endIndex=51, @@ -338,7 +335,7 @@ Field( item="20", name="FAMILY_CASH_RESOURCES", - friendly_name="family cash resources", + friendly_name="Amount of the Family's Cash Resources", type="number", startIndex=51, endIndex=55, @@ -350,7 +347,7 @@ Field( item="21A", name="CASH_AMOUNT", - friendly_name="cash amount", + friendly_name="Cash and Cash Equivalents", type="number", startIndex=55, endIndex=59, @@ -362,7 +359,7 @@ Field( item="21B", name="NBR_MONTHS", - friendly_name="number of months", + friendly_name="Number of Months", type="number", startIndex=59, endIndex=62, @@ -374,7 +371,7 @@ Field( item="22A", name="CC_AMOUNT", - friendly_name="child care amount", + friendly_name="TANF Child Care Care Amount", type="number", startIndex=62, endIndex=66, @@ -386,7 +383,7 @@ Field( item="22B", name="CHILDREN_COVERED", - friendly_name="children covered", + friendly_name="TANF Child Care Number of Children Covered", type="number", startIndex=66, endIndex=68, @@ -398,7 +395,7 @@ Field( item="22C", name="CC_NBR_MONTHS", - friendly_name="child care number of months", + friendly_name="TANF Child Care Number of Months", type="number", startIndex=68, endIndex=71, @@ -410,7 +407,7 @@ Field( item="23A", name="TRANSP_AMOUNT", - friendly_name="transportation amount", + friendly_name="Transportation Amount", type="number", startIndex=71, endIndex=75, @@ -422,7 +419,7 @@ Field( item="23B", name="TRANSP_NBR_MONTHS", - friendly_name="transportation number of months", + friendly_name="Transportation Number of Months", type="number", startIndex=75, endIndex=78, @@ -434,7 +431,7 @@ Field( item="24A", name="TRANSITION_SERVICES_AMOUNT", - friendly_name="transition services amount", + friendly_name="Transitional Services Amount", type="number", startIndex=78, endIndex=82, @@ -446,7 +443,7 @@ Field( item="24B", name="TRANSITION_NBR_MONTHS", - friendly_name="transition services number of months", + friendly_name="Transitional Services Number of Months", type="number", startIndex=82, endIndex=85, @@ -458,7 +455,7 @@ Field( item="25A", name="OTHER_AMOUNT", - friendly_name="other amount", + friendly_name="Other Amount", type="number", startIndex=85, endIndex=89, @@ -470,7 +467,7 @@ Field( item="25B", name="OTHER_NBR_MONTHS", - friendly_name="other number of months", + friendly_name="Other Number of Months", type="number", startIndex=89, endIndex=92, @@ -482,7 +479,7 @@ Field( item="26AI", name="SANC_REDUCTION_AMT", - friendly_name="sanction reduction amount", + friendly_name="Total Dollar Amount of Reductions due to Sanctions", type="number", startIndex=92, endIndex=96, @@ -494,7 +491,7 @@ Field( item="26AII", name="WORK_REQ_SANCTION", - friendly_name="work requirement sanction", + friendly_name="Work Requirements Sanction", type="number", startIndex=96, endIndex=97, @@ -506,7 +503,7 @@ Field( item="26AIII", name="FAMILY_SANC_ADULT", - friendly_name="family sanction adult", + friendly_name="Family Sanction for an Adult with No High School Diploma orEquivalent:", type="number", startIndex=97, endIndex=98, @@ -518,7 +515,7 @@ Field( item="26AIV", name="SANC_TEEN_PARENT", - friendly_name="sanctioned teen parent", + friendly_name="Sanction for Teen Parent not Attending School", type="number", startIndex=98, endIndex=99, @@ -530,7 +527,7 @@ Field( item="26AV", name="NON_COOPERATION_CSE", - friendly_name="non-cooperation with child support", + friendly_name="Non-Cooperation with Child Support", type="number", startIndex=99, endIndex=100, @@ -542,7 +539,7 @@ Field( item="26AVI", name="FAILURE_TO_COMPLY", - friendly_name="failure to comply", + friendly_name="Failure to comply with an Individual Responsibility Plan", type="number", startIndex=100, endIndex=101, @@ -554,7 +551,7 @@ Field( item="26AVII", name="OTHER_SANCTION", - friendly_name="other, sanction", + friendly_name="Other Sanction", type="number", startIndex=101, endIndex=102, @@ -566,7 +563,7 @@ Field( item="26B", name="RECOUPMENT_PRIOR_OVRPMT", - friendly_name="recoupment prior overpayment", + friendly_name="Recoupment of Prior Overpayment", type="number", startIndex=102, endIndex=106, @@ -578,7 +575,7 @@ Field( item="26CI", name="OTHER_TOTAL_REDUCTIONS", - friendly_name="other total reductions", + friendly_name="Total Dollar Amount of Reductions due to Other Reasons", type="number", startIndex=106, endIndex=110, @@ -590,7 +587,7 @@ Field( item="26CII", name="FAMILY_CAP", - friendly_name="family cap", + friendly_name="Family Cap", type="number", startIndex=110, endIndex=111, @@ -602,7 +599,7 @@ Field( item="26CIII", name="REDUCTIONS_ON_RECEIPTS", - friendly_name="reductions on receipts", + friendly_name="Reduction Based on Length of Receipt of Assistance", type="number", startIndex=111, endIndex=112, @@ -614,7 +611,7 @@ Field( item="26CIV", name="OTHER_NON_SANCTION", - friendly_name="other, non-sanction", + friendly_name="Other, Non-sanction", type="number", startIndex=112, endIndex=113, @@ -636,7 +633,7 @@ Field( item="28", name="FAMILY_EXEMPT_TIME_LIMITS", - friendly_name="family exempt time limits", + friendly_name="Exempt during the reporting month from theTribal Time-Limit Provisions", type="number", startIndex=114, endIndex=116, @@ -646,7 +643,7 @@ Field( item="29", name="FAMILY_NEW_CHILD", - friendly_name="family new child", + friendly_name="A New Child-Only Family", type="number", startIndex=116, endIndex=117, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 32de69f7a..0f8e3f402 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -14,10 +14,6 @@ document=Tribal_TANF_T2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(122), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -120,7 +116,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -130,7 +126,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -143,7 +139,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -153,7 +149,7 @@ Field( item="30", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -163,7 +159,7 @@ Field( item="31", name="NONCUSTODIAL_PARENT", - friendly_name="noncustodial parent", + friendly_name="Noncustodial Parent Indicator", type="number", startIndex=20, endIndex=21, @@ -173,22 +169,21 @@ Field( item="32", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=21, endIndex=29, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="33", name="SSN", - friendly_name="social security number - ssn", + friendly_name="Social Security Number", type="string", startIndex=29, endIndex=38, @@ -199,7 +194,7 @@ Field( item="34A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=38, endIndex=39, @@ -209,7 +204,7 @@ Field( item="34B", name="RACE_AMER_INDIAN", - friendly_name="race american-indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Native", type="number", startIndex=39, endIndex=40, @@ -219,7 +214,7 @@ Field( item="34C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=40, endIndex=41, @@ -229,7 +224,7 @@ Field( item="34D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=41, endIndex=42, @@ -239,7 +234,7 @@ Field( item="34E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", type="number", startIndex=42, endIndex=43, @@ -249,7 +244,7 @@ Field( item="34F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Ethnicity/Race: White", type="number", startIndex=43, endIndex=44, @@ -259,7 +254,7 @@ Field( item="35", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=44, endIndex=45, @@ -271,7 +266,7 @@ Field( item="36A", name="FED_OASDI_PROGRAM", - friendly_name="federal old age survivors and disability insurance program", + friendly_name="Receives Disability Benefits: OASDI Program", type="number", startIndex=45, endIndex=46, @@ -281,7 +276,7 @@ Field( item="36B", name="FED_DISABILITY_STATUS", - friendly_name="federal disability status", + friendly_name="Receives Disability Benefits: Federal Disability Status", type="number", startIndex=46, endIndex=47, @@ -291,8 +286,7 @@ Field( item="36C", name="DISABLED_TITLE_XIVAPDT", - friendly_name="Receives Aid to the Permanently and Totally Disabled" + - " Under Title XIV-APDT of the Social Security Act", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type="string", startIndex=47, endIndex=48, @@ -306,7 +300,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="receives from the aid to the aged, blind, and disabled program", + friendly_name="Receives Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", type="number", startIndex=48, endIndex=49, @@ -318,7 +312,7 @@ Field( item="36E", name="RECEIVE_SSI", - friendly_name="receives social security income", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=49, endIndex=50, @@ -330,7 +324,7 @@ Field( item="37", name="MARITAL_STATUS", - friendly_name="marital status", + friendly_name="Marital Status", type="number", startIndex=50, endIndex=51, @@ -342,7 +336,7 @@ Field( item="38", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=51, endIndex=53, @@ -354,7 +348,7 @@ Field( item="39", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent With Minor Child in the Family", type="number", startIndex=53, endIndex=54, @@ -366,7 +360,7 @@ Field( item="40", name="NEEDS_PREGNANT_WOMAN", - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type="number", startIndex=54, endIndex=55, @@ -378,7 +372,7 @@ Field( item="41", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=55, endIndex=57, @@ -393,7 +387,7 @@ Field( item="42", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Alienage", type="number", startIndex=57, endIndex=58, @@ -403,7 +397,7 @@ Field( item="43", name="COOPERATION_CHILD_SUPPORT", - friendly_name="cooperation with child support", + friendly_name="Cooperation with Child Support", type="number", startIndex=58, endIndex=59, @@ -415,7 +409,7 @@ Field( item="44", name="MONTHS_FED_TIME_LIMIT", - friendly_name="countable months toward federal time limit", + friendly_name="Number of Months Countable toward Tribal Time Limit", type="string", startIndex=59, endIndex=62, @@ -427,7 +421,7 @@ Field( item="45", name="MONTHS_STATE_TIME_LIMIT", - friendly_name="months of state time limit", + friendly_name="Number of Countable Months Remaining Under the Tribe's Time Limit", type="string", startIndex=62, endIndex=64, @@ -439,7 +433,7 @@ Field( item="46", name="CURRENT_MONTH_STATE_EXEMPT", - friendly_name="current month state exempt", + friendly_name="Is Current Month Exempt from the State's (Tribe's) Time Limit", type="number", startIndex=64, endIndex=65, @@ -451,7 +445,7 @@ Field( item="47", name="EMPLOYMENT_STATUS", - friendly_name="employment status", + friendly_name="Employment Status", type="number", startIndex=65, endIndex=66, @@ -463,7 +457,7 @@ Field( item="48", name="WORK_PART_STATUS", - friendly_name="work participation status", + friendly_name="Work Participation Status", type="string", startIndex=66, endIndex=68, @@ -480,7 +474,7 @@ Field( item="49", name="UNSUB_EMPLOYMENT", - friendly_name="unsubsidized employment", + friendly_name="Unsubsidized Employment", type="string", startIndex=68, endIndex=70, @@ -492,7 +486,7 @@ Field( item="50", name="SUB_PRIVATE_EMPLOYMENT", - friendly_name="subsidized private employment", + friendly_name="Subsidized Private-Sector Employment.", type="string", startIndex=70, endIndex=72, @@ -504,7 +498,7 @@ Field( item="51", name="SUB_PUBLIC_EMPLOYMENT", - friendly_name="subsidized public employment", + friendly_name="Subsidized Public-Sector Employment", type="string", startIndex=72, endIndex=74, @@ -516,7 +510,7 @@ Field( item="52", name="WORK_EXPERIENCE", - friendly_name="work experience", + friendly_name="Work Experience", type="string", startIndex=74, endIndex=76, @@ -528,7 +522,7 @@ Field( item="53", name="OJT", - friendly_name="on the job training", + friendly_name="On-the-job Training", type="string", startIndex=76, endIndex=78, @@ -540,7 +534,7 @@ Field( item="54", name="JOB_SEARCH", - friendly_name="job search", + friendly_name="Job Search and Job Readiness Assistance", type="string", startIndex=78, endIndex=80, @@ -552,7 +546,7 @@ Field( item="55", name="COMM_SERVICES", - friendly_name="community service", + friendly_name="Community Service Programs", type="string", startIndex=80, endIndex=82, @@ -564,7 +558,7 @@ Field( item="56", name="VOCATIONAL_ED_TRAINING", - friendly_name="vocational education", + friendly_name="Vocational Educational Training", type="string", startIndex=82, endIndex=84, @@ -576,7 +570,7 @@ Field( item="57", name="JOB_SKILLS_TRAINING", - friendly_name="job skills training", + friendly_name="Job Skills Training Directly Related to Employment", type="string", startIndex=84, endIndex=86, @@ -588,7 +582,7 @@ Field( item="58", name="ED_NO_HIGH_SCHOOL_DIPLOMA", - friendly_name="education no high school diploma", + friendly_name="Education Directly Related to Employment for Individualswith no High School Diploma or Certificate of High SchoolEquivalency", type="string", startIndex=86, endIndex=88, @@ -600,7 +594,7 @@ Field( item="59", name="SCHOOL_ATTENDENCE", - friendly_name="school attendance", + friendly_name="Satisfactory School Attendance for Individuals with No HighSchool Diploma or Certificate of High School Equivalency", type="string", startIndex=88, endIndex=90, @@ -612,7 +606,7 @@ Field( item="60", name="PROVIDE_CC", - friendly_name="provide child care", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program", type="string", startIndex=90, endIndex=92, @@ -624,7 +618,7 @@ Field( item="61", name="ADD_WORK_ACTIVITIES", - friendly_name="additional work activities", + friendly_name="Additional Work Activities", type="string", startIndex=92, endIndex=94, @@ -636,7 +630,7 @@ Field( item="62", name="OTHER_WORK_ACTIVITIES", - friendly_name="other work activities", + friendly_name="Other Work Activities", type="string", startIndex=94, endIndex=96, @@ -648,7 +642,7 @@ Field( item="63", name="REQ_HRS_WAIVER_DEMO", - friendly_name="required hours under waiver demo", + friendly_name="Required Hours of Work under Waiver Demonstration", type="string", startIndex=96, endIndex=98, @@ -660,7 +654,7 @@ Field( item="64", name="EARNED_INCOME", - friendly_name="earned income", + friendly_name="Amount of Earned Income", type="string", startIndex=98, endIndex=102, @@ -672,7 +666,7 @@ Field( item="65A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="unearned income tax credit", + friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", type="string", startIndex=102, endIndex=106, @@ -684,7 +678,7 @@ Field( item="65B", name="UNEARNED_SOCIAL_SECURITY", - friendly_name="unearned social security", + friendly_name="Amount of Unearned Income: Social Security", type="string", startIndex=106, endIndex=110, @@ -696,7 +690,7 @@ Field( item="65C", name="UNEARNED_SSI", - friendly_name="unearned ssi benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=110, endIndex=114, @@ -708,7 +702,7 @@ Field( item="65D", name="UNEARNED_WORKERS_COMP", - friendly_name="unearned workers compensation", + friendly_name="Amount of Unearned Income: Worker's Compensation", type="string", startIndex=114, endIndex=118, @@ -720,7 +714,7 @@ Field( item="65E", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=118, endIndex=122, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index 7e1cb4634..26d0d32e1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -97,7 +97,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -107,7 +107,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -117,7 +117,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -127,7 +127,7 @@ Field( item="66", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -137,7 +137,7 @@ Field( item="67", name="DATE_OF_BIRTH", - friendly_name="date of birth", + friendly_name="Date of Birth", type="string", startIndex=20, endIndex=28, @@ -152,7 +152,7 @@ transform_func=tanf_ssn_decryption_func, item="68", name="SSN", - friendly_name="social security number - ssn", + friendly_name="Social Security Number", type="string", startIndex=28, endIndex=37, @@ -163,7 +163,7 @@ Field( item="69A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -173,7 +173,7 @@ Field( item="69B", name="RACE_AMER_INDIAN", - friendly_name="race american indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -183,7 +183,7 @@ Field( item="69C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=39, endIndex=40, @@ -193,7 +193,7 @@ Field( item="69D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=40, endIndex=41, @@ -203,7 +203,7 @@ Field( item="69E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Native Hawaiian or Other Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -213,7 +213,7 @@ Field( item="69F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="White", type="number", startIndex=42, endIndex=43, @@ -223,7 +223,7 @@ Field( item="70", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=43, endIndex=44, @@ -233,7 +233,7 @@ Field( item="71A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="receives non-social security act benefits", + friendly_name="Receives Benefits Based on Federal Disability Status under Non-Social Security Act Programs", type="number", startIndex=44, endIndex=45, @@ -243,7 +243,7 @@ Field( item="71B", name="RECEIVE_SSI", - friendly_name="receives social security income", + friendly_name="Receives Supplemental Security Income under Title XVI-SSI of the Social Security Act", type="number", startIndex=45, endIndex=46, @@ -253,7 +253,7 @@ Field( item="72", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=46, endIndex=48, @@ -263,7 +263,7 @@ Field( item="73", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent With Minor Child in the Family", type="number", startIndex=48, endIndex=49, @@ -273,7 +273,7 @@ Field( item="74", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=49, endIndex=51, @@ -288,7 +288,7 @@ Field( item="75", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Alienage", type="number", startIndex=51, endIndex=52, @@ -298,7 +298,7 @@ Field( item="76A", name="UNEARNED_SSI", - friendly_name="unearned ssi benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=52, endIndex=56, @@ -308,7 +308,7 @@ Field( item="76B", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=56, endIndex=60, @@ -408,7 +408,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -418,7 +418,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -428,7 +428,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -438,7 +438,7 @@ Field( item="66", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=60, endIndex=61, @@ -448,7 +448,7 @@ Field( item="67", name="DATE_OF_BIRTH", - friendly_name="date of birth", + friendly_name="Date of Birth", type="string", startIndex=61, endIndex=69, @@ -463,7 +463,7 @@ transform_func=tanf_ssn_decryption_func, item="68", name="SSN", - friendly_name="social security number - ssn", + friendly_name="Social Security Number", type="string", startIndex=69, endIndex=78, @@ -474,7 +474,7 @@ Field( item="69A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=78, endIndex=79, @@ -484,7 +484,7 @@ Field( item="69B", name="RACE_AMER_INDIAN", - friendly_name="race american indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Natve", type="number", startIndex=79, endIndex=80, @@ -494,7 +494,7 @@ Field( item="69C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=80, endIndex=81, @@ -504,7 +504,7 @@ Field( item="69D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=81, endIndex=82, @@ -514,7 +514,7 @@ Field( item="69E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", type="number", startIndex=82, endIndex=83, @@ -524,7 +524,7 @@ Field( item="69F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Ethnicity/Race: White", type="number", startIndex=83, endIndex=84, @@ -534,7 +534,7 @@ Field( item="70", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=84, endIndex=85, @@ -544,7 +544,7 @@ Field( item="71A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="receives non-ssa benefits", + friendly_name="Receives Disability Benefits: Federal Disability Status", type="number", startIndex=85, endIndex=86, @@ -554,7 +554,7 @@ Field( item="71B", name="RECEIVE_SSI", - friendly_name="receives SSI", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=86, endIndex=87, @@ -564,7 +564,7 @@ Field( item="72", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=87, endIndex=89, @@ -574,7 +574,7 @@ Field( item="73", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent With Minor Child in the Family", type="number", startIndex=89, endIndex=90, @@ -584,7 +584,7 @@ Field( item="74", name="EDUCATION_LEVEL", - friendly_name="education level", + friendly_name="Educational Level", type="string", startIndex=90, endIndex=92, @@ -598,7 +598,7 @@ Field( item="75", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Alienage", type="number", startIndex=92, endIndex=93, @@ -608,7 +608,7 @@ Field( item="76A", name="UNEARNED_SSI", - friendly_name="unearned ssi benefit", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=93, endIndex=97, @@ -618,7 +618,7 @@ Field( item="76B", name="OTHER_UNEARNED_INCOME", - friendly_name="other unearned income", + friendly_name="Amount of Unearned Income: Other", type="string", startIndex=97, endIndex=101, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index a9530beef..4209c1fa3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -12,10 +12,6 @@ document=Tribal_TANF_T4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[], @@ -23,7 +19,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -33,7 +29,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -46,7 +42,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -56,7 +52,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="county fips code", + friendly_name="County FIPS Code", type="string", startIndex=19, endIndex=22, @@ -66,7 +62,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=22, endIndex=24, @@ -76,7 +72,7 @@ Field( item="7", name="ZIP_CODE", - friendly_name="zip code", + friendly_name="Zip Code", type="string", startIndex=24, endIndex=29, @@ -86,7 +82,7 @@ Field( item="8", name="DISPOSITION", - friendly_name="disposition", + friendly_name="Disposition", type="number", startIndex=29, endIndex=30, @@ -96,7 +92,7 @@ Field( item="9", name="CLOSURE_REASON", - friendly_name="closure reason", + friendly_name="Reason for Closure", type="string", startIndex=30, endIndex=32, @@ -110,7 +106,7 @@ Field( item="10", name="REC_SUB_HOUSING", - friendly_name="receives subsidized housing", + friendly_name="Received Subsidized Housing", type="number", startIndex=32, endIndex=33, @@ -120,7 +116,7 @@ Field( item="11", name="REC_MED_ASSIST", - friendly_name="receives medical assistance", + friendly_name="Received Medical Assistance:", type="number", startIndex=33, endIndex=34, @@ -130,7 +126,7 @@ Field( item="12", name="REC_FOOD_STAMPS", - friendly_name="receives food stamps", + friendly_name="Received Food Stamps", type="number", startIndex=34, endIndex=35, @@ -140,7 +136,7 @@ Field( item="13", name="REC_SUB_CC", - friendly_name="receives subsidized child care", + friendly_name="Received Subsidized Child Care", type="number", startIndex=35, endIndex=36, @@ -150,7 +146,7 @@ Field( item="14", name="BLANK", - friendly_name="blank", + friendly_name="Family Affiliation:", type="string", startIndex=36, endIndex=71, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index feaff651f..93dc07b8f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -14,10 +14,6 @@ document=Tribal_TANF_T5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), - validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), - ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -103,7 +99,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -113,7 +109,7 @@ Field( item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=8, @@ -126,7 +122,7 @@ Field( item="6", name="CASE_NUMBER", - friendly_name="case number", + friendly_name="Case Number--TANF", type="string", startIndex=8, endIndex=19, @@ -136,7 +132,7 @@ Field( item="14", name="FAMILY_AFFILIATION", - friendly_name="family affiliation", + friendly_name="Family Affiliation", type="number", startIndex=19, endIndex=20, @@ -146,22 +142,21 @@ Field( item="15", name="DATE_OF_BIRTH", - friendly_name="date of birth", - type="string", + friendly_name="Date of Birth", + type="number", startIndex=20, endIndex=28, required=True, - validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators=[ + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + ], ), TransformField( transform_func=tanf_ssn_decryption_func, item="16", name="SSN", - friendly_name="social security number - ssn", + friendly_name="Social Security Number", type="string", startIndex=28, endIndex=37, @@ -172,7 +167,7 @@ Field( item="17A", name="RACE_HISPANIC", - friendly_name="race hispanic", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -182,7 +177,7 @@ Field( item="17B", name="RACE_AMER_INDIAN", - friendly_name="race american indian", + friendly_name="Ethnicity/Race: American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -192,7 +187,7 @@ Field( item="17C", name="RACE_ASIAN", - friendly_name="race asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=39, endIndex=40, @@ -202,7 +197,7 @@ Field( item="17D", name="RACE_BLACK", - friendly_name="race black", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=40, endIndex=41, @@ -212,7 +207,7 @@ Field( item="17E", name="RACE_HAWAIIAN", - friendly_name="race hawaiian", + friendly_name="Ethnicity/Race: Hawaiian or Other Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -222,7 +217,7 @@ Field( item="17F", name="RACE_WHITE", - friendly_name="race white", + friendly_name="Ethnicity/Race: White", type="number", startIndex=42, endIndex=43, @@ -232,7 +227,7 @@ Field( item="18", name="GENDER", - friendly_name="gender", + friendly_name="Gender", type="number", startIndex=43, endIndex=44, @@ -242,7 +237,7 @@ Field( item="19A", name="REC_OASDI_INSURANCE", - friendly_name="receives old-age survivors and disability insurance", + friendly_name="Receives Disability Benefits: OASDI Program", type="number", startIndex=44, endIndex=45, @@ -252,7 +247,7 @@ Field( item="19B", name="REC_FEDERAL_DISABILITY", - friendly_name="receives federal disability", + friendly_name="Received Disability Benefits: Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -262,7 +257,7 @@ Field( item="19C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="receives aid for totally disabled", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", type="number", startIndex=46, endIndex=47, @@ -272,7 +267,7 @@ Field( item="19D", name="REC_AID_AGED_BLIND", - friendly_name="receives from the aid to the aged, blind, and disabled program", + friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", type="number", startIndex=47, endIndex=48, @@ -282,7 +277,7 @@ Field( item="19E", name="REC_SSI", - friendly_name="receives social security income", + friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", type="number", startIndex=48, endIndex=49, @@ -292,7 +287,7 @@ Field( item="20", name="MARITAL_STATUS", - friendly_name="marital status", + friendly_name="Marital Status", type="number", startIndex=49, endIndex=50, @@ -302,7 +297,7 @@ Field( item="21", name="RELATIONSHIP_HOH", - friendly_name="relationship to head of household", + friendly_name="Relationship to Head-of-Household", type="string", startIndex=50, endIndex=52, @@ -312,7 +307,7 @@ Field( item="22", name="PARENT_MINOR_CHILD", - friendly_name="parent of minor child", + friendly_name="Parent With Minor Child in the Family", type="number", startIndex=52, endIndex=53, @@ -322,7 +317,7 @@ Field( item="23", name="NEEDS_OF_PREGNANT_WOMAN", - friendly_name="needs of pregnant woman", + friendly_name="Needs of a Pregnant Woman", type="number", startIndex=53, endIndex=54, @@ -332,7 +327,7 @@ Field( item="24", name="EDUCATION_LEVEL", - friendly_name="educational level", + friendly_name="Educational Level", type="string", startIndex=54, endIndex=56, @@ -347,7 +342,7 @@ Field( item="25", name="CITIZENSHIP_STATUS", - friendly_name="citizenship status", + friendly_name="Citizenship/Alienage", type="number", startIndex=56, endIndex=57, @@ -361,7 +356,7 @@ Field( item="26", name="COUNTABLE_MONTH_FED_TIME", - friendly_name="countable months toward federal time", + friendly_name="Number of Months Countable toward Tribal Time Limit:", type="string", startIndex=57, endIndex=60, @@ -371,7 +366,7 @@ Field( item="27", name="COUNTABLE_MONTHS_STATE_TRIBE", - friendly_name="countable months remaining under state tribe", + friendly_name="Number of Countable Months Remaining Under Tribe's Time Limit", type="string", startIndex=60, endIndex=62, @@ -381,7 +376,7 @@ Field( item="28", name="EMPLOYMENT_STATUS", - friendly_name="employment status", + friendly_name="Employment Status", type="number", startIndex=62, endIndex=63, @@ -391,7 +386,7 @@ Field( item="29", name="AMOUNT_EARNED_INCOME", - friendly_name="amount earnedof income", + friendly_name="Amount of Earned Income", type="string", startIndex=63, endIndex=67, @@ -401,7 +396,7 @@ Field( item="30", name="AMOUNT_UNEARNED_INCOME", - friendly_name="amount of unearned income", + friendly_name="Amount of Unearned Income", type="string", startIndex=67, endIndex=71, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index e946f158a..b641b7983 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -12,7 +12,6 @@ document=Tribal_TANF_T6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -27,7 +26,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -37,7 +36,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -51,7 +50,7 @@ calendar_quarter_to_rpt_month_year(0), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -64,7 +63,7 @@ Field( item="4A", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applications", type="number", startIndex=7, endIndex=15, @@ -74,7 +73,7 @@ Field( item="5A", name="NUM_APPROVED", - friendly_name="total number of approved applications", + friendly_name="Total Number of Approved Applications", type="number", startIndex=31, endIndex=39, @@ -84,7 +83,7 @@ Field( item="6A", name="NUM_DENIED", - friendly_name="total number of denied applications", + friendly_name="Total Number of Denied Applications", type="number", startIndex=55, endIndex=63, @@ -94,7 +93,7 @@ Field( item="7A", name="ASSISTANCE", - friendly_name="total amount of assistance", + friendly_name="Total Amount of Assistance", type="number", startIndex=79, endIndex=91, @@ -104,7 +103,7 @@ Field( item="8A", name="NUM_FAMILIES", - friendly_name="total number of families", + friendly_name="Total Number of Families", type="number", startIndex=115, endIndex=123, @@ -114,7 +113,7 @@ Field( item="9A", name="NUM_2_PARENTS", - friendly_name="total number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=139, endIndex=147, @@ -124,7 +123,7 @@ Field( item="10A", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=163, endIndex=171, @@ -134,7 +133,7 @@ Field( item="11A", name="NUM_NO_PARENTS", - friendly_name="total number of no parent families", + friendly_name="Total Number of No Parent Families", type="number", startIndex=187, endIndex=195, @@ -144,7 +143,7 @@ Field( item="12A", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=211, endIndex=219, @@ -154,7 +153,7 @@ Field( item="13A", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=235, endIndex=243, @@ -164,7 +163,7 @@ Field( item="14A", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=259, endIndex=267, @@ -174,7 +173,7 @@ Field( item="15A", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents participating in work activities", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=283, endIndex=291, @@ -184,7 +183,7 @@ Field( item="16A", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=307, endIndex=315, @@ -194,7 +193,7 @@ Field( item="17A", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out-of-wedlock births", + friendly_name="Total Number of Out-of-Wedlock Births", type="number", startIndex=331, endIndex=339, @@ -204,7 +203,7 @@ Field( item="18A", name="NUM_CLOSED_CASES", - friendly_name="total number of closed cases", + friendly_name="Total Number of Closed Cases", type="number", startIndex=355, endIndex=363, @@ -216,10 +215,8 @@ s2 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -234,7 +231,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -244,7 +241,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -255,7 +252,7 @@ calendar_quarter_to_rpt_month_year(1), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -265,7 +262,7 @@ Field( item="4B", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applicants", type="number", startIndex=15, endIndex=23, @@ -275,7 +272,7 @@ Field( item="5B", name="NUM_APPROVED", - friendly_name="total number of approved applications", + friendly_name="Total Number of Approved Applications", type="number", startIndex=39, endIndex=47, @@ -285,7 +282,7 @@ Field( item="6B", name="NUM_DENIED", - friendly_name="total number of denied applications", + friendly_name="Total Number of Denied Applications", type="number", startIndex=63, endIndex=71, @@ -295,7 +292,7 @@ Field( item="7B", name="ASSISTANCE", - friendly_name="assistance", + friendly_name="Total Amount of Assistance", type="number", startIndex=91, endIndex=103, @@ -305,7 +302,7 @@ Field( item="8B", name="NUM_FAMILIES", - friendly_name="total number of families", + friendly_name="Total Number of Families", type="number", startIndex=123, endIndex=131, @@ -315,7 +312,7 @@ Field( item="9B", name="NUM_2_PARENTS", - friendly_name="total number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=147, endIndex=155, @@ -325,7 +322,7 @@ Field( item="10B", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of One-Parent Families", type="number", startIndex=171, endIndex=179, @@ -335,7 +332,7 @@ Field( item="11B", name="NUM_NO_PARENTS", - friendly_name="total number of no parent families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=195, endIndex=203, @@ -345,7 +342,7 @@ Field( item="12B", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=219, endIndex=227, @@ -355,7 +352,7 @@ Field( item="13B", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=243, endIndex=251, @@ -365,7 +362,7 @@ Field( item="14B", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=267, endIndex=275, @@ -375,7 +372,7 @@ Field( item="15B", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=291, endIndex=299, @@ -385,7 +382,7 @@ Field( item="16B", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=315, endIndex=323, @@ -395,7 +392,7 @@ Field( item="17B", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out wedlock births", + friendly_name="Total Number of Out-of-Wedlock Births", type="number", startIndex=339, endIndex=347, @@ -405,7 +402,7 @@ Field( item="18B", name="NUM_CLOSED_CASES", - friendly_name="total number of closed assets", + friendly_name="Total Number of Closed Cases", type="number", startIndex=363, endIndex=371, @@ -417,10 +414,8 @@ s3 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), - quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -435,7 +430,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -445,7 +440,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -456,7 +451,7 @@ calendar_quarter_to_rpt_month_year(2), item="4", name="RPT_MONTH_YEAR", - friendly_name="reporting month and year", + friendly_name="Reporitng Year and Month", type="number", startIndex=2, endIndex=7, @@ -466,7 +461,7 @@ Field( item="4C", name="NUM_APPLICATIONS", - friendly_name="total number of applications", + friendly_name="Total Number of Applications", type="number", startIndex=23, endIndex=31, @@ -476,7 +471,7 @@ Field( item="5C", name="NUM_APPROVED", - friendly_name="total number of approved applications", + friendly_name="Total Number of Approved Applications", type="number", startIndex=47, endIndex=55, @@ -486,7 +481,7 @@ Field( item="6C", name="NUM_DENIED", - friendly_name="total number of denied applications", + friendly_name="Total Number of Denied Applications", type="number", startIndex=71, endIndex=79, @@ -496,7 +491,7 @@ Field( item="7C", name="ASSISTANCE", - friendly_name="total amount of assistance", + friendly_name="Total Amount of Assistance", type="number", startIndex=103, endIndex=115, @@ -506,7 +501,7 @@ Field( item="8C", name="NUM_FAMILIES", - friendly_name="total number of families", + friendly_name="Total Number of Families", type="number", startIndex=131, endIndex=139, @@ -516,7 +511,7 @@ Field( item="9C", name="NUM_2_PARENTS", - friendly_name="total number of two-parent families", + friendly_name="Total Number of Two-parent Families", type="number", startIndex=155, endIndex=163, @@ -526,7 +521,7 @@ Field( item="10C", name="NUM_1_PARENTS", - friendly_name="total number of one-parent families", + friendly_name="Total Number of One-parent Families", type="number", startIndex=179, endIndex=187, @@ -536,7 +531,7 @@ Field( item="11C", name="NUM_NO_PARENTS", - friendly_name="total number of no parent families", + friendly_name="Total Number of No parent Families", type="number", startIndex=203, endIndex=211, @@ -546,7 +541,7 @@ Field( item="12C", name="NUM_RECIPIENTS", - friendly_name="total number of recipients", + friendly_name="Total Number of Recipients", type="number", startIndex=227, endIndex=235, @@ -556,7 +551,7 @@ Field( item="13C", name="NUM_ADULT_RECIPIENTS", - friendly_name="total number of adult recipients", + friendly_name="Total Number of Adult Recipients", type="number", startIndex=251, endIndex=259, @@ -566,7 +561,7 @@ Field( item="14C", name="NUM_CHILD_RECIPIENTS", - friendly_name="total number of child recipients", + friendly_name="Total Number of Child Recipients", type="number", startIndex=275, endIndex=283, @@ -576,7 +571,7 @@ Field( item="15C", name="NUM_NONCUSTODIALS", - friendly_name="total number of noncustodial parents", + friendly_name="Total Number of Noncustodial Parents Participating in Work Activities", type="number", startIndex=299, endIndex=307, @@ -586,7 +581,7 @@ Field( item="16C", name="NUM_BIRTHS", - friendly_name="total number of births", + friendly_name="Total Number of Births", type="number", startIndex=323, endIndex=331, @@ -596,7 +591,7 @@ Field( item="17C", name="NUM_OUTWEDLOCK_BIRTHS", - friendly_name="total number of out wedlock births", + friendly_name="Total Number of Out-of-Wedlock Births", type="number", startIndex=347, endIndex=355, @@ -606,7 +601,7 @@ Field( item="18C", name="NUM_CLOSED_CASES", - friendly_name="total number of closed cases", + friendly_name="Total Number of Closed Cases", type="number", startIndex=371, endIndex=379, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index b6ec33538..67b34aa63 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -25,14 +25,13 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), - validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -42,7 +41,7 @@ Field( item="3", name="CALENDAR_QUARTER", - friendly_name="calendar quarter", + friendly_name="Calendar Quarter", type="number", startIndex=2, endIndex=7, @@ -56,7 +55,7 @@ transform_func=calendar_quarter_to_rpt_month_year(month_index), item="3A", name="RPT_MONTH_YEAR", - friendly_name="reporting month year", + friendly_name="Reporting Year and Month", type="number", startIndex=2, endIndex=7, @@ -69,7 +68,7 @@ Field( item="4", name="TDRS_SECTION_IND", - friendly_name="tdrs section indicator", + friendly_name="TDRS Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -79,7 +78,7 @@ Field( item="5", name="STRATUM", - friendly_name="stratum", + friendly_name="Stratum", type="string", startIndex=stratum_index, endIndex=stratum_index + 2, @@ -89,7 +88,7 @@ Field( item=families_value_item_number, name="FAMILIES_MONTH", - friendly_name="families month", + friendly_name="Families Month", type="number", startIndex=families_index, endIndex=families_index + 7, From 7388f80bd222577a4c8511043767e62ff195cf5b Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Mon, 8 Apr 2024 08:49:38 -0700 Subject: [PATCH 002/105] Updating validators Adding back validators that got removed on previous commit --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 4 +++ .../tdpservice/parsers/schema_defs/ssp/m2.py | 4 +++ .../tdpservice/parsers/schema_defs/ssp/m3.py | 4 +++ .../tdpservice/parsers/schema_defs/ssp/m4.py | 4 +++ .../tdpservice/parsers/schema_defs/ssp/m5.py | 4 +++ .../tdpservice/parsers/schema_defs/ssp/m6.py | 1 + .../tdpservice/parsers/schema_defs/ssp/m7.py | 1 + .../tdpservice/parsers/schema_defs/tanf/t1.py | 4 +++ .../tdpservice/parsers/schema_defs/tanf/t2.py | 10 ++++-- .../tdpservice/parsers/schema_defs/tanf/t3.py | 36 ++++++++++++------- .../tdpservice/parsers/schema_defs/tanf/t4.py | 4 +++ .../tdpservice/parsers/schema_defs/tanf/t5.py | 9 +++-- .../tdpservice/parsers/schema_defs/tanf/t6.py | 5 +++ .../tdpservice/parsers/schema_defs/tanf/t7.py | 1 + .../parsers/schema_defs/tribal_tanf/t1.py | 4 +++ .../parsers/schema_defs/tribal_tanf/t2.py | 9 +++-- .../parsers/schema_defs/tribal_tanf/t4.py | 4 +++ .../parsers/schema_defs/tribal_tanf/t5.py | 9 +++-- .../parsers/schema_defs/tribal_tanf/t6.py | 5 +++ .../parsers/schema_defs/tribal_tanf/t7.py | 1 + 20 files changed, 102 insertions(+), 21 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index e6465ae27..294a71d44 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -12,6 +12,10 @@ document=SSP_M1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(150), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 352b6d3fe..e68d8b8a1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -14,6 +14,10 @@ document=SSP_M2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(150), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 039b7b938..132e27e8a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -11,6 +11,10 @@ document=SSP_M3DataSubmissionDocument(), preparsing_validators=[ validators.notEmpty(start=19, end=60), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 89f916984..cc86e033b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -12,6 +12,10 @@ document=SSP_M4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(66), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 35f5f0121..f3fa86e95 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -14,6 +14,10 @@ document=SSP_M5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(66), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index f7b686caa..bff13272d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -11,6 +11,7 @@ document=SSP_M6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(259), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index b2d31e994..af1b2c015 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -25,6 +25,7 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 946e3e692..de181f04d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -12,6 +12,10 @@ document=TANF_T1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(156), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 36fba6621..44d2d4fcf 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -14,6 +14,10 @@ document=TANF_T2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(156), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -122,6 +126,7 @@ result_field="WORK_PART_STATUS", result_function=validators.notMatches("99"), ), + validators.validate__WORK_ELIGIBLE_INDICATOR__HOH__AGE(), ], fields=[ Field( @@ -185,10 +190,11 @@ startIndex=21, endIndex=29, required=True, - validators=[ + validators=[validators.intHasLength(8), validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(), - ], + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index b2212d5d3..502695148 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -12,6 +12,10 @@ document=TANF_T3DataSubmissionDocument(), preparsing_validators=[ validators.notEmpty(start=19, end=60), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -137,10 +141,11 @@ startIndex=20, endIndex=28, required=True, - validators=[ - validators.dateYearIsLargerThan(1950), - validators.dateMonthIsValid(), - ], + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ], ), TransformField( transform_func=tanf_ssn_decryption_func, @@ -317,6 +322,10 @@ quiet_preparser_errors=True, preparsing_validators=[ validators.notEmpty(start=60, end=101), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -442,10 +451,11 @@ startIndex=61, endIndex=69, required=True, - validators=[ - validators.dateYearIsLargerThan(1950), + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(), - ], + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, @@ -462,7 +472,7 @@ Field( item="70A", name="RACE_HISPANIC", - friendly_name="Ethnicity: Hispanic or Latino", + friendly_name="Ethnicity/Race: Hispanic or Latino", type="number", startIndex=78, endIndex=79, @@ -472,7 +482,7 @@ Field( item="70B", name="RACE_AMER_INDIAN", - friendly_name="Race: American Indian or Alaska Native", + friendly_name="Ethnicity/Race: American Indian or Alaska Native", type="number", startIndex=79, endIndex=80, @@ -482,7 +492,7 @@ Field( item="70C", name="RACE_ASIAN", - friendly_name="Race: Asian", + friendly_name="Ethnicity/Race: Asian", type="number", startIndex=80, endIndex=81, @@ -492,7 +502,7 @@ Field( item="70D", name="RACE_BLACK", - friendly_name="Race: Black or African American", + friendly_name="Ethnicity/Race: Black or African American", type="number", startIndex=81, endIndex=82, @@ -502,7 +512,7 @@ Field( item="70E", name="RACE_HAWAIIAN", - friendly_name="Race: Native Hawaiian or Other Pacific Islander", + friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", type="number", startIndex=82, endIndex=83, @@ -512,7 +522,7 @@ Field( item="70F", name="RACE_WHITE", - friendly_name="Race: White", + friendly_name="Ethnicity/Race: White", type="number", startIndex=83, endIndex=84, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 88185ab84..c14d7b870 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -13,6 +13,10 @@ document=TANF_T4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index b87a3af6a..5b5f13e70 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -14,6 +14,10 @@ document=TANF_T5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -153,10 +157,11 @@ startIndex=20, endIndex=28, required=True, - validators=[ + validators=[validators.intHasLength(8), validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(), - ], + validators.dateDayIsValid() + ], ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index a03ea79de..34fd1ba46 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -12,6 +12,7 @@ document=TANF_T6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -227,8 +228,10 @@ s2 = RowSchema( document=TANF_T6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( @@ -438,8 +441,10 @@ s3 = RowSchema( document=TANF_T6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index edf7299f2..48ec6e654 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -25,6 +25,7 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index f36411a72..18b05e50a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -12,6 +12,10 @@ document=Tribal_TANF_T1DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(122), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 0f8e3f402..48599ae0b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -14,6 +14,10 @@ document=Tribal_TANF_T2DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(122), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -174,10 +178,11 @@ startIndex=21, endIndex=29, required=True, - validators=[ + validators=[validators.intHasLength(8), validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(), - ], + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 4209c1fa3..9e41d3343 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -12,6 +12,10 @@ document=Tribal_TANF_T4DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 93dc07b8f..e419c8930 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -14,6 +14,10 @@ document=Tribal_TANF_T5DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(71), + validators.or_priority_validators([ + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), + ]), validators.notEmpty(8, 19) ], postparsing_validators=[ @@ -147,10 +151,11 @@ startIndex=20, endIndex=28, required=True, - validators=[ + validators=[validators.intHasLength(8), validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(), - ], + validators.dateDayIsValid() + ], ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index b641b7983..d2baefdc0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -12,6 +12,7 @@ document=Tribal_TANF_T6DataSubmissionDocument(), preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -215,8 +216,10 @@ s2 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -414,8 +417,10 @@ s3 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.hasLength(379), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 67b34aa63..61636ef47 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -25,6 +25,7 @@ validators.hasLength(247), validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), + validators.field_year_month_with_header_year_quarter(), ], postparsing_validators=[], fields=[ From 1ea3c8a2abd1c993d9e693fcbb4fb894b90596f9 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 8 Apr 2024 09:58:19 -0600 Subject: [PATCH 003/105] - update incorrect name --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 4209c1fa3..cc19dd1d2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -145,7 +145,7 @@ ), Field( item="14", - name="BLANK", + name="FAMILY_AFFILIATION", friendly_name="Family Affiliation:", type="string", startIndex=36, From 84e0965e2984dec76c4a89ab01572fa47eda222c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 8 Apr 2024 10:42:18 -0600 Subject: [PATCH 004/105] - Fix merge conflicts and update tests --- tdrs-backend/tdpservice/data_files/test/test_api.py | 4 ++-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 4 ++-- .../tdpservice/parsers/schema_defs/ssp/m2.py | 6 +++--- .../tdpservice/parsers/schema_defs/tanf/t2.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t3.py | 4 ++-- .../tdpservice/parsers/schema_defs/tanf/t5.py | 2 +- .../tdpservice/parsers/schema_defs/tribal_tanf/t2.py | 2 +- .../tdpservice/parsers/schema_defs/tribal_tanf/t5.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_parse.py | 12 ++++++------ tdrs-backend/tdpservice/settings/common.py | 2 +- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index bfcc3f2c5..b61b848e5 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -102,8 +102,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1: 0 is not larger than 0." + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if Cash and Cash Equivalents: Amount :873 " + \ + "validator1 passed then Cash and Cash Equivalents: Number of Months T1: 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index c1260a172..62f56f578 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -12,12 +12,12 @@ record_type="M1", document=SSP_M1DataSubmissionDocument(), preparsing_validators=[ - validators.hasLength(150), + validators.recordHasLength(150), + validators.caseNumberNotEmpty(8, 19), validators.or_priority_validators([ validators.field_year_month_with_header_year_quarter(), validators.validateRptMonthYear(), ]), - validators.notEmpty(8, 19) ], postparsing_validators=[ validators.if_then_validator( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 8926d7698..2f5ead8ca 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -14,12 +14,12 @@ record_type="M2", document=SSP_M2DataSubmissionDocument(), preparsing_validators=[ - validators.hasLength(150), + validators.recordHasLength(150), + validators.caseNumberNotEmpty(8, 19), validators.or_priority_validators([ validators.field_year_month_with_header_year_quarter(), validators.validateRptMonthYear(), ]), - validators.notEmpty(8, 19) ], postparsing_validators=[ validators.validate__FAM_AFF__SSN(), @@ -184,7 +184,7 @@ item="28", name='DATE_OF_BIRTH', friendly_name="Date of Birth", - type='number', + type='string', startIndex=21, endIndex=29, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 78bb268d0..6c24fd251 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -187,7 +187,7 @@ item="32", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=21, endIndex=29, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 29a1ea66c..0875b3911 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -138,7 +138,7 @@ item="68", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=20, endIndex=28, required=True, @@ -449,7 +449,7 @@ item="68", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=61, endIndex=69, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index d756df167..de691c34d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -154,7 +154,7 @@ item="15", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=20, endIndex=28, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 19a2f0b01..b19c4914a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -175,7 +175,7 @@ item="32", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=21, endIndex=29, required=True, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 294cabae2..065918b5c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -148,7 +148,7 @@ item="15", name="DATE_OF_BIRTH", friendly_name="Date of Birth", - type="number", + type="string", startIndex=20, endIndex=28, required=True, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 5177c306a..cbe87af90 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -183,9 +183,9 @@ def test_parse_big_file(test_big_file, dfs): dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'months': [ - {'month': 'Oct', 'accepted_without_errors': 25, 'accepted_with_errors': 245}, - {'month': 'Nov', 'accepted_without_errors': 18, 'accepted_with_errors': 255}, - {'month': 'Dec', 'accepted_without_errors': 27, 'accepted_with_errors': 245}], + {'month': 'Oct', 'accepted_without_errors': 129, 'accepted_with_errors': 141}, + {'month': 'Nov', 'accepted_without_errors': 143, 'accepted_with_errors': 130}, + {'month': 'Dec', 'accepted_without_errors': 131, 'accepted_with_errors': 141}], 'rejected': 0} assert TANF_T1.objects.count() == expected_t1_record_count @@ -616,11 +616,11 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): parse.parse_datafile(small_tanf_section1_datafile, dfs) dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.status == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'months': [ - {'month': 'Oct', 'accepted_without_errors': 4, 'accepted_with_errors': 1}, + {'month': 'Oct', 'accepted_without_errors': 5, 'accepted_with_errors': 0}, {'month': 'Nov', 'accepted_without_errors': 0, 'accepted_with_errors': 0}, {'month': 'Dec', 'accepted_without_errors': 0, 'accepted_with_errors': 0}], 'rejected': 0} @@ -843,7 +843,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel parse.parse_datafile(bad_ssp_s1__row_missing_required_field, dfs) parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) - assert parser_errors.count() == 7 + assert parser_errors.count() == 6 row_2_error = parser_errors.get( row_number=2, diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index ec6b75c59..0407b2cf0 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -194,7 +194,7 @@ class Common(Configuration): # Logging # set level as 'INFO' if env var is not set - LOGGING_LEVEL = os.getenv('LOGGING_LEVEL', 'INFO') + LOGGING_LEVEL = 'WARNING'#os.getenv('LOGGING_LEVEL', 'INFO') LOGGING = { "version": 1, "disable_existing_loggers": False, From 1a27754c9beb8071d8f59ab8c994d8427647416a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 8 Apr 2024 10:59:22 -0600 Subject: [PATCH 005/105] - fix lint --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 4 +- .../tdpservice/parsers/schema_defs/ssp/m2.py | 31 +++++++++------ .../tdpservice/parsers/schema_defs/ssp/m3.py | 10 +++-- .../tdpservice/parsers/schema_defs/ssp/m4.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t2.py | 38 ++++++++++++------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 8 ++-- .../tdpservice/parsers/schema_defs/tanf/t5.py | 8 ++-- .../parsers/schema_defs/tribal_tanf/t2.py | 17 +++++---- .../parsers/schema_defs/tribal_tanf/t5.py | 8 ++-- 9 files changed, 76 insertions(+), 52 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 62f56f578..f7569a9b1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -15,8 +15,8 @@ validators.recordHasLength(150), validators.caseNumberNotEmpty(8, 19), validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), ]), ], postparsing_validators=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 2f5ead8ca..82702565a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -17,8 +17,8 @@ validators.recordHasLength(150), validators.caseNumberNotEmpty(8, 19), validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), ]), ], postparsing_validators=[ @@ -625,7 +625,8 @@ Field( item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=102, endIndex=104, @@ -635,7 +636,8 @@ Field( item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=104, endIndex=106, @@ -645,7 +647,8 @@ Field( item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Holidays", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate of High School Equivalency: Hours of Holidays", type='number', startIndex=106, endIndex=108, @@ -655,7 +658,8 @@ Field( item="53A", name='SCHOOL_ATTENDENCE_HOP', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma " + + "or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=108, endIndex=110, @@ -665,7 +669,8 @@ Field( item="53B", name='SCHOOL_ATTENDENCE_EA', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or " + + "Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=110, endIndex=112, @@ -675,7 +680,8 @@ Field( item="53C", name='SCHOOL_ATTENDENCE_HOL', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Holidays", + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma " + + "or Certificate: Hours of Holidays", type='number', startIndex=112, endIndex=114, @@ -685,7 +691,8 @@ Field( item="54A", name='PROVIDE_CC_HOP', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Participation", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + + "Community Service Program: Hours of Participation", type='number', startIndex=114, endIndex=116, @@ -695,7 +702,8 @@ Field( item="54B", name='PROVIDE_CC_EA', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Excused Absences", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + + "Community Service Program: Hours of Excused Absences", type='number', startIndex=116, endIndex=118, @@ -705,7 +713,8 @@ Field( item="54C", name='PROVIDE_CC_HOL', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Holidays", + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + + "Community Service Program: Hours of Holidays", type='number', startIndex=118, endIndex=120, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index af6d56d7c..df45b998a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,8 +13,8 @@ preparsing_validators=[ validators.notEmpty(start=19, end=60), validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), ]), validators.notEmpty(8, 19) ], @@ -244,7 +244,8 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or " + + "Aged, Blind, and Disabled Under Title XVI-AABD", type='number', startIndex=45, endIndex=46, @@ -557,7 +558,8 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or " + + "Aged, Blind, and Disabled Under Title XVI-AABD", type='number', startIndex=86, endIndex=87, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 613b28193..7fca57786 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -15,8 +15,8 @@ validators.recordHasLength(66), validators.caseNumberNotEmpty(8, 19), validators.or_priority_validators([ - validators.field_year_month_with_header_year_quarter(), - validators.validateRptMonthYear(), + validators.field_year_month_with_header_year_quarter(), + validators.validateRptMonthYear(), ]), ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 6c24fd251..e53e87b42 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -192,10 +192,10 @@ endIndex=29, required=True, validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, @@ -319,7 +319,8 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under Title XVI-AABD ", + friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under " + + "Title XVI-AABD ", type="number", startIndex=48, endIndex=49, @@ -745,7 +746,8 @@ Field( item="59A", name="ED_NO_HIGH_SCHOOL_DIPL_HOP", - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate of High School Equivalency: Hours of Participation", type="string", startIndex=108, endIndex=110, @@ -757,7 +759,8 @@ Field( item="59B", name="ED_NO_HIGH_SCHOOL_DIPL_EA", - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate: Hours of Excused Absences", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate: Hours of Excused Absences", type="string", startIndex=110, endIndex=112, @@ -769,7 +772,8 @@ Field( item="59C", name="ED_NO_HIGH_SCHOOL_DIPL_HOL", - friendly_name="Education Directly Related to Employment for an Individual with NO High School Diploma or Certificate: Hours of Holidays", + friendly_name="Education Directly Related to Employment for an Individual with NO High " + + "School Diploma or Certificate: Hours of Holidays", type="string", startIndex=112, endIndex=114, @@ -781,7 +785,8 @@ Field( item="60A", name="SCHOOL_ATTENDENCE_HOP", - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Participation", + friendly_name="Satisfactory School Attendance for Individuals with No High School " + + "Diploma or Certificate: Hours of Participation", type="string", startIndex=114, endIndex=116, @@ -793,7 +798,8 @@ Field( item="60B", name="SCHOOL_ATTENDENCE_EA", - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Ecused Absences", + friendly_name="Satisfactory School Attendance for Individuals with No High School " + + "Diploma or Certificate: Hours of Ecused Absences", type="string", startIndex=116, endIndex=118, @@ -805,7 +811,8 @@ Field( item="60C", name="SCHOOL_ATTENDENCE_HOL", - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or Certificate: Hours of Holidays", + friendly_name="Satisfactory School Attendance for Individuals with No High School " + + "Diploma or Certificate: Hours of Holidays", type="string", startIndex=118, endIndex=120, @@ -817,7 +824,8 @@ Field( item="61A", name="PROVIDE_CC_HOP", - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Participation", + friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + "in a Community Service Program: Hours of Participation", type="string", startIndex=120, endIndex=122, @@ -829,7 +837,8 @@ Field( item="61B", name="PROVIDE_CC_EA", - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Excused Absences", + friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + "in a Community Service Program: Hours of Excused Absences", type="string", startIndex=122, endIndex=124, @@ -841,7 +850,8 @@ Field( item="61C", name="PROVIDE_CC_HOL", - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program: Hours of Holidays", + friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + "in a Community Service Program: Hours of Holidays", type="string", startIndex=124, endIndex=126, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 0875b3911..561ffde95 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -454,10 +454,10 @@ endIndex=69, required=True, validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index de691c34d..f6fdd9d5a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -159,10 +159,10 @@ endIndex=28, required=True, validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ], + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ], ), TransformField( transform_func=tanf_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index b19c4914a..52def22f2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -180,10 +180,10 @@ endIndex=29, required=True, validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ] + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ] ), TransformField( transform_func=tanf_ssn_decryption_func, @@ -588,7 +588,8 @@ Field( item="58", name="ED_NO_HIGH_SCHOOL_DIPLOMA", - friendly_name="Education Directly Related to Employment for Individualswith no High School Diploma or Certificate of High SchoolEquivalency", + friendly_name="Education Directly Related to Employment for Individuals with no High " + + "School Diploma or Certificate of High SchoolEquivalency", type="string", startIndex=86, endIndex=88, @@ -600,7 +601,8 @@ Field( item="59", name="SCHOOL_ATTENDENCE", - friendly_name="Satisfactory School Attendance for Individuals with No HighSchool Diploma or Certificate of High School Equivalency", + friendly_name="Satisfactory School Attendance for Individuals with No High School " + + "Diploma or Certificate of High School Equivalency", type="string", startIndex=88, endIndex=90, @@ -612,7 +614,8 @@ Field( item="60", name="PROVIDE_CC", - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a Community Service Program", + friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + "in a Community Service Program", type="string", startIndex=90, endIndex=92, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 065918b5c..9d4ce11df 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -153,10 +153,10 @@ endIndex=28, required=True, validators=[validators.intHasLength(8), - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - validators.dateDayIsValid() - ], + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ], ), TransformField( transform_func=tanf_ssn_decryption_func, From 0fe6d97ad0d2542ac6c795b7d18a0571d15571dd Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 12 Apr 2024 11:59:51 -0600 Subject: [PATCH 006/105] - Re-added validators --- .../tdpservice/parsers/schema_defs/ssp/m2.py | 5 ++++- .../tdpservice/parsers/schema_defs/ssp/m3.py | 18 ++++++++++-------- .../tdpservice/parsers/schema_defs/ssp/m5.py | 9 +++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 82702565a..9659c5f8a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -188,7 +188,10 @@ startIndex=21, endIndex=29, required=True, - validators=[validators.isLargerThan(0)] + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid()] ), TransformField( transform_func=ssp_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index df45b998a..a67b904c7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -144,10 +144,11 @@ startIndex=20, endIndex=28, required=True, - validators=[ - validators.dateYearIsLargerThan(1998), - validators.dateMonthIsValid(), - ] + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ] ), TransformField( transform_func=ssp_ssn_decryption_func, @@ -458,10 +459,11 @@ startIndex=61, endIndex=69, required=True, - validators=[ - validators.dateYearIsLargerThan(1998), - validators.dateMonthIsValid(), - ] + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ] ), TransformField( transform_func=ssp_ssn_decryption_func, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index c7ee96882..aa6fd467f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -157,10 +157,11 @@ startIndex=20, endIndex=28, required=True, - validators=[ - validators.dateYearIsLargerThan(1900), - validators.dateMonthIsValid(), - ], + validators=[validators.intHasLength(8), + validators.dateYearIsLargerThan(1900), + validators.dateMonthIsValid(), + validators.dateDayIsValid() + ], ), TransformField( transform_func=ssp_ssn_decryption_func, From 8cd6068f463260e3567f59717ed13965645a1cfa Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 18 Apr 2024 07:54:03 -0600 Subject: [PATCH 007/105] - remvoe trailing colon --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index dd0292c1e..17107db03 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -151,7 +151,7 @@ Field( item="14", name="FAMILY_AFFILIATION", - friendly_name="Family Affiliation:", + friendly_name="Family Affiliation", type="string", startIndex=36, endIndex=71, From 6e983aad390532812c134d22b6dbc6ed295d2bc8 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Wed, 24 Apr 2024 12:41:42 -0700 Subject: [PATCH 008/105] Updates to the friendly names added code no longer in use where indicated and text edits --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py | 4 ++-- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 4 ++-- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 8 ++++---- .../tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 4 ++-- .../tdpservice/parsers/schema_defs/tribal_tanf/t5.py | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index f7569a9b1..43a981f99 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -246,7 +246,7 @@ Field( item="15", name='RECEIVES_SUB_CC', - friendly_name="Receives Subsidized Child Care:", + friendly_name="Receives Subsidized Child Care: Code no longer in use", type='number', startIndex=41, endIndex=42, @@ -516,7 +516,7 @@ Field( item="25", name='WAIVER_EVAL_CONTROL_GRPS', - friendly_name="Waiver Evaluation Experimental and Control Groups", + friendly_name="Waiver Evaluation Experimental and Control Groups: Code no longer in use", type='number', startIndex=112, endIndex=113, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 9659c5f8a..a5a29dec3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -787,7 +787,7 @@ Field( item="59C", name='UNEARNED_SSI', - friendly_name="Amount of Unearned Income: Social Security: SSI Benefit", + friendly_name="Amount of Unearned Income: SSI Benefit", type='number', startIndex=138, endIndex=142, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index aa6fd467f..c546abdf3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -388,7 +388,7 @@ Field( item="27", name="AMOUNT_UNEARNED_INCOME", - friendly_name="Amount of Earned Income", + friendly_name="Amount of Unearned Income", type="string", startIndex=62, endIndex=66, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 444bdbbc3..be641cb8c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -127,7 +127,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -651,7 +651,7 @@ Field( item="29", name="FAMILY_NEW_CHILD", - friendly_name="TANF Family a New Child-Only Family", + friendly_name="TANF Family a New Child-Only Family: Code no longer in use", type="number", startIndex=116, endIndex=117, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index e53e87b42..94da2d34d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -319,7 +319,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under " + + friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under: Code no longer in use" + "Title XVI-AABD ", type="number", startIndex=48, @@ -441,7 +441,7 @@ Field( item="45", name="MONTHS_STATE_TIME_LIMIT", - friendly_name="Number of Countable Months Remaining Under State's Time Limit", + friendly_name="Number of Countable Months Remaining Under State’s Time Limit: Code no longer in use", type="string", startIndex=62, endIndex=64, @@ -863,7 +863,7 @@ Field( item="62", name="OTHER_WORK_ACTIVITIES", - friendly_name="Hours of Other Work Activities ", + friendly_name="Hours of Other Work Activities", type="string", startIndex=126, endIndex=128, @@ -911,7 +911,7 @@ Field( item="66A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", + friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC): Code no longer in use", type="string", startIndex=136, endIndex=140, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 17107db03..eecfd3ce6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -77,7 +77,7 @@ Field( item="7", name="ZIP_CODE", - friendly_name="Zip Code", + friendly_name="ZIP Code", type="string", startIndex=24, endIndex=29, @@ -121,7 +121,7 @@ Field( item="11", name="REC_MED_ASSIST", - friendly_name="Received Medical Assistance:", + friendly_name="Received Medical Assistance", type="number", startIndex=33, endIndex=34, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 9d4ce11df..c90db77e5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -243,7 +243,7 @@ Field( item="19A", name="REC_OASDI_INSURANCE", - friendly_name="Receives Disability Benefits: OASDI Program", + friendly_name="Received Disability Benefits: OASDI Program", type="number", startIndex=44, endIndex=45, @@ -253,7 +253,7 @@ Field( item="19B", name="REC_FEDERAL_DISABILITY", - friendly_name="Received Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Federal Disability Status", type="number", startIndex=45, endIndex=46, From 025c4e79b98cfe9eab121838c556929a9f2b12be Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Wed, 24 Apr 2024 18:23:03 -0700 Subject: [PATCH 009/105] Update t2.py address linting errors to make less than 120 characters --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 94da2d34d..0dbb1cd00 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -319,8 +319,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under: Code no longer in use" + - "Title XVI-AABD ", + friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under: Code no longer in use", type="number", startIndex=48, endIndex=49, @@ -441,7 +440,7 @@ Field( item="45", name="MONTHS_STATE_TIME_LIMIT", - friendly_name="Number of Countable Months Remaining Under State’s Time Limit: Code no longer in use", + friendly_name="Months Remaining Under State's Time Limit: Code no longer in use", type="string", startIndex=62, endIndex=64, From 300ffb66484fc9d6be6a62e0ac83e7b7b6ada611 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Thu, 25 Apr 2024 08:32:47 -0700 Subject: [PATCH 010/105] t2.py linting fixes addressing character count --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 0dbb1cd00..5ae06e9ed 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -319,7 +319,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Aid to the Aged, Blind, and Disabled Under: Code no longer in use", + friendly_name="Receives Disability Benefits: Code no longer in use", type="number", startIndex=48, endIndex=49, From dbb1e23b2042ac9413bd04c81530db6e65df08a0 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:17:52 -0400 Subject: [PATCH 011/105] clean option string util --- .../tdpservice/parsers/test/test_util.py | 16 +++++++++++++++- tdrs-backend/tdpservice/parsers/util.py | 6 ++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index cd50f9c4a..54fbc73b5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -3,7 +3,7 @@ import pytest from ..fields import Field from ..row_schema import RowSchema, SchemaManager -from ..util import make_generate_parser_error, create_test_datafile +from ..util import make_generate_parser_error, create_test_datafile, clean_options_string def passing_validator(): @@ -525,3 +525,17 @@ def postparse_validator(): assert is_valid is False assert errors[0].fields_json == {'friendly_name': {'FIRST': 'first', 'SECOND': 'second'}} assert errors[0].error_message == "an Error" + + +@pytest.mark.parametrize('options, expected', [ + ([1, 2, 3, 4], '[1, 2, 3, 4]'), + (['1', '2', '3', '4'], '[1, 2, 3, 4]'), + (['a', 'b', 'c', 'd'], '[a, b, c, d]'), + (('a', 'b', 'c', 'd'), '[a, b, c, d]'), + (["'a'", "'b'", "'c'", "'d'"], "['a', 'b', 'c', 'd']"), + (['words', 'are very', 'weird'], '[words, are very, weird]'), +]) +def test_clean_options_string(options, expected): + """Test `clean_options_string` util func.""" + result = clean_options_string(options) + assert result == expected diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 3a5528391..bcb163c6f 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -92,6 +92,12 @@ def contains_encrypted_indicator(line, encryption_field): return False +def clean_options_string(options, remove=['\'', '"', ' ']): + """Return a prettied-up version of an options array.""" + options_str = ', '.join(str(o) for o in options) + return f'[{options_str}]' + + ''' text -> section YES text -> models{} YES From c45c6cbddeb0bf6254cbf7a9397c1c82c00c6310 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:18:09 -0400 Subject: [PATCH 012/105] add friendly name and item num to cat 2 --- .../tdpservice/data_files/test/test_api.py | 10 +- .../tdpservice/parsers/test/test_parse.py | 34 ++--- .../parsers/test/test_validators.py | 90 +++++++----- tdrs-backend/tdpservice/parsers/validators.py | 132 +++++++++--------- 4 files changed, 148 insertions(+), 118 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index bfcc3f2c5..62fb7dad4 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -103,7 +103,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1: 0 is not larger than 0." + + " then number of months T1 Item -1 number of months: 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -114,7 +114,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod @@ -134,8 +134,10 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1: 0 is not larger than 0.") + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ( + "if CASH_AMOUNT :873 validator1 passed then " + "NBR_MONTHS T1 Item -1 NBR_MONTHS: 0 is not larger than 0." + ) @staticmethod def assert_data_file_exists(data_file_data, version, user): diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 1de564969..d3ae3f829 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -232,7 +232,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -261,7 +261,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -343,7 +343,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -354,7 +354,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER record length is 11 characters but must be 23.', + 'TRAILER: record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -368,7 +368,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -400,7 +400,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -412,7 +412,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_2_error.error_message == 'T1 record length is 117 characters but must be 156.' + assert row_2_error.error_message == 'T1: record length is 117 characters but must be 156.' assert row_2_error.content_type is None assert row_2_error.object_id is None @@ -424,10 +424,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1 record length is 7 characters but must be 156.', + 'T1: record length is 7 characters but must be 156.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER record length is 7 characters but must be 23.', + 'TRAILER: record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -446,7 +446,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None @@ -509,7 +509,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -590,7 +590,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == 'M1: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert err.error_message == 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' assert err.content_type is not None assert err.object_id is not None assert parser_errors.count() == 32486 @@ -884,7 +884,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER record length is 15 characters but must be 23.' + error_message='TRAILER: record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1571,9 +1571,9 @@ def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): year_error = parser_errors[1] digits_error = parser_errors[0] - assert month_error.error_message == "T2: $9 is not a valid month." - assert year_error.error_message == "T2: Year Q897 must be larger than 1900." - assert digits_error.error_message == "T2: Q897$9 3 does not have exactly 8 digits." + assert month_error.error_message == "T2 Item 32 date of birth: $9 is not a valid month." + assert year_error.error_message == "T2 Item 32 date of birth: Year Q897 must be larger than 1900." + assert digits_error.error_message == "T2 Item 32 date of birth: Q897$9 3 does not have exactly 8 digits." @pytest.mark.django_db @@ -1703,7 +1703,7 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file error = parser_errors.first() assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_message == "HEADER update indicator: U does not match D." + assert error.error_message == "HEADER Item 10 update indicator: U does not match D." @pytest.fixture def tribal_section_4_bad_quarter(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 8f77292f5..76a9a6cf7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -48,8 +48,8 @@ def test_or_validators(): assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("5", RowSchema(), "friendly_name", "item_no") == (False, - "T1 friendly_name: 5 does not match 2. or " - "T1 friendly_name: 5 does not " + "T1 Item item_no friendly_name: 5 does not match 2. or " + "T1 Item item_no friendly_name: 5 does not " "match 3.") validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), @@ -64,16 +64,16 @@ def test_or_validators(): value = "5" assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - 'T1 friendly_name: 5 does not match 2. or ' - 'T1 friendly_name: 5 does not match 3. or ' - 'T1 friendly_name: 5 does not match 4.') + 'T1 Item item_no friendly_name: 5 does not match 2. or ' + 'T1 Item item_no friendly_name: 5 does not match 3. or ' + 'T1 Item item_no friendly_name: 5 does not match 4.') validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, - "T1 friendly_name: 1 does not match 2. " - "or T1 friendly_name: 1 does not " - "match 3. or T1: 1 is not larger than 4.") + "T1 Item item_no friendly_name: 1 does not match 2. " + "or T1 Item item_no friendly_name: 1 does not " + "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4.") def test_if_validators(): """Test `if_then_validator` gives a valid result.""" @@ -89,14 +89,14 @@ def test_if_validators(): result_field_name="Field2", result_function=validators.matches('1'), ) result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Field2: 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 Field2: 2 does not match 1.', ['Field1', 'Field2']) def test_and_validators(): """Test `and_validators` gives a valid result.""" validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1: 1 is not larger than 2.') + assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1 Item item_no friendly_name: 1 is not larger than 2.') assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -133,7 +133,7 @@ def test_quarterIsValid(value, valid): val = validators.quarterIsValid() result = val(value, RowSchema(), "friendly_name", "item_no") - errorText = None if valid else f"T1: {value[-1:]} is not a valid quarter." + errorText = None if valid else f"T1 Item item_no friendly_name: {value[-1:]} is not a valid quarter." assert result == (valid, errorText) def test_validateSSN(): @@ -146,7 +146,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: {value} is in {options}.") + assert result == (False, f"T1 Item item_no friendly_name: {value} is in {options}.") def test_validateRace(): """Test `validateRace`.""" @@ -157,7 +157,7 @@ def test_validateRace(): value = 3 result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: {value} is not greater than or equal to 0 or smaller than or equal to 2.") + assert result == (False, f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2.") def test_validateRptMonthYear(): """Test `validateRptMonthYear`.""" @@ -168,17 +168,17 @@ def test_validateRptMonthYear(): value = "T1 " result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") value = "T1189912" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") value = "T1202013" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") def test_matches_returns_valid(): @@ -200,7 +200,7 @@ def test_matches_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 friendly_name: TEST does not match test.' + assert error == 'T1 Item item_no friendly_name: TEST does not match test.' def test_oneOf_returns_valid(): @@ -224,7 +224,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 64 is not in [17, 24, 36].' + assert error == 'T1 Item item_no friendly_name: 64 is not in [17, 24, 36].' def test_between_returns_valid(): @@ -257,7 +257,31 @@ def test_between_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 47 is not between 48 and 400.' + assert error == 'T1 Item item_no friendly_name: 47 is not between 48 and 400.' + + +@pytest.mark.parametrize('value, expected_is_valid, expected_error', [ + (7, True, None), + (77731, True, None), + ('7', True, None), + ('234897', True, None), + ('a', False, 'T1 Item item_no friendly_name: a is not a number.'), + ('houston, we have a problem', False, 'T1 Item item_no friendly_name: houston, we have a problem is not a number.'), + (' test', False, 'T1 Item item_no friendly_name: test is not a number.'), + (' 7 ', True, None), + (' 8388323', True, None), + ('87932875 ', True, None), + (' 00 ', True, None), + (' 88 ', True, None), + (' 088 ', True, None), + (' 8 8 ', False, 'T1 Item item_no friendly_name: 8 8 is not a number.'), +]) +def test_isNumber(value, expected_is_valid, expected_error): + """Test `isNumber` validator.""" + validator = validators.isNumber() + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + assert is_valid == expected_is_valid + assert error == expected_error def test_date_month_is_valid_returns_valid(): @@ -275,7 +299,7 @@ def test_date_month_is_valid_returns_invalid(): validator = validators.dateMonthIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 13 is not a valid month.' + assert error == 'T1 Item item_no friendly_name: 13 is not a valid month.' def test_date_day_is_valid_returns_valid(): @@ -293,7 +317,7 @@ def test_date_day_is_valid_returns_invalid(): validator = validators.dateDayIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 32 is not a valid day.' + assert error == 'T1 Item item_no friendly_name: 32 is not a valid day.' def test_olderThan(): @@ -305,7 +329,7 @@ def test_olderThan(): value = 20240101 result = validator(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, (f"T1: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " + assert result == (False, (f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " "to meet the minimum age requirement.")) @@ -318,7 +342,7 @@ def test_dateYearIsLargerThan(): value = 18990101 assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - f"T1: Year {str(value)[:4]} must be larger " + f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger " f"than {year}.") @@ -330,7 +354,7 @@ def test_between_returns_invalid_for_string_value(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 047 is not between 100 and 400.' + assert error == 'T1 Item item_no friendly_name: 047 is not between 100 and 400.' def test_recordHasLength_returns_valid(): @@ -352,7 +376,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 record length is 7 characters but must be 22.' + assert error == 'T1 Item item_no friendly_name: record length is 7 characters but must be 22.' def test_intHasLength_returns_valid(): @@ -374,7 +398,7 @@ def test_intHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 1a3 does not have exactly 22 digits.' + assert error == 'T1 Item item_no friendly_name: 1a3 does not have exactly 22 digits.' def test_contains_returns_valid(): @@ -396,7 +420,7 @@ def test_contains_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 12345abcde does not contain 6789.' + assert error == 'T1 Item item_no friendly_name: 12345abcde does not contain 6789.' def test_startsWith_returns_valid(): @@ -418,7 +442,7 @@ def test_startsWith_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 12345abcde does not start with abc.' + assert error == 'T1 Item item_no friendly_name: 12345abcde does not start with abc.' def test_notEmpty_returns_valid_full_string(): @@ -440,7 +464,7 @@ def test_notEmpty_returns_invalid_full_string(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: contains blanks between positions 0 and 9.' + assert error == 'T1 Item item_no friendly_name: contains blanks between positions 0 and 9.' def test_notEmpty_returns_valid_substring(): @@ -462,7 +486,7 @@ def test_notEmpty_returns_invalid_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1: 111 333 contains blanks between positions 3 and 5." + assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 3 and 5." def test_notEmpty_returns_nonexistent_substring(): @@ -473,7 +497,7 @@ def test_notEmpty_returns_nonexistent_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1: 111 333 contains blanks between positions 10 and 12." + assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 10 and 12." @pytest.mark.parametrize("test_input", [1, 2, 3, 4]) @@ -493,7 +517,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1: {test_input} is not a valid quarter." + assert error == f"T1 Item item_no friendly_name: {test_input} is not a valid quarter." @pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"]) def test_calendarQuarterIsValid_returns_invalid(value): @@ -502,7 +526,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error_msg == f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ + assert error_msg == f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ "representing the Calendar Year and Quarter formatted as YYYYQ" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 94f44288b..f4ea25cc6 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -1,7 +1,7 @@ """Generic parser validator functions for use in schema definitions.""" from .models import ParserErrorCategoryChoices -from .util import fiscal_to_calendar, year_month_to_year_quarter +from .util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string import datetime import logging @@ -217,16 +217,53 @@ def sumIsLargerFunc(value, row_schema): return sumIsLargerFunc +def recordHasLength(length): + """Validate that value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + ) + + +def caseNumberNotEmpty(start=0, end=None): + """Validate that string value isn't only blanks.""" + return make_validator( + lambda value: not _is_empty(value, start, end), + lambda value, row_schema, + friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + ) + + +def calendarQuarterIsValid(start=0, end=None): + """Validate that the calendar quarter value is valid.""" + return make_validator( + lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 + and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " + "representing the Calendar Year and Quarter formatted as YYYYQ", + ) + + # generic validators +def format_error_context(row_schema, friendly_name, item_num): + return f'{row_schema.record_type} Item {item_num} {friendly_name}' + + def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, lambda value, row_schema, friendly_name, item_num: error_func(option) if error_func - else f"{row_schema.record_type} {friendly_name}: {value} does not match {option}.", + else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not match {option}.", ) @@ -234,7 +271,7 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." ) @@ -242,7 +279,7 @@ def oneOf(options=[]): """Validate that value does not exist in the provided options array.""" return make_validator( lambda value: value in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not in {options}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in {clean_options_string(options)}." ) @@ -250,7 +287,7 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {clean_options_string(options)}." ) @@ -259,18 +296,7 @@ def between(min, max): return make_validator( lambda value: int(value) > min and int(value) < max, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not between {min} and {max}.", - ) - - -def recordHasLength(length): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type} record length is {len(value)} characters but must be {length}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) @@ -279,7 +305,7 @@ def intHasLength(num_digits): return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} does not have exactly {num_digits} digits.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not have exactly {num_digits} digits.", ) @@ -288,7 +314,7 @@ def contains(substring): return make_validator( lambda value: value.find(substring) != -1, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} does not contain {substring}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", ) @@ -298,15 +324,15 @@ def startsWith(substring, error_func=None): lambda value: value.startswith(substring), lambda value, row_schema, friendly_name, item_num: error_func(substring) if error_func - else f"{row_schema.record_type}: {value} does not start with {substring}.", + else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not start with {substring}.", ) def isNumber(): """Validate that value can be casted to a number.""" return make_validator( - lambda value: value.isnumeric(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not a number." + lambda value: str(value).strip().isnumeric(), + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." ) @@ -314,7 +340,7 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." ) @@ -322,7 +348,7 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." ) @@ -331,7 +357,7 @@ def isInStringRange(lower, upper): return make_validator( lambda value: int(value) >= lower and int(value) <= upper, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not in range [{lower}, {upper}].", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", ) @@ -340,7 +366,7 @@ def isStringLargerThan(val): return make_validator( lambda value: int(value) > val, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {val}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", ) @@ -356,39 +382,17 @@ def notEmpty(start=0, end=None): return make_validator( lambda value: not _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: {str(value)} contains blanks between positions {start} ' + friendly_name, item_num: f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks between positions {start} ' f'and {end if end else len(str(value))}.' ) -def caseNumberNotEmpty(start=0, end=None): - """Validate that string value isn't only blanks.""" - return make_validator( - lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' - ) - - -def calendarQuarterIsValid(start=0, end=None): - """Validate that the calendar quarter value is valid.""" - return make_validator( - lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 - and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " - "representing the Calendar Year and Quarter formatted as YYYYQ", - ) - - def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{value} is not blank between positions {start} and {end if end else len(value)}.' + friendly_name, item_num: f'{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not blank between positions {start} and {end if end else len(value)}.' ) @@ -396,7 +400,7 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is zero." + lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is zero." ) @@ -405,7 +409,7 @@ def isLargerThan(LowerBound): return make_validator( lambda value: float(value) > LowerBound if value is not None else False, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {LowerBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", ) @@ -414,7 +418,7 @@ def isSmallerThan(UpperBound): return make_validator( lambda value: value < UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not smaller than {UpperBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", ) @@ -423,7 +427,7 @@ def isLargerThanOrEqualTo(LowerBound): return make_validator( lambda value: value >= LowerBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {LowerBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", ) @@ -432,7 +436,7 @@ def isSmallerThanOrEqualTo(UpperBound): return make_validator( lambda value: value <= UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not smaller than {UpperBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", ) @@ -441,7 +445,7 @@ def isInLimits(LowerBound, UpperBound): return make_validator( lambda value: value >= LowerBound and value <= UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger or equal to {LowerBound} and " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger or equal to {LowerBound} and " f"smaller or equal to {UpperBound}." ) @@ -453,7 +457,7 @@ def dateMonthIsValid(): return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[4:6]} is not a valid month.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): @@ -461,7 +465,7 @@ def dateDayIsValid(): return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[6:]} is not a valid day.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[6:]} is not a valid day.", ) @@ -470,7 +474,7 @@ def olderThan(min_age): return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, lambda value, row_schema, - friendly_name, item_num: (f"{row_schema.record_type}: {str(value)[:4]} must be less than or equal to " + friendly_name, item_num: (f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[:4]} must be less than or equal to " f"{datetime.date.today().year - min_age} to meet the minimum age requirement.") ) @@ -480,7 +484,7 @@ def dateYearIsLargerThan(year): return make_validator( lambda value: int(str(value)[:4]) > year, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: Year {str(value)[:4]} must be larger than {year}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: Year {str(value)[:4]} must be larger than {year}.", ) @@ -489,7 +493,7 @@ def quarterIsValid(): return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[-1]} is not a valid quarter.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[-1]} is not a valid quarter.", ) @@ -498,7 +502,7 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is in {options}." ) @@ -507,7 +511,7 @@ def validateRace(): return make_validator( lambda value: value >= 0 and value <= 2, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not greater than or equal to 0 " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -519,7 +523,7 @@ def validateRptMonthYear(): "06", "07", "08", "09", "10", "11", "12"}, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: The value: {value[2:8]}, does not follow the YYYYMM " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: The value: {value[2:8]}, does not follow the YYYYMM " "format for Reporting Year and Month.", ) From 7c6e05581265cec97397ea7b96af529d21759f61 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 26 Apr 2024 10:19:29 -0600 Subject: [PATCH 013/105] making a dummy change --- tdrs-backend/tdpservice/dummy.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tdrs-backend/tdpservice/dummy.txt diff --git a/tdrs-backend/tdpservice/dummy.txt b/tdrs-backend/tdpservice/dummy.txt new file mode 100644 index 000000000..121361e5c --- /dev/null +++ b/tdrs-backend/tdpservice/dummy.txt @@ -0,0 +1 @@ +this is a dummy change \ No newline at end of file From 992d503c1f2b6eef9b58d4ab9f82cd7e0978471d Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 26 Apr 2024 10:29:38 -0600 Subject: [PATCH 014/105] Revert "making a dummy change" This reverts commit 7c6e05581265cec97397ea7b96af529d21759f61. --- tdrs-backend/tdpservice/dummy.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tdrs-backend/tdpservice/dummy.txt diff --git a/tdrs-backend/tdpservice/dummy.txt b/tdrs-backend/tdpservice/dummy.txt deleted file mode 100644 index 121361e5c..000000000 --- a/tdrs-backend/tdpservice/dummy.txt +++ /dev/null @@ -1 +0,0 @@ -this is a dummy change \ No newline at end of file From a9789001e7037c13b4a1865392512118cd48da4f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:51:21 -0400 Subject: [PATCH 015/105] fix linting --- .../tdpservice/parsers/test/test_parse.py | 4 +- .../parsers/test/test_validators.py | 78 ++++++++---- tdrs-backend/tdpservice/parsers/validators.py | 116 ++++++++++-------- 3 files changed, 120 insertions(+), 78 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index d3ae3f829..8106b261e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -590,7 +590,9 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert err.error_message == ( + 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + ) assert err.content_type is not None assert err.object_id is not None assert parser_errors.count() == 32486 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 76a9a6cf7..03ba4aab7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -47,10 +47,11 @@ def test_or_validators(): validator = validators.or_validators(validators.matches(("2")), validators.matches(("3"))) assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("5", RowSchema(), "friendly_name", "item_no") == (False, - "T1 Item item_no friendly_name: 5 does not match 2. or " - "T1 Item item_no friendly_name: 5 does not " - "match 3.") + assert validator("5", RowSchema(), "friendly_name", "item_no") == ( + False, + "T1 Item item_no friendly_name: 5 does not match 2. or " + "T1 Item item_no friendly_name: 5 does not match 3." + ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), validators.matches(("4"))) @@ -63,17 +64,21 @@ def test_or_validators(): assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) value = "5" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - 'T1 Item item_no friendly_name: 5 does not match 2. or ' - 'T1 Item item_no friendly_name: 5 does not match 3. or ' - 'T1 Item item_no friendly_name: 5 does not match 4.') + assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + False, + 'T1 Item item_no friendly_name: 5 does not match 2. or ' + 'T1 Item item_no friendly_name: 5 does not match 3. or ' + 'T1 Item item_no friendly_name: 5 does not match 4.' + ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, - "T1 Item item_no friendly_name: 1 does not match 2. " - "or T1 Item item_no friendly_name: 1 does not " - "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4.") + assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + False, + "T1 Item item_no friendly_name: 1 does not match 2. " + "or T1 Item item_no friendly_name: 1 does not " + "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4." + ) def test_if_validators(): """Test `if_then_validator` gives a valid result.""" @@ -96,7 +101,10 @@ def test_if_validators(): def test_and_validators(): """Test `and_validators` gives a valid result.""" validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1 Item item_no friendly_name: 1 is not larger than 2.') + assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + False, + 'T1 Item item_no friendly_name: 1 is not larger than 2.' + ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -157,7 +165,10 @@ def test_validateRace(): value = 3 result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2.") + assert result == ( + False, + f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2." + ) def test_validateRptMonthYear(): """Test `validateRptMonthYear`.""" @@ -168,18 +179,27 @@ def test_validateRptMonthYear(): value = "T1 " result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not " + "follow the YYYYMM format for Reporting Year and Month." + ) value = "T1189912" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow " + "the YYYYMM format for Reporting Year and Month." + ) value = "T1202013" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does " + "not follow the YYYYMM format for Reporting Year and Month." + ) def test_matches_returns_valid(): """Test `matches` gives a valid result.""" @@ -329,8 +349,11 @@ def test_olderThan(): value = 20240101 result = validator(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, (f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " - "to meet the minimum age requirement.")) + assert result == ( + False, + f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to " + f"{date.today().year - min_age} to meet the minimum age requirement." + ) def test_dateYearIsLargerThan(): @@ -341,9 +364,10 @@ def test_dateYearIsLargerThan(): assert validator(value) == (True, None) value = 18990101 - assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger " - f"than {year}.") + assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + False, + f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger than {year}." + ) def test_between_returns_invalid_for_string_value(): @@ -526,8 +550,10 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error_msg == f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ + assert error_msg == ( + f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" + ) @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index f4ea25cc6..fc17998c4 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -254,6 +254,7 @@ def calendarQuarterIsValid(start=0, end=None): def format_error_context(row_schema, friendly_name, item_num): + """Format the error message for consistency across cat2 validators.""" return f'{row_schema.record_type} Item {item_num} {friendly_name}' @@ -271,7 +272,8 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." ) @@ -279,7 +281,9 @@ def oneOf(options=[]): """Validate that value does not exist in the provided options array.""" return make_validator( lambda value: value in options, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in {clean_options_string(options)}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} is not in {clean_options_string(options)}." ) @@ -287,7 +291,9 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {clean_options_string(options)}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} is in {clean_options_string(options)}." ) @@ -295,8 +301,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) @@ -304,8 +310,9 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not have exactly {num_digits} digits.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} does not have exactly {num_digits} digits.", ) @@ -313,8 +320,8 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", ) @@ -332,7 +339,8 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." ) @@ -340,7 +348,8 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." ) @@ -348,7 +357,8 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." ) @@ -356,8 +366,8 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", ) @@ -365,8 +375,8 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", ) @@ -381,9 +391,9 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks between positions {start} ' - f'and {end if end else len(str(value))}.' + lambda value, row_schema, friendly_name, item_num: + f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks ' + f'between positions {start} and {end if end else len(str(value))}.' ) @@ -391,8 +401,9 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not blank between positions {start} and {end if end else len(value)}.' + lambda value, row_schema, friendly_name, item_num: + f'{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank ' + f'between positions {start} and {end if end else len(value)}.' ) @@ -400,7 +411,8 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is zero." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is zero." ) @@ -408,8 +420,8 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", ) @@ -417,8 +429,8 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", ) @@ -426,8 +438,8 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", ) @@ -435,8 +447,8 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", ) @@ -444,9 +456,9 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: value >= LowerBound and value <= UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger or equal to {LowerBound} and " - f"smaller or equal to {UpperBound}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " + f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -456,16 +468,16 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[4:6]} is not a valid month.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[6:]} is not a valid day.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[6:]} is not a valid day.", ) @@ -473,9 +485,9 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, - friendly_name, item_num: (f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[:4]} must be less than or equal to " - f"{datetime.date.today().year - min_age} to meet the minimum age requirement.") + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[:4]} must be less " + f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -483,8 +495,9 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: Year {str(value)[:4]} must be larger than {year}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"Year {str(value)[:4]} must be larger than {year}.", ) @@ -492,8 +505,8 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[-1]} is not a valid quarter.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[-1]} is not a valid quarter.", ) @@ -502,7 +515,8 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {options}." ) @@ -510,9 +524,9 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not greater than or equal to 0 " - "or smaller than or equal to 2." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not greater than or equal to 0 " + "or smaller than or equal to 2." ) @@ -522,9 +536,9 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: The value: {value[2:8]}, does not follow the YYYYMM " - "format for Reporting Year and Month.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: The value: {value[2:8]}, " + "does not follow the YYYYMM format for Reporting Year and Month.", ) From 977a0e8faa418fb9ec264e725e3b636ff37e13e6 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:52:37 -0400 Subject: [PATCH 016/105] fix a couple tests --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 03ba4aab7..6a128c606 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -400,7 +400,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: record length is 7 characters but must be 22.' + assert error == 'T1: record length is 7 characters but must be 22.' def test_intHasLength_returns_valid(): @@ -551,7 +551,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): assert is_valid is False assert error_msg == ( - f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" ) From af563c81e87976851239acf4fe9fc20504beee74 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 2 May 2024 14:13:06 -0400 Subject: [PATCH 017/105] add parens to friendly names --- .../tdpservice/data_files/test/test_api.py | 4 +- .../tdpservice/parsers/test/test_parse.py | 10 +-- .../parsers/test/test_validators.py | 70 +++++++++---------- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 62fb7dad4..3ce127db9 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -103,7 +103,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 number of months: 0 is not larger than 0." + + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -136,7 +136,7 @@ def assert_error_report_file_content_matches_without_friendly_names(response): + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 NBR_MONTHS: 0 is not larger than 0." + "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." ) @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8106b261e..562aa9723 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -591,7 +591,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert err.error_message == ( - 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + 'M1 Item 11 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' ) assert err.content_type is not None assert err.object_id is not None @@ -1573,9 +1573,9 @@ def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): year_error = parser_errors[1] digits_error = parser_errors[0] - assert month_error.error_message == "T2 Item 32 date of birth: $9 is not a valid month." - assert year_error.error_message == "T2 Item 32 date of birth: Year Q897 must be larger than 1900." - assert digits_error.error_message == "T2 Item 32 date of birth: Q897$9 3 does not have exactly 8 digits." + assert month_error.error_message == "T2 Item 32 (date of birth): $9 is not a valid month." + assert year_error.error_message == "T2 Item 32 (date of birth): Year Q897 must be larger than 1900." + assert digits_error.error_message == "T2 Item 32 (date of birth): Q897$9 3 does not have exactly 8 digits." @pytest.mark.django_db @@ -1705,7 +1705,7 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file error = parser_errors.first() assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_message == "HEADER Item 10 update indicator: U does not match D." + assert error.error_message == "HEADER Item 10 (update indicator): U does not match D." @pytest.fixture def tribal_section_4_bad_quarter(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 6a128c606..3d341a743 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -49,8 +49,8 @@ def test_or_validators(): assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("5", RowSchema(), "friendly_name", "item_no") == ( False, - "T1 Item item_no friendly_name: 5 does not match 2. or " - "T1 Item item_no friendly_name: 5 does not match 3." + "T1 Item item_no (friendly_name): 5 does not match 2. or " + "T1 Item item_no (friendly_name): 5 does not match 3." ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), @@ -66,18 +66,18 @@ def test_or_validators(): value = "5" assert validator(value, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no friendly_name: 5 does not match 2. or ' - 'T1 Item item_no friendly_name: 5 does not match 3. or ' - 'T1 Item item_no friendly_name: 5 does not match 4.' + 'T1 Item item_no (friendly_name): 5 does not match 2. or ' + 'T1 Item item_no (friendly_name): 5 does not match 3. or ' + 'T1 Item item_no (friendly_name): 5 does not match 4.' ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - "T1 Item item_no friendly_name: 1 does not match 2. " - "or T1 Item item_no friendly_name: 1 does not " - "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4." + "T1 Item item_no (friendly_name): 1 does not match 2. " + "or T1 Item item_no (friendly_name): 1 does not " + "match 3. or T1 Item item_no (friendly_name): 1 is not larger than 4." ) def test_if_validators(): @@ -94,7 +94,7 @@ def test_if_validators(): result_field_name="Field2", result_function=validators.matches('1'), ) result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 Field2: 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 (Field2): 2 does not match 1.', ['Field1', 'Field2']) @@ -103,7 +103,7 @@ def test_and_validators(): validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no friendly_name: 1 is not larger than 2.' + 'T1 Item item_no (friendly_name): 1 is not larger than 2.' ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -141,7 +141,7 @@ def test_quarterIsValid(value, valid): val = validators.quarterIsValid() result = val(value, RowSchema(), "friendly_name", "item_no") - errorText = None if valid else f"T1 Item item_no friendly_name: {value[-1:]} is not a valid quarter." + errorText = None if valid else f"T1 Item item_no (friendly_name): {value[-1:]} is not a valid quarter." assert result == (valid, errorText) def test_validateSSN(): @@ -154,7 +154,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: {value} is in {options}.") + assert result == (False, f"T1 Item item_no (friendly_name): {value} is in {options}.") def test_validateRace(): """Test `validateRace`.""" @@ -167,7 +167,7 @@ def test_validateRace(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2." + f"T1 Item item_no (friendly_name): {value} is not greater than or equal to 0 or smaller than or equal to 2." ) def test_validateRptMonthYear(): @@ -181,7 +181,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not " "follow the YYYYMM format for Reporting Year and Month." ) @@ -189,7 +189,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not follow " "the YYYYMM format for Reporting Year and Month." ) @@ -197,7 +197,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does " "not follow the YYYYMM format for Reporting Year and Month." ) @@ -220,7 +220,7 @@ def test_matches_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: TEST does not match test.' + assert error == 'T1 Item item_no (friendly_name): TEST does not match test.' def test_oneOf_returns_valid(): @@ -244,7 +244,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 64 is not in [17, 24, 36].' + assert error == 'T1 Item item_no (friendly_name): 64 is not in [17, 24, 36].' def test_between_returns_valid(): @@ -277,7 +277,7 @@ def test_between_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 47 is not between 48 and 400.' + assert error == 'T1 Item item_no (friendly_name): 47 is not between 48 and 400.' @pytest.mark.parametrize('value, expected_is_valid, expected_error', [ @@ -285,16 +285,16 @@ def test_between_returns_invalid(): (77731, True, None), ('7', True, None), ('234897', True, None), - ('a', False, 'T1 Item item_no friendly_name: a is not a number.'), - ('houston, we have a problem', False, 'T1 Item item_no friendly_name: houston, we have a problem is not a number.'), - (' test', False, 'T1 Item item_no friendly_name: test is not a number.'), + ('a', False, 'T1 Item item_no (friendly_name): a is not a number.'), + ('houston, we have a problem', False, 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.'), + (' test', False, 'T1 Item item_no (friendly_name): test is not a number.'), (' 7 ', True, None), (' 8388323', True, None), ('87932875 ', True, None), (' 00 ', True, None), (' 88 ', True, None), (' 088 ', True, None), - (' 8 8 ', False, 'T1 Item item_no friendly_name: 8 8 is not a number.'), + (' 8 8 ', False, 'T1 Item item_no (friendly_name): 8 8 is not a number.'), ]) def test_isNumber(value, expected_is_valid, expected_error): """Test `isNumber` validator.""" @@ -319,7 +319,7 @@ def test_date_month_is_valid_returns_invalid(): validator = validators.dateMonthIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 13 is not a valid month.' + assert error == 'T1 Item item_no (friendly_name): 13 is not a valid month.' def test_date_day_is_valid_returns_valid(): @@ -337,7 +337,7 @@ def test_date_day_is_valid_returns_invalid(): validator = validators.dateDayIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 32 is not a valid day.' + assert error == 'T1 Item item_no (friendly_name): 32 is not a valid day.' def test_olderThan(): @@ -351,7 +351,7 @@ def test_olderThan(): result = validator(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to " + f"T1 Item item_no (friendly_name): {str(value)[:4]} must be less than or equal to " f"{date.today().year - min_age} to meet the minimum age requirement." ) @@ -366,7 +366,7 @@ def test_dateYearIsLargerThan(): value = 18990101 assert validator(value, RowSchema(), "friendly_name", "item_no") == ( False, - f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger than {year}." + f"T1 Item item_no (friendly_name): Year {str(value)[:4]} must be larger than {year}." ) @@ -378,7 +378,7 @@ def test_between_returns_invalid_for_string_value(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 047 is not between 100 and 400.' + assert error == 'T1 Item item_no (friendly_name): 047 is not between 100 and 400.' def test_recordHasLength_returns_valid(): @@ -422,7 +422,7 @@ def test_intHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 1a3 does not have exactly 22 digits.' + assert error == 'T1 Item item_no (friendly_name): 1a3 does not have exactly 22 digits.' def test_contains_returns_valid(): @@ -444,7 +444,7 @@ def test_contains_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 12345abcde does not contain 6789.' + assert error == 'T1 Item item_no (friendly_name): 12345abcde does not contain 6789.' def test_startsWith_returns_valid(): @@ -466,7 +466,7 @@ def test_startsWith_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 12345abcde does not start with abc.' + assert error == 'T1 Item item_no (friendly_name): 12345abcde does not start with abc.' def test_notEmpty_returns_valid_full_string(): @@ -488,7 +488,7 @@ def test_notEmpty_returns_invalid_full_string(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: contains blanks between positions 0 and 9.' + assert error == 'T1 Item item_no (friendly_name): contains blanks between positions 0 and 9.' def test_notEmpty_returns_valid_substring(): @@ -510,7 +510,7 @@ def test_notEmpty_returns_invalid_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 3 and 5." + assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 3 and 5." def test_notEmpty_returns_nonexistent_substring(): @@ -521,7 +521,7 @@ def test_notEmpty_returns_nonexistent_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 10 and 12." + assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 10 and 12." @pytest.mark.parametrize("test_input", [1, 2, 3, 4]) @@ -541,7 +541,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1 Item item_no friendly_name: {test_input} is not a valid quarter." + assert error == f"T1 Item item_no (friendly_name): {test_input} is not a valid quarter." @pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"]) def test_calendarQuarterIsValid_returns_invalid(value): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index fc17998c4..41f966885 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -255,7 +255,7 @@ def calendarQuarterIsValid(start=0, end=None): def format_error_context(row_schema, friendly_name, item_num): """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} {friendly_name}' + return f'{row_schema.record_type} Item {item_num} ({friendly_name})' def matches(option, error_func=None): From 01af1d2dc1f7cc5153aeb5bb48e26e24238a9274 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 2 May 2024 14:24:04 -0400 Subject: [PATCH 018/105] lint --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 3d341a743..3433e8d9e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -286,7 +286,10 @@ def test_between_returns_invalid(): ('7', True, None), ('234897', True, None), ('a', False, 'T1 Item item_no (friendly_name): a is not a number.'), - ('houston, we have a problem', False, 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.'), + ( + 'houston, we have a problem', False, + 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.' + ), (' test', False, 'T1 Item item_no (friendly_name): test is not a number.'), (' 7 ', True, None), (' 8388323', True, None), From 97aa0ef47ca59d24625ebe86e4a65b76b69a8459 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 11:10:33 -0400 Subject: [PATCH 019/105] fix tests --- .../tdpservice/parsers/test/test_parse.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index ffe0d2396..a807d303e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -236,7 +236,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -347,7 +347,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -358,7 +358,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER: record length is 11 characters but must be 23.', + 'TRAILER record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -372,7 +372,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -404,7 +404,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -416,7 +416,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_2_error.error_message == 'T1: record length is 117 characters but must be 156.' + assert row_2_error.error_message == 'T1 record length is 117 characters but must be 156.' assert row_2_error.content_type is None assert row_2_error.object_id is None @@ -428,10 +428,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1: record length is 7 characters but must be 156.', + 'T1 record length is 7 characters but must be 156.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER: record length is 7 characters but must be 23.', + 'TRAILER record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -450,7 +450,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None @@ -513,7 +513,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -885,7 +885,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER: record length is 15 characters but must be 23.' + error_message='TRAILER record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1685,7 +1685,7 @@ def t3_file_zero_filled_second(): ('t3_file', True, 0), ('t3_file_two_child', True, 1), ('t3_file_two_child_with_space_filled', True, 0), - ('two_child_second_filled', True, 9), + ('two_child_second_filled', True, 8), ('t3_file_zero_filled_second', True, 0)]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, request, dfs): From ad87ab166f276dd582031d5588723d0ed47a4fba Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 11:25:14 -0400 Subject: [PATCH 020/105] fix more tests --- tdrs-backend/tdpservice/data_files/test/test_api.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_validators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index faacac4d8..6045dc555 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -112,7 +112,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ "but must be 23." @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 13c5c0b65..044c9d864 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -403,7 +403,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: record length is 7 characters but must be 22.' + assert error == 'T1 record length is 7 characters but must be 22.' def test_hasLengthGreaterThan_returns_valid(): """Test `hasLengthGreaterThan` gives a valid result.""" From c2c465f8ce06417cdf962c16dfedbcd8d23af7f3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 16:40:59 -0400 Subject: [PATCH 021/105] fix merge duplication --- tdrs-backend/tdpservice/parsers/validators.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index ed30a19a5..a4ec1c5f2 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -318,28 +318,6 @@ def hasLengthGreaterThan(val, error_func=None): ) -def recordHasLength(length): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type} record length is {len(value)} characters but must be {length}.", - ) - - -def hasLengthGreaterThan(val, error_func=None): - """Validate that value (string or array) has a length greater than val.""" - return make_validator( - lambda value: len(value) >= val, - lambda value, - row_schema, - friendly_name, - item_num: f"Value length {len(value)} is not greater than {val}.", - ) - - def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( From 7713c20241a75e66f6482fb99bf45f52dbde3bd4 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Thu, 16 May 2024 23:15:45 -0700 Subject: [PATCH 022/105] Friendly Name Updates Shortened friendly names, removed "code not in use", removed "race/ethnicity", shortened benefits being received --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 14 +++---- .../tdpservice/parsers/schema_defs/ssp/m2.py | 38 +++++++++--------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 31 +++++++------- .../tdpservice/parsers/schema_defs/ssp/m4.py | 4 +- .../tdpservice/parsers/schema_defs/ssp/m5.py | 20 +++++----- .../tdpservice/parsers/schema_defs/ssp/m7.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 18 ++++----- .../tdpservice/parsers/schema_defs/tanf/t2.py | 40 +++++++++---------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 20 +++++----- .../tdpservice/parsers/schema_defs/tanf/t4.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t5.py | 20 +++++----- .../tdpservice/parsers/schema_defs/tanf/t7.py | 4 +- .../parsers/schema_defs/tribal_tanf/t1.py | 6 +-- .../parsers/schema_defs/tribal_tanf/t2.py | 35 ++++++++-------- .../parsers/schema_defs/tribal_tanf/t3.py | 30 +++++++------- .../parsers/schema_defs/tribal_tanf/t4.py | 2 +- .../parsers/schema_defs/tribal_tanf/t5.py | 22 +++++----- .../parsers/schema_defs/tribal_tanf/t7.py | 4 +- 18 files changed, 156 insertions(+), 160 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 43a981f99..dc79a60d3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -136,7 +136,7 @@ Field( item="2", name='COUNTY_FIPS_CODE', - friendly_name="County FIPS Code", + friendly_name="County FIPS code", type='string', startIndex=19, endIndex=22, @@ -226,7 +226,7 @@ Field( item="13", name='RECEIVES_FOOD_STAMPS', - friendly_name="Receives Assistance from the Supplemental Nutrition Assistance Program (SNAP)", + friendly_name="Receives Assistance from SNAP", type='number', startIndex=36, endIndex=37, @@ -236,7 +236,7 @@ Field( item="14", name='AMT_FOOD_STAMP_ASSISTANCE', - friendly_name="Amount of Supplemental Nutrition Assistance Program (SNAP) Benefits", + friendly_name="Amount of SNAP Benefits", type='number', startIndex=37, endIndex=41, @@ -246,7 +246,7 @@ Field( item="15", name='RECEIVES_SUB_CC', - friendly_name="Receives Subsidized Child Care: Code no longer in use", + friendly_name="Receives Subsidized Child Care", type='number', startIndex=41, endIndex=42, @@ -416,7 +416,7 @@ Field( item="24AIII", name='FAMILY_SANC_ADULT', - friendly_name="Sanctions: Code no longer in use", + friendly_name="Family Sanctions", type='number', startIndex=96, endIndex=97, @@ -506,7 +506,7 @@ Field( item="24CIV", name='OTHER_NON_SANCTION', - friendly_name="Other: Non-Sanction, Non-Recoupment ", + friendly_name="Other: Non-Sanction, Non-Recoupment", type='number', startIndex=111, endIndex=112, @@ -516,7 +516,7 @@ Field( item="25", name='WAIVER_EVAL_CONTROL_GRPS', - friendly_name="Waiver Evaluation Experimental and Control Groups: Code no longer in use", + friendly_name="Waiver Evaluation Experimental and Control Groups", type='number', startIndex=112, endIndex=113, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index a5a29dec3..b212be5f3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -209,7 +209,7 @@ item="30A", name='RACE_HISPANIC', type='number', - friendly_name="Race/Ethnicity: Hispanic or Latino", + friendly_name="Hispanic or Latino", startIndex=38, endIndex=39, required=False, @@ -218,7 +218,7 @@ Field( item="30B", name='RACE_AMER_INDIAN', - friendly_name="Race/Ethnicity: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type='number', startIndex=39, endIndex=40, @@ -228,7 +228,7 @@ Field( item="30C", name='RACE_ASIAN', - friendly_name="Race/Ethnicity: Asian", + friendly_name="Asian", type='number', startIndex=40, endIndex=41, @@ -238,7 +238,7 @@ Field( item="30D", name='RACE_BLACK', - friendly_name="Race/Ethnicity: Black or African American", + friendly_name="Black or African American", type='number', startIndex=41, endIndex=42, @@ -248,7 +248,7 @@ Field( item="30E", name='RACE_HAWAIIAN', - friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Other Pacific Islander", type='number', startIndex=42, endIndex=43, @@ -258,7 +258,7 @@ Field( item="30F", name='RACE_WHITE', - friendly_name="Race/Ethnicity: White", + friendly_name="White", type='number', startIndex=43, endIndex=44, @@ -288,7 +288,7 @@ Field( item="32B", name='FED_DISABILITY_STATUS', - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type='number', startIndex=46, endIndex=47, @@ -298,7 +298,7 @@ Field( item="32C", name='DISABLED_TITLE_XIVAPDT', - friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled", type='number', startIndex=47, endIndex=48, @@ -308,7 +308,7 @@ Field( item="32D", name='AID_AGED_BLIND', - friendly_name="Receives Disability Benefits: Code no longer in use.", + friendly_name="Receives Disability Benefit: AABD", type='number', startIndex=48, endIndex=49, @@ -318,7 +318,7 @@ Field( item="32E", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI ", + friendly_name="Receives Disability Benefits: SSI ", type='number', startIndex=49, endIndex=50, @@ -498,7 +498,7 @@ Field( item="47", name='OJT', - friendly_name="On-the-job Training ", + friendly_name="On-the-job Training", type='number', startIndex=76, endIndex=78, @@ -628,7 +628,7 @@ Field( item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', - friendly_name="Education Directly Related to Employment for an Individual with NO High " + + friendly_name="Education Directly Related to Employment for an Individual with NO High" + "School Diploma or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=102, @@ -639,7 +639,7 @@ Field( item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', - friendly_name="Education Directly Related to Employment for an Individual with NO High " + + friendly_name="Education Directly Related to Employment for an Individual with NO High" + "School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=104, @@ -650,7 +650,7 @@ Field( item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', - friendly_name="Education Directly Related to Employment for an Individual with NO High " + + friendly_name="Education Directly Related to Employment for an Individual with NO High" + "School Diploma or Certificate of High School Equivalency: Hours of Holidays", type='number', startIndex=106, @@ -661,7 +661,7 @@ Field( item="53A", name='SCHOOL_ATTENDENCE_HOP', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma " + + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma" + "or Certificate of High School Equivalency: Hours of Participation", type='number', startIndex=108, @@ -672,7 +672,7 @@ Field( item="53B", name='SCHOOL_ATTENDENCE_EA', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or " + + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or" + "Certificate of High School Equivalency: Hours of Excused Absences", type='number', startIndex=110, @@ -683,7 +683,7 @@ Field( item="53C", name='SCHOOL_ATTENDENCE_HOL', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma " + + friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma" + "or Certificate: Hours of Holidays", type='number', startIndex=112, @@ -694,7 +694,7 @@ Field( item="54A", name='PROVIDE_CC_HOP', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a" + "Community Service Program: Hours of Participation", type='number', startIndex=114, @@ -716,7 +716,7 @@ Field( item="54C", name='PROVIDE_CC_HOL', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + + friendly_name="Providing Child Care Services to an Individual Who Is Participating in a" + "Community Service Program: Hours of Holidays", type='number', startIndex=118, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index a67b904c7..7c7f2ac5d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -165,7 +165,7 @@ Field( item="63A", name='RACE_HISPANIC', - friendly_name="Race/Ethnicity: Hispanic or Latino", + friendly_name="Hispanic or Latino", type='number', startIndex=37, endIndex=38, @@ -175,7 +175,7 @@ Field( item="63B", name='RACE_AMER_INDIAN', - friendly_name="Race/Ethnicity: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type='number', startIndex=38, endIndex=39, @@ -185,7 +185,7 @@ Field( item="63C", name='RACE_ASIAN', - friendly_name="Race/Ethnicity: Asian", + friendly_name="Asian", type='number', startIndex=39, endIndex=40, @@ -195,7 +195,7 @@ Field( item="63D", name='RACE_BLACK', - friendly_name="Race/Ethnicity: Black or African American", + friendly_name="Black or African American", type='number', startIndex=40, endIndex=41, @@ -205,7 +205,7 @@ Field( item="63E", name='RACE_HAWAIIAN', - friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type='number', startIndex=41, endIndex=42, @@ -215,7 +215,7 @@ Field( item="63F", name='RACE_WHITE', - friendly_name="Race/Ethnicity: White", + friendly_name="White", type='number', startIndex=42, endIndex=43, @@ -245,7 +245,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or " + + friendly_name="Receives Disability Benefits: SSI Under Title XVI-SSI or " + "Aged, Blind, and Disabled Under Title XVI-AABD", type='number', startIndex=45, @@ -480,7 +480,7 @@ Field( item="63A", name='RACE_HISPANIC', - friendly_name="Race/Ethnicity: Hispanic or Latino", + friendly_name="Hispanic or Latino", type='number', startIndex=78, endIndex=79, @@ -490,7 +490,7 @@ Field( item="63B", name='RACE_AMER_INDIAN', - friendly_name="Race/Ethnicity: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type='number', startIndex=79, endIndex=80, @@ -500,7 +500,7 @@ Field( item="63C", name='RACE_ASIAN', - friendly_name="Race/Ethnicity: Asian", + friendly_name="Asian", type='number', startIndex=80, endIndex=81, @@ -510,7 +510,7 @@ Field( item="63D", name='RACE_BLACK', - friendly_name="Race/Ethnicity: Black or African American", + friendly_name="Black or African American", type='number', startIndex=81, endIndex=82, @@ -520,7 +520,7 @@ Field( item="63E", name='RACE_HAWAIIAN', - friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type='number', startIndex=82, endIndex=83, @@ -530,7 +530,7 @@ Field( item="63F", name='RACE_WHITE', - friendly_name="Race/Ethnicity: White", + friendly_name="White", type='number', startIndex=83, endIndex=84, @@ -550,7 +550,7 @@ Field( item="65A", name='RECEIVE_NONSSI_BENEFITS', - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type='number', startIndex=85, endIndex=86, @@ -560,8 +560,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI or " + - "Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: SSI or AABD", type='number', startIndex=86, endIndex=87, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 7fca57786..c5a140aa6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -57,7 +57,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="County FIPS Code", + friendly_name="County FIPS code", type="string", startIndex=19, endIndex=22, @@ -132,7 +132,7 @@ Field( item="11", name="REC_FOOD_STAMPS", - friendly_name="Received Assistance from the Supplemental Nutrition Assistance Program (SNAP)", + friendly_name="Received Assistance from SNAP", type="number", startIndex=34, endIndex=35, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index c546abdf3..a09ac1eea 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -178,7 +178,7 @@ Field( item="16A", name="RACE_HISPANIC", - friendly_name="Race/Ethnicity: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -188,7 +188,7 @@ Field( item="16B", name="RACE_AMER_INDIAN", - friendly_name="Race/Ethnicity: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -198,7 +198,7 @@ Field( item="16C", name="RACE_ASIAN", - friendly_name="Race/Ethnicity: Asian", + friendly_name="Asian", type="number", startIndex=39, endIndex=40, @@ -208,7 +208,7 @@ Field( item="16D", name="RACE_BLACK", - friendly_name="Race/Ethnicity: Black or African American", + friendly_name="Black or African American", type="number", startIndex=40, endIndex=41, @@ -218,7 +218,7 @@ Field( item="16E", name="RACE_HAWAIIAN", - friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -228,7 +228,7 @@ Field( item="16F", name="RACE_WHITE", - friendly_name="Race/Ethnicity: White", + friendly_name="White", type="number", startIndex=42, endIndex=43, @@ -258,7 +258,7 @@ Field( item="18B", name="REC_FEDERAL_DISABILITY", - friendly_name="Received Disability Benefits: Federal Disability Status", + friendly_name="Received Disability Benefits: Other Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -268,7 +268,7 @@ Field( item="18C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled", type="number", startIndex=46, endIndex=47, @@ -278,7 +278,7 @@ Field( item="18D", name="REC_AID_AGED_BLIND", - friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Received Disability Benefits: AABD", type="number", startIndex=47, endIndex=48, @@ -288,7 +288,7 @@ Field( item="18E", name="REC_SSI", - friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Received Disability Benefits: SSI", type="number", startIndex=48, endIndex=49, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 869c113dc..f67dacce1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -70,7 +70,7 @@ Field( item="3", name="TDRS_SECTION_IND", - friendly_name="TDRS Section Indicator", + friendly_name="SDR Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -90,7 +90,7 @@ Field( item=families_item_numbers[i - 1], name="FAMILIES_MONTH", - friendly_name="Families Month", + friendly_name="Number of Families", type="number", startIndex=families_index, endIndex=families_index + 7, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index be641cb8c..2a2cc5c1a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -280,7 +280,7 @@ Field( item="15", name="RECEIVES_FOOD_STAMPS", - friendly_name="Receives Assistance from the Supplemental Nutrition Assistance Program (SNAP)", + friendly_name="Receives Assistance from the SNAP", type="number", startIndex=37, endIndex=38, @@ -292,7 +292,7 @@ Field( item="16", name="AMT_FOOD_STAMP_ASSISTANCE", - friendly_name="Amount of Supplemental Nutrition Assistance Program (SNAP) Benefits", + friendly_name="Amount of SNAP Benefits", type="number", startIndex=38, endIndex=42, @@ -352,7 +352,7 @@ Field( item="21A", name="CASH_AMOUNT", - friendly_name="Cash and Cash Equivalents: Amount", + friendly_name="Cash Amount", type="number", startIndex=55, endIndex=59, @@ -376,7 +376,7 @@ Field( item="22A", name="CC_AMOUNT", - friendly_name="TANF Child Care: Amount", + friendly_name="Child Care Amount", type="number", startIndex=62, endIndex=66, @@ -484,7 +484,7 @@ Field( item="26AI", name="SANC_REDUCTION_AMT", - friendly_name="Total Dollar Amount of Reductions due to Sanctions", + friendly_name="Total Dollar Amount of Reductions Due to Sanctions", type="number", startIndex=92, endIndex=96, @@ -508,7 +508,7 @@ Field( item="26AIII", name="FAMILY_SANC_ADULT", - friendly_name="Sanctions: Code no longer in use", + friendly_name="Family Sanction", type="number", startIndex=97, endIndex=98, @@ -616,7 +616,7 @@ Field( item="26CIV", name="OTHER_NON_SANCTION", - friendly_name="Other: Non-Sanction, Non-Recoupment", + friendly_name="Non-Sanction", type="number", startIndex=112, endIndex=113, @@ -628,7 +628,7 @@ Field( item="27", name="WAIVER_EVAL_CONTROL_GRPS", - friendly_name="Waiver Evaluation Experimental and Control Groups", + friendly_name="Waiver Evaluation", type="string", startIndex=113, endIndex=114, @@ -651,7 +651,7 @@ Field( item="29", name="FAMILY_NEW_CHILD", - friendly_name="TANF Family a New Child-Only Family: Code no longer in use", + friendly_name="TANF Family a New Child-Only Family", type="number", startIndex=116, endIndex=117, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 5ae06e9ed..62b19a15c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -133,7 +133,7 @@ Field( item="0", name="RecordType", - friendly_name="record type", + friendly_name="Record Type", type="string", startIndex=0, endIndex=2, @@ -212,7 +212,7 @@ Field( item="34A", name="RACE_HISPANIC", - friendly_name="Race/Ethnicity: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=38, endIndex=39, @@ -222,7 +222,7 @@ Field( item="34B", name="RACE_AMER_INDIAN", - friendly_name="Race/Ethnicity: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=39, endIndex=40, @@ -232,7 +232,7 @@ Field( item="34C", name="RACE_ASIAN", - friendly_name="Race/Ethnicity: Asian", + friendly_name="Asian", type="number", startIndex=40, endIndex=41, @@ -242,7 +242,7 @@ Field( item="34D", name="RACE_BLACK", - friendly_name="Race/Ethnicity: Black or African American", + friendly_name="Black or African American", type="number", startIndex=41, endIndex=42, @@ -252,7 +252,7 @@ Field( item="34E", name="RACE_HAWAIIAN", - friendly_name="Race/Ethnicity: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=42, endIndex=43, @@ -262,7 +262,7 @@ Field( item="34F", name="RACE_WHITE", - friendly_name="Race/Ethnicity: White", + friendly_name="White", type="number", startIndex=43, endIndex=44, @@ -284,7 +284,7 @@ Field( item="36A", name="FED_OASDI_PROGRAM", - friendly_name="Receives Disability Benefits: OASDI Program", + friendly_name="Receives Disability Benefits: SSDI", type="number", startIndex=45, endIndex=46, @@ -294,7 +294,7 @@ Field( item="36B", name="FED_DISABILITY_STATUS", - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type="number", startIndex=46, endIndex=47, @@ -304,7 +304,7 @@ Field( item="36C", name="DISABLED_TITLE_XIVAPDT", - friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled", type="string", startIndex=47, endIndex=48, @@ -319,7 +319,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Code no longer in use", + friendly_name="Receives Disability Benefits: AABD", type="number", startIndex=48, endIndex=49, @@ -331,7 +331,7 @@ Field( item="36E", name="RECEIVE_SSI", - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Receives Disability Benefits: SSI", type="number", startIndex=49, endIndex=50, @@ -440,7 +440,7 @@ Field( item="45", name="MONTHS_STATE_TIME_LIMIT", - friendly_name="Months Remaining Under State's Time Limit: Code no longer in use", + friendly_name="Months Remaining Under State's Time Limit", type="string", startIndex=62, endIndex=64, @@ -541,7 +541,7 @@ Field( item="52", name="SUB_PUBLIC_EMPLOYMENT", - friendly_name="Subsidized Public-Sector Employment ", + friendly_name="Subsidized Public-Sector Employment", type="string", startIndex=74, endIndex=76, @@ -797,7 +797,7 @@ Field( item="60B", name="SCHOOL_ATTENDENCE_EA", - friendly_name="Satisfactory School Attendance for Individuals with No High School " + + friendly_name="Satisfactory School Attendance for Individuals with No High School" + "Diploma or Certificate: Hours of Ecused Absences", type="string", startIndex=116, @@ -810,7 +810,7 @@ Field( item="60C", name="SCHOOL_ATTENDENCE_HOL", - friendly_name="Satisfactory School Attendance for Individuals with No High School " + + friendly_name="Satisfactory School Attendance for Individuals with No High School" + "Diploma or Certificate: Hours of Holidays", type="string", startIndex=118, @@ -823,7 +823,7 @@ Field( item="61A", name="PROVIDE_CC_HOP", - friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + friendly_name="Providing Child Care for an Individual Who Is Participating" + "in a Community Service Program: Hours of Participation", type="string", startIndex=120, @@ -836,7 +836,7 @@ Field( item="61B", name="PROVIDE_CC_EA", - friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + friendly_name="Providing Child Care for an Individual Who Is Participating" + "in a Community Service Program: Hours of Excused Absences", type="string", startIndex=122, @@ -849,7 +849,7 @@ Field( item="61C", name="PROVIDE_CC_HOL", - friendly_name="Providing Child Care Services to an Individual Who Is Participating " + + friendly_name="Providing Child Care Services for an Individual Who Is Participating" + "in a Community Service Program: Hours of Holidays", type="string", startIndex=124, @@ -910,7 +910,7 @@ Field( item="66A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC): Code no longer in use", + friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", type="string", startIndex=136, endIndex=140, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 561ffde95..f96398783 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -263,7 +263,7 @@ Field( item="74", name="PARENT_MINOR_CHILD", - friendly_name="Parental status of minor who is not a head-of-household or spouse of the head-of-household", + friendly_name="Parental Status of Minor", type="number", startIndex=48, endIndex=49, @@ -474,7 +474,7 @@ Field( item="70A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=78, endIndex=79, @@ -484,7 +484,7 @@ Field( item="70B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=79, endIndex=80, @@ -494,7 +494,7 @@ Field( item="70C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=80, endIndex=81, @@ -504,7 +504,7 @@ Field( item="70D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=81, endIndex=82, @@ -514,7 +514,7 @@ Field( item="70E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=82, endIndex=83, @@ -524,7 +524,7 @@ Field( item="70F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=83, endIndex=84, @@ -544,7 +544,7 @@ Field( item="72A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type="number", startIndex=85, endIndex=86, @@ -554,7 +554,7 @@ Field( item="72B", name="RECEIVE_SSI", - friendly_name="Receives Disability Benefits: SSI Under Title XVI-SSI or AABD Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: SSI or AABD", type="number", startIndex=86, endIndex=87, @@ -574,7 +574,7 @@ Field( item="74", name="PARENT_MINOR_CHILD", - friendly_name="Parental status of minor who is not a head-of-household or spouse of the head-of-household", + friendly_name="Parental Status of Minor", type="number", startIndex=89, endIndex=90, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index bc1941d41..e2e3953d8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -58,7 +58,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="County FIPS Code", + friendly_name="County FIPS code", type="string", startIndex=19, endIndex=22, @@ -133,7 +133,7 @@ Field( item="12", name="REC_FOOD_STAMPS", - friendly_name="Received Assistance from the Supplemental Nutrition Assistance Program (SNAP)", + friendly_name="Received Assistance from SNAP", type="number", startIndex=34, endIndex=35, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index f6fdd9d5a..2f98c6220 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -179,7 +179,7 @@ Field( item="17A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hisapic or Latino", + friendly_name="Hisapic or Latino", type="number", startIndex=37, endIndex=38, @@ -189,7 +189,7 @@ Field( item="17B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native ", + friendly_name="American Indian or Alaska Native ", type="number", startIndex=38, endIndex=39, @@ -199,7 +199,7 @@ Field( item="17C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=39, endIndex=40, @@ -209,7 +209,7 @@ Field( item="17D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=40, endIndex=41, @@ -219,7 +219,7 @@ Field( item="17E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -229,7 +229,7 @@ Field( item="17F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=42, endIndex=43, @@ -259,7 +259,7 @@ Field( item="19B", name="REC_FEDERAL_DISABILITY", - friendly_name="Received Disability Benefits: Federal Disability Status", + friendly_name="Received Disability Benefits: Other Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -269,7 +269,7 @@ Field( item="19C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled", type="number", startIndex=46, endIndex=47, @@ -279,7 +279,7 @@ Field( item="19D", name="REC_AID_AGED_BLIND", - friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Received Disability Benefits: AABD", type="number", startIndex=47, endIndex=48, @@ -289,7 +289,7 @@ Field( item="19E", name="REC_SSI", - friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Received Disability Benefits: SSI", type="number", startIndex=48, endIndex=49, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index 2a55e291b..61a6b5cf6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -70,7 +70,7 @@ Field( item="4", name="TDRS_SECTION_IND", - friendly_name="TDRS Section Indicator", + friendly_name="TDR Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -90,7 +90,7 @@ Field( item=families_value_item_number, name="FAMILIES_MONTH", - friendly_name="Families Month", + friendly_name="Number of Families", type="number", startIndex=families_index, endIndex=families_index + 7, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index c5f819430..6bffd5553 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -352,7 +352,7 @@ Field( item="21A", name="CASH_AMOUNT", - friendly_name="Cash and Cash Equivalents", + friendly_name="Cash Amount", type="number", startIndex=55, endIndex=59, @@ -508,7 +508,7 @@ Field( item="26AIII", name="FAMILY_SANC_ADULT", - friendly_name="Family Sanction for an Adult with No High School Diploma orEquivalent:", + friendly_name="Family Sanction for an Adult with No High School Diploma or Equivalent:", type="number", startIndex=97, endIndex=98, @@ -638,7 +638,7 @@ Field( item="28", name="FAMILY_EXEMPT_TIME_LIMITS", - friendly_name="Exempt during the reporting month from theTribal Time-Limit Provisions", + friendly_name="Exempt during reporting month from the Tribal Time-Limit Provisions", type="number", startIndex=114, endIndex=116, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 52def22f2..830c0b59c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -200,7 +200,7 @@ Field( item="34A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=38, endIndex=39, @@ -210,7 +210,7 @@ Field( item="34B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=39, endIndex=40, @@ -220,7 +220,7 @@ Field( item="34C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=40, endIndex=41, @@ -230,7 +230,7 @@ Field( item="34D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=41, endIndex=42, @@ -240,7 +240,7 @@ Field( item="34E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=42, endIndex=43, @@ -250,7 +250,7 @@ Field( item="34F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=43, endIndex=44, @@ -282,7 +282,7 @@ Field( item="36B", name="FED_DISABILITY_STATUS", - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type="number", startIndex=46, endIndex=47, @@ -292,7 +292,7 @@ Field( item="36C", name="DISABLED_TITLE_XIVAPDT", - friendly_name="Receives Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Receives Disability Benefits: Permanently and Totally Disabled", type="string", startIndex=47, endIndex=48, @@ -306,7 +306,7 @@ Field( item="36D", name="AID_AGED_BLIND", - friendly_name="Receives Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: AABD", type="number", startIndex=48, endIndex=49, @@ -318,7 +318,7 @@ Field( item="36E", name="RECEIVE_SSI", - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Receives Disability Benefits: SSI", type="number", startIndex=49, endIndex=50, @@ -439,7 +439,7 @@ Field( item="46", name="CURRENT_MONTH_STATE_EXEMPT", - friendly_name="Is Current Month Exempt from the State's (Tribe's) Time Limit", + friendly_name="Current Month Exempt from the (Tribe's) Time Limit", type="number", startIndex=64, endIndex=65, @@ -588,8 +588,7 @@ Field( item="58", name="ED_NO_HIGH_SCHOOL_DIPLOMA", - friendly_name="Education Directly Related to Employment for Individuals with no High " + - "School Diploma or Certificate of High SchoolEquivalency", + friendly_name="Education Directly Related to Employment", type="string", startIndex=86, endIndex=88, @@ -601,8 +600,7 @@ Field( item="59", name="SCHOOL_ATTENDENCE", - friendly_name="Satisfactory School Attendance for Individuals with No High School " + - "Diploma or Certificate of High School Equivalency", + friendly_name="Satisfactory School Attendance", type="string", startIndex=88, endIndex=90, @@ -614,8 +612,7 @@ Field( item="60", name="PROVIDE_CC", - friendly_name="Providing Child Care Services to an Individual Who Is Participating " + - "in a Community Service Program", + friendly_name="Providing Child Care", type="string", startIndex=90, endIndex=92, @@ -675,7 +672,7 @@ Field( item="65A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", + friendly_name="Amount of Unearned Income", type="string", startIndex=102, endIndex=106, @@ -687,7 +684,7 @@ Field( item="65B", name="UNEARNED_SOCIAL_SECURITY", - friendly_name="Amount of Unearned Income: Social Security", + friendly_name="Amount of Unearned Income: SSI", type="string", startIndex=106, endIndex=110, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index cfb839b0f..eacccf5b0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -163,7 +163,7 @@ Field( item="69A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -173,7 +173,7 @@ Field( item="69B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -183,7 +183,7 @@ Field( item="69C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=39, endIndex=40, @@ -193,7 +193,7 @@ Field( item="69D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=40, endIndex=41, @@ -203,7 +203,7 @@ Field( item="69E", name="RACE_HAWAIIAN", - friendly_name="Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -233,7 +233,7 @@ Field( item="71A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="Receives Benefits Based on Federal Disability Status under Non-Social Security Act Programs", + friendly_name="Received Disability Benefits: Other Federal Disability Status", type="number", startIndex=44, endIndex=45, @@ -243,7 +243,7 @@ Field( item="71B", name="RECEIVE_SSI", - friendly_name="Receives Supplemental Security Income under Title XVI-SSI of the Social Security Act", + friendly_name="Receives Supplemental Security Income: SSI", type="number", startIndex=45, endIndex=46, @@ -474,7 +474,7 @@ Field( item="69A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=78, endIndex=79, @@ -484,7 +484,7 @@ Field( item="69B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Natve", + friendly_name="American Indian or Alaska Natve", type="number", startIndex=79, endIndex=80, @@ -494,7 +494,7 @@ Field( item="69C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=80, endIndex=81, @@ -504,7 +504,7 @@ Field( item="69D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=81, endIndex=82, @@ -514,7 +514,7 @@ Field( item="69E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Native Hawaiian or Other Pacific Islander", + friendly_name="Native Hawaiian or Other Pacific Islander", type="number", startIndex=82, endIndex=83, @@ -524,7 +524,7 @@ Field( item="69F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=83, endIndex=84, @@ -544,7 +544,7 @@ Field( item="71A", name="RECEIVE_NONSSA_BENEFITS", - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type="number", startIndex=85, endIndex=86, @@ -554,7 +554,7 @@ Field( item="71B", name="RECEIVE_SSI", - friendly_name="Receives Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Receives Disability Benefits: SSI", type="number", startIndex=86, endIndex=87, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index eecfd3ce6..ca6d9a9c8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -57,7 +57,7 @@ Field( item="2", name="COUNTY_FIPS_CODE", - friendly_name="County FIPS Code", + friendly_name="County FIPS code", type="string", startIndex=19, endIndex=22, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index c90db77e5..dd2f887d5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -173,7 +173,7 @@ Field( item="17A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -183,7 +183,7 @@ Field( item="17B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native", + friendly_name="American Indian or Alaska Native", type="number", startIndex=38, endIndex=39, @@ -193,7 +193,7 @@ Field( item="17C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=39, endIndex=40, @@ -203,7 +203,7 @@ Field( item="17D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=40, endIndex=41, @@ -213,7 +213,7 @@ Field( item="17E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Hawaiian or Other Pacific Islander", + friendly_name="Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -223,7 +223,7 @@ Field( item="17F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=42, endIndex=43, @@ -253,7 +253,7 @@ Field( item="19B", name="REC_FEDERAL_DISABILITY", - friendly_name="Receives Disability Benefits: Federal Disability Status", + friendly_name="Receives Disability Benefits: Other Federal Disability Status", type="number", startIndex=45, endIndex=46, @@ -263,7 +263,7 @@ Field( item="19C", name="REC_AID_TOTALLY_DISABLED", - friendly_name="Received Disability Benefits: Permanently and Totally Disabled Under Title XIV-APDT", + friendly_name="Received Disability Benefits: Permanently and Totally Disabled", type="number", startIndex=46, endIndex=47, @@ -273,7 +273,7 @@ Field( item="19D", name="REC_AID_AGED_BLIND", - friendly_name="Received Disability Benefits: Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Received Disability Benefits: AABD", type="number", startIndex=47, endIndex=48, @@ -283,7 +283,7 @@ Field( item="19E", name="REC_SSI", - friendly_name="Received Disability Benefits: Supplemental Security Income Under Title XVI-SSI", + friendly_name="Received Disability Benefits: SSI", type="number", startIndex=48, endIndex=49, @@ -362,7 +362,7 @@ Field( item="26", name="COUNTABLE_MONTH_FED_TIME", - friendly_name="Number of Months Countable toward Tribal Time Limit:", + friendly_name="Number of Months Countable Toward Tribal Time Limit", type="string", startIndex=57, endIndex=60, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 6e06f9f0b..dbadc1c5a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -70,7 +70,7 @@ Field( item="4", name="TDRS_SECTION_IND", - friendly_name="TDRS Section Indicator", + friendly_name="Section Indicator", type="string", startIndex=section_ind_index, endIndex=section_ind_index + 1, @@ -90,7 +90,7 @@ Field( item=families_value_item_number, name="FAMILIES_MONTH", - friendly_name="Families Month", + friendly_name="Number of Families", type="number", startIndex=families_index, endIndex=families_index + 7, From 8c3a4a197d5c35785a71918f2d45e6d31e1f966d Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Fri, 17 May 2024 12:35:25 -0700 Subject: [PATCH 023/105] rewrote 54 job search --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 830c0b59c..5360a944b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -540,7 +540,7 @@ Field( item="54", name="JOB_SEARCH", - friendly_name="Job Search and Job Readiness Assistance", + friendly_name="Job Search and Job Readiness", type="string", startIndex=78, endIndex=80, @@ -576,7 +576,7 @@ Field( item="57", name="JOB_SKILLS_TRAINING", - friendly_name="Job Skills Training Directly Related to Employment", + friendly_name="Job Skills Training", type="string", startIndex=84, endIndex=86, From d44eb3e529ba2fff3a2e343a4c9e92777fbf9138 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Fri, 17 May 2024 12:40:02 -0700 Subject: [PATCH 024/105] finished resolving conflicts --- .../tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index d8521d35f..ef45c6ee0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -149,15 +149,9 @@ validators=[validators.isInLimits(1, 3)], ), Field( -<<<<<<< HEAD - item="14", - name="FAMILY_AFFILIATION", - friendly_name="Family Affiliation", -======= item="-1", name="BLANK", friendly_name="blank", ->>>>>>> 9bcf9b93bc83220f701ab924a2bdf0b6f5066393 type="string", startIndex=36, endIndex=71, From f19bb83ec65a7f942169aaeee7d7a2bd51d8a641 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Fri, 17 May 2024 13:07:00 -0700 Subject: [PATCH 025/105] updated test_api.py finding lint issues --- tdrs-backend/tdpservice/data_files/test/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 6eb0b7a55..2e0783555 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,8 +100,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if Cash and Cash Equivalents: Amount :873 " + \ - "validator1 passed then Cash and Cash Equivalents: Number of Months T1: 0 is not larger than 0." + assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \ + "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): From 68789c3d31ebbe35bd566c0374207ac26b7e0365 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 20 May 2024 13:11:04 -0400 Subject: [PATCH 026/105] fix tests --- .../tdpservice/data_files/test/test_api.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 27 ++++++++++--------- .../parsers/test/test_validators.py | 4 +-- tdrs-backend/tdpservice/parsers/validators.py | 21 ++++++++------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 6045dc555..faacac4d8 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -112,7 +112,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index df78d6aff..7ffcb71a1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -236,7 +236,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -347,7 +347,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -358,7 +358,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER record length is 11 characters but must be 23.', + 'TRAILER: record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -372,7 +372,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == "T1 record length of 7 characters is not in the range [117, 156]." + assert length_error.error_message == "T1: record length of 7 characters is not in the range [117, 156]." assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -404,7 +404,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -416,7 +416,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == 'T1: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert row_2_error.error_message == 'T1 Item 13 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' # catch-rpt-month-year-mismatches row_3_errors = parser_errors.filter(row_number=3) @@ -426,10 +426,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1 record length of 7 characters is not in the range [117, 156].', + 'T1: record length of 7 characters is not in the range [117, 156].', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER record length is 7 characters but must be 23.', + 'TRAILER: record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -448,7 +448,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length of 7 characters is not in the range [117, 156].' + assert length_error.error_message == 'T1: record length of 7 characters is not in the range [117, 156].' assert length_error.content_type is None assert length_error.object_id is None @@ -511,7 +511,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -883,7 +883,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER record length is 15 characters but must be 23.' + error_message='TRAILER: record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1705,7 +1705,7 @@ def t3_file_zero_filled_second(): 'The second child record is too short at 97 characters' + ' and must be at least 101 characters.'), ('t3_file_two_child_with_space_filled', 2, 0, ''), - ('two_child_second_filled', 2, 9, 'T3: Year 6 must be larger than 1900.'), + ('two_child_second_filled', 2, 8, 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), ('t3_file_zero_filled_second', 1, 0, '')]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, error_message, request, dfs): @@ -1721,6 +1721,7 @@ def test_misformatted_multi_records(file_fixture, result, number_of_errors, erro assert parser_errors.count() == number_of_errors if number_of_errors > 0: error_messages = [parser_error.error_message for parser_error in parser_errors] + print(error_messages) assert error_message in error_messages parser_errors = ParserError.objects.all().exclude( diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 66b019226..bf819f1f4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -403,7 +403,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 record length is 7 characters but must be 22.' + assert error == 'T1: record length is 7 characters but must be 22.' def test_hasLengthGreaterThan_returns_valid(): """Test `hasLengthGreaterThan` gives a valid result.""" @@ -449,7 +449,7 @@ def test_recordHasLengthBetween_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1 record length of {len(value)} characters is not in the range [{lower}, {upper}]." + assert error == f"T1: record length of {len(value)} characters is not in the range [{lower}, {upper}]." def test_intHasLength_returns_valid(): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 32a6de39a..a0baf23fd 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -229,6 +229,17 @@ def recordHasLength(length): ) +def recordHasLengthBetween(lower, upper, error_func=None): + """Validate that value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) >= lower and len(value) <= upper, + lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) + if error_func + else + f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", + ) + + def caseNumberNotEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( @@ -306,16 +317,6 @@ def between(min, max): f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) -def recordHasLengthBetween(lower, upper, error_func=None): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) - if error_func - else - f"{row_schema.record_type} record length of {len(value)} characters is not in the range [{lower}, {upper}].", - ) - def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( From 06677a0466a3ba1037139f92ed0a6ab15645c381 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 20 May 2024 13:12:27 -0400 Subject: [PATCH 027/105] lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 7ffcb71a1..6993726fb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -416,7 +416,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == 'T1 Item 13 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' + assert row_2_error.error_message == ( + 'T1 Item 13 (receives subsidized housing): 3 is not ' + 'larger or equal to 1 and smaller or equal to 2.' + ) # catch-rpt-month-year-mismatches row_3_errors = parser_errors.filter(row_number=3) @@ -1705,7 +1708,8 @@ def t3_file_zero_filled_second(): 'The second child record is too short at 97 characters' + ' and must be at least 101 characters.'), ('t3_file_two_child_with_space_filled', 2, 0, ''), - ('two_child_second_filled', 2, 8, 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), + ('two_child_second_filled', 2, 8, + 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), ('t3_file_zero_filled_second', 1, 0, '')]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, error_message, request, dfs): From 484b655a2e532d943ebfeebf543952bbe7458a70 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Mon, 20 May 2024 12:40:07 -0700 Subject: [PATCH 028/105] corrected message --- tdrs-backend/tdpservice/data_files/test/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 2e0783555..57fda6182 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,8 +100,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \ - "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER." + assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if Cash Amount :873 validator1 passed then " + \ + "Cash and Cash Equivalents: Number of Months T1: 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): From 2ca2b17a5da9e1362da53407de2f47284bf09dea Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Mon, 20 May 2024 12:40:39 -0700 Subject: [PATCH 029/105] updated friendly names --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 2 +- .../tdpservice/parsers/schema_defs/ssp/m2.py | 42 ++++++++-------- .../tdpservice/parsers/schema_defs/ssp/m4.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t2.py | 50 +++++++++---------- .../tdpservice/parsers/schema_defs/tanf/t4.py | 2 +- .../parsers/schema_defs/tribal_tanf/t4.py | 2 +- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index dc79a60d3..feecada7b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -226,7 +226,7 @@ Field( item="13", name='RECEIVES_FOOD_STAMPS', - friendly_name="Receives Assistance from SNAP", + friendly_name="Receives SNAP Assistance", type='number', startIndex=36, endIndex=37, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index b212be5f3..d3ca888bc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -508,7 +508,7 @@ Field( item="48A", name='JOB_SEARCH_HOP', - friendly_name="Job Search and Job Readiness Assistance: Hours of Participation", + friendly_name="Job Search and Job Readiness: Hours of Participation", type='number', startIndex=78, endIndex=80, @@ -518,7 +518,7 @@ Field( item="48B", name='JOB_SEARCH_EA', - friendly_name="Job Search and Job Readiness Assistance: Hours of Excused Absences", + friendly_name="Job Search and Job Readiness: Hours of Excused Absences", type='number', startIndex=80, endIndex=82, @@ -528,7 +528,7 @@ Field( item="48C", name='JOB_SEARCH_HOL', - friendly_name="Job Search and Job Readiness Assistance: Hours of Holidays", + friendly_name="Job Search and Job Readiness: Hours of Holidays", type='number', startIndex=82, endIndex=84, @@ -628,8 +628,8 @@ Field( item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', - friendly_name="Education Directly Related to Employment for an Individual with NO High" + - "School Diploma or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Education Directly Related to Employment: " + + "Hours of Participation", type='number', startIndex=102, endIndex=104, @@ -639,8 +639,8 @@ Field( item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', - friendly_name="Education Directly Related to Employment for an Individual with NO High" + - "School Diploma or Certificate of High School Equivalency: Hours of Excused Absences", + friendly_name="Education Directly Related to Employment: " + + "Hours of Excused Absences", type='number', startIndex=104, endIndex=106, @@ -650,8 +650,8 @@ Field( item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', - friendly_name="Education Directly Related to Employment for an Individual with NO High" + - "School Diploma or Certificate of High School Equivalency: Hours of Holidays", + friendly_name="Education Directly Related to Employment: " + + "Hours of Holidays", type='number', startIndex=106, endIndex=108, @@ -661,8 +661,8 @@ Field( item="53A", name='SCHOOL_ATTENDENCE_HOP', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma" + - "or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Satisfactory School Attendance: " + + "Hours of Participation", type='number', startIndex=108, endIndex=110, @@ -672,8 +672,8 @@ Field( item="53B", name='SCHOOL_ATTENDENCE_EA', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma or" + - "Certificate of High School Equivalency: Hours of Excused Absences", + friendly_name="Satisfactory School Attendance: " + + "Hours of Excused Absences", type='number', startIndex=110, endIndex=112, @@ -683,8 +683,8 @@ Field( item="53C", name='SCHOOL_ATTENDENCE_HOL', - friendly_name="Satisfactory School Attendance for Individuals with No High School Diploma" + - "or Certificate: Hours of Holidays", + friendly_name="Satisfactory School Attendance: " + + "Hours of Holidays", type='number', startIndex=112, endIndex=114, @@ -694,8 +694,8 @@ Field( item="54A", name='PROVIDE_CC_HOP', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a" + - "Community Service Program: Hours of Participation", + friendly_name="Providing Child Care Services: " + + "Hours of Participation", type='number', startIndex=114, endIndex=116, @@ -705,8 +705,8 @@ Field( item="54B", name='PROVIDE_CC_EA', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a " + - "Community Service Program: Hours of Excused Absences", + friendly_name="Providing Child Care Services: " + + "Hours of Excused Absences", type='number', startIndex=116, endIndex=118, @@ -716,8 +716,8 @@ Field( item="54C", name='PROVIDE_CC_HOL', - friendly_name="Providing Child Care Services to an Individual Who Is Participating in a" + - "Community Service Program: Hours of Holidays", + friendly_name="Providing Child Care Services: " + + "Hours of Holidays", type='number', startIndex=118, endIndex=120, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index c5a140aa6..91fb4c65b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -132,7 +132,7 @@ Field( item="11", name="REC_FOOD_STAMPS", - friendly_name="Received Assistance from SNAP", + friendly_name="Received SNAP Assistance", type="number", startIndex=34, endIndex=35, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 2a2cc5c1a..48848fa14 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -280,7 +280,7 @@ Field( item="15", name="RECEIVES_FOOD_STAMPS", - friendly_name="Receives Assistance from the SNAP", + friendly_name="Receives SNAP Assistance", type="number", startIndex=37, endIndex=38, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 62b19a15c..5b2e59ec9 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -601,7 +601,7 @@ Field( item="55A", name="JOB_SEARCH_HOP", - friendly_name="Job Search and Job Readiness Assistance: Hours of Participation", + friendly_name="Job Search and Job Readiness: Hours of Participation", type="string", startIndex=84, endIndex=86, @@ -613,7 +613,7 @@ Field( item="55B", name="JOB_SEARCH_EA", - friendly_name="Job Search and Job Readiness Assistance: Hours of Excused Absences", + friendly_name="Job Search and Job Readiness: Hours of Excused Absences", type="string", startIndex=86, endIndex=88, @@ -625,7 +625,7 @@ Field( item="55C", name="JOB_SEARCH_HOL", - friendly_name="Job Search and Job Readiness Assistance: Hours of Holidays", + friendly_name="Job Search and Job Readiness: Hours of Holidays", type="string", startIndex=88, endIndex=90, @@ -709,7 +709,7 @@ Field( item="58A", name="JOB_SKILLS_TRAINING_HOP", - friendly_name="Job Skills Training Directly Related to Employment: Hours of Participation", + friendly_name="Job Skills Training: Hours of Participation", type="string", startIndex=102, endIndex=104, @@ -721,7 +721,7 @@ Field( item="58B", name="JOB_SKILLS_TRAINING_EA", - friendly_name="Job Skills Training Directly Related to Employment: Hours of Excused Absences", + friendly_name="Job Skills Training: Hours of Excused Absences", type="string", startIndex=104, endIndex=106, @@ -733,7 +733,7 @@ Field( item="58C", name="JOB_SKILLS_TRAINING_HOL", - friendly_name="Job Skills Training Directly Related to Employment: Hours of Holidays", + friendly_name="Job Skills Training: Hours of Holidays", type="string", startIndex=106, endIndex=108, @@ -745,8 +745,8 @@ Field( item="59A", name="ED_NO_HIGH_SCHOOL_DIPL_HOP", - friendly_name="Education Directly Related to Employment for an Individual with NO High " + - "School Diploma or Certificate of High School Equivalency: Hours of Participation", + friendly_name="Education Directly Related to Employment: " + + "Hours of Participation", type="string", startIndex=108, endIndex=110, @@ -758,8 +758,8 @@ Field( item="59B", name="ED_NO_HIGH_SCHOOL_DIPL_EA", - friendly_name="Education Directly Related to Employment for an Individual with NO High " + - "School Diploma or Certificate: Hours of Excused Absences", + friendly_name="Education Directly Related to Employment: " + + "Hours of Excused Absences", type="string", startIndex=110, endIndex=112, @@ -771,8 +771,8 @@ Field( item="59C", name="ED_NO_HIGH_SCHOOL_DIPL_HOL", - friendly_name="Education Directly Related to Employment for an Individual with NO High " + - "School Diploma or Certificate: Hours of Holidays", + friendly_name="Education Directly Related to Employment: " + + "Hours of Holidays", type="string", startIndex=112, endIndex=114, @@ -784,8 +784,8 @@ Field( item="60A", name="SCHOOL_ATTENDENCE_HOP", - friendly_name="Satisfactory School Attendance for Individuals with No High School " + - "Diploma or Certificate: Hours of Participation", + friendly_name="Satisfactory School Attendance: " + + "Hours of Participation", type="string", startIndex=114, endIndex=116, @@ -797,8 +797,8 @@ Field( item="60B", name="SCHOOL_ATTENDENCE_EA", - friendly_name="Satisfactory School Attendance for Individuals with No High School" + - "Diploma or Certificate: Hours of Ecused Absences", + friendly_name="Satisfactory School Attendance: " + + "Hours of Ecused Absences", type="string", startIndex=116, endIndex=118, @@ -810,8 +810,8 @@ Field( item="60C", name="SCHOOL_ATTENDENCE_HOL", - friendly_name="Satisfactory School Attendance for Individuals with No High School" + - "Diploma or Certificate: Hours of Holidays", + friendly_name="Satisfactory School Attendance: " + + "Hours of Holidays", type="string", startIndex=118, endIndex=120, @@ -823,8 +823,8 @@ Field( item="61A", name="PROVIDE_CC_HOP", - friendly_name="Providing Child Care for an Individual Who Is Participating" + - "in a Community Service Program: Hours of Participation", + friendly_name="Providing Child Care: " + + "Hours of Participation", type="string", startIndex=120, endIndex=122, @@ -836,8 +836,8 @@ Field( item="61B", name="PROVIDE_CC_EA", - friendly_name="Providing Child Care for an Individual Who Is Participating" + - "in a Community Service Program: Hours of Excused Absences", + friendly_name="Providing Child Care: " + + "Hours of Excused Absences", type="string", startIndex=122, endIndex=124, @@ -849,8 +849,8 @@ Field( item="61C", name="PROVIDE_CC_HOL", - friendly_name="Providing Child Care Services for an Individual Who Is Participating" + - "in a Community Service Program: Hours of Holidays", + friendly_name="Providing Child Care Services: " + + "Hours of Holidays", type="string", startIndex=124, endIndex=126, @@ -910,7 +910,7 @@ Field( item="66A", name="UNEARNED_INCOME_TAX_CREDIT", - friendly_name="Amount of Unearned Income: Earned Income Tax Credit (EITC)", + friendly_name="Amount of Unearned Income: EITC", type="string", startIndex=136, endIndex=140, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 337ec4b2f..3fa6718d4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -133,7 +133,7 @@ Field( item="12", name="REC_FOOD_STAMPS", - friendly_name="Received Assistance from SNAP", + friendly_name="Received SNAP Assistance", type="number", startIndex=34, endIndex=35, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index ef45c6ee0..0a9144737 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -131,7 +131,7 @@ Field( item="12", name="REC_FOOD_STAMPS", - friendly_name="Received Food Stamps", + friendly_name="Received SNAP Assistance", type="number", startIndex=34, endIndex=35, From f3f8fbca575ef3bf89bd5b7db8a563adab0b8e61 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 21 May 2024 15:27:39 -0400 Subject: [PATCH 030/105] - getting test case back its expected state --- tdrs-backend/tdpservice/parsers/test/data/bad_trailer_2.txt | 2 +- tdrs-backend/tdpservice/parsers/test/test_parse.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/data/bad_trailer_2.txt b/tdrs-backend/tdpservice/parsers/test/data/bad_trailer_2.txt index 9060606ce..0ed792a8a 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/bad_trailer_2.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/bad_trailer_2.txt @@ -1,3 +1,3 @@ HEADER20204A06 TAN1 D -T12020101111111111223003403361110213120000300000000000008730010000000000000000000000000000000000222222000000002229012 +T12020101111111111223003403361110213320000300000000000008730010000000000000000000000000000000000222222000000002229012 T1trash \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b36e975db..a8ae171b2 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -428,7 +428,6 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): assert row_3_error.error_message in { 'T1 record length of 7 characters is not in the range [117, 156].', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', - 'T1trash does not start with TRAILER.', 'TRAILER record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} From 9a96a9338437cbac7b600822440620ebb445dc1b Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 22 May 2024 15:02:35 -0400 Subject: [PATCH 031/105] fix tets --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 10 ++++++++-- .../tdpservice/parsers/test/test_validators.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 117c5656b..9423d74b3 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1015,7 +1015,10 @@ def test_parse_tanf_section2_file(tanf_section2_file, dfs): err = parser_errors.first() assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == "T4: 3 is not larger or equal to 1 and smaller or equal to 2." + assert err.error_message == ( + "T4 Item 10 (receives subsidized housing): 3 " + "is not larger or equal to 1 and smaller or equal to 2." + ) assert err.content_type.model == "tanf_t4" assert err.object_id is not None @@ -1768,7 +1771,10 @@ def test_empty_t4_t5_values(t4_t5_empty_values, dfs): assert t4[0].STRATUM is None logger.info(t4[0].__dict__) assert t5.count() == 1 - assert parser_errors[0].error_message == "T4: 3 is not larger or equal to 1 and smaller or equal to 2." + assert parser_errors[0].error_message == ( + "T4 Item 10 (receives subsidized housing): 3 is " + "not larger or equal to 1 and smaller or equal to 2." + ) @pytest.mark.django_db() diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 28b15b1ae..0ee8815e4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -262,7 +262,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ + assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ '29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55].' From 725a959384c48f0c913f6b4ba5b14075290dbb00 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 29 May 2024 10:26:56 -0400 Subject: [PATCH 032/105] udpate required validator --- tdrs-backend/tdpservice/parsers/row_schema.py | 7 +++++-- tdrs-backend/tdpservice/parsers/test/test_util.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index c16718383..7500ab593 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -1,7 +1,7 @@ """Row schema for datafile.""" from .models import ParserErrorCategoryChoices from .fields import Field, TransformField -from .validators import value_is_empty +from .validators import value_is_empty, format_error_context import logging logger = logging.getLogger(__name__) @@ -150,7 +150,10 @@ def run_field_validators(self, instance, generate_error): generate_error( schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, - error_message=f"{field.name} is required but a value was not provided.", + error_message=( + f"{format_error_context(self, field.friendly_name, field.item)}: " + "field is required but a value was not provided." + ), record=instance, field=field ) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index a36dacf27..dd4465e9c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -297,8 +297,8 @@ def test_field_validators_blank_and_required_returns_error(first, second): is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == [ - 'first is required but a value was not provided.', - 'second is required but a value was not provided.' + 'T1 Item 1 (first): field is required but a value was not provided.', + 'T1 Item 2 (second): field is required but a value was not provided.' ] From 3ac7d5f092b42bd9d7d4aa5ad911d6baa70f4e06 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Sun, 2 Jun 2024 16:55:48 -0400 Subject: [PATCH 033/105] add upcoming data deadline notification --- tdrs-backend/tdpservice/email/email_enums.py | 1 + tdrs-backend/tdpservice/email/tasks.py | 78 +++++++++ .../upcoming-submission-deadline.html | 27 ++++ .../test/test_upcoming_deadline_email.py | 148 ++++++++++++++++++ tdrs-backend/tdpservice/settings/common.py | 40 +++++ 5 files changed, 294 insertions(+) create mode 100644 tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html create mode 100644 tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py index 2f56ee6c5..3b7028681 100644 --- a/tdrs-backend/tdpservice/email/email_enums.py +++ b/tdrs-backend/tdpservice/email/email_enums.py @@ -13,3 +13,4 @@ class EmailType(Enum): REQUEST_DENIED = 'request-denied.html' DEACTIVATION_WARNING = 'account-deactivation-warning.html' ACCOUNT_DEACTIVATED = 'account-deactivated.html' + UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html' diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 3da03d93d..3167180bd 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -11,6 +11,10 @@ import logging from tdpservice.email.helpers.account_access_requests import send_num_access_requests_email from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email +from tdpservice.stts.models import STT +from tdpservice.data_files.models import DataFile +from .email import automated_email +from .email_enums import EmailType logger = logging.getLogger(__name__) @@ -69,3 +73,77 @@ def email_admin_num_access_requests(): subject, email_context, ) + + +@shared_task +def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): + """Send all Data Analysts a reminder to submit if they have not already.""" + def get_fiscal_year(calendar_year, fiscal_quarter): + return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year + + now = datetime.now() + fiscal_year = get_fiscal_year(now.year, fiscal_quarter) + + all_locations = STT.objects.all() + + # + reminder_locations = [] + for loc in all_locations: + submitted_sections = DataFile.objects.all().filter( + stt=loc, + year=fiscal_year, + quarter=fiscal_quarter, + # version=?? + ) + required_sections = loc.filenames.keys() + + submitted_all_sections = True + for s in required_sections: + if not submitted_sections.filter(section=s).exists(): + submitted_all_sections = False + + if not submitted_all_sections: + reminder_locations.append(loc) + + # + # locations_with_no_submission_for_data_period = all_locations.filter() # -> ? + # spike ticket for improvement and/or iterate + + template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value + text_message = 'Your datafiles are due in five days.' + subject = f'Upcoming submission deadline: {due_date}' + + for loc in reminder_locations: + recipients = User.objects.filter( + stt=loc, + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='Data Analyst') + ) + + for rec in recipients: + context = { + 'first_name': rec.first_name, + 'fiscal_year': fiscal_year, + 'fiscal_quarter': fiscal_quarter, + 'submission_deadline': due_date, + 'url': settings.FRONTEND_BASE_URL, + 'subject': subject + } + + logger_context = { + 'user_id': rec.id, + 'object_id': rec.id, + 'object_repr': rec.username, + } + + automated_email( + email_path=template_path, + recipient_email=rec.username, + subject=subject, + email_context=context, + text_message=text_message, + logger_context=logger_context + ) + + if len(recipients) == 0: + print(f'{loc.name} needs a reminder email but has no recipients') diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html new file mode 100644 index 000000000..72ac24b57 --- /dev/null +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -0,0 +1,27 @@ +{% extends 'base.html' %} +{% block content %} + + +

Hello {{ first_name }},

+

This is a friendly reminder that your data files are due in 5 days.

+

Please sign in to the TANF Data Portal +to upload and submit data files for Fiscal Year {{ fiscal_year }} - {{ fiscal_quarter}} by {{ submission_deadline }}.

+ + + + +

Submit your data files

+ +Need help?
+We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions. +

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team by replying to this email and we will follow up with you directly.

+

Thank you,

+

The TDP Team

+ + + + + + +{% endblock %} diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py new file mode 100644 index 000000000..eb40906c3 --- /dev/null +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -0,0 +1,148 @@ +"""Test function for sending upcoming data deadline reminders.""" +import pytest +from django.core import mail +from tdpservice.email.tasks import send_data_submission_reminder +from datetime import datetime +from django.contrib.auth.models import Group + +from tdpservice.stts.models import STT +from tdpservice.users.models import User +from tdpservice.data_files.models import DataFile + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_sends_no_sections_submitted( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function runs when no sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 1 + assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_sends_some_sections_submitted( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function runs when some sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + now = datetime.now() + fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year + + data_file = DataFile.create_new_version({ + "section": 'Active Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 1 + assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + + +"""Test function for sending upcoming data deadline reminders.""" +import pytest +from django.core import mail +from tdpservice.email.tasks import send_data_submission_reminder +from django.contrib.auth.models import Group + +from tdpservice.stts.models import STT +from tdpservice.users.models import User + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_no_send_when_all_sections_complete( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function does not run when all sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + now = datetime.now() + fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year + + data_file = DataFile.create_new_version({ + "section": 'Active Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + data_file2 = DataFile.create_new_version({ + "section": 'Closed Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 0 diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 6ca924fe0..13875a52d 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -489,6 +489,46 @@ class Common(Configuration): 'task': 'tdpservice.email.tasks.email_admin_num_access_requests', 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) }, + 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='2', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'February 14th', + 'reporting_period': 'Oct - Dec', + 'fiscal_quarter': 'Q1', + } + }, + 'Email Data Analyst Q2 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='5', day_of_month='10', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'May 15th', + 'reporting_period': 'Jan - Mar', + 'fiscal_quarter': 'Q2', + } + }, + 'Email Data Analyst Q3 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='8', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'August 14th', + 'reporting_period': 'Apr - Jun', + 'fiscal_quarter': 'Q3', + } + }, + 'Email Data Analyst Q4 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='11', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'November 14th', + 'reporting_period': 'Jul - Sep', + 'fiscal_quarter': 'Q4', + } + }, } CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None) From 24bceb9d5e6794a7e6b76f3bcb20790f2f8739f5 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Sun, 2 Jun 2024 17:06:10 -0400 Subject: [PATCH 034/105] lint --- .../test/test_upcoming_deadline_email.py | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py index eb40906c3..b6511505d 100644 --- a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -1,10 +1,9 @@ """Test function for sending upcoming data deadline reminders.""" import pytest +from datetime import datetime from django.core import mail from tdpservice.email.tasks import send_data_submission_reminder -from datetime import datetime from django.contrib.auth.models import Group - from tdpservice.stts.models import STT from tdpservice.users.models import User from tdpservice.data_files.models import DataFile @@ -20,7 +19,6 @@ def test_upcoming_deadline_sends_no_sections_submitted( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function runs when no sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -53,7 +51,6 @@ def test_upcoming_deadline_sends_no_sections_submitted( def test_upcoming_deadline_sends_some_sections_submitted( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function runs when some sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -73,7 +70,7 @@ def test_upcoming_deadline_sends_some_sections_submitted( now = datetime.now() fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year - data_file = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Active Case Data', "quarter": fiscal_quarter, "year": fiscal_year, @@ -87,16 +84,6 @@ def test_upcoming_deadline_sends_some_sections_submitted( assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" -"""Test function for sending upcoming data deadline reminders.""" -import pytest -from django.core import mail -from tdpservice.email.tasks import send_data_submission_reminder -from django.contrib.auth.models import Group - -from tdpservice.stts.models import STT -from tdpservice.users.models import User - - @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ ('February 14', 'Oct - Dec', 'Q1'), ('May 15th', 'Jan - Mar', 'Q2'), @@ -107,7 +94,6 @@ def test_upcoming_deadline_sends_some_sections_submitted( def test_upcoming_deadline_no_send_when_all_sections_complete( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function does not run when all sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -127,7 +113,7 @@ def test_upcoming_deadline_no_send_when_all_sections_complete( now = datetime.now() fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year - data_file = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Active Case Data', "quarter": fiscal_quarter, "year": fiscal_year, @@ -135,7 +121,7 @@ def test_upcoming_deadline_no_send_when_all_sections_complete( "user": data_analyst, }) - data_file2 = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Closed Case Data', "quarter": fiscal_quarter, "year": fiscal_year, From fd351e30b3324556eb61d524fadbefca13c1bfd9 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 13:52:15 -0400 Subject: [PATCH 035/105] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 3167180bd..c9d6c5e21 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -94,12 +94,12 @@ def get_fiscal_year(calendar_year, fiscal_quarter): year=fiscal_year, quarter=fiscal_quarter, # version=?? - ) + ).values_list('section', flat=True).distinct() required_sections = loc.filenames.keys() submitted_all_sections = True for s in required_sections: - if not submitted_sections.filter(section=s).exists(): + if s not in submitted_sections: submitted_all_sections = False if not submitted_all_sections: From c34dd0411dc8761a466344707ca6c6a5395437ec Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:52:13 -0400 Subject: [PATCH 036/105] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index c9d6c5e21..dd5fbed18 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -88,13 +88,13 @@ def get_fiscal_year(calendar_year, fiscal_quarter): # reminder_locations = [] + year_quarter_files = DataFile.objects.all().filter( + year=fiscal_year, + quarter=fiscal_quarter, + ) + for loc in all_locations: - submitted_sections = DataFile.objects.all().filter( - stt=loc, - year=fiscal_year, - quarter=fiscal_quarter, - # version=?? - ).values_list('section', flat=True).distinct() + submitted_sections = year_quarter_files.filter(stt=loc).values_list('section', flat=True).distinct() required_sections = loc.filenames.keys() submitted_all_sections = True From cb8b37a81a24402819a39062521ab9efb61cad1f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:52:19 -0400 Subject: [PATCH 037/105] clarity comment --- tdrs-backend/tdpservice/settings/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 13875a52d..4a12a855c 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -491,6 +491,7 @@ class Common(Configuration): }, 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Feb 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='2', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -501,6 +502,7 @@ class Common(Configuration): }, 'Email Data Analyst Q2 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # May 10 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='5', day_of_month='10', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -511,6 +513,7 @@ class Common(Configuration): }, 'Email Data Analyst Q3 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Aug 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='8', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -521,6 +524,7 @@ class Common(Configuration): }, 'Email Data Analyst Q4 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Nov 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='11', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { From e1296c74c65a59898005546a374afb13435e79b8 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:55:54 -0400 Subject: [PATCH 038/105] add no recips logentry --- tdrs-backend/tdpservice/email/tasks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index dd5fbed18..20216ba68 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,7 +13,7 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from .email import automated_email +from .email import automated_email, log from .email_enums import EmailType @@ -146,4 +146,13 @@ def get_fiscal_year(calendar_year, fiscal_quarter): ) if len(recipients) == 0: - print(f'{loc.name} needs a reminder email but has no recipients') + system_user, created = User.objects.get_or_create(username='system') + if created: + log('Created reserved system user.') + + logger_context = { + 'user_id': system_user.pk, + 'object_id': loc.id, + 'object_repr': loc.name, + } + log(f"{loc.name} has no recipients for data submission deadline reminder.", logger_context=logger_context) From 29bc6697c199bdeae7f3b9f5dc70bde376f24449 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:20:00 -0400 Subject: [PATCH 039/105] rm comments --- tdrs-backend/tdpservice/email/tasks.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 20216ba68..fe0aafd22 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -86,7 +86,6 @@ def get_fiscal_year(calendar_year, fiscal_quarter): all_locations = STT.objects.all() - # reminder_locations = [] year_quarter_files = DataFile.objects.all().filter( year=fiscal_year, @@ -105,13 +104,9 @@ def get_fiscal_year(calendar_year, fiscal_quarter): if not submitted_all_sections: reminder_locations.append(loc) - # - # locations_with_no_submission_for_data_period = all_locations.filter() # -> ? - # spike ticket for improvement and/or iterate - template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value - text_message = 'Your datafiles are due in five days.' - subject = f'Upcoming submission deadline: {due_date}' + text_message = f'Your datafiles are due by {due_date}.' + subject = f'Action Requested: Please submit your {TANF/SSP} data files' for loc in reminder_locations: recipients = User.objects.filter( From 5c52b060283c874820189cce844f3d5f30767781 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:22:53 -0400 Subject: [PATCH 040/105] update email subject to include ssp/tanf --- tdrs-backend/tdpservice/email/tasks.py | 4 +++- .../tdpservice/email/test/test_upcoming_deadline_email.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index fe0aafd22..4abd58c89 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -106,9 +106,11 @@ def get_fiscal_year(calendar_year, fiscal_quarter): template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value text_message = f'Your datafiles are due by {due_date}.' - subject = f'Action Requested: Please submit your {TANF/SSP} data files' for loc in reminder_locations: + tanf_ssp_label = 'SSP' if loc.ssp else 'TANF' + subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' + recipients = User.objects.filter( stt=loc, account_approval_status=AccountApprovalStatusChoices.APPROVED, diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py index b6511505d..1c7b224dd 100644 --- a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -38,7 +38,7 @@ def test_upcoming_deadline_sends_no_sections_submitted( send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) assert len(mail.outbox) == 1 - assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ @@ -81,7 +81,7 @@ def test_upcoming_deadline_sends_some_sections_submitted( send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) assert len(mail.outbox) == 1 - assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ From 9d9b1412f99154adc319152f77101e8ea9087c08 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:26:36 -0400 Subject: [PATCH 041/105] SSP -> TANF and SSP --- tdrs-backend/tdpservice/email/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 4abd58c89..92ff0b9e8 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -108,7 +108,7 @@ def get_fiscal_year(calendar_year, fiscal_quarter): text_message = f'Your datafiles are due by {due_date}.' for loc in reminder_locations: - tanf_ssp_label = 'SSP' if loc.ssp else 'TANF' + tanf_ssp_label = 'TANF and SSP' if loc.ssp else 'TANF' subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' recipients = User.objects.filter( From 3be46db469d66c6fb7c5dc6a223312e6b22ebdaf Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 4 Jun 2024 11:35:29 -0400 Subject: [PATCH 042/105] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 92ff0b9e8..480c971f7 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -107,15 +107,16 @@ def get_fiscal_year(calendar_year, fiscal_quarter): template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value text_message = f'Your datafiles are due by {due_date}.' + all_data_analysts = User.objects.all().filter( + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='Data Analyst') + ) + for loc in reminder_locations: tanf_ssp_label = 'TANF and SSP' if loc.ssp else 'TANF' subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' - recipients = User.objects.filter( - stt=loc, - account_approval_status=AccountApprovalStatusChoices.APPROVED, - groups=Group.objects.get(name='Data Analyst') - ) + recipients = all_data_analysts.filter(stt=loc) for rec in recipients: context = { From 799ce28e96edc215d5f054d8e5ae7d9bfeac19e0 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 12:18:13 -0400 Subject: [PATCH 043/105] Merge branch 'develop' into 2693-cat2-messaging-cleanup --- ...4-configuration-by-environment-variable.md | 4 +- scripts/deploy-backend.sh | 1 + tdrs-backend/docs/api/authentication.md | 11 ++ .../tdpservice/data_files/test/test_api.py | 24 ++-- tdrs-backend/tdpservice/data_files/util.py | 38 ++++-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 13 +- .../tdpservice/parsers/schema_defs/ssp/m2.py | 10 +- .../tdpservice/parsers/schema_defs/ssp/m3.py | 14 +-- .../tdpservice/parsers/schema_defs/ssp/m4.py | 9 +- .../tdpservice/parsers/schema_defs/ssp/m5.py | 8 +- .../tdpservice/parsers/schema_defs/ssp/m6.py | 8 +- .../tdpservice/parsers/schema_defs/ssp/m7.py | 8 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 10 +- .../tdpservice/parsers/schema_defs/tanf/t2.py | 1 - .../tdpservice/parsers/schema_defs/tanf/t3.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t4.py | 9 +- .../tdpservice/parsers/schema_defs/tanf/t5.py | 1 - .../parsers/schema_defs/tribal_tanf/t1.py | 14 +-- .../parsers/schema_defs/tribal_tanf/t2.py | 17 +-- .../parsers/schema_defs/tribal_tanf/t3.py | 8 +- .../parsers/schema_defs/tribal_tanf/t4.py | 12 +- .../parsers/schema_defs/tribal_tanf/t5.py | 10 +- .../tdpservice/parsers/test/conftest.py | 64 ++++++++++ .../parsers/test/data/ADS.E2J.FTP2.TS142.txt | 12 +- .../tdpservice/parsers/test/test_parse.py | 84 ++++++++++++- .../parsers/test/test_transforms.py | 22 +++- .../parsers/test/test_validators.py | 28 ----- tdrs-backend/tdpservice/parsers/transforms.py | 6 + tdrs-backend/tdpservice/parsers/validators.py | 112 ++---------------- .../0028_education_level_to_string.py | 23 ++++ .../tdpservice/search_indexes/models/ssp.py | 4 +- .../tdpservice/security/test/test_views.py | 66 +++++++++++ tdrs-backend/tdpservice/security/urls.py | 12 ++ tdrs-backend/tdpservice/security/utils.py | 43 +++++++ tdrs-backend/tdpservice/security/views.py | 36 ++++++ tdrs-backend/tdpservice/settings/common.py | 4 +- tdrs-backend/tdpservice/urls.py | 1 + .../SubmissionHistory/CaseAggregatesTable.jsx | 2 +- .../SubmissionHistory/SubmissionHistory.jsx | 36 ++++-- .../SubmissionHistory.test.js | 4 +- .../TotalAggregatesTable.jsx | 2 +- 41 files changed, 544 insertions(+), 251 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/conftest.py create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py create mode 100644 tdrs-backend/tdpservice/security/test/test_views.py create mode 100644 tdrs-backend/tdpservice/security/urls.py create mode 100644 tdrs-backend/tdpservice/security/utils.py create mode 100644 tdrs-backend/tdpservice/security/views.py diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md index dbd6920c1..95c1a4604 100644 --- a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md +++ b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md @@ -9,9 +9,11 @@ Accepted Applications need to be configured differently depending on where they are running. For example, the backend running locally will have different configuration then the backend running in production. +Further, environment variables can be designated "secret" or not; the term "secret key" is often used in place of secret environment variables. Secret keys are sometimes (but not always) shared between different deployment environments, which makes it useful to have a central "single source of truth" where a secret key can be kept and copied out to different environments. CircleCI solves this use case for us, allowing secret keys to be managed by the project's Environment Variables, and accessed in the deployment process to write to cloud.gov applications. + ## Decision -We will use environment variables to configure applications. +We will use environment variables to configure applications. We will use Environment Variables in CircleCI to store and manage secret keys. ## Consequences diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 7a742ad79..3f53b6b59 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -62,6 +62,7 @@ set_cf_envs() "REDIS_URI" "JWT_KEY" "STAGING_JWT_KEY" + "SENDGRID_API_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" diff --git a/tdrs-backend/docs/api/authentication.md b/tdrs-backend/docs/api/authentication.md index e51f2174d..177c23213 100644 --- a/tdrs-backend/docs/api/authentication.md +++ b/tdrs-backend/docs/api/authentication.md @@ -4,3 +4,14 @@ For clients to authenticate, they have to authenticate with Login.gov via the ba This will allow the backend to identify the browser which requested access and authorize them based on the cookie they provide in their API calls. The secured portion of this authorization is due to the httpOnly cookie being inaccessible to the client's local browser. + +# Generating API token + +In order to use APIs, an activated `OFA Sys Admin` user has to generate a new token and use it in the API request following these steps: +1. User has to first login using frontend and going through the normal login process +2. After user is logged in, user can grab a token at `/v1/security/get-token` +3. The token then can be used in authorization header. As an example: + +```curl --location 'http://{host}/v1/users/' --header 'Authorization: Token {token}'``` + +Note: the authentication token is available for 24 hours by default but this can be overridden using the `TOKEN_EXPIRATION_HOURS` environment variable. diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index faacac4d8..d08ce3f1d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -96,11 +96,11 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." @staticmethod @@ -108,11 +108,11 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod @@ -128,11 +128,11 @@ def assert_error_report_file_content_matches_without_friendly_names(response): wb = openpyxl.load_workbook('mycls.xlsx') ws = wb.get_sheet_by_name('Sheet1') - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ( + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." ) diff --git a/tdrs-backend/tdpservice/data_files/util.py b/tdrs-backend/tdpservice/data_files/util.py index 3c5470c13..17beb90aa 100644 --- a/tdrs-backend/tdpservice/data_files/util.py +++ b/tdrs-backend/tdpservice/data_files/util.py @@ -36,27 +36,51 @@ def format_error_msg(x): output = BytesIO() workbook = xlsxwriter.Workbook(output) worksheet = workbook.add_worksheet() + report_columns = [ ('case_number', lambda x: x['case_number']), ('year', lambda x: str(x['rpt_month_year'])[0:4] if x['rpt_month_year'] else None), ('month', lambda x: calendar.month_name[ int(str(x['rpt_month_year'])[4:]) ] if x['rpt_month_year'] else None), - ('error_type', lambda x: x['error_type']), ('error_message', lambda x: format_error_msg(chk(x))), ('item_number', lambda x: x['item_number']), ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])), ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])), ('row_number', lambda x: x['row_number']), - ('column_number', lambda x: x['column_number']) ] # write beta banner - worksheet.write(row, col, - "Error reporting in TDP is still in development." + - "We'll be in touch when it's ready to use!" + - "For now please refer to the reports you receive via email") + worksheet.write( + row, col, + "Please refer to the most recent versions of the coding " + + "instructions (linked below) when looking up items " + + "and allowable values during the data revision process" + ) + + row, col = 1, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/tribal-tanf-data-coding-instructions', + string='For Tribal TANF data reports: Tribal TANF Instructions', + ) + row, col = 2, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/acf-ofa-pi-23-04', + string='For TANF and SSP-MOE data reports: TANF / SSP-MOE (ACF-199 / ACF-209) Instructions' + ) + + row, col = 3, 0 + worksheet.write_url( + row, col, + 'https://tdp-project-updates.app.cloud.gov/knowledge-center/viewing-error-reports.html', + string='Visit the Knowledge Center for further guidance on reviewing error reports' + ) + + row, col = 5, 0 + # write csv header bold = workbook.add_format({'bold': True}) @@ -68,7 +92,7 @@ def format_header(header_list: list): [worksheet.write(row, col, format_header(key[0]), bold) for col, key in enumerate(report_columns)] [ - worksheet.write(row + 3, col, key[1](data_i)) for col, key in enumerate(report_columns) + worksheet.write(row + 6, col, key[1](data_i)) for col, key in enumerate(report_columns) for row, data_i in enumerate(data) ] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 5c2ac878c..338a97216 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument @@ -133,15 +133,16 @@ required=True, validators=[validators.notEmpty()] ), - Field( + TransformField( + zero_pad(3), item="2", - name='COUNTY_FIPS_CODE', + name="COUNTY_FIPS_CODE", friendly_name="county fips code", - type='string', + type="string", startIndex=19, endIndex=22, required=True, - validators=[validators.isNumber(),] + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 0f72a48e1..c44bfc76a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -370,14 +370,14 @@ item="37", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=55, endIndex=57, required=False, validators=[ validators.or_validators( - validators.isInLimits(0, 16), validators.isInLimits(98, 99) - ) + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) + ), ] ), Field( @@ -388,7 +388,7 @@ startIndex=57, endIndex=58, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="39", @@ -398,7 +398,7 @@ startIndex=58, endIndex=59, required=False, - validators=[validators.oneOf([0, 1, 2, 9])] + validators=[validators.oneOf([1, 2, 9])] ), Field( item="40", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 19069089b..12376bbfc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -279,15 +279,15 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=49, endIndex=51, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) - ) + ), ] ), Field( @@ -298,7 +298,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", @@ -593,13 +593,13 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=90, endIndex=92, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ) ] @@ -612,7 +612,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 792ac2ba6..705e2592a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument @@ -54,7 +54,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +63,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(0, 999)], + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 7ba6aa2c6..080716cb0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -347,7 +347,8 @@ validators.or_validators( validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - ) + ), + validators.notMatches("00") ], ), Field( @@ -359,10 +360,7 @@ endIndex=57, required=False, validators=[ - validators.or_validators( - validators.isInLimits(0, 3), - validators.matches(9) - ) + validators.oneOf([1, 2, 3, 9]), ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index e85bc8fb1..69d1bda7a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -1,10 +1,10 @@ """Schema for HEADER row of all submission types.""" -from ...transforms import calendar_quarter_to_rpt_month_year -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument s1 = RowSchema( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 8d6664a43..39ecf8f84 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -1,9 +1,9 @@ """Schema for TANF T7 Row.""" -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ...transforms import calendar_quarter_to_rpt_month_year -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument schemas = [] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index dc5d153cb..371b29df6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -1,6 +1,7 @@ """Schema for t1 record types.""" -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument @@ -155,7 +156,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -163,9 +165,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 47b6e9144..683649a9b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -98,7 +98,6 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 4bc9f6195..5cf53bc6a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -295,7 +295,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", @@ -608,7 +608,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 2828e2a8f..2de7ea71c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -1,7 +1,7 @@ """Schema for HEADER row of all submission types.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument @@ -55,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -63,7 +64,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(1, 999)], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index fa0e1792c..df9bf9ce2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -92,7 +92,6 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), validators.if_then_validator( condition_field_name="DATE_OF_BIRTH", condition_function=validators.olderThan(18), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 221739dd9..a5d4a45a7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T1 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument t1 = SchemaManager( @@ -156,7 +157,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -164,9 +166,7 @@ startIndex=19, endIndex=22, required=False, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 396418cba..e815ac849 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T2 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func, zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T2DataSubmissionDocument @@ -98,7 +98,7 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), @@ -347,9 +347,9 @@ type="string", startIndex=51, endIndex=53, - required=False, + required=True, validators=[ - validators.isInStringRange(0, 10), + validators.isInStringRange(1, 10), ], ), Field( @@ -622,7 +622,8 @@ validators.isInStringRange(0, 99), ], ), - Field( + TransformField( + zero_pad(2), item="61", name="ADD_WORK_ACTIVITIES", friendly_name="additional work activities", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index 7acbdadaa..4e03bbe61 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T3 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T3DataSubmissionDocument FIRST_CHILD = 1 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index e59cecade..9f1f53802 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T4 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument @@ -54,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +64,7 @@ startIndex=19, endIndex=22, required=False, - validators=[validators.matches("000")], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 553197447..22ea004a8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T5 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T5DataSubmissionDocument @@ -92,7 +92,7 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.matches(1), diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py new file mode 100644 index 000000000..1754e66e3 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -0,0 +1,64 @@ +"""Fixtures for parsing integration tests.""" +import pytest +from .factories import ParsingFileFactory + +@pytest.fixture +def t3_cat2_invalid_citizenship_file(): + """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A06 TAN1ED\n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' + b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' + b'1100000099998888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m2_cat2_invalid_37_38_39_file(): + """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m2_cat2_invalid_37_38_39_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' + b'00000000000000000000000000000225300000000000000000000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.fixture +def m3_cat2_invalid_68_69_file(): + """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m3_cat2_invalid_68_69_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' + b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' + b'8888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m5_cat2_invalid_23_24_file(): + """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m5_cat2_invalid_23_24_file.txt', + section='SSP Closed Case Data', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' + b'TRAILER0000001 ') + ) + return parsing_file diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt index 7ef4b46b0..4078f53ee 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt @@ -1,21 +1,21 @@ HEADER20194C00142TAN1ED -T420191011111111762255 0402451153123 +T420191011111111762 1 0402451153123 T520191011111111762120160102WTTTTT@YB2122222222221 822981 0 03 0 0 T520191011111111762120170526WTTTTTZPW2122221222221 822981 0 03 0 0 T520191011111111762319880112WTTTTTTY#2222212222222 122161 1591 0 0 T520191011111111762319610502WTTTTTT##2222212222222 222161 0601 0 0 -T420191011111112343255 0402451 91113 +T420191011111112343 1 0402451 91113 T520191011111112343119860308WTTTTTTTY2122222222221 122111 44162 0 0 -T420191011111112970255 0403561 91112 +T420191011111112970 1 0403561 91112 T520191011111112970119940807WTTTTT@#Z2122221222221 122121 10501 0 0 -T420191111111111339255 0403561 83113 +T420191111111111339 1 0403561 83113 T520191111111111339119880402WTTTTTZ#B2122221222223 221121 3571 0 0 T520191111111111339119970502WTTTTTTYB2122222222221 111111 8522 0 0 -T420191111111112073255 0403561151123 +T420191111111112073 1 0403561151123 T520191111111112073319900312WTTTTTT0@2122222222222 122121 0601 0 0 T520191111111112073319920507WTTTTT@B02122221222222 222121 0601 0 0 T520191111111112073120090514WTTTTT@@@2122222222221 822 11 0 03 0 0 -T420191111111112472255 0403561183113 +T420191111111112472 1 0403561183113 T520191111111112472120140814WTTTTTZZ02222212222221 422981 0 03 0 0 T520191111111112472119840305WTTTTT90W2122221222221 122101 40202 0 0 TRAILER 19 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 9423d74b3..ef2132b65 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1091,7 +1091,7 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks, dfs): parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) - assert parser_errors.count() == 23 + assert parser_errors.count() == 22 # Should only be cat3 validator errors for error in parser_errors: @@ -1979,3 +1979,85 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): "representing the Calendar Year and Quarter formatted as YYYYQ" Tribal_TANF_T7.objects.count() == 0 + +@pytest.mark.django_db() +def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): + """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" + dfs.datafile = t3_cat2_invalid_citizenship_file + t3_cat2_invalid_citizenship_file.year = 2021 + t3_cat2_invalid_citizenship_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(t3_cat2_invalid_citizenship_file, dfs) + + parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + for e in parser_errors: + assert e.error_message == "T3: 0 is not in [1, 2, 9]." + + +@pytest.mark.django_db() +def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): + """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + dfs.datafile = m2_cat2_invalid_37_38_39_file + m2_cat2_invalid_37_38_39_file.year = 2024 + m2_cat2_invalid_37_38_39_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m2_cat2_invalid_37_38_39_file, dfs) + + parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 3 + + error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", + "M2: 0 is not in [1, 2, 3, 9].", + "M2: 0 is not in [1, 2, 9]."} + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): + """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m3_cat2_invalid_68_69_file + m3_cat2_invalid_68_69_file.year = 2024 + m3_cat2_invalid_68_69_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m3_cat2_invalid_68_69_file, dfs) + + parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 4 + + error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): + """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m5_cat2_invalid_23_24_file + m5_cat2_invalid_23_24_file.year = 2019 + m5_cat2_invalid_23_24_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) + + parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + error_msgs = {"M5: 00 matches 00.", + "M5: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs diff --git a/tdrs-backend/tdpservice/parsers/test/test_transforms.py b/tdrs-backend/tdpservice/parsers/test/test_transforms.py index ff1188785..1aabe69db 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_transforms.py +++ b/tdrs-backend/tdpservice/parsers/test/test_transforms.py @@ -1,11 +1,31 @@ -"""Tests for the transforms module.""" +"""Test for Transforms.""" +import pytest +import tdpservice.parsers.transforms as transforms from tdpservice.parsers.transforms import ( tanf_ssn_decryption_func, ssp_ssn_decryption_func, ) +@pytest.mark.parametrize("value,digits,expected", [ + ("1", 3, "001"), + ("10", 3, "010"), + ("100", 3, "100"), + ("1000", 3, "1000"), + ("1 ", 3, "01 "), + ("1 ", 3, "1 "), + ("1", 0, "1"), + ("1", -1, "1") +]) +def test_zero_pad(value, digits, expected): + """Test zero_pad returns valid value.""" + transform = transforms.zero_pad(digits) + result = transform(value) + + assert result == expected + + def test_tanf_ssn_decryption_func(): """Test the TANF SSN decryption function.""" assert tanf_ssn_decryption_func(None) is None diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 0ee8815e4..94045acfb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -922,19 +922,6 @@ def test_validate_cooperation_with_child_support(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_months_federal_time_limit(self, record): - """Test cat3 validator for federal time limit.""" - val = validators.validate__FAM_AFF__HOH__Fed_Time() - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'MONTHS_FED_TIME_LIMIT']) - - record.FAMILY_AFFILIATION = 1 - record.MONTHS_FED_TIME_LIMIT = "000" - record.RELATIONSHIP_HOH = "01" - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_employment_status(self, record): """Test cat3 validator for employment status.""" val = validators.if_then_validator( @@ -1233,21 +1220,6 @@ def test_validate_citizenship_status(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_hoh_fed_time(self, record): - """Test cat3 validator for federal disability.""" - val = validators.validate__FAM_AFF__HOH__Count_Fed_Time() - - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'COUNTABLE_MONTH_FED_TIME']) - - record.FAMILY_AFFILIATION = 1 - record.RELATIONSHIP_HOH = 1 - record.COUNTABLE_MONTH_FED_TIME = 0 - - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_oasdi_insurance(self, record): """Test cat3 validator for OASDI insurance.""" val = validators.if_then_validator( diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 78d15dcbb..cd51e2012 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -36,3 +36,9 @@ def ssp_ssn_decryption_func(value, **kwargs): decryption_table = str.maketrans(decryption_dict) return value.translate(decryption_table) return value + +def zero_pad(digits): + """Zero pad a string.""" + def transform(value, **kwargs): + return value.lstrip().zfill(digits) + return transform diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 54bf98704..9f99e928c 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -330,6 +330,18 @@ def between(min, max): f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) + +def fieldHasLength(length): + """Validate that the field value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + ) + + def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( @@ -490,7 +502,7 @@ def isSmallerThanOrEqualTo(UpperBound): def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( - lambda value: value >= LowerBound and value <= UpperBound, + lambda value: int(value) >= LowerBound and int(value) <= UpperBound, lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." @@ -614,104 +626,6 @@ def validate(instance, row_schema): return validate - -def validate__FAM_AFF__HOH__Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then MONTHS_FED_TIME_LIMIT >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "MONTHS_FED_TIME_LIMIT >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - MONTHS_FED_TIME_LIMIT = ( - instance["MONTHS_FED_TIME_LIMIT"] - if type(instance) is dict - else getattr(instance, "MONTHS_FED_TIME_LIMIT") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if MONTHS_FED_TIME_LIMIT is None or int(MONTHS_FED_TIME_LIMIT) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "MONTHS_FED_TIME_LIMIT": MONTHS_FED_TIME_LIMIT} - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - -def validate__FAM_AFF__HOH__Count_Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then COUNTABLE_MONTH_FED_TIME >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "COUNTABLE_MONTH_FED_TIME >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - COUNTABLE_MONTH_FED_TIME = ( - instance["COUNTABLE_MONTH_FED_TIME"] - if type(instance) is dict - else getattr(instance, "COUNTABLE_MONTH_FED_TIME") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if int(COUNTABLE_MONTH_FED_TIME) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "COUNTABLE_MONTH_FED_TIME": COUNTABLE_MONTH_FED_TIME - } - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Count_Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - def validate_header_section_matches_submission(datafile, section, generate_error): """Validate header section matches submission section.""" is_valid = datafile.section == section diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py new file mode 100644 index 000000000..4a7cf36b4 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.15 on 2024-04-29 18:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0027_tribal_ssp_tanf_dob_to_string'), + ] + + operations = [ + migrations.AlterField( + model_name='ssp_m2', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + migrations.AlterField( + model_name='ssp_m3', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index b11e6fff5..bb5840323 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -113,7 +113,7 @@ class SSP_M2(models.Model): RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False) EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) @@ -194,7 +194,7 @@ class SSP_M3(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/security/test/test_views.py b/tdrs-backend/tdpservice/security/test/test_views.py new file mode 100644 index 000000000..b602ca577 --- /dev/null +++ b/tdrs-backend/tdpservice/security/test/test_views.py @@ -0,0 +1,66 @@ +"""Tests for the views in the security app.""" + +import pytest +import logging +from rest_framework.authtoken.models import Token +from tdpservice.users.models import User, AccountApprovalStatusChoices +from tdpservice.security.views import token_is_valid +from django.test import Client +from django.urls import reverse +from django.contrib.auth.models import Group + +client = Client() + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def token(): + """Return a DRF token.""" + user = User.objects.create(username="testuser") + token = Token.objects.create(user=user) + return token + + +@pytest.mark.django_db +def test_token_is_valid(token): + """Test token_is_valid function.""" + logger.info(token.__dict__) + assert token_is_valid(token) is True + token.created = token.created.replace(year=2000) + # token.save() + assert token_is_valid(token) is False + + +@pytest.mark.django_db +def test_generate_new_token(client): + """Test generate_new_token function.""" + url = reverse("get-new-token") + # assert if user is not authenticated + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not ofa_sys_admin + user = User.objects.create_user(username="testuser", password="testpassword") + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not approved + user.account_approval_status = AccountApprovalStatusChoices.PENDING + user.groups.add(Group.objects.get(name="OFA System Admin")) + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if token is valid + user.account_approval_status = AccountApprovalStatusChoices.APPROVED + user.save() + + client.login(username="testuser", password="testpassword") + url = reverse("get-new-token") + response = client.get(url) + assert response.status_code == 200 + assert response.data == str(Token.objects.get(user=user)) diff --git a/tdrs-backend/tdpservice/security/urls.py b/tdrs-backend/tdpservice/security/urls.py new file mode 100644 index 000000000..ef62d5e44 --- /dev/null +++ b/tdrs-backend/tdpservice/security/urls.py @@ -0,0 +1,12 @@ +"""URL patterns for the security app.""" + +from . import views +from django.urls import path + +urlpatterns = [ + path( + "get-token", + views.generate_new_token, + name="get-new-token", + ), +] diff --git a/tdrs-backend/tdpservice/security/utils.py b/tdrs-backend/tdpservice/security/utils.py new file mode 100644 index 000000000..873dc3aa4 --- /dev/null +++ b/tdrs-backend/tdpservice/security/utils.py @@ -0,0 +1,43 @@ +"""Utility classes and functions for security.""" + +from rest_framework import exceptions +from rest_framework.authentication import TokenAuthentication +from django.utils.translation import gettext_lazy as _ +from datetime import datetime +import pytz +from datetime import timedelta +from django.conf import settings +import logging + +logger = logging.getLogger(__name__) + +def token_is_valid(token): + """Check if token is valid.""" + utc_now = datetime.now() + utc_now = utc_now.replace(tzinfo=pytz.utc) + if token.created < (utc_now - timedelta(hours=settings.TOKEN_EXPIRATION_HOURS)): + logger.info("API auth Token expired") + return False + return token is not None + +# have to use ExpTokenAuthentication in settings.py instead of TokenAuthentication +class ExpTokenAuthentication(TokenAuthentication): + """Custom token authentication class that checks if token is expired.""" + + # see https://github.com/encode/django-rest-framework/blob/master/rest_framework/authentication.py + + def authenticate_credentials(self, key): + """Authenticate the credentials.""" + model = self.get_model() + try: + token = model.objects.select_related("user").get(key=key) + except model.DoesNotExist: + raise exceptions.AuthenticationFailed(_("Invalid token.")) + + if not token.user.is_active: + raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) + + if not token_is_valid(token): + raise exceptions.AuthenticationFailed(_("Token expired.")) + + return (token.user, token) diff --git a/tdrs-backend/tdpservice/security/views.py b/tdrs-backend/tdpservice/security/views.py new file mode 100644 index 000000000..64377332c --- /dev/null +++ b/tdrs-backend/tdpservice/security/views.py @@ -0,0 +1,36 @@ +"""Views for the security app.""" + +from rest_framework.decorators import api_view +from rest_framework.response import Response +from django.contrib.auth.decorators import user_passes_test +from tdpservice.users.models import User, AccountApprovalStatusChoices +from rest_framework.authtoken.models import Token +from tdpservice.security.utils import token_is_valid + +import logging + +logger = logging.getLogger(__name__) + + +def can_get_new_token(user): + """Check if user can get a new token.""" + return ( + user.is_authenticated + and user.is_ofa_sys_admin + and user.account_approval_status == AccountApprovalStatusChoices.APPROVED + ) + + +@user_passes_test(can_get_new_token, login_url="/login/") +@api_view(["GET"]) +def generate_new_token(request): + """Generate new token for the API user.""" + if request.method == "GET": + user = User.objects.get(username=request.user) + token, created = Token.objects.get_or_create(user=user) + if token_is_valid(token): + return Response(str(token)) + else: + token.delete() + token = Token.objects.create(user=user) + return Response(str(token)) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 50bc6cb90..6ca924fe0 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -297,7 +297,7 @@ class Common(Configuration): "DEFAULT_AUTHENTICATION_CLASSES": ( "tdpservice.users.authentication.CustomAuthentication", "rest_framework.authentication.SessionAuthentication", - "rest_framework.authentication.TokenAuthentication", + "tdpservice.security.utils.ExpTokenAuthentication", ), "DEFAULT_FILTER_BACKENDS": [ "django_filters.rest_framework.DjangoFilterBackend", @@ -311,6 +311,8 @@ class Common(Configuration): "django.contrib.auth.backends.ModelBackend", ) + TOKEN_EXPIRATION_HOURS = int(os.getenv("TOKEN_EXPIRATION_HOURS", 24)) + # CORS CORS_ALLOW_CREDENTIALS = True diff --git a/tdrs-backend/tdpservice/urls.py b/tdrs-backend/tdpservice/urls.py index f19e26b98..d40e64651 100755 --- a/tdrs-backend/tdpservice/urls.py +++ b/tdrs-backend/tdpservice/urls.py @@ -38,6 +38,7 @@ path("stts/", include("tdpservice.stts.urls")), path("data_files/", include("tdpservice.data_files.urls")), path("logs/", write_logs), + path("security/", include("tdpservice.security.urls")), ] if settings.DEBUG: diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 9800206c6..3ddfe7365 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -121,7 +121,7 @@ export const CaseAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index d61ce85ec..654339ed6 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -72,17 +72,31 @@ const SubmissionHistory = ({ filterValues }) => { }, [hasFetchedFiles, files, dispatch, filterValues]) return ( -
- {fileUploadSections.map((section, index) => ( - f.section.includes(section))} - /> - ))} -
+ <> + +
+ {fileUploadSections.map((section, index) => ( + f.section.includes(section))} + /> + ))} +
+ ) } diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 06baeb730..325c7d898 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -241,9 +241,7 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).not.toBeInTheDocument() expect(screen.queryByText('test6.txt')).toBeInTheDocument() - expect( - screen.queryByText('Error Reports (In development)') - ).toBeInTheDocument() + expect(screen.queryByText('Error Reports')).toBeInTheDocument() }) it('Shows SSP results when SSP-MOE file type selected', () => { diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 8a8e6fd5c..3f4ba24a4 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -109,7 +109,7 @@ export const TotalAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports From 20021d4d36bd85c0dbcb58ae3d51cf9ac114edf1 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 14:26:39 -0400 Subject: [PATCH 044/105] fix tests --- .../tdpservice/parsers/test/test_parse.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index ef2132b65..dc5e75567 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1463,15 +1463,15 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ - {'accepted_without_errors': 0, - 'accepted_with_errors': 3, 'month': 'Oct'}, - {'accepted_without_errors': 0, - 'accepted_with_errors': 3, 'month': 'Nov'}, + {'accepted_without_errors': 3, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 3, + 'accepted_with_errors': 0, 'month': 'Nov'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED assert Tribal_TANF_T4.objects.all().count() == 6 assert Tribal_TANF_T5.objects.all().count() == 13 @@ -1996,7 +1996,7 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs assert parser_errors.count() == 2 for e in parser_errors: - assert e.error_message == "T3: 0 is not in [1, 2, 9]." + assert e.error_message == "T3 Item 76 (citizenship status): 0 is not in [1, 2, 9]." @pytest.mark.django_db() @@ -2014,9 +2014,9 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", - "M2: 0 is not in [1, 2, 3, 9].", - "M2: 0 is not in [1, 2, 9]."} + error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", + "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -2035,8 +2035,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", - "M3: 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -2056,8 +2056,8 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert parser_errors.count() == 2 - error_msgs = {"M5: 00 matches 00.", - "M5: 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M5 Item 23 (education level): 00 matches 00.", + "M5 Item 24 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs From e2f13fac8c039c8696989cf58ade54949479bcfb Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 14:35:18 -0400 Subject: [PATCH 045/105] fix test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dc5e75567..563c179a1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2035,7 +2035,7 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: From 6c50bdc434cf67ac2a2ffd2e61617dcf354691f6 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 12:14:38 -0400 Subject: [PATCH 046/105] abs import --- tdrs-backend/tdpservice/email/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 480c971f7..972c7eef6 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,8 +13,8 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from .email import automated_email, log -from .email_enums import EmailType +from tdpservice.email import automated_email, log +from tdpservice.email.email_enums import EmailType logger = logging.getLogger(__name__) From cb82c56dbd882af6ace3d1caf05720cc0c340d03 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 12:16:15 -0400 Subject: [PATCH 047/105] move calendar_to_fiscal to util --- tdrs-backend/tdpservice/email/tasks.py | 5 ++--- tdrs-backend/tdpservice/parsers/util.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 972c7eef6..15b255ad8 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -15,6 +15,7 @@ from tdpservice.data_files.models import DataFile from tdpservice.email import automated_email, log from tdpservice.email.email_enums import EmailType +from tdpservice.parsers.util import calendar_to_fiscal logger = logging.getLogger(__name__) @@ -78,11 +79,9 @@ def email_admin_num_access_requests(): @shared_task def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): """Send all Data Analysts a reminder to submit if they have not already.""" - def get_fiscal_year(calendar_year, fiscal_quarter): - return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year now = datetime.now() - fiscal_year = get_fiscal_year(now.year, fiscal_quarter) + fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter) all_locations = STT.objects.all() diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 287c58cff..839f9f142 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -130,6 +130,11 @@ def fiscal_to_calendar(year, fiscal_quarter): ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter + +def calendar_to_fiscal(calendar_year, fiscal_quarter): + return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year + + def transform_to_months(quarter): """Return a list of months in a quarter depending the quarter's format.""" match quarter: From dd65bbe0e4761187883da905228539010f59ecd3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 14:35:21 -0400 Subject: [PATCH 048/105] fix tests --- tdrs-backend/tdpservice/email/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 15b255ad8..84e5f1415 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,7 +13,7 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from tdpservice.email import automated_email, log +from tdpservice.email.email import automated_email, log from tdpservice.email.email_enums import EmailType from tdpservice.parsers.util import calendar_to_fiscal From 2cfcfb30ef5c47b6965d13934a6c5b78840e3383 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 14:45:56 -0400 Subject: [PATCH 049/105] lint --- tdrs-backend/tdpservice/email/tasks.py | 1 - tdrs-backend/tdpservice/parsers/util.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 84e5f1415..73b2158e9 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -79,7 +79,6 @@ def email_admin_num_access_requests(): @shared_task def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): """Send all Data Analysts a reminder to submit if they have not already.""" - now = datetime.now() fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 839f9f142..76c29dd37 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -132,6 +132,7 @@ def fiscal_to_calendar(year, fiscal_quarter): def calendar_to_fiscal(calendar_year, fiscal_quarter): + """Decrement the calendar year if in Q1.""" return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year From 884da8e0df87306380d4f089854ba7c604f4865f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 12 Jun 2024 09:37:07 -0400 Subject: [PATCH 050/105] pass error_context_format for different formatting cat 3 messages --- tdrs-backend/tdpservice/parsers/row_schema.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 4 +- .../parsers/test/test_validators.py | 952 +++++++++++++++--- tdrs-backend/tdpservice/parsers/validators.py | 192 ++-- 4 files changed, 916 insertions(+), 234 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 7500ab593..39f3814f3 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -151,7 +151,7 @@ def run_field_validators(self, instance, generate_error): schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, error_message=( - f"{format_error_context(self, field.friendly_name, field.item)}: " + f"{format_error_context(self, field.friendly_name, field.item)} " "field is required but a value was not provided." ), record=instance, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 563c179a1..b9a5eb895 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2014,7 +2014,7 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 (education level) 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -2035,7 +2035,7 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", + error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 (education level) 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 94045acfb..4f6bd0772 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -6,6 +6,7 @@ from .. import validators from .. import schema_defs, util from ..row_schema import RowSchema +from ..fields import Field from tdpservice.parsers.test.factories import TanfT1Factory, TanfT2Factory, TanfT3Factory, TanfT5Factory, TanfT6Factory from tdpservice.parsers.test.factories import SSPM5Factory @@ -45,39 +46,39 @@ def test_or_validators(): """Test `or_validators` gives a valid result.""" value = "2" validator = validators.or_validators(validators.matches(("2")), validators.matches(("3"))) - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("5", RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator("3", RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator("5", RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - "T1 Item item_no (friendly_name): 5 does not match 2. or " - "T1 Item item_no (friendly_name): 5 does not match 3." + "Item item_no (friendly_name) 5 does not match 2. or " + "Item item_no (friendly_name) 5 does not match 3." ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), validators.matches(("4"))) - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "3" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "4" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "5" - assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - 'T1 Item item_no (friendly_name): 5 does not match 2. or ' - 'T1 Item item_no (friendly_name): 5 does not match 3. or ' - 'T1 Item item_no (friendly_name): 5 does not match 4.' + "Item item_no (friendly_name) 5 does not match 2. or " + "Item item_no (friendly_name) 5 does not match 3. or " + "Item item_no (friendly_name) 5 does not match 4." ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) - assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + assert validator(5, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator(1, RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - "T1 Item item_no (friendly_name): 1 does not match 2. " - "or T1 Item item_no (friendly_name): 1 does not " - "match 3. or T1 Item item_no (friendly_name): 1 is not larger than 4." + "Item item_no (friendly_name) 1 does not match 2. or " + "Item item_no (friendly_name) 1 does not match 3. or " + "Item item_no (friendly_name) 1 is not larger than 4." ) def test_if_validators(): @@ -87,14 +88,28 @@ def test_if_validators(): condition_field_name="Field1", condition_function=validators.matches('1'), result_field_name="Field2", result_function=validators.matches('2'), ) - assert validator(value, RowSchema()) == (True, None, ['Field1', 'Field2']) + assert validator(value, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='Field1', friendly_name='field 1'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='Field2', friendly_name='field 2'), + ] + )) == (True, None, ['Field1', 'Field2']) validator = validator = validators.if_then_validator( condition_field_name="Field1", condition_function=validators.matches('1'), result_field_name="Field2", result_function=validators.matches('1'), ) - result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 (Field2): 2 does not match 1.', + result = validator(value, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='Field1', friendly_name='field 1'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='Field2', friendly_name='field 2'), + ] + )) + assert result == (False, 'if Field1 :1 validator1 passed then Field2 Item 2 (field 2) 2 does not match 1.', ['Field1', 'Field2']) @@ -103,7 +118,7 @@ def test_and_validators(): validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no (friendly_name): 1 is not larger than 2.' + 'Item item_no (friendly_name) 1 is not larger than 2.' ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -139,7 +154,7 @@ def test_validate__FAM_AFF__SSN(): def test_quarterIsValid(value, valid): """Test `quarterIsValid`.""" val = validators.quarterIsValid() - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) errorText = None if valid else f"T1 Item item_no (friendly_name): {value[-1:]} is not a valid quarter." assert result == (valid, errorText) @@ -153,7 +168,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == (False, f"T1 Item item_no (friendly_name): {value} is in {options}.") def test_validateRace(): @@ -164,7 +179,7 @@ def test_validateRace(): assert result == (True, None) value = 3 - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): {value} is not greater than or equal to 0 or smaller than or equal to 2." @@ -178,7 +193,7 @@ def test_validateRptMonthYear(): assert result == (True, None) value = "T1 " - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not " @@ -186,7 +201,7 @@ def test_validateRptMonthYear(): ) value = "T1189912" - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not follow " @@ -194,7 +209,7 @@ def test_validateRptMonthYear(): ) value = "T1202013" - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does " @@ -217,7 +232,7 @@ def test_matches_returns_invalid(): value = 'TEST' validator = validators.matches('test') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): TEST does not match test.' @@ -238,7 +253,7 @@ def test_oneOf_returns_valid(): options = ["17-55"] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -250,7 +265,7 @@ def test_oneOf_returns_invalid(): options = [17, 24, 36] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 64 is not in [17, 24, 36].' @@ -259,7 +274,7 @@ def test_oneOf_returns_invalid(): options = ["17-55"] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ @@ -293,7 +308,7 @@ def test_between_returns_invalid(): value = 47 validator = validators.between(48, 400) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 47 is not between 48 and 400.' @@ -321,7 +336,7 @@ def test_between_returns_invalid(): def test_isNumber(value, expected_is_valid, expected_error): """Test `isNumber` validator.""" validator = validators.isNumber() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid == expected_is_valid assert error == expected_error @@ -339,7 +354,7 @@ def test_date_month_is_valid_returns_invalid(): """Test `dateMonthIsValid` gives an invalid result.""" value = '20191327' validator = validators.dateMonthIsValid() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 13 is not a valid month.' @@ -357,7 +372,7 @@ def test_date_day_is_valid_returns_invalid(): """Test `dateDayIsValid` gives an invalid result.""" value = '20191132' validator = validators.dateDayIsValid() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 32 is not a valid day.' @@ -370,7 +385,7 @@ def test_olderThan(): assert validator(value) == (True, None) value = 20240101 - result = validator(value, RowSchema(), "friendly_name", "item_no") + result = validator(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): {str(value)[:4]} must be less than or equal to " @@ -386,7 +401,7 @@ def test_dateYearIsLargerThan(): assert validator(value) == (True, None) value = 18990101 - assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", None) == ( False, f"T1 Item item_no (friendly_name): Year {str(value)[:4]} must be larger than {year}." ) @@ -397,7 +412,7 @@ def test_between_returns_invalid_for_string_value(): value = '047' validator = validators.between(100, 400) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 047 is not between 100 and 400.' @@ -419,7 +434,7 @@ def test_recordHasLength_returns_invalid(): value = 'abcd123' validator = validators.recordHasLength(22) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1: record length is 7 characters but must be 22.' @@ -429,7 +444,7 @@ def test_hasLengthGreaterThan_returns_valid(): value = 'abcd123' validator = validators.hasLengthGreaterThan(6) - is_valid, error = validator(value, None, "friendly_name", "item_no") + is_valid, error = validator(value, None, "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -452,7 +467,7 @@ def test_recordHasLengthBetween_returns_valid(): upper = 15 validator = validators.recordHasLengthBetween(lower, upper) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -465,7 +480,7 @@ def test_recordHasLengthBetween_returns_invalid(): upper = 1 validator = validators.recordHasLengthBetween(lower, upper) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == f"T1: record length of {len(value)} characters is not in the range [{lower}, {upper}]." @@ -487,7 +502,7 @@ def test_intHasLength_returns_invalid(): value = '1a3' validator = validators.intHasLength(22) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 1a3 does not have exactly 22 digits.' @@ -509,7 +524,7 @@ def test_contains_returns_invalid(): value = '12345abcde' validator = validators.contains('6789') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 12345abcde does not contain 6789.' @@ -531,7 +546,7 @@ def test_startsWith_returns_invalid(): value = '12345abcde' validator = validators.startsWith('abc') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 12345abcde does not start with abc.' @@ -553,7 +568,7 @@ def test_notEmpty_returns_invalid_full_string(): value = ' ' validator = validators.notEmpty() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): contains blanks between positions 0 and 9.' @@ -575,7 +590,7 @@ def test_notEmpty_returns_invalid_substring(): value = '111 333' validator = validators.notEmpty(start=3, end=5) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 3 and 5." @@ -586,7 +601,7 @@ def test_notEmpty_returns_nonexistent_substring(): value = '111 333' validator = validators.notEmpty(start=10, end=12) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 10 and 12." @@ -596,7 +611,7 @@ def test_notEmpty_returns_nonexistent_substring(): def test_quarterIsValid_returns_true_if_valid(test_input): """Test `quarterIsValid` gives a valid result for values 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -606,7 +621,7 @@ def test_quarterIsValid_returns_true_if_valid(test_input): def test_quarterIsValid_returns_false_if_invalid(test_input): """Test `quarterIsValid` gives an invalid result for values not 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no", 'prefix') assert is_valid is False assert error == f"T1 Item item_no (friendly_name): {test_input} is not a valid quarter." @@ -615,7 +630,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): def test_calendarQuarterIsValid_returns_invalid(value): """Test `calendarQuarterIsValid` returns false on invalid input.""" val = validators.calendarQuarterIsValid(2, 7) - is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error_msg == ( @@ -628,7 +643,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): def test_calendarQuarterIsValid_returns_valid(value): """Test `calendarQuarterIsValid` returns false on invalid input.""" val = validators.calendarQuarterIsValid(2, 7) - is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error_msg is None @@ -662,11 +677,25 @@ def test_validate_food_stamps(self, record): ) record.RECEIVES_FOOD_STAMPS = 1 record.AMT_FOOD_STAMP_ASSISTANCE = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_FOOD_STAMPS', friendly_name='receives food stamps'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamps'), + ] + )) assert result == (True, None, ['RECEIVES_FOOD_STAMPS', 'AMT_FOOD_STAMP_ASSISTANCE']) record.AMT_FOOD_STAMP_ASSISTANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_FOOD_STAMPS', friendly_name='receives food stamps'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamps'), + ] + )) assert result[0] is False def test_validate_subsidized_child_care(self, record): @@ -677,12 +706,26 @@ def test_validate_subsidized_child_care(self, record): ) record.RECEIVES_SUB_CC = 4 record.AMT_SUB_CC = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_SUB_CC', friendly_name='receives sub cc'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + ] + )) assert result == (True, None, ['RECEIVES_SUB_CC', 'AMT_SUB_CC']) record.RECEIVES_SUB_CC = 4 record.AMT_SUB_CC = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_SUB_CC', friendly_name='receives sub cc'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + ] + )) assert result[0] is False def test_validate_cash_amount_and_nbr_months(self, record): @@ -691,12 +734,26 @@ def test_validate_cash_amount_and_nbr_months(self, record): condition_field_name='CASH_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CASH_AMOUNT', friendly_name='cash amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NBR_MONTHS', friendly_name='nbr months'), + ] + )) assert result == (True, None, ['CASH_AMOUNT', 'NBR_MONTHS']) record.CASH_AMOUNT = 1 record.NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CASH_AMOUNT', friendly_name='cash amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NBR_MONTHS', friendly_name='nbr months'), + ] + )) assert result[0] is False def test_validate_child_care(self, record): @@ -705,12 +762,26 @@ def test_validate_child_care(self, record): condition_field_name='CC_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='CHILDREN_COVERED', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CHILDREN_COVERED', friendly_name='chldrn coverd'), + ] + )) assert result == (True, None, ['CC_AMOUNT', 'CHILDREN_COVERED']) record.CC_AMOUNT = 1 record.CHILDREN_COVERED = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CHILDREN_COVERED', friendly_name='chldrn coverd'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -719,7 +790,14 @@ def test_validate_child_care(self, record): ) record.CC_AMOUNT = 10 record.CC_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CC_NBR_MONTHS', friendly_name='cc nbr mnths'), + ] + )) assert result[0] is False def test_validate_transportation(self, record): @@ -728,12 +806,26 @@ def test_validate_transportation(self, record): condition_field_name='TRANSP_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='TRANSP_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSP_NBR_MONTHS', friendly_name='transp nbr months'), + ] + )) assert result == (True, None, ['TRANSP_AMOUNT', 'TRANSP_NBR_MONTHS']) record.TRANSP_AMOUNT = 1 record.TRANSP_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSP_NBR_MONTHS', friendly_name='transp nbr months'), + ] + )) assert result[0] is False def test_validate_transitional_services(self, record): @@ -742,12 +834,26 @@ def test_validate_transitional_services(self, record): condition_field_name='TRANSITION_SERVICES_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='TRANSITION_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSITION_NBR_MONTHS', friendly_name='transition nbr months'), + ] + )) assert result == (True, None, ['TRANSITION_SERVICES_AMOUNT', 'TRANSITION_NBR_MONTHS']) record.TRANSITION_SERVICES_AMOUNT = 1 record.TRANSITION_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSITION_NBR_MONTHS', friendly_name='transition nbr months'), + ] + )) assert result[0] is False def test_validate_other(self, record): @@ -756,12 +862,26 @@ def test_validate_other(self, record): condition_field_name='OTHER_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='OTHER_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='OTHER_NBR_MONTHS', friendly_name='other nbr months'), + ] + )) assert result == (True, None, ['OTHER_AMOUNT', 'OTHER_NBR_MONTHS']) record.OTHER_AMOUNT = 1 record.OTHER_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='OTHER_NBR_MONTHS', friendly_name='other nbr months'), + ] + )) assert result[0] is False def test_validate_reasons_for_amount_of_assistance_reductions(self, record): @@ -771,12 +891,26 @@ def test_validate_reasons_for_amount_of_assistance_reductions(self, record): result_field_name='WORK_REQ_SANCTION', result_function=validators.oneOf((1, 2)), ) record.SANC_REDUCTION_AMT = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='SANC_REDUCTION_AMT', friendly_name='sanc reduction amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_REQ_SANCTION', friendly_name='work req sanction'), + ] + )) assert result == (True, None, ['SANC_REDUCTION_AMT', 'WORK_REQ_SANCTION']) record.SANC_REDUCTION_AMT = 10 record.WORK_REQ_SANCTION = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='SANC_REDUCTION_AMT', friendly_name='sanc reduction amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_REQ_SANCTION', friendly_name='work req sanction'), + ] + )) assert result[0] is False def test_validate_sum(self, record): @@ -793,7 +927,22 @@ def test_validate_sum(self, record): record.TRANSP_AMOUNT = 0 record.TRANSITION_SERVICES_AMOUNT = 0 record.OTHER_AMOUNT = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamp assis'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=5, startIndex=6, endIndex=7, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=6, startIndex=7, endIndex=8, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + ] + )) assert result[0] is False @@ -816,12 +965,26 @@ def test_validate_ssn(self, record): ) record.SSN = "999989999" record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='ssn'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.FAMILY_AFFILIATION = 1 record.SSN = "999999999" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='ssn'), + ] + )) assert result[0] is False def test_validate_race_ethnicity(self, record): @@ -833,7 +996,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2, 3)), result_field_name=race, result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 0 @@ -842,7 +1012,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2, 3)), result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_validate_marital_status(self, record): @@ -852,12 +1029,26 @@ def test_validate_marital_status(self, record): result_field_name='MARITAL_STATUS', result_function=validators.isInLimits(1, 5), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='married?'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.FAMILY_AFFILIATION = 3 record.MARITAL_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='married?'), + ] + )) assert result[0] is False def test_validate_parent_with_minor(self, record): @@ -866,11 +1057,25 @@ def test_validate_parent_with_minor(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_validate_education_level(self, record): @@ -884,12 +1089,26 @@ def test_validate_education_level(self, record): "98", "99")), ) record.FAMILY_AFFILIATION = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 1 record.EDUCATION_LEVEL = "00" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_validate_citizenship(self, record): @@ -899,12 +1118,26 @@ def test_validate_citizenship(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_validate_cooperation_with_child_support(self, record): @@ -914,12 +1147,26 @@ def test_validate_cooperation_with_child_support(self, record): result_field_name='COOPERATION_CHILD_SUPPORT', result_function=validators.oneOf((1, 2, 9)), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='COOPERATION_CHILD_SUPPORT', friendly_name='cooperation child support'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'COOPERATION_CHILD_SUPPORT']) record.FAMILY_AFFILIATION = 1 record.COOPERATION_CHILD_SUPPORT = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='COOPERATION_CHILD_SUPPORT', friendly_name='cooperation child support'), + ] + )) assert result[0] is False def test_validate_employment_status(self, record): @@ -929,12 +1176,26 @@ def test_validate_employment_status(self, record): result_field_name='EMPLOYMENT_STATUS', result_function=validators.isInLimits(1, 3), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EMPLOYMENT_STATUS', friendly_name='employment status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EMPLOYMENT_STATUS']) record.FAMILY_AFFILIATION = 3 record.EMPLOYMENT_STATUS = 4 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EMPLOYMENT_STATUS', friendly_name='employment status'), + ] + )) assert result[0] is False def test_validate_work_eligible_indicator(self, record): @@ -947,12 +1208,26 @@ def test_validate_work_eligible_indicator(self, record): ), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'WORK_ELIGIBLE_INDICATOR']) record.FAMILY_AFFILIATION = 1 record.WORK_ELIGIBLE_INDICATOR = "00" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + ] + )) assert result[0] is False def test_validate_work_participation(self, record): @@ -964,12 +1239,26 @@ def test_validate_work_participation(self, record): '19', '99']), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'WORK_PART_STATUS']) record.FAMILY_AFFILIATION = 2 record.WORK_PART_STATUS = "04" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -980,7 +1269,14 @@ def test_validate_work_participation(self, record): ) record.WORK_PART_STATUS = "99" record.WORK_ELIGIBLE_INDICATOR = "01" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result[0] is False @@ -1000,12 +1296,26 @@ def test_validate_ssn(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.matches(1), result_field_name='SSN', result_function=validators.notOneOf(("999999999", "000000000")), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.FAMILY_AFFILIATION = 1 record.SSN = "999999999" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False def test_validate_t3_race_ethnicity(self, record): @@ -1017,7 +1327,14 @@ def test_validate_t3_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), result_field_name=race, result_function=validators.oneOf((1, 2)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 0 @@ -1026,7 +1343,14 @@ def test_validate_t3_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), result_field_name=race, result_function=validators.oneOf((1, 2)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_validate_relationship_hoh(self, record): @@ -1037,12 +1361,26 @@ def test_validate_relationship_hoh(self, record): ) record.FAMILY_AFFILIATION = 0 record.RELATIONSHIP_HOH = "04" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='RELATIONSHIP_HOH', friendly_name='relationship hoh'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH']) record.FAMILY_AFFILIATION = 1 record.RELATIONSHIP_HOH = "01" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='RELATIONSHIP_HOH', friendly_name='relationship hoh'), + ] + )) assert result[0] is False def test_validate_t3_education_level(self, record): @@ -1052,12 +1390,26 @@ def test_validate_t3_education_level(self, record): result_field_name='EDUCATION_LEVEL', result_function=validators.notMatches("99"), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='ed lev'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 1 record.EDUCATION_LEVEL = "99" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='ed lev'), + ] + )) assert result[0] is False def test_validate_t3_citizenship(self, record): @@ -1067,12 +1419,26 @@ def test_validate_t3_citizenship(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -1081,7 +1447,14 @@ def test_validate_t3_citizenship(self, record): ) record.FAMILY_AFFILIATION = 2 record.CITIZENSHIP_STATUS = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False @@ -1100,26 +1473,58 @@ def test_validate_ssn(self, record): result_field_name='SSN', result_function=validators.isNumber() ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.SSN = "abc" record.FAMILY_AFFILIATION = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False def test_validate_ssn_citizenship(self, record): """Test cat3 validator for SSN/citizenship.""" val = validators.validate__FAM_AFF__SSN() - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN']) record.FAMILY_AFFILIATION = 2 record.SSN = "000000000" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False def test_validate_race_ethnicity(self, record): @@ -1129,10 +1534,17 @@ def test_validate_race_ethnicity(self, record): for race in races: val = validators.if_then_validator( condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), - result_field_name='RACE_HISPANIC', result_function=validators.isInLimits(1, 2) + result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RACE_HISPANIC']) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) + assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 1 record.RACE_HISPANIC = 0 @@ -1146,7 +1558,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) assert result[0] is False def test_validate_marital_status(self, record): @@ -1157,13 +1576,27 @@ def test_validate_marital_status(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.FAMILY_AFFILIATION = 2 record.MARITAL_STATUS = 6 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result[0] is False def test_validate_parent_minor(self, record): @@ -1174,13 +1607,27 @@ def test_validate_parent_minor(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.FAMILY_AFFILIATION = 2 record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_validate_education(self, record): @@ -1194,13 +1641,27 @@ def test_validate_education(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 2 record.EDUCATION_LEVEL = "0" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_validate_citizenship_status(self, record): @@ -1211,13 +1672,27 @@ def test_validate_citizenship_status(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_validate_oasdi_insurance(self, record): @@ -1228,13 +1703,27 @@ def test_validate_oasdi_insurance(self, record): ) record.DATE_OF_BIRTH = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result == (True, None, ['DATE_OF_BIRTH', 'REC_OASDI_INSURANCE']) record.DATE_OF_BIRTH = 200001 record.REC_OASDI_INSURANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result[0] is False def test_validate_federal_disability(self, record): @@ -1245,13 +1734,27 @@ def test_validate_federal_disability(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'REC_FEDERAL_DISABILITY']) record.FAMILY_AFFILIATION = 1 record.REC_FEDERAL_DISABILITY = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result[0] is False @@ -1268,12 +1771,30 @@ def test_sum_of_applications(self, record): val = validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]) record.NUM_APPLICATIONS = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_APPLICATIONS', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_APPROVED', friendly_name='num approved'), + Field(item=2, startIndex=4, endIndex=5, type='string', + name='NUM_DENIED', friendly_name='num denied'), + ] + )) assert result == (True, None, ['NUM_APPLICATIONS', 'NUM_APPROVED', 'NUM_DENIED']) record.NUM_APPLICATIONS = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_APPLICATIONS', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_APPROVED', friendly_name='num approved'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_DENIED', friendly_name='num denied'), + ] + )) assert result[0] is False @@ -1282,12 +1803,34 @@ def test_sum_of_families(self, record): val = validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]) record.NUM_FAMILIES = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_FAMILIES', friendly_name='num fam'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_2_PARENTS', friendly_name='num 2 parent'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_1_PARENTS', friendly_name='num 2 parent'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='NUM_NO_PARENTS', friendly_name='num 0 parent'), + ] + )) assert result == (True, None, ['NUM_FAMILIES', 'NUM_2_PARENTS', 'NUM_1_PARENTS', 'NUM_NO_PARENTS']) record.NUM_FAMILIES = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_FAMILIES', friendly_name='num fam'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_2_PARENTS', friendly_name='num 2 parent'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_1_PARENTS', friendly_name='num 2 parent'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='NUM_NO_PARENTS', friendly_name='num 0 parent'), + ] + )) assert result[0] is False @@ -1296,12 +1839,30 @@ def test_sum_of_recipients(self, record): val = validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]) record.NUM_RECIPIENTS = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_RECIPIENTS', friendly_name='num recip'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_ADULT_RECIPIENTS', friendly_name='num adult recip'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_CHILD_RECIPIENTS', friendly_name='num child recip'), + ] + )) assert result == (True, None, ['NUM_RECIPIENTS', 'NUM_ADULT_RECIPIENTS', 'NUM_CHILD_RECIPIENTS']) record.NUM_RECIPIENTS = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_RECIPIENTS', friendly_name='num recip'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_ADULT_RECIPIENTS', friendly_name='num adult recip'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_CHILD_RECIPIENTS', friendly_name='num child recip'), + ] + )) assert result[0] is False @@ -1320,11 +1881,25 @@ def test_fam_affil_ssn(self, record): result_field_name='SSN', result_function=validators.validateSSN(), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ["FAMILY_AFFILIATION", "SSN"]) record.SSN = '111111111' - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False @@ -1336,7 +1911,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name=race, result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_fam_affil_marital_stat(self, record): @@ -1346,12 +1928,26 @@ def test_fam_affil_marital_stat(self, record): result_field_name='MARITAL_STATUS', result_function=validators.isInLimits(1, 5), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.MARITAL_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result[0] is False def test_fam_affil_parent_with_minor(self, record): @@ -1361,12 +1957,26 @@ def test_fam_affil_parent_with_minor(self, record): result_field_name='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_fam_affil_ed_level(self, record): @@ -1377,12 +1987,26 @@ def test_fam_affil_ed_level(self, record): validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.EDUCATION_LEVEL = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_fam_affil_citz_stat(self, record): @@ -1392,12 +2016,26 @@ def test_fam_affil_citz_stat(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_dob_oasdi_insur(self, record): @@ -1407,12 +2045,26 @@ def test_dob_oasdi_insur(self, record): result_field_name='REC_OASDI_INSURANCE', result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result == (True, None, ['DATE_OF_BIRTH', 'REC_OASDI_INSURANCE']) record.REC_OASDI_INSURANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result[0] is False def test_fam_affil_fed_disability(self, record): @@ -1422,11 +2074,25 @@ def test_fam_affil_fed_disability(self, record): result_field_name='REC_FEDERAL_DISABILITY', result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'REC_FEDERAL_DISABILITY']) record.REC_FEDERAL_DISABILITY = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result[0] is False def test_is_quiet_preparser_errors(): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 9f99e928c..d8583cd90 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -22,27 +22,41 @@ def value_is_empty(value, length, extra_vals={}): return value is None or value in empty_values + +def format_error_context(row_schema, friendly_name, item_num, format='prefix'): + """Format the error message for consistency across cat2 validators.""" + match format: + case 'inline': + return f'Item {item_num} ({friendly_name})' + + case 'prefix' | _: + return f'{row_schema.record_type} Item {item_num} ({friendly_name}):' + + # higher order validator functions def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - def validator(value, row_schema=None, friendly_name=None, item_num=None): + + # struct + # or kwargs + def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): try: if validator_func(value): return (True, None) - return (False, error_func(value, row_schema, friendly_name, item_num)) + return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) except Exception as e: logger.debug(f"Caught exception in validator. Exception: {e}") - return (False, error_func(value, row_schema, friendly_name, item_num)) + return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) return validator def or_validators(*args, **kwargs): """Return a validator that is true only if one of the validators is true.""" return ( - lambda value, row_schema, friendly_name, item_num: (True, None) - if any([validator(value, row_schema, friendly_name, item_num)[0] for validator in args]) - else (False, " or ".join([validator(value, row_schema, friendly_name, item_num)[1] for validator in args])) + lambda value, row_schema, friendly_name, item_num, error_context_format='inline': (True, None) + if any([validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args]) + else (False, " or ".join([validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args])) ) @@ -50,14 +64,14 @@ def and_validators(validator1, validator2): """Return a validator that is true only if both validators are true.""" return ( lambda value, row_schema, friendly_name, item_num: (True, None) - if (validator1(value, row_schema, friendly_name, item_num)[0] and validator2(value, row_schema, - friendly_name, item_num)[0]) + if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] and validator2(value, row_schema, + friendly_name, item_num, 'inline')[0]) else ( False, - (validator1(value, row_schema, friendly_name, item_num)[1]) - if validator1(value, row_schema, friendly_name, item_num)[1] is not None + (validator1(value, row_schema, friendly_name, item_num, 'inline')[1]) + if validator1(value, row_schema, friendly_name, item_num, 'inline')[1] is not None else "" + " and " + validator2(value)[1] - if validator2(value, row_schema, friendly_name, item_num)[1] is not None + if validator2(value, row_schema, friendly_name, item_num, 'inline')[1] is not None else "", ) ) @@ -69,9 +83,9 @@ def or_priority_validators(validators=[]): """ def or_priority_validators_func(value, rows_schema, friendly_name=None, item_num=None): for validator in validators: - if not validator(value, rows_schema, friendly_name, item_num)[0]: + if not validator(value, rows_schema, friendly_name, item_num, 'inline')[0]: return (False, validator(value, rows_schema, - friendly_name, item_num)[1]) + friendly_name, item_num, 'inline')[1]) return (True, None) return or_priority_validators_func @@ -80,14 +94,14 @@ def or_priority_validators_func(value, rows_schema, friendly_name=None, item_num def extended_and_validators(*args, **kwargs): """Return a validator that is true only if all validators are true.""" def returned_func(value, row_schema, friendly_name, item_num): - if all([validator(value, row_schema, friendly_name, item_num)[0] for validator in args]): + if all([validator(value, row_schema, friendly_name, item_num, 'inline')[0] for validator in args]): return (True, None) else: return (False, "".join( [ " and " + validator(value, row_schema, - friendly_name, item_num)[1] if validator(value, row_schema, - friendly_name, item_num)[0] else "" + friendly_name, item_num, 'inline')[1] if validator(value, row_schema, + friendly_name, item_num, 'inline')[0] else "" for validator in args ] )) @@ -115,9 +129,12 @@ def if_then_validator_func(value, row_schema): value[result_field_name] if type(value) is dict else getattr(value, result_field_name) ) + condition_field = row_schema.get_field_by_name(condition_field_name) + result_field = row_schema.get_field_by_name(result_field_name) + # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields - validator1_result = condition_function(value1, row_schema, condition_field_name, "-1") - validator2_result = result_function(value2, row_schema, result_field_name, "-1") + validator1_result = condition_function(value1, row_schema, condition_field.friendly_name, condition_field.item, 'inline') + validator2_result = result_function(value2, row_schema, result_field.friendly_name, result_field.item, 'inline') if not validator1_result[0]: returned_value = (True, None, [condition_field_name, result_field_name]) @@ -179,7 +196,7 @@ def sumIsEqualFunc(value, row_schema): def field_year_month_with_header_year_quarter(): """Validate that the field year and month match the header year and quarter.""" - def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num): + def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num, error_context_format=None): field_month_year = row_schema.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR') df_quarter = row_schema.datafile.quarter @@ -225,7 +242,8 @@ def recordHasLength(length): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + item_num, + error_context_format: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", ) @@ -233,7 +251,7 @@ def recordHasLengthBetween(lower, upper, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(value, lower, upper) if error_func else f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", @@ -245,7 +263,7 @@ def caseNumberNotEmpty(start=0, end=None): return make_validator( lambda value: not _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + friendly_name, item_num, error_context_format: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' ) @@ -257,7 +275,8 @@ def calendarQuarterIsValid(start=0, end=None): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " + item_num, + error_context_format: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ", ) @@ -265,18 +284,13 @@ def calendarQuarterIsValid(start=0, end=None): # generic validators -def format_error_context(row_schema, friendly_name, item_num): - """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} ({friendly_name})' - - def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, - lambda value, row_schema, friendly_name, item_num: error_func(option) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(option) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not match {option}.", + else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not match {option}.", ) @@ -284,8 +298,8 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} matches {option}." ) @@ -306,8 +320,8 @@ def check_option(value, options): return make_validator( lambda value: check_option(value, options), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} is not in {clean_options_string(options)}." ) @@ -316,8 +330,8 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} is in {clean_options_string(options)}." ) @@ -326,8 +340,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not between {min} and {max}.", ) @@ -338,7 +352,8 @@ def fieldHasLength(length): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + item_num, + error_context_format: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", ) @@ -349,7 +364,8 @@ def hasLengthGreaterThan(val, error_func=None): lambda value, row_schema, friendly_name, - item_num: f"Value length {len(value)} is not greater than {val}.", + item_num, + error_context_format: f"Value length {len(value)} is not greater than {val}.", ) @@ -357,8 +373,8 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} does not have exactly {num_digits} digits.", ) @@ -367,8 +383,8 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not contain {substring}.", ) @@ -376,9 +392,9 @@ def startsWith(substring, error_func=None): """Validate that string value starts with the given substring param.""" return make_validator( lambda value: value.startswith(substring), - lambda value, row_schema, friendly_name, item_num: error_func(substring) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(substring) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not start with {substring}.", + else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not start with {substring}.", ) @@ -386,8 +402,8 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not a number." ) @@ -395,8 +411,8 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not alphanumeric." ) @@ -404,8 +420,8 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank." ) @@ -413,8 +429,8 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not in range [{lower}, {upper}].", ) @@ -422,8 +438,8 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {val}.", ) @@ -438,8 +454,8 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num: - f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks ' + lambda value, row_schema, friendly_name, item_num, error_context_format: + f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)} contains blanks ' f'between positions {start} and {end if end else len(str(value))}.' ) @@ -448,8 +464,8 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num: - f'{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank ' + lambda value, row_schema, friendly_name, item_num, error_context_format: + f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank ' f'between positions {start} and {end if end else len(value)}.' ) @@ -458,8 +474,8 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is zero." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is zero." ) @@ -467,8 +483,8 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", ) @@ -476,8 +492,8 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", ) @@ -485,8 +501,8 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", ) @@ -494,8 +510,8 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", ) @@ -503,8 +519,8 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: int(value) >= LowerBound and int(value) <= UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -514,16 +530,16 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[4:6]} is not a valid month.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[6:]} is not a valid day.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[6:]} is not a valid day.", ) @@ -531,8 +547,8 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[:4]} must be less " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[:4]} must be less " f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -541,8 +557,8 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"Year {str(value)[:4]} must be larger than {year}.", ) @@ -551,8 +567,8 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[-1]} is not a valid quarter.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[-1]} is not a valid quarter.", ) @@ -561,8 +577,8 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is in {options}." ) @@ -570,8 +586,8 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not greater than or equal to 0 " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -582,8 +598,8 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: The value: {value[2:8]}, " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} The value: {value[2:8]}, " "does not follow the YYYYMM format for Reporting Year and Month.", ) From 1ecc38ceaf2874ba2c8885008aceb8059768215a Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:08:08 -0400 Subject: [PATCH 051/105] create ValidationErrorArgs dto --- tdrs-backend/tdpservice/parsers/row_schema.py | 12 +- tdrs-backend/tdpservice/parsers/validators.py | 181 ++++++++---------- 2 files changed, 94 insertions(+), 99 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 39f3814f3..01a87f046 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -1,7 +1,7 @@ """Row schema for datafile.""" from .models import ParserErrorCategoryChoices from .fields import Field, TransformField -from .validators import value_is_empty, format_error_context +from .validators import value_is_empty, format_error_context, ValidationErrorArgs import logging logger = logging.getLogger(__name__) @@ -146,12 +146,20 @@ def run_field_validators(self, instance, generate_error): ) elif field.required: is_valid = False + eargs = ValidationErrorArgs( + value=value, + row_schema=self, + friendly_name=field.friendly_name, + item_num=field.item, + error_context_format='prefix' + ) + errors.append( generate_error( schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, error_message=( - f"{format_error_context(self, field.friendly_name, field.item)} " + f"{format_error_context(eargs)} " "field is required but a value was not provided." ), record=instance, diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d8583cd90..f330670a1 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -1,9 +1,12 @@ """Generic parser validator functions for use in schema definitions.""" -from .models import ParserErrorCategoryChoices -from .util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string import datetime import logging +from dataclasses import dataclass +from typing import Any +# from tdpservice.parsers.row_schema import RowSchema +from tdpservice.parsers.models import ParserErrorCategoryChoices +from tdpservice.parsers.util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string logger = logging.getLogger(__name__) @@ -23,31 +26,49 @@ def value_is_empty(value, length, extra_vals={}): return value is None or value in empty_values -def format_error_context(row_schema, friendly_name, item_num, format='prefix'): +@dataclass +class ValidationErrorArgs: + value: Any + row_schema: object # RowSchema causes circular import + friendly_name: str + item_num: str + error_context_format: str = 'prefix' + + +def format_error_context(eargs: ValidationErrorArgs): """Format the error message for consistency across cat2 validators.""" - match format: + match eargs.error_context_format: case 'inline': - return f'Item {item_num} ({friendly_name})' + return f'Item {eargs.item_num} ({eargs.friendly_name})' case 'prefix' | _: - return f'{row_schema.record_type} Item {item_num} ({friendly_name}):' + return f'{eargs.row_schema.record_type} Item {eargs.item_num} ({eargs.friendly_name}):' # higher order validator functions + def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" # struct # or kwargs def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): + eargs = ValidationErrorArgs( + value=value, + row_schema=row_schema, + friendly_name=friendly_name, + item_num=item_num, + error_context_format=error_context_format + ) + try: if validator_func(value): return (True, None) - return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) + return (False, error_func(eargs)) except Exception as e: logger.debug(f"Caught exception in validator. Exception: {e}") - return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) + return (False, error_func(eargs)) return validator @@ -239,11 +260,8 @@ def recordHasLength(length): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + lambda eargs: f"{eargs.row_schema.record_type}: record length is " + f"{len(eargs.value)} characters but must be {length}.", ) @@ -251,10 +269,11 @@ def recordHasLengthBetween(lower, upper, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(value, lower, upper) + lambda eargs: error_func(eargs.value, lower, upper) if error_func else - f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", + f"{eargs.row_schema.record_type}: record length of {len(eargs.value)} " + f"characters is not in the range [{lower}, {upper}].", ) @@ -262,8 +281,7 @@ def caseNumberNotEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num, error_context_format: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + lambda eargs: f'{eargs.row_schema.record_type}: Case number {str(eargs.value)} cannot contain blanks.' ) @@ -272,12 +290,8 @@ def calendarQuarterIsValid(start=0, end=None): return make_validator( lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " - "representing the Calendar Year and Quarter formatted as YYYYQ", + lambda eargs: f"{eargs.row_schema.record_type}: {eargs.value[start:end]} is invalid. " + "Calendar Quarter must be a numeric representing the Calendar Year and Quarter formatted as YYYYQ", ) @@ -288,9 +302,9 @@ def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(option) + lambda eargs: error_func(option) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not match {option}.", + else f"{format_error_context(eargs)} {eargs.value} does not match {option}.", ) @@ -298,8 +312,7 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} matches {option}." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} matches {option}." ) @@ -320,9 +333,8 @@ def check_option(value, options): return make_validator( lambda value: check_option(value, options), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} is not in {clean_options_string(options)}." + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not in {clean_options_string(options)}." ) @@ -330,9 +342,8 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} is in {clean_options_string(options)}." + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is in {clean_options_string(options)}." ) @@ -340,8 +351,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not between {min} and {max}.", + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not between {min} and {max}.", ) @@ -349,11 +360,8 @@ def fieldHasLength(length): """Validate that the field value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + lambda eargs: + f"{eargs.row_schema.record_type} field length is {len(eargs.value)} characters but must be {length}.", ) @@ -361,11 +369,8 @@ def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( lambda value: len(value) >= val, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"Value length {len(value)} is not greater than {val}.", + lambda eargs: + f"Value length {len(eargs.value)} is not greater than {val}.", ) @@ -373,9 +378,8 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} does not have exactly {num_digits} digits.", + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} does not have exactly {num_digits} digits.", ) @@ -383,8 +387,7 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not contain {substring}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} does not contain {substring}.", ) @@ -392,9 +395,9 @@ def startsWith(substring, error_func=None): """Validate that string value starts with the given substring param.""" return make_validator( lambda value: value.startswith(substring), - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(substring) + lambda eargs: error_func(substring) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not start with {substring}.", + else f"{format_error_context(eargs)} {eargs.value} does not start with {substring}.", ) @@ -402,8 +405,7 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not a number." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not a number." ) @@ -411,8 +413,7 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not alphanumeric." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not alphanumeric." ) @@ -420,8 +421,7 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not blank." ) @@ -429,8 +429,7 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not in range [{lower}, {upper}].", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not in range [{lower}, {upper}].", ) @@ -438,8 +437,7 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {val}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {val}.", ) @@ -454,9 +452,9 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)} contains blanks ' - f'between positions {start} and {end if end else len(str(value))}.' + lambda eargs: + f'{format_error_context(eargs)} {str(eargs.value)} contains blanks ' + f'between positions {start} and {end if end else len(str(eargs.value))}.' ) @@ -464,9 +462,9 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank ' - f'between positions {start} and {end if end else len(value)}.' + lambda eargs: + f'{format_error_context(eargs)} {eargs.value} is not blank ' + f'between positions {start} and {end if end else len(eargs.value)}.' ) @@ -474,8 +472,7 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is zero." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is zero." ) @@ -483,8 +480,7 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {LowerBound}.", ) @@ -492,8 +488,7 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not smaller than {UpperBound}.", ) @@ -501,8 +496,7 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {LowerBound}.", ) @@ -510,8 +504,7 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not smaller than {UpperBound}.", ) @@ -519,8 +512,8 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: int(value) >= LowerBound and int(value) <= UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger or equal " + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -530,16 +523,14 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[4:6]} is not a valid month.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[6:]} is not a valid day.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[6:]} is not a valid day.", ) @@ -547,8 +538,8 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[:4]} must be less " + lambda eargs: + f"{format_error_context(eargs)} {str(eargs.value)[:4]} must be less " f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -557,9 +548,7 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"Year {str(value)[:4]} must be larger than {year}.", + lambda eargs: f"{format_error_context(eargs)} Year {str(eargs.value)[:4]} must be larger than {year}.", ) @@ -567,8 +556,7 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[-1]} is not a valid quarter.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[-1]} is not a valid quarter.", ) @@ -577,8 +565,7 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is in {options}." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is in {options}." ) @@ -586,8 +573,8 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not greater than or equal to 0 " + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -598,8 +585,8 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} The value: {value[2:8]}, " + lambda eargs: + f"{format_error_context(eargs)} The value: {eargs.value[2:8]}, " "does not follow the YYYYMM format for Reporting Year and Month.", ) From a67c232e4fbd1bbe7509aac0e2fe262a02b63ae4 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:08:11 -0400 Subject: [PATCH 052/105] fix test --- tdrs-backend/tdpservice/data_files/test/test_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index d08ce3f1d..500fabc94 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -97,11 +97,13 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): ws = DataFileAPITestBase.get_spreadsheet(response) COL_ERROR_MESSAGE = 4 + COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." + + " then number of months Item 21B (number of months) 0 is not larger than 0." + assert ws.cell(row=8, column=COL_ITEM_NAME).value == 'number of months,cash amount' @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -129,13 +131,15 @@ def assert_error_report_file_content_matches_without_friendly_names(response): ws = wb.get_sheet_by_name('Sheet1') COL_ERROR_MESSAGE = 4 + COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." + "NBR_MONTHS Item 21B (number of months) 0 is not larger than 0." ) + assert ws.cell(row=8, column=COL_ITEM_NAME).value == '[CASH_AMOUNT(55-59), NBR_MONTHS(59-62)]' @staticmethod def assert_data_file_exists(data_file_data, version, user): From 887c17005664b807b38700021d620bcd599351ec Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:27:28 -0400 Subject: [PATCH 053/105] lint --- .../tdpservice/parsers/test/test_parse.py | 6 ++- tdrs-backend/tdpservice/parsers/validators.py | 44 +++++++++++++------ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b9a5eb895..7dac97e8b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2014,7 +2014,8 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 (education level) 00 is not in range [98, 99].", + error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 " + "(education level) 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -2035,7 +2036,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 (education level) 00 is not in range [98, 99].", + error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 " + "(education level) 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index f330670a1..5d05e2ea1 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -28,6 +28,8 @@ def value_is_empty(value, length, extra_vals={}): @dataclass class ValidationErrorArgs: + """Dataclass for args to `make_validator` `error_func`s.""" + value: Any row_schema: object # RowSchema causes circular import friendly_name: str @@ -50,9 +52,6 @@ def format_error_context(eargs: ValidationErrorArgs): def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - - # struct - # or kwargs def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): eargs = ValidationErrorArgs( value=value, @@ -75,9 +74,14 @@ def validator(value, row_schema=None, friendly_name=None, item_num=None, error_c def or_validators(*args, **kwargs): """Return a validator that is true only if one of the validators is true.""" return ( - lambda value, row_schema, friendly_name, item_num, error_context_format='inline': (True, None) - if any([validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args]) - else (False, " or ".join([validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args])) + lambda value, row_schema, friendly_name, + item_num, error_context_format='inline': (True, None) + if any([ + validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args + ]) + else (False, " or ".join([ + validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args + ])) ) @@ -85,8 +89,8 @@ def and_validators(validator1, validator2): """Return a validator that is true only if both validators are true.""" return ( lambda value, row_schema, friendly_name, item_num: (True, None) - if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] and validator2(value, row_schema, - friendly_name, item_num, 'inline')[0]) + if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] + and validator2(value, row_schema, friendly_name, item_num, 'inline')[0]) else ( False, (validator1(value, row_schema, friendly_name, item_num, 'inline')[1]) @@ -120,9 +124,8 @@ def returned_func(value, row_schema, friendly_name, item_num): else: return (False, "".join( [ - " and " + validator(value, row_schema, - friendly_name, item_num, 'inline')[1] if validator(value, row_schema, - friendly_name, item_num, 'inline')[0] else "" + " and " + validator(value, row_schema, friendly_name, item_num, 'inline')[1] + if validator(value, row_schema, friendly_name, item_num, 'inline')[0] else "" for validator in args ] )) @@ -154,8 +157,20 @@ def if_then_validator_func(value, row_schema): result_field = row_schema.get_field_by_name(result_field_name) # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields - validator1_result = condition_function(value1, row_schema, condition_field.friendly_name, condition_field.item, 'inline') - validator2_result = result_function(value2, row_schema, result_field.friendly_name, result_field.item, 'inline') + validator1_result = condition_function( + value1, + row_schema, + condition_field.friendly_name, + condition_field.item, + 'inline' + ) + validator2_result = result_function( + value2, + row_schema, + result_field.friendly_name, + result_field.item, + 'inline' + ) if not validator1_result[0]: returned_value = (True, None, [condition_field_name, result_field_name]) @@ -217,7 +232,8 @@ def sumIsEqualFunc(value, row_schema): def field_year_month_with_header_year_quarter(): """Validate that the field year and month match the header year and quarter.""" - def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num, error_context_format=None): + def validate_reporting_month_year_fields_header( + line, row_schema, friendly_name, item_num, error_context_format=None): field_month_year = row_schema.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR') df_quarter = row_schema.datafile.quarter From c0e95bd189595df33f9913a225e249c769f69d66 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 18 Jun 2024 15:21:46 -0400 Subject: [PATCH 054/105] - Updated stt filter to display stt name instead of code to make it easier to filter --- .../tdpservice/search_indexes/admin/filters.py | 2 +- .../tdpservice/search_indexes/admin/mixins.py | 8 ++++++-- .../tdpservice/search_indexes/admin/ssp.py | 14 +++++++------- .../tdpservice/search_indexes/admin/tanf.py | 14 +++++++------- .../tdpservice/search_indexes/admin/tribal.py | 14 +++++++------- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index 1777d4dab..bf08e995e 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -97,7 +97,7 @@ def queryset(self, request, queryset): class STTFilter(MultipleChoiceListFilter): """Simple filter class to show records based on stt.""" - title = _('STT Code') + title = _('STT') parameter_name = 'stt_code' diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py index 9a817672a..7642f4a6c 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py @@ -70,20 +70,24 @@ def export_as_csv(self, request, queryset): export_as_csv.short_description = "Export Selected as CSV" -class SttCodeMixin: +class SttMixin: """Mixin class to display a record's associated stt code.""" def stt_code(self, obj): """Return stt code.""" return obj.datafile.stt.stt_code + def stt_name(self, obj): + """Return stt name.""" + return obj.datafile.stt.name + class AdminModelMixin(admin.ModelAdmin): """Base class for all mixin classes needing to modify ModelAdmin methods. Needed to satisfy Python MRO.""" pass -class CsvExportAdminMixin(AdminModelMixin, ExportCsvMixin, SttCodeMixin): +class CsvExportAdminMixin(AdminModelMixin, ExportCsvMixin, SttMixin): """Class to encapsulate CSV related mixins.""" actions = ["export_as_csv"] diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index f8b210691..e4ad06965 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -10,7 +10,7 @@ class SSP_M1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -27,7 +27,7 @@ class SSP_M2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -44,7 +44,7 @@ class SSP_M3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -60,7 +60,7 @@ class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -76,7 +76,7 @@ class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -93,7 +93,7 @@ class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'CALENDAR_QUARTER', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -113,7 +113,7 @@ class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'STRATUM', 'FAMILIES_MONTH', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 203652323..d215b89ad 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -10,7 +10,7 @@ class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -27,7 +27,7 @@ class TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -44,7 +44,7 @@ class TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -61,7 +61,7 @@ class TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -78,7 +78,7 @@ class TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -95,7 +95,7 @@ class TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'CALENDAR_QUARTER', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -116,7 +116,7 @@ class TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'STRATUM', 'FAMILIES_MONTH', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py index 704b07893..058c2e10a 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py @@ -10,7 +10,7 @@ class Tribal_TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -27,7 +27,7 @@ class Tribal_TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -44,7 +44,7 @@ class Tribal_TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -60,7 +60,7 @@ class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -75,7 +75,7 @@ class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'RecordType', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -92,7 +92,7 @@ class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'CALENDAR_QUARTER', 'RPT_MONTH_YEAR', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ @@ -112,7 +112,7 @@ class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): 'STRATUM', 'FAMILIES_MONTH', 'datafile', - 'stt_code', + 'stt_name', ] list_filter = [ From b12c695286ea13533f16a89c162c6f85d3581964 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jun 2024 11:01:01 -0400 Subject: [PATCH 055/105] - Remove custom filter in favor of dependency - Update STTFilter to leverage dependency filter --- tdrs-backend/Pipfile | 1 + tdrs-backend/Pipfile.lock | 273 +++++++++--------- .../search_indexes/admin/filters.py | 86 +----- .../tdpservice/search_indexes/admin/ssp.py | 14 +- .../tdpservice/search_indexes/admin/tanf.py | 16 +- .../tdpservice/search_indexes/admin/tribal.py | 14 +- .../templates/multiselectlistfilter.html | 24 -- tdrs-backend/tdpservice/settings/common.py | 1 + 8 files changed, 183 insertions(+), 246 deletions(-) delete mode 100644 tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 117e86c75..51a998b7e 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -32,6 +32,7 @@ django-configurations = "==2.2" django-cors-headers = "==3.12.0" django-extensions = "==3.1.3" django-filter = "==21.1" +django-more-admin-filters = "==1.8" django-model-utils = "==4.1.1" django-storages = "==1.12.3" django-unique-upload = "==0.2.1" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index c2cdbfdc0..0ca355085 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a082fb8d3118128843dec21e83b70a4ee5d9743a2e869918452d1b8c47533edc" + "sha256": "2dd2adca467bcb7a6281923765737b5b0b52101a30efc80401e5552109874674" }, "pipfile-spec": 6, "requires": { @@ -83,11 +83,11 @@ }, "certifi": { "hashes": [ - "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", - "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" + "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", + "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" ], "markers": "python_version >= '3.6'", - "version": "==2024.2.2" + "version": "==2024.6.2" }, "cffi": { "hashes": [ @@ -366,6 +366,14 @@ "index": "pypi", "version": "==4.1.1" }, + "django-more-admin-filters": { + "hashes": [ + "sha256:2d5dd9e8b55d85638d5e260dfb694b1903288b61c37e655b9443b70a5f36833f", + "sha256:fc4d3a3bf0367763a887dceca4b469e467ad062a9e8da1c29b6d6137c5b0e3cd" + ], + "index": "pypi", + "version": "==1.8" + }, "django-nine": { "hashes": [ "sha256:304e0f83cea5a35359375fc919d00f9917b655c1d388244cbfc7363f59489177", @@ -508,11 +516,11 @@ }, "ipython": { "hashes": [ - "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501", - "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3" + "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab", + "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716" ], "markers": "python_version >= '3.7'", - "version": "==8.24.0" + "version": "==8.25.0" }, "itypes": { "hashes": [ @@ -655,11 +663,11 @@ }, "packaging": { "hashes": [ - "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" + "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", + "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" ], - "markers": "python_version >= '3.7'", - "version": "==24.0" + "markers": "python_version >= '3.8'", + "version": "==24.1" }, "parso": { "hashes": [ @@ -762,11 +770,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d", - "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6" + "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10", + "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.0.43" + "version": "==3.0.47" }, "psycopg2": { "hashes": [ @@ -974,11 +982,11 @@ }, "setuptools": { "hashes": [ - "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" + "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", + "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" ], "markers": "python_version >= '3.8'", - "version": "==70.0.0" + "version": "==70.1.0" }, "six": { "hashes": [ @@ -1019,20 +1027,20 @@ }, "tornado": { "hashes": [ - "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0", - "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63", - "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263", - "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052", - "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f", - "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee", - "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78", - "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579", - "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212", - "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e", - "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2" + "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8", + "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f", + "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4", + "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3", + "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14", + "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842", + "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9", + "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698", + "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7", + "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d", + "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4" ], "markers": "python_version >= '3.8'", - "version": "==6.4" + "version": "==6.4.1" }, "traitlets": { "hashes": [ @@ -1044,11 +1052,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0", - "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a" + "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], "markers": "python_version < '3.11'", - "version": "==4.11.0" + "version": "==4.12.2" }, "uritemplate": { "hashes": [ @@ -1060,11 +1068,11 @@ }, "urllib3": { "hashes": [ - "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", - "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", + "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.18" + "version": "==1.26.19" }, "vine": { "hashes": [ @@ -1232,61 +1240,61 @@ "toml" ], "hashes": [ - "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de", - "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661", - "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26", - "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41", - "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d", - "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981", - "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2", - "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34", - "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f", - "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a", - "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35", - "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223", - "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1", - "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746", - "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90", - "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c", - "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca", - "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8", - "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596", - "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e", - "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd", - "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e", - "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3", - "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e", - "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312", - "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7", - "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572", - "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428", - "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f", - "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07", - "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e", - "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4", - "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136", - "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5", - "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8", - "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d", - "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228", - "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206", - "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa", - "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e", - "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be", - "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5", - "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668", - "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601", - "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057", - "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146", - "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f", - "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8", - "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7", - "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987", - "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19", - "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece" + "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523", + "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f", + "sha256:05ac5f60faa0c704c0f7e6a5cbfd6f02101ed05e0aee4d2822637a9e672c998d", + "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb", + "sha256:1d2a830ade66d3563bb61d1e3c77c8def97b30ed91e166c67d0632c018f380f0", + "sha256:239a4e75e09c2b12ea478d28815acf83334d32e722e7433471fbf641c606344c", + "sha256:244f509f126dc71369393ce5fea17c0592c40ee44e607b6d855e9c4ac57aac98", + "sha256:25a5caf742c6195e08002d3b6c2dd6947e50efc5fc2c2205f61ecb47592d2d83", + "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8", + "sha256:2e079c9ec772fedbade9d7ebc36202a1d9ef7291bc9b3a024ca395c4d52853d7", + "sha256:33ca90a0eb29225f195e30684ba4a6db05dbef03c2ccd50b9077714c48153cac", + "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84", + "sha256:341dd8f61c26337c37988345ca5c8ccabeff33093a26953a1ac72e7d0103c4fb", + "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3", + "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884", + "sha256:38a3b98dae8a7c9057bd91fbf3415c05e700a5114c5f1b5b0ea5f8f429ba6614", + "sha256:3d5a67f0da401e105753d474369ab034c7bae51a4c31c77d94030d59e41df5bd", + "sha256:50084d3516aa263791198913a17354bd1dc627d3c1639209640b9cac3fef5807", + "sha256:55f689f846661e3f26efa535071775d0483388a1ccfab899df72924805e9e7cd", + "sha256:5bc5a8c87714b0c67cfeb4c7caa82b2d71e8864d1a46aa990b5588fa953673b8", + "sha256:62bda40da1e68898186f274f832ef3e759ce929da9a9fd9fcf265956de269dbc", + "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db", + "sha256:75e3f4e86804023e991096b29e147e635f5e2568f77883a1e6eed74512659ab0", + "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08", + "sha256:7cec2af81f9e7569280822be68bd57e51b86d42e59ea30d10ebdbb22d2cb7232", + "sha256:8383a6c8cefba1b7cecc0149415046b6fc38836295bc4c84e820872eb5478b3d", + "sha256:8c836309931839cca658a78a888dab9676b5c988d0dd34ca247f5f3e679f4e7a", + "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1", + "sha256:923b7b1c717bd0f0f92d862d1ff51d9b2b55dbbd133e05680204465f454bb286", + "sha256:990fb20b32990b2ce2c5f974c3e738c9358b2735bc05075d50a6f36721b8f303", + "sha256:9aad68c3f2566dfae84bf46295a79e79d904e1c21ccfc66de88cd446f8686341", + "sha256:a5812840d1d00eafae6585aba38021f90a705a25b8216ec7f66aebe5b619fb84", + "sha256:a6519d917abb15e12380406d721e37613e2a67d166f9fb7e5a8ce0375744cd45", + "sha256:ab0b028165eea880af12f66086694768f2c3139b2c31ad5e032c8edbafca6ffc", + "sha256:aea7da970f1feccf48be7335f8b2ca64baf9b589d79e05b9397a06696ce1a1ec", + "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd", + "sha256:b368e1aee1b9b75757942d44d7598dcd22a9dbb126affcbba82d15917f0cc155", + "sha256:bde997cac85fcac227b27d4fb2c7608a2c5f6558469b0eb704c5726ae49e1c52", + "sha256:c4c2872b3c91f9baa836147ca33650dc5c172e9273c808c3c3199c75490e709d", + "sha256:c59d2ad092dc0551d9f79d9d44d005c945ba95832a6798f98f9216ede3d5f485", + "sha256:d1da0a2e3b37b745a2b2a678a4c796462cf753aebf94edcc87dcc6b8641eae31", + "sha256:d8b7339180d00de83e930358223c617cc343dd08e1aa5ec7b06c3a121aec4e1d", + "sha256:dd4b3355b01273a56b20c219e74e7549e14370b31a4ffe42706a8cda91f19f6d", + "sha256:e08c470c2eb01977d221fd87495b44867a56d4d594f43739a8028f8646a51e0d", + "sha256:f5102a92855d518b0996eb197772f5ac2a527c0ec617124ad5242a3af5e25f85", + "sha256:f542287b1489c7a860d43a7d8883e27ca62ab84ca53c965d11dac1d3a1fab7ce", + "sha256:f78300789a708ac1f17e134593f577407d52d0417305435b134805c4fb135adb", + "sha256:f81bc26d609bf0fbc622c7122ba6307993c83c795d2d6f6f6fd8c000a770d974", + "sha256:f836c174c3a7f639bded48ec913f348c4761cbf49de4a20a956d3431a7c9cb24", + "sha256:fa21a04112c59ad54f69d80e376f7f9d0f5f9123ab87ecd18fbb9ec3a2beed56", + "sha256:fcf7d1d6f5da887ca04302db8e0e0cf56ce9a5e05f202720e49b3e8157ddb9a9", + "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35" ], "markers": "python_version >= '3.8'", - "version": "==7.5.1" + "version": "==7.5.3" }, "docutils": { "hashes": [ @@ -1315,11 +1323,11 @@ }, "faker": { "hashes": [ - "sha256:45b84f47ff1ef86e3d1a8d11583ca871ecf6730fad0660edadc02576583a2423", - "sha256:cfe97c4857c4c36ee32ea4aaabef884895992e209bae4cbd26807cf3e05c6918" + "sha256:4c40b34a9c569018d4f9d6366d71a4da8a883d5ddf2b23197be5370f29b7e1b6", + "sha256:bdec5f2fb057d244ebef6e0ed318fea4dcbdf32c3a1a010766fc45f5d68fc68d" ], "markers": "python_version >= '3.8'", - "version": "==25.2.0" + "version": "==25.8.0" }, "flake8": { "hashes": [ @@ -1495,11 +1503,11 @@ }, "packaging": { "hashes": [ - "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" + "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", + "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" ], - "markers": "python_version >= '3.7'", - "version": "==24.0" + "markers": "python_version >= '3.8'", + "version": "==24.1" }, "pluggy": { "hashes": [ @@ -1717,54 +1725,57 @@ }, "typing-extensions": { "hashes": [ - "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0", - "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a" + "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], "markers": "python_version < '3.11'", - "version": "==4.11.0" + "version": "==4.12.2" }, "urllib3": { "hashes": [ - "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", - "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", + "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.18" + "version": "==1.26.19" }, "watchdog": { "hashes": [ - "sha256:11e12fafb13372e18ca1bbf12d50f593e7280646687463dd47730fd4f4d5d257", - "sha256:2895bf0518361a9728773083908801a376743bcc37dfa252b801af8fd281b1ca", - "sha256:39cb34b1f1afbf23e9562501673e7146777efe95da24fab5707b88f7fb11649b", - "sha256:45cc09cc4c3b43fb10b59ef4d07318d9a3ecdbff03abd2e36e77b6dd9f9a5c85", - "sha256:4986db5e8880b0e6b7cd52ba36255d4793bf5cdc95bd6264806c233173b1ec0b", - "sha256:5369136a6474678e02426bd984466343924d1df8e2fd94a9b443cb7e3aa20d19", - "sha256:557ba04c816d23ce98a06e70af6abaa0485f6d94994ec78a42b05d1c03dcbd50", - "sha256:6a4db54edea37d1058b08947c789a2354ee02972ed5d1e0dca9b0b820f4c7f92", - "sha256:6a80d5cae8c265842c7419c560b9961561556c4361b297b4c431903f8c33b269", - "sha256:6a9c71a0b02985b4b0b6d14b875a6c86ddea2fdbebd0c9a720a806a8bbffc69f", - "sha256:6c47bdd680009b11c9ac382163e05ca43baf4127954c5f6d0250e7d772d2b80c", - "sha256:6e949a8a94186bced05b6508faa61b7adacc911115664ccb1923b9ad1f1ccf7b", - "sha256:73c7a935e62033bd5e8f0da33a4dcb763da2361921a69a5a95aaf6c93aa03a87", - "sha256:76ad8484379695f3fe46228962017a7e1337e9acadafed67eb20aabb175df98b", - "sha256:8350d4055505412a426b6ad8c521bc7d367d1637a762c70fdd93a3a0d595990b", - "sha256:87e9df830022488e235dd601478c15ad73a0389628588ba0b028cb74eb72fed8", - "sha256:8f9a542c979df62098ae9c58b19e03ad3df1c9d8c6895d96c0d51da17b243b1c", - "sha256:8fec441f5adcf81dd240a5fe78e3d83767999771630b5ddfc5867827a34fa3d3", - "sha256:9a03e16e55465177d416699331b0f3564138f1807ecc5f2de9d55d8f188d08c7", - "sha256:ba30a896166f0fee83183cec913298151b73164160d965af2e93a20bbd2ab605", - "sha256:c17d98799f32e3f55f181f19dd2021d762eb38fdd381b4a748b9f5a36738e935", - "sha256:c522392acc5e962bcac3b22b9592493ffd06d1fc5d755954e6be9f4990de932b", - "sha256:d0f9bd1fd919134d459d8abf954f63886745f4660ef66480b9d753a7c9d40927", - "sha256:d18d7f18a47de6863cd480734613502904611730f8def45fc52a5d97503e5101", - "sha256:d31481ccf4694a8416b681544c23bd271f5a123162ab603c7d7d2dd7dd901a07", - "sha256:e3e7065cbdabe6183ab82199d7a4f6b3ba0a438c5a512a68559846ccb76a78ec", - "sha256:eed82cdf79cd7f0232e2fdc1ad05b06a5e102a43e331f7d041e5f0e0a34a51c4", - "sha256:f970663fa4f7e80401a7b0cbeec00fa801bf0287d93d48368fc3e6fa32716245", - "sha256:f9b2fdca47dc855516b2d66eef3c39f2672cbf7e7a42e7e67ad2cbfcd6ba107d" + "sha256:0144c0ea9997b92615af1d94afc0c217e07ce2c14912c7b1a5731776329fcfc7", + "sha256:03e70d2df2258fb6cb0e95bbdbe06c16e608af94a3ffbd2b90c3f1e83eb10767", + "sha256:093b23e6906a8b97051191a4a0c73a77ecc958121d42346274c6af6520dec175", + "sha256:123587af84260c991dc5f62a6e7ef3d1c57dfddc99faacee508c71d287248459", + "sha256:17e32f147d8bf9657e0922c0940bcde863b894cd871dbb694beb6704cfbd2fb5", + "sha256:206afc3d964f9a233e6ad34618ec60b9837d0582b500b63687e34011e15bb429", + "sha256:4107ac5ab936a63952dea2a46a734a23230aa2f6f9db1291bf171dac3ebd53c6", + "sha256:4513ec234c68b14d4161440e07f995f231be21a09329051e67a2118a7a612d2d", + "sha256:611be3904f9843f0529c35a3ff3fd617449463cb4b73b1633950b3d97fa4bfb7", + "sha256:62c613ad689ddcb11707f030e722fa929f322ef7e4f18f5335d2b73c61a85c28", + "sha256:667f3c579e813fcbad1b784db7a1aaa96524bed53437e119f6a2f5de4db04235", + "sha256:6e8c70d2cd745daec2a08734d9f63092b793ad97612470a0ee4cbb8f5f705c57", + "sha256:7577b3c43e5909623149f76b099ac49a1a01ca4e167d1785c76eb52fa585745a", + "sha256:998d2be6976a0ee3a81fb8e2777900c28641fb5bfbd0c84717d89bca0addcdc5", + "sha256:a3c2c317a8fb53e5b3d25790553796105501a235343f5d2bf23bb8649c2c8709", + "sha256:ab998f567ebdf6b1da7dc1e5accfaa7c6992244629c0fdaef062f43249bd8dee", + "sha256:ac7041b385f04c047fcc2951dc001671dee1b7e0615cde772e84b01fbf68ee84", + "sha256:bca36be5707e81b9e6ce3208d92d95540d4ca244c006b61511753583c81c70dd", + "sha256:c9904904b6564d4ee8a1ed820db76185a3c96e05560c776c79a6ce5ab71888ba", + "sha256:cad0bbd66cd59fc474b4a4376bc5ac3fc698723510cbb64091c2a793b18654db", + "sha256:d10a681c9a1d5a77e75c48a3b8e1a9f2ae2928eda463e8d33660437705659682", + "sha256:d4925e4bf7b9bddd1c3de13c9b8a2cdb89a468f640e66fbfabaf735bd85b3e35", + "sha256:d7b9f5f3299e8dd230880b6c55504a1f69cf1e4316275d1b215ebdd8187ec88d", + "sha256:da2dfdaa8006eb6a71051795856bedd97e5b03e57da96f98e375682c48850645", + "sha256:dddba7ca1c807045323b6af4ff80f5ddc4d654c8bce8317dde1bd96b128ed253", + "sha256:e7921319fe4430b11278d924ef66d4daa469fafb1da679a2e48c935fa27af193", + "sha256:e93f451f2dfa433d97765ca2634628b789b49ba8b504fdde5837cdcf25fdb53b", + "sha256:eebaacf674fa25511e8867028d281e602ee6500045b57f43b08778082f7f8b44", + "sha256:ef0107bbb6a55f5be727cfc2ef945d5676b97bffb8425650dadbb184be9f9a2b", + "sha256:f0de0f284248ab40188f23380b03b59126d1479cd59940f2a34f8852db710625", + "sha256:f27279d060e2ab24c0aa98363ff906d2386aa6c4dc2f1a374655d4e02a6c5e5e", + "sha256:f8affdf3c0f0466e69f5b3917cdd042f89c8c63aebdb9f7c078996f607cdb0f5" ], "markers": "python_version >= '3.8'", - "version": "==4.0.0" + "version": "==4.0.1" } } } diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index bf08e995e..23ba044ec 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -1,63 +1,11 @@ """Filter classes.""" from django.utils.translation import ugettext_lazy as _ from django.contrib.admin import SimpleListFilter +from more_admin_filters import MultiSelectDropdownFilter from tdpservice.stts.models import STT import datetime -class MultipleChoiceListFilter(SimpleListFilter): - """Filter class allowing multiple filter options.""" - - template = 'multiselectlistfilter.html' - - def lookups(self, request, model_admin): - """Must be overridden to return a list of tuples (value, verbose value).""" - raise NotImplementedError( - 'The MultipleChoiceListFilter.lookups() method must be overridden to ' - 'return a list of tuples (value, verbose value).' - ) - - def queryset(self, request, queryset): - """Return queryset based on selected parameters.""" - if request.GET.get(self.parameter_name): - kwargs = {self.parameter_name: request.GET[self.parameter_name].split(',')} - queryset = queryset.filter(**kwargs) - return queryset - - def value_as_list(self): - """Convert multiple filter fields to list.""" - return self.value().split(',') if self.value() else [] - - def choices(self, changelist): - """Overriden choices method.""" - def amend_query_string(include=None, exclude=None): - selections = self.value_as_list() - if include and include not in selections: - selections.append(include) - if exclude and exclude in selections: - selections.remove(exclude) - if selections: - csv = ','.join(selections) - return changelist.get_query_string({self.parameter_name: csv}) - else: - return changelist.get_query_string(remove=[self.parameter_name]) - - yield { - 'selected': self.value() is None, - 'query_string': changelist.get_query_string(remove=[self.parameter_name]), - 'display': 'All', - 'reset': True, - } - for lookup, title in self.lookup_choices: - yield { - 'selected': str(lookup) in self.value_as_list(), - 'query_string': changelist.get_query_string({self.parameter_name: lookup}), - 'include_query_string': amend_query_string(include=str(lookup)), - 'exclude_query_string': amend_query_string(exclude=str(lookup)), - 'display': title, - } - - class CreationDateFilter(SimpleListFilter): """Simple filter class to show newest created datafile records.""" @@ -94,26 +42,24 @@ def queryset(self, request, queryset): return queryset -class STTFilter(MultipleChoiceListFilter): - """Simple filter class to show records based on stt.""" +class STTFilter(MultiSelectDropdownFilter): + """Multi-select dropdown filter class to filter records based on stt.""" + def __init__(self, field, request, params, model, model_admin, field_path): + super(MultiSelectDropdownFilter, self).__init__(field, request, params, model, model_admin, field_path) + self.lookup_choices = (self._get_lookup_choices(request, model_admin.get_queryset(request))) - title = _('STT') + def _get_lookup_choices(self, request, queryset): + """Filter queryset to show records matching selected fiscal year.""" + record_type = str(request.path).split('/')[-2] + if 'tribal' in record_type: + queryset = queryset.filter(datafile__stt__type=STT.EntityType.TRIBE) + elif 'ssp' in record_type: + queryset = queryset.filter(datafile__stt__ssp=True) + else: + queryset = queryset.filter(datafile__stt__type=STT.EntityType.STATE) - parameter_name = 'stt_code' + return queryset.distinct().order_by('datafile__stt__name').values_list('datafile__stt__name', flat=True) - def lookups(self, request, model_admin): - """Available options in dropdown.""" - options = [] - for obj in STT.objects.all(): - options.append((obj.stt_code, _(obj.name))) - return options - - def queryset(self, request, queryset): - """Return queryset of records based on stt code(s).""" - if self.value() is not None and queryset.exists(): - stts = self.value().split(',') - queryset = queryset.filter(datafile__stt__stt_code__in=stts) - return queryset class FiscalPeriodFilter(SimpleListFilter): """Simple filter class to filter records based on datafile fiscal year.""" diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index e4ad06965..16fef2df0 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -16,7 +16,7 @@ class SSP_M1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -33,7 +33,7 @@ class SSP_M2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -50,7 +50,7 @@ class SSP_M3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -66,7 +66,7 @@ class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -82,7 +82,7 @@ class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -99,7 +99,7 @@ class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -119,5 +119,5 @@ class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index d215b89ad..9a6025c5c 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -1,6 +1,8 @@ """ModelAdmin classes for parsed TANF data files.""" from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter from .mixins import CsvExportAdminMixin, ReadOnlyAdminMixin +from more_admin_filters import MultiSelectDropdownFilter + class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -16,7 +18,7 @@ class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -33,7 +35,7 @@ class TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -50,7 +52,7 @@ class TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -67,7 +69,7 @@ class TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -84,7 +86,7 @@ class TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -101,7 +103,7 @@ class TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -122,5 +124,5 @@ class TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py index 058c2e10a..020f470d1 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py @@ -16,7 +16,7 @@ class Tribal_TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -33,7 +33,7 @@ class Tribal_TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] @@ -50,7 +50,7 @@ class Tribal_TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -66,7 +66,7 @@ class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): """ModelAdmin class for parsed Tribal_T5 data files.""" @@ -81,7 +81,7 @@ class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -98,7 +98,7 @@ class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): @@ -118,5 +118,5 @@ class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): list_filter = [ FiscalPeriodFilter, CreationDateFilter, - STTFilter, + ('datafile__stt__name', STTFilter), ] diff --git a/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html b/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html deleted file mode 100644 index d6b4a45c6..000000000 --- a/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html +++ /dev/null @@ -1,24 +0,0 @@ -{% load i18n %} -

{% blocktrans with filter_title=title %} By {{ filter_title }} {% endblocktrans %}

-
    -{% for choice in choices %} - {% if choice.reset %} - - {{ choice.display }} - - {% endif %} -{% endfor %} -{% for choice in choices %} - {% if not choice.reset %} - - {{ choice.display }} - {% if choice.selected and choice.exclude_query_string %} - (exclude) - {% endif %} - {% if not choice.selected and choice.include_query_string %} - (include) - {% endif %} - - {% endif %} -{% endfor %} -
\ No newline at end of file diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index d4f6e0c13..da5ebd35b 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -53,6 +53,7 @@ class Common(Configuration): "storages", "django_elasticsearch_dsl", "django_elasticsearch_dsl_drf", + "more_admin_filters", # Local apps "tdpservice.core.apps.CoreConfig", "tdpservice.users", From 9a4408031e67e97a8717ccb6699a2ffaa98ef973 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jun 2024 11:05:40 -0400 Subject: [PATCH 056/105] - update doc string --- tdrs-backend/tdpservice/search_indexes/admin/filters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index 23ba044ec..d5981a14b 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -44,12 +44,13 @@ def queryset(self, request, queryset): class STTFilter(MultiSelectDropdownFilter): """Multi-select dropdown filter class to filter records based on stt.""" + def __init__(self, field, request, params, model, model_admin, field_path): super(MultiSelectDropdownFilter, self).__init__(field, request, params, model, model_admin, field_path) self.lookup_choices = (self._get_lookup_choices(request, model_admin.get_queryset(request))) def _get_lookup_choices(self, request, queryset): - """Filter queryset to show records matching selected fiscal year.""" + """Filter queryset to guarentee lookup_choices only has STTs associated with the record type.""" record_type = str(request.path).split('/')[-2] if 'tribal' in record_type: queryset = queryset.filter(datafile__stt__type=STT.EntityType.TRIBE) From b829bcdac778fe9b6530cb000cc7cc0ae75a3981 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jun 2024 11:06:21 -0400 Subject: [PATCH 057/105] - fix lint --- tdrs-backend/tdpservice/search_indexes/admin/tanf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 9a6025c5c..c1521162b 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -1,8 +1,6 @@ """ModelAdmin classes for parsed TANF data files.""" from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter from .mixins import CsvExportAdminMixin, ReadOnlyAdminMixin -from more_admin_filters import MultiSelectDropdownFilter - class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin): From a1589e261ef83140dbb8f5f484a849bcb5c6038e Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 21 Jun 2024 12:25:36 -0400 Subject: [PATCH 058/105] - Updated fiscal year filter to not list `All` twice - Updated STT filter lookup options --- .../tdpservice/search_indexes/admin/filters.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index d5981a14b..766e3744d 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -47,19 +47,20 @@ class STTFilter(MultiSelectDropdownFilter): def __init__(self, field, request, params, model, model_admin, field_path): super(MultiSelectDropdownFilter, self).__init__(field, request, params, model, model_admin, field_path) - self.lookup_choices = (self._get_lookup_choices(request, model_admin.get_queryset(request))) + self.lookup_choices = self._get_lookup_choices(request) - def _get_lookup_choices(self, request, queryset): + def _get_lookup_choices(self, request): """Filter queryset to guarentee lookup_choices only has STTs associated with the record type.""" record_type = str(request.path).split('/')[-2] + queryset = STT.objects.all() if 'tribal' in record_type: - queryset = queryset.filter(datafile__stt__type=STT.EntityType.TRIBE) + queryset = queryset.filter(type=STT.EntityType.TRIBE) elif 'ssp' in record_type: - queryset = queryset.filter(datafile__stt__ssp=True) + queryset = queryset.filter(ssp=True) else: - queryset = queryset.filter(datafile__stt__type=STT.EntityType.STATE) + queryset = queryset.filter(type=STT.EntityType.STATE) - return queryset.distinct().order_by('datafile__stt__name').values_list('datafile__stt__name', flat=True) + return (queryset.distinct().order_by('name').values_list('name', flat=True)) class FiscalPeriodFilter(SimpleListFilter): @@ -76,7 +77,7 @@ def lookups(self, request, model_admin): quarters = [1, 2, 3, 4] months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"] years = [year for year in range(current_year, 2020, -1)] - options = [(None, _('All'))] + options = [] for year in years: for qtr, month in zip(quarters, months): From 3b0a1b862cd7a408354143eee3e70aa080b64c2e Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 24 Jun 2024 10:06:59 -0400 Subject: [PATCH 059/105] rm comment --- tdrs-backend/tdpservice/parsers/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 5d05e2ea1..5e617c8c7 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -156,7 +156,6 @@ def if_then_validator_func(value, row_schema): condition_field = row_schema.get_field_by_name(condition_field_name) result_field = row_schema.get_field_by_name(result_field_name) - # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields validator1_result = condition_function( value1, row_schema, From 640fe342eeee997bf2430caa9db95287f7b82721 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:14:23 -0400 Subject: [PATCH 060/105] fix tests --- tdrs-backend/tdpservice/data_files/test/test_api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index d08ce3f1d..5f177721d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,8 +100,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \ + "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -132,10 +132,9 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." - ) + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("Every T1 record should have at least one " + "corresponding T2 or T3 record with the same " + "RPT_MONTH_YEAR and CASE_NUMBER.") @staticmethod def assert_data_file_exists(data_file_data, version, user): From 89c0f2da05baad42c382442485be6efcf41126d3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:15:57 -0400 Subject: [PATCH 061/105] fix lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 29b3b9176..c354cf11f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1646,7 +1646,8 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. " + "or M2 Item 37 (education level): 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -1669,7 +1670,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. " + "or M3 Item 68 (education level): 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: From fb0e539298344d06526afc0c015b81f2c3a18cf7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:30:15 -0400 Subject: [PATCH 062/105] lint --- tdrs-backend/tdpservice/data_files/test/test_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 1910fa581..5f177721d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -97,7 +97,6 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): ws = DataFileAPITestBase.get_spreadsheet(response) COL_ERROR_MESSAGE = 4 - COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" @@ -130,7 +129,6 @@ def assert_error_report_file_content_matches_without_friendly_names(response): ws = wb.get_sheet_by_name('Sheet1') COL_ERROR_MESSAGE = 4 - COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" From 637bb0e3dd8658b19655ab0490d66aa7bc726fe2 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 1 Jul 2024 07:33:42 -0400 Subject: [PATCH 063/105] - Updated tests to use new friendly names --- tdrs-backend/tdpservice/parsers/test/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py index 3dedf650e..9fab9e8d2 100644 --- a/tdrs-backend/tdpservice/parsers/test/conftest.py +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -711,21 +711,21 @@ def tanf_s4_partial_dup_file(): def partial_dup_t1_err_msg(): """Fixture for t1 record partial duplicate error.""" return ("Partial duplicate record detected with record type {record_type} at line 3. Record is a partial " - "duplicate of the record at line number 2. Duplicated fields causing error: record type, " - "reporting month and year, and case number.") + "duplicate of the record at line number 2. Duplicated fields causing error: Record Type, " + "Reporting Year and Month, and Case Number.") @pytest.fixture def partial_dup_t5_err_msg(): """Fixture for t5 record partial duplicate error.""" return ("Partial duplicate record detected with record type {record_type} at line 3. Record is a partial " - "duplicate of the record at line number 2. Duplicated fields causing error: record type, " - "reporting month and year, case number, family affiliation, date of birth, and social security number.") + "duplicate of the record at line number 2. Duplicated fields causing error: Record Type, " + "Reporting Year and Month, Case Number, Family Affiliation, Date of Birth, and Social Security Number.") @pytest.fixture def partial_dup_s3_s4_err_msg(): """Fixture for t7 record partial duplicate error.""" return ("Partial duplicate record detected with record type {record_type} at line 3. Record is a partial " - "duplicate of the record at line number 2. Duplicated fields causing error: record type.") + "duplicate of the record at line number 2. Duplicated fields causing error: Record Type.") @pytest.fixture def cat4_edge_case_file(stt_user, stt): From 1fd0aa37ae954342704caa74189455ad2e27df1f Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:00:03 -0400 Subject: [PATCH 064/105] Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index d1c8522ea..4690e269c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -275,7 +275,7 @@ Field( item="7B", name='NUM_RECIPIENTS', - friendly_name='SSP-MOERecipients', + friendly_name='SSP-MOE Recipients', type='number', startIndex=111, endIndex=119, From 4e530df1938b9c6c3fc14c8971e92d5221c5720b Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:00:19 -0400 Subject: [PATCH 065/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index e72ad08ea..2d0551aa5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -801,7 +801,7 @@ item="60B", name="SCHOOL_ATTENDENCE_EA", friendly_name="Satisfactory School Attendance: " + - "Hours of Ecused Absences", + "Hours of Excused Absences", type="string", startIndex=116, endIndex=118, From 49a311b93d700943a4f1413ccca84592b965f04f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 12:01:28 -0400 Subject: [PATCH 066/105] - remove 19 --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index e72ad08ea..0211649cd 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -511,7 +511,6 @@ "16", "17", "18", - "19", "99", ] ) From ca400427fed50d41390980275a9fc51d224c8588 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:01:49 -0400 Subject: [PATCH 067/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index 20edec16f..37fa131f4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -107,7 +107,7 @@ Field( item="7A", name="ASSISTANCE", - friendly_name="Total Number of Cash Assistance", + friendly_name="Total Amount of Cash Assistance", type="number", startIndex=79, endIndex=91, From ce38703ea40ce2d370626d1d59bf96804dbdef2c Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:02:11 -0400 Subject: [PATCH 068/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index 37fa131f4..e772f2819 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -517,7 +517,7 @@ Field( item="5C", name="NUM_APPROVED", - friendly_name="Total Number of Applications", + friendly_name="Total Number of Approved Applications", type="number", startIndex=47, endIndex=55, From 6238578e9bf1783dcb51c07b0ebe21b1a5c68e45 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:02:21 -0400 Subject: [PATCH 069/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index e772f2819..4b355c4ed 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -537,7 +537,7 @@ Field( item="7C", name="ASSISTANCE", - friendly_name="Total Number of Cash Assistance", + friendly_name="Total Amount of Cash Assistance", type="number", startIndex=103, endIndex=115, From 5a1a0201984b09c5180cd875280ed3399f40352e Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:02:35 -0400 Subject: [PATCH 070/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index d00040a19..3a8610dc0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -125,7 +125,7 @@ Field( item="10A", name="NUM_1_PARENTS", - friendly_name="Total Number of No-Parent Families", + friendly_name="Total Number of One-Parent Families", type="number", startIndex=163, endIndex=171, From ac537a87be216ae7a22ceb42a25efa4b9abd541a Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:02:49 -0400 Subject: [PATCH 071/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 3a8610dc0..30bb14d28 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -135,7 +135,7 @@ Field( item="11A", name="NUM_NO_PARENTS", - friendly_name="Total Number of No Parent Families", + friendly_name="Total Number of No-Parent Families", type="number", startIndex=187, endIndex=195, From 3c478478248dce57c9a0270eed78b233144ae9fc Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:03:05 -0400 Subject: [PATCH 072/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 30bb14d28..4d1ad7739 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -268,7 +268,7 @@ Field( item="4B", name="NUM_APPLICATIONS", - friendly_name="Total Number of Applicants", + friendly_name="Total Number of Applications", type="number", startIndex=15, endIndex=23, From d85d13735a1cd3782fe55e0acf8c58005deb7f23 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:03:17 -0400 Subject: [PATCH 073/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 4d1ad7739..a85ca325e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -541,7 +541,7 @@ Field( item="11C", name="NUM_NO_PARENTS", - friendly_name="Total Number of No parent Families", + friendly_name="Total Number of No-parent Families", type="number", startIndex=203, endIndex=211, From d53d4f6ef21b06ec76e542b239f73660fc49782c Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:03:58 -0400 Subject: [PATCH 074/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 313898ede..d7846fc1b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -121,7 +121,6 @@ "CASH_AMOUNT", "CC_AMOUNT", "TRANSP_AMOUNT", - "TRANSITION_SERVICES_AMOUNT", "OTHER_AMOUNT", ), 0, From a5eb48dff207c2c0c857f4d5de5b33ad263fa0de Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:04:25 -0400 Subject: [PATCH 075/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index d7846fc1b..1bf5a8c62 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -121,7 +121,6 @@ "CASH_AMOUNT", "CC_AMOUNT", "TRANSP_AMOUNT", - "OTHER_AMOUNT", ), 0, ), From ba065060bd1325a718e54d967236031a260debeb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 12:05:00 -0400 Subject: [PATCH 076/105] - revert debug comment --- tdrs-backend/tdpservice/settings/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 4631f4cc4..ad7c4379d 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -194,7 +194,7 @@ class Common(Configuration): # Logging # set level as 'INFO' if env var is not set - LOGGING_LEVEL = 'WARNING'#os.getenv('LOGGING_LEVEL', 'INFO') + LOGGING_LEVEL = os.getenv('LOGGING_LEVEL', 'INFO') LOGGING = { "version": 1, "disable_existing_loggers": False, From 080a2da7daffeabbd2045ac67b3e79559612196b Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:05:10 -0400 Subject: [PATCH 077/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 1bf5a8c62..84a5fd2da 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -203,7 +203,7 @@ endIndex=30, required=True, validators=[ - validators.isInLimits(1, 3), + validators.isInLimits(1, 2), ], ), Field( From 0f49dea9f9b6c96a2d7d8d78e8411ad955d70c65 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 12:06:11 -0400 Subject: [PATCH 078/105] - revert validator changes --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 313898ede..64ed26c28 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -217,7 +217,7 @@ endIndex=31, required=True, validators=[ - validators.oneOf([1, 2]), + validators.matches(1), ], ), Field( @@ -265,7 +265,7 @@ endIndex=36, required=True, validators=[ - validators.isInLimits(1, 3), + validators.isInLimits(1, 2), ], ), Field( From 111628e51ff2ccc5fcd7ae70872eb3de9e4c9dc3 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:06:23 -0400 Subject: [PATCH 079/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 175f9e85c..a067b58ce 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -689,7 +689,7 @@ Field( item="65B", name="UNEARNED_SOCIAL_SECURITY", - friendly_name="Amount of Unearned Income: SSI", + friendly_name="Amount of Unearned Income: Social Security", type="string", startIndex=106, endIndex=110, From ea27763bbb5bc9b1d36996ea6a8ec68994d174ba Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 12:51:18 -0400 Subject: [PATCH 080/105] - Fixing unresolved merge conflict --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index d8a37092c..a2054cb78 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -138,9 +138,9 @@ def test_parse_big_file(big_file, dfs): dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'months': [ - {'month': 'Oct', 'accepted_without_errors': 129, 'accepted_with_errors': 141}, - {'month': 'Nov', 'accepted_without_errors': 143, 'accepted_with_errors': 130}, - {'month': 'Dec', 'accepted_without_errors': 131, 'accepted_with_errors': 141}], + {'month': 'Oct', 'accepted_without_errors': 25, 'accepted_with_errors': 245}, + {'month': 'Nov', 'accepted_without_errors': 18, 'accepted_with_errors': 255}, + {'month': 'Dec', 'accepted_without_errors': 27, 'accepted_with_errors': 245}], 'rejected': 0} assert TANF_T1.objects.count() == expected_t1_record_count From a93705bb6414a3a93121a275b1bcac2a85258e68 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 13:10:49 -0400 Subject: [PATCH 081/105] - resolved merge conflicts causing failure --- .../tdpservice/parsers/test/test_parse.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index a2054cb78..652638c35 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -313,7 +313,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): errors = parse.parse_datafile(bad_trailer_file_2, dfs) parser_errors = ParserError.objects.filter(file=bad_trailer_file_2) - assert parser_errors.count() == 8 + assert parser_errors.count() == 9 trailer_errors = list(parser_errors.filter(row_number=3).order_by('id')) @@ -329,7 +329,8 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): assert trailer_error_2.content_type is None assert trailer_error_2.object_id is None - row_2_error = parser_errors.get(row_number=2) + row_2_errors = parser_errors.filter(row_number=2).order_by('id') + row_2_error = row_2_errors.first() assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert row_2_error.error_message == 'T1: 3 is not larger or equal to 1 and smaller or equal to 2.' @@ -352,8 +353,9 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): errors_2_0 = errors["2_0"] errors_3_0 = errors["3_0"] error_trailer = errors["trailer"] + row_2_error_set = set(row_2_errors) for error_2_0 in errors_2_0: - assert error_2_0 in [row_2_error] + assert error_2_0 in row_2_error_set for error_3_0 in errors_3_0: assert error_3_0 in row_3_error_list assert error_trailer == [trailer_error_1, trailer_error_2] @@ -366,14 +368,6 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): assert length_error.content_type is None assert length_error.object_id is None - errors_2_0 = errors["2_0"] - errors_3_0 = errors["3_0"] - error_trailer = errors["trailer"] - for error_2_0 in errors_2_0: - assert error_2_0 in [row_2_error] - for error_3_0 in errors_3_0: - assert error_3_0 in row_3_error_list - assert error_trailer == [trailer_error_1, trailer_error_2] trailer_error_3 = trailer_errors[3] assert trailer_error_3.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error_3.error_message == 'T1: Case number T1trash cannot contain blanks.' From 1b706e6c7114a2f572b2cc3bcfd27d85477074ec Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 2 Jul 2024 13:12:26 -0400 Subject: [PATCH 082/105] - resolve merge conflict causing failure --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 652638c35..230738c8f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -506,11 +506,11 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): parse.parse_datafile(small_tanf_section1_datafile, dfs) dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'months': [ - {'month': 'Oct', 'accepted_without_errors': 5, 'accepted_with_errors': 0}, + {'month': 'Oct', 'accepted_without_errors': 4, 'accepted_with_errors': 1}, {'month': 'Nov', 'accepted_without_errors': 0, 'accepted_with_errors': 0}, {'month': 'Dec', 'accepted_without_errors': 0, 'accepted_with_errors': 0}], 'rejected': 0} From bf1ad61deeb754e738872599b022296c41596833 Mon Sep 17 00:00:00 2001 From: Victoria Amoroso <106103383+victoriaatraft@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:19:44 -0700 Subject: [PATCH 083/105] Update tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index f10d865d1..471a067cc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -182,7 +182,7 @@ Field( item="17A", name="RACE_HISPANIC", - friendly_name="Hisapic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=37, endIndex=38, From 8cc001d9e9c0195d94f8545a44982565f6a8a3ec Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Fri, 5 Jul 2024 08:52:29 -0700 Subject: [PATCH 084/105] Update t3.py Removed Ethnicity/Race --- .../tdpservice/parsers/schema_defs/tanf/t3.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 8dcb220d0..901271a89 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -169,7 +169,7 @@ Field( item="70A", name="RACE_HISPANIC", - friendly_name="Ethnicity/Race: Hispanic or Latino", + friendly_name="Hispanic or Latino", type="number", startIndex=37, endIndex=38, @@ -179,7 +179,7 @@ Field( item="70B", name="RACE_AMER_INDIAN", - friendly_name="Ethnicity/Race: American Indian or Alaska Native ", + friendly_name="American Indian or Alaska Native ", type="number", startIndex=38, endIndex=39, @@ -189,7 +189,7 @@ Field( item="70C", name="RACE_ASIAN", - friendly_name="Ethnicity/Race: Asian", + friendly_name="Asian", type="number", startIndex=39, endIndex=40, @@ -199,7 +199,7 @@ Field( item="70D", name="RACE_BLACK", - friendly_name="Ethnicity/Race: Black or African American", + friendly_name="Black or African American", type="number", startIndex=40, endIndex=41, @@ -209,7 +209,7 @@ Field( item="70E", name="RACE_HAWAIIAN", - friendly_name="Ethnicity/Race: Native Hawaiian or Pacific Islander", + friendly_name="Native Hawaiian or Pacific Islander", type="number", startIndex=41, endIndex=42, @@ -219,7 +219,7 @@ Field( item="70F", name="RACE_WHITE", - friendly_name="Ethnicity/Race: White", + friendly_name="White", type="number", startIndex=42, endIndex=43, From b15077cd09db00fa5ac44dca5154c8a8ce0211c2 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 5 Jul 2024 12:25:54 -0400 Subject: [PATCH 085/105] - resolved merge conflicts --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index db4cfa75b..478b0d497 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -333,7 +333,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = row_2_errors.first() assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert row_2_error.error_message == ( - 'T1 Item 13 (receives subsidized housing): 3 is not ' + 'T1 Item 13 (Receives Subsidized Housing): 3 is not ' 'larger or equal to 1 and smaller or equal to 2.' ) @@ -347,7 +347,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): assert row_3_error.error_message in { 'T1: record length of 7 characters is not in the range [117, 156].', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', - 'TRAILER record length is 7 characters but must be 23.', + 'TRAILER: record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None From 880f13b5032f0e178ea45c1e9831279654434fcb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 5 Jul 2024 14:33:59 -0400 Subject: [PATCH 086/105] - fixed failing tests --- .../tdpservice/parsers/test/test_parse.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 478b0d497..0c3cf0113 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -482,7 +482,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert err.error_message == ( - 'M1 Item 11 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' + 'M1 Item 11 (Receives Subsidized Housing): 3 is not larger or equal to 1 and smaller or equal to 2.' ) assert err.content_type is not None assert err.object_id is not None @@ -882,7 +882,7 @@ def test_parse_tanf_section2_file(tanf_section2_file, dfs): err = parser_errors.first() assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert err.error_message == ( - "T4 Item 10 (receives subsidized housing): 3 " + "T4 Item 10 (Received Subsidized Housing): 3 " "is not larger or equal to 1 and smaller or equal to 2." ) assert err.content_type.model == "tanf_t4" @@ -1377,7 +1377,7 @@ def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): ' and must be at least 101 characters.'), ('t3_file_two_child_with_space_filled', 2, 0, ''), ('two_child_second_filled', 2, 8, - 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), + 'T3 Item 68 (Date of Birth): Year 6 must be larger than 1900.'), ('t3_file_zero_filled_second', 1, 0, '')]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, error_message, request, dfs): @@ -1393,7 +1393,6 @@ def test_misformatted_multi_records(file_fixture, result, number_of_errors, erro assert parser_errors.count() == number_of_errors if number_of_errors > 0: error_messages = [parser_error.error_message for parser_error in parser_errors] - print(error_messages) assert error_message in error_messages parser_errors = ParserError.objects.all().exclude( @@ -1417,7 +1416,7 @@ def test_empty_t4_t5_values(t4_t5_empty_values, dfs): logger.info(t4[0].__dict__) assert t5.count() == 1 assert parser_errors[0].error_message == ( - "T4 Item 10 (receives subsidized housing): 3 is " + "T4 Item 10 (Received Subsidized Housing): 3 is " "not larger or equal to 1 and smaller or equal to 2." ) @@ -1438,9 +1437,9 @@ def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): year_error = parser_errors[1] digits_error = parser_errors[0] - assert month_error.error_message == "T2 Item 32 (date of birth): $9 is not a valid month." - assert year_error.error_message == "T2 Item 32 (date of birth): Year Q897 must be larger than 1900." - assert digits_error.error_message == "T2 Item 32 (date of birth): Q897$9 3 does not have exactly 8 digits." + assert month_error.error_message == "T2 Item 32 (Date of Birth): $9 is not a valid month." + assert year_error.error_message == "T2 Item 32 (Date of Birth): Year Q897 must be larger than 1900." + assert digits_error.error_message == "T2 Item 32 (Date of Birth): Q897$9 3 does not have exactly 8 digits." @pytest.mark.django_db def test_bulk_create_returns_rollback_response_on_bulk_index_exception(small_correct_file, mocker, dfs): @@ -1619,7 +1618,7 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs assert parser_errors.count() == 2 for e in parser_errors: - assert e.error_message == "T3 Item 76 (citizenship status): 0 is not in [1, 2, 9]." + assert e.error_message == "T3 Item 76 (Citizenship/Immigration Status): 0 is not in [1, 2, 9]." @pytest.mark.django_db() @@ -1639,10 +1638,10 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. " - "or M2 Item 37 (education level): 00 is not in range [98, 99].", - "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", - "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} + error_msgs = {"M2 Item 37 (Educational Level): 00 is not in range [1, 16]. or M2 Item 37 (Educational Level): " + + "00 is not in range [98, 99].", + "M2 Item 38 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", + "M2 Item 39 (Cooperated with Child Support): 0 is not in [1, 2, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -1663,9 +1662,12 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. " - "or M3 Item 68 (education level): 00 is not in range [98, 99].", - "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + "00 is not in range [98, 99].", + "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", + "M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + "00 is not in range [98, 99].", + "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -1687,8 +1689,8 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert parser_errors.count() == 2 - error_msgs = {"M5 Item 23 (education level): 00 matches 00.", - "M5 Item 24 (citizenship status): 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M5 Item 23 (Educational Level): 00 matches 00.", + "M5 Item 24 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs From c39438adb3b57638b48151a55f07885fab472140 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 5 Jul 2024 15:06:23 -0400 Subject: [PATCH 087/105] - Added missing territories --- tdrs-backend/tdpservice/search_indexes/admin/filters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index 766e3744d..517d27e48 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -1,6 +1,7 @@ """Filter classes.""" from django.utils.translation import ugettext_lazy as _ from django.contrib.admin import SimpleListFilter +from django.db.models import Q as Query from more_admin_filters import MultiSelectDropdownFilter from tdpservice.stts.models import STT import datetime @@ -58,7 +59,8 @@ def _get_lookup_choices(self, request): elif 'ssp' in record_type: queryset = queryset.filter(ssp=True) else: - queryset = queryset.filter(type=STT.EntityType.STATE) + type_query = Query(type=STT.EntityType.STATE) | Query(type=STT.EntityType.TERRITORY) + queryset = queryset.filter(type_query) return (queryset.distinct().order_by('name').values_list('name', flat=True)) From 9b76e219477a34a857049a255c964299c785d161 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 8 Jul 2024 10:54:21 -0400 Subject: [PATCH 088/105] fix confl --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 4 ---- tdrs-backend/tdpservice/parsers/validators.py | 5 ----- 2 files changed, 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 2b2c575f0..c39ff545b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -279,7 +279,6 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False - assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ '29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55].' @@ -648,13 +647,10 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False - assert error_msg == ( - f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " assert error_msg == ( f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" ) - ) @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index e56b7f550..029317be4 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -313,11 +313,6 @@ def calendarQuarterIsValid(start=0, end=None): # generic validators -def format_error_context(row_schema, friendly_name, item_num): - """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} ({friendly_name})' - - def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( From 1c15c36ee1aca41037568e2675b385593d13cf18 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 8 Jul 2024 10:55:57 -0400 Subject: [PATCH 089/105] lint --- tdrs-backend/tdpservice/parsers/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 029317be4..5e617c8c7 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -371,7 +371,6 @@ def between(min, max): ) - def fieldHasLength(length): """Validate that the field value (string or array) has a length matching length param.""" return make_validator( From 63492d188d898c25091dca5255a2eef0e91fa72e Mon Sep 17 00:00:00 2001 From: Victoria Amoroso Date: Tue, 9 Jul 2024 07:25:05 -0700 Subject: [PATCH 090/105] Updated mp3.py Shortened Receives Disability Benefits: SSI or AABD --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index e7abaa583..72fabf613 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -253,8 +253,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: SSI Under Title XVI-SSI or " + - "Aged, Blind, and Disabled Under Title XVI-AABD", + friendly_name="Receives Disability Benefits: SSI or AABD" type='number', startIndex=45, endIndex=46, From 3af861bb66606e530067e8740cd0e413e083a025 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 9 Jul 2024 12:20:56 -0400 Subject: [PATCH 091/105] - fixed missed comma error --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 72fabf613..9705145c9 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -253,7 +253,7 @@ Field( item="65B", name='RECEIVE_SSI', - friendly_name="Receives Disability Benefits: SSI or AABD" + friendly_name="Receives Disability Benefits: SSI or AABD", type='number', startIndex=45, endIndex=46, From de09be43dd25c071a40b7f6764ef0fb9f84c8cbb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 11 Jul 2024 14:33:40 -0400 Subject: [PATCH 092/105] - fixed issues --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 2 ++ tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 4690e269c..43d9ec7f5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -174,6 +174,7 @@ s2 = RowSchema( record_type="M6", document=SSP_M6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.recordHasLength(259), validators.field_year_month_with_header_year_quarter(), @@ -338,6 +339,7 @@ s3 = RowSchema( record_type="M6", document=SSP_M6DataSubmissionDocument(), + quiet_preparser_errors=True, preparsing_validators=[ validators.recordHasLength(259), validators.field_year_month_with_header_year_quarter(), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 35f240594..78d83709e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -498,7 +498,7 @@ type="string", startIndex=68, endIndex=70, - required=False, + required=True, validators=[ validators.oneOf( [ From afd1e632c4dc1b9c7cac7fd4ec0afeda97dbd29c Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:55:02 -0400 Subject: [PATCH 093/105] Upgrade Postgres to v15.x (#3066) * - Upgraded local to postrgres 15.7 * - updating version on terraform * - disabling tests for now * - adding timeouts to TF for safety * - Deleting DB from TF state * - re-add DB * - remove deployment until DB is ready * - updated apt.yml to have pg15 client - updated deploy to only run apt commands * - re-enable deply dev * - UPdate client version - re-enable migrations * - really disable tests * - Added instructions for DB upgrade - updated settings to use nice DB name in all envs except prod * - Updated TF scripts to actually work * - remove prefix * - Made scripts a little smarter * - re-enable tests * - small add to readme * - fixed tests * - updated with friendly names * - Updated ENV to APP where appropriate * Update docs/Technical-Documentation/cloud-foundry-db-upgrade.md Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * - hide user in error message --------- Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- .../cloud-foundry-db-upgrade.md | 120 ++++++++++++++++++ tdrs-backend/apt.yml | 4 +- tdrs-backend/docker-compose.yml | 2 +- .../tdpservice/data_files/test/test_api.py | 16 ++- .../tdpservice/scheduling/db_backup.py | 2 +- tdrs-backend/tdpservice/settings/cloudgov.py | 9 +- terraform/README.md | 46 +++---- terraform/create_backend_vars.sh | 21 ++- terraform/create_tf_vars.sh | 16 ++- terraform/dev/main.tf | 7 +- terraform/production/main.tf | 7 +- terraform/staging/main.tf | 7 +- 12 files changed, 206 insertions(+), 51 deletions(-) create mode 100644 docs/Technical-Documentation/cloud-foundry-db-upgrade.md diff --git a/docs/Technical-Documentation/cloud-foundry-db-upgrade.md b/docs/Technical-Documentation/cloud-foundry-db-upgrade.md new file mode 100644 index 000000000..466b562f6 --- /dev/null +++ b/docs/Technical-Documentation/cloud-foundry-db-upgrade.md @@ -0,0 +1,120 @@ +# Cloud Foundry, Cloud.gov AWS RDS Database Upgrade + +## Process + +If you are performing this process for the staging or production, you need to ensure you are performing the changes through the [HHS](https://github.com/HHS/TANF-app) repo and not the [Raft](https://github.com/raft-tech/TANF-app) repo. +
+ +### 1. SSH into a backend app in your desired environment +```bash +cf ssh tdp-backend- +``` +
+ +### 2. Create a backup of all the databases in the ENV's RDS instance +Note: you can get the required field values from `VCAP_SERVICES`. +```bash +/home/vcap/deps/0/apt/usr/lib/postgresql//bin/pg_dump -h -p -d -U -F c --no-acl --no-owner -f .pg +``` +
+ +### 3. Copy the backup(s) to your local machine +Note: This assumes you ran the backup command above in the home directory of the app. As an added bonus for later steps, you should execute this command from somewhere within `tdrs-backend` directory! Make sure not to commit the files/directories that are copied to your local directory. +```bash +cf ssh tdp-backend-- -c 'tar cfz - ~/app/*.pg' | tar xfz - -C . +``` +
+ +### 4. Verify backup file size(s) match the backup size(s) in the app +```bash +ls -lh /home/vcap/app +``` +As an added verification step, you should consider restoring the backups into a local server and verifying the contents with `psql` or `pgAdmin`. +

+ +### 5. Update the `version` key in the `json_params` item in the `database` resource in the `main.tf` file in the environment(s) you're upgrading with the new database server version +```yaml +json_params = "{\"version\": \"\"}" +``` +
+ +### 6. Update the `postgresql-client` version to the new version in `tdrs-backend/apt.yml` +```yaml +- postgresql-client- +``` +Note: if the underlying OS for CloudFoundry is no longer `cflinuxfs4` you may also need to update the repo we point to for the postgres client binaries. +

+ +### 7. Update the postgres container version in `tdrs-backend/docker-compose.yml` +```yaml +postgres: +image: postgres: +``` +
+ +### 8. Update Terraform state to delete then re-create RDS instance +Follow the instuctions in the `terraform/README.md` and proceed from there. Modify the `main.tf` file in the `terraform/` directory to inform TF of the changes. To delete the existing RDS instance you can simply comment out the whole database `resource` in the file (even though you made changes in the steps above). TF will see that the resource is no longer there, delete it, and appropriately update it's state. Then you simply re-comment the database `resource` back in with the changes you made in previous steps. TF will create the new RDS instance with your new updates, and also update the state in S3. +

+ +### 9. Bind backend to the new RDS instance to get credentials +```bash +cf bind-service tdp-backend- tdp-db- +``` +Be sure to re-stage the app when prompted +

+ +### 10. Apply the backend manifest to begin the restore process +If you copied the backups as mentioned in the note from step 3, the backups will be copied for you to the app instance in the command below. If not, you will need to use `scp` to copy the backups to the app instance after running the command below. +```bash +cf push tdp-backend- --no-route -f manifest.buildpack.yml -t 180 --strategy rolling +``` +
+ +### 11. SSH into the app you just pushed +```bash +cf ssh tdp-backend- +``` +
+ +### 12. Create the appropriate database(s) in the new RDS server +Note: you can get the required field values from `VCAP_SERVICES`. +```bash +/home/vcap/deps/0/apt/usr/lib/postgresql//bin/createdb -U -h +``` +
+ +### 13. Restore the backup(s) to the appropriate database(s) +Note: you can get the required field values from `VCAP_SERVICES`. +```bash +/home/vcap/deps/0/apt/usr/lib/postgresql//bin/pg_restore -p -h -U -d .pg +``` +During this step, you may see errors similar to the message below. Note `` is imputed in the message to avoid leaking environment specific usernames/roles. +```bash +pg_restore: from TOC entry 215; 1259 17313 SEQUENCE users_user_user_permissions_id_seq +pg_restore: error: could not execute query: ERROR: role "" does not exist +Command was: ALTER TABLE public.users_user_user_permissions_id_seq OWNER TO ; +``` +and the result and total amount of these errors should be: +```bash +pg_restore: warning: errors ignored on restore: 68 +``` +If this is what you see, everything is OK. This happens because the `pg_dump` doesn't remove owner associations on sequences for some reason. But you will see in the blocks above that `pg_restore` correctly alters the sequence owner to the new database user. +

+ +### 14. Use `psql` to get into the database to check state +Note: you can get the required field values from `VCAP_SERVICES`. +```bash +/home/vcap/deps/0/apt/usr/lib/postgresql//bin/psql +``` +
+ +### 15. Re-deploy or Re-stage the backend and frontend apps +Pending your environment you can do this GitHub labels or you can re-stage the apps from Cloud.gov. +

+ +### 16. Access the re-deployed/re-staged apps and run a smoke test +- Log in +- Submit a few datafiles +- Make sure new and existing submission histories populate correctly +- Checkout the DACs data +
diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index f07aee4a3..cbcb0edf4 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -2,8 +2,8 @@ cleancache: true keys: - https://www.postgresql.org/media/keys/ACCC4CF8.asc repos: - - deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main + - deb http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main packages: - - postgresql-client-12 + - postgresql-client-15 - libjemalloc-dev - redis diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 07d014fb5..81d7065c4 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -12,7 +12,7 @@ services: - ../scripts/zap-hook.py:/zap/scripts/zap-hook.py:ro postgres: - image: postgres:11.6 + image: postgres:15.7 environment: - PGDATA=/var/lib/postgresql/data/ - POSTGRES_DB=tdrs_test diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 5f177721d..9ae1a408e 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,8 +100,10 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \ - "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER." + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if Cash Amount :873 validator1 passed then Cash and " + "Cash Equivalents: Number of Months T1 Item -1 (Cash " + "and Cash Equivalents: Number of Months): 0 is not " + "larger than 0.") @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -112,8 +114,8 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ - "but must be 23." + assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == ("M1 Item 11 (Receives Subsidized Housing): 3 is " + "not larger or equal to 1 and smaller or equal to 2.") @staticmethod def assert_error_report_file_content_matches_without_friendly_names(response): @@ -132,9 +134,9 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("Every T1 record should have at least one " - "corresponding T2 or T3 record with the same " - "RPT_MONTH_YEAR and CASE_NUMBER.") + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " + "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not " + "larger than 0.") @staticmethod def assert_data_file_exists(data_file_data, version, user): diff --git a/tdrs-backend/tdpservice/scheduling/db_backup.py b/tdrs-backend/tdpservice/scheduling/db_backup.py index 05f51ad09..48d0da749 100644 --- a/tdrs-backend/tdpservice/scheduling/db_backup.py +++ b/tdrs-backend/tdpservice/scheduling/db_backup.py @@ -29,7 +29,7 @@ def get_system_values(): sys_values['SPACE'] = json.loads(OS_ENV['VCAP_APPLICATION'])['space_name'] # Postgres client pg_dump directory - sys_values['POSTGRES_CLIENT_DIR'] = "/home/vcap/deps/0/apt/usr/lib/postgresql/12/bin/" + sys_values['POSTGRES_CLIENT_DIR'] = "/home/vcap/deps/0/apt/usr/lib/postgresql/15/bin/" # If the client directory and binaries don't exist, we need to find them. if not (os.path.exists(sys_values['POSTGRES_CLIENT_DIR']) and diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 67f2c5b60..0da7a63d0 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -44,9 +44,8 @@ class CloudGov(Common): cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev') cloudgov_space_suffix = cloudgov_space.strip('tanf-') cloudgov_name = cloudgov_app.get('name').split("-")[-1] # converting "tdp-backend-name" to just "name" - services_basename = cloudgov_name if ( - cloudgov_name == "develop" and cloudgov_space_suffix == "staging" - ) else cloudgov_space_suffix + # TODO: does this break prod? + services_basename = cloudgov_space_suffix database_creds = get_cloudgov_service_creds_by_instance_name( cloudgov_services['aws-rds'], @@ -68,10 +67,10 @@ class CloudGov(Common): ### # Dynamic Database configuration based on cloud.gov services # - env_based_db_name = f'tdp_db_{cloudgov_space_suffix}_{cloudgov_name}' + env_based_db_name = f'tdp_db_{cloudgov_name}' logger.debug("css: " + cloudgov_space_suffix) - if (cloudgov_space_suffix in ["prod", "staging"]): + if (cloudgov_space_suffix == "prod"): db_name = database_creds['db_name'] else: db_name = env_based_db_name diff --git a/terraform/README.md b/terraform/README.md index 72bec143b..7513b17b5 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -28,7 +28,7 @@ We use an S3 bucket created by Cloud Foundry in Cloud.gov as our remote backend Note that a single S3 bucket maintains the Terraform State for both the development and staging environments, and this instance is deployed in the development space. -| | development | staging | production | +| | development | staging | production | |---|---|---|---| | S3 Key | `terraform.tfstate.dev` | `terraform.tfstate.staging` | `terraform.tfstate.prod` | | Service Space | `tanf-dev` | `tanf-dev` | `tanf-prod` | @@ -45,11 +45,11 @@ Sometimes a developer will need to run Terraform locally to perform manual opera 1. **Install Cloud Foundry CLI** - On macOS: `brew install cloudfoundry/tap/cf-cli` - On other platforms: [Download and install cf][cf-install] - + 1. **Install CircleCI local CLI** - On macOS: `brew install circleci` - On other platforms: [Download and install circleci][circleci] - + 1. **Install jq CLI** - On macOS: `brew install jq` - On other platforms: [Download and install jq][jq] @@ -59,10 +59,10 @@ Sometimes a developer will need to run Terraform locally to perform manual opera # login cf login -a api.fr.cloud.gov --sso # Follow temporary authorization code prompt. - - # Select the target org (probably `hhs-acf-prototyping`), + + # Select the target org (probably `hhs-acf-prototyping`), # and the space within which you want to provision infrastructure. - + # Spaces: # dev = tanf-dev # staging = tanf-staging @@ -75,7 +75,7 @@ Sometimes a developer will need to run Terraform locally to perform manual opera ```bash ./create_tf_vars.sh - + # Should generate a file `variables.tfvars` in the `/terraform/dev` directory. # Your file should look something like this: # @@ -83,32 +83,32 @@ Sometimes a developer will need to run Terraform locally to perform manual opera # cf_password = "some-dev-password" # cf_space_name = "tanf-dev" ``` - + ## Test Deployment in Development 1. Follow the instructions above and ensure the `variables.tfvars` file has been generated with proper values. 1. `cd` into `/terraform/dev` 1. Prepare terraform backend: - + **Remote vs. Local Backend:** - + If you merely wish to test some new changes without regards to the currently deployed state stored in the remote TF state S3 bucket, you may want to use a "local" backend with Terraform. ```terraform terraform { backend "local" {} } ``` - + With this change to `main.tf`, you should be able to run `terraform init` successfully. **Get Remote S3 Credentials:** - + In the `/terraform` directory, you can run the `create_backend_vars.sh` script which can be modified with details of your current environment, and will yield a `backend_config.tfvars` file which must be later passed in to Terraform. For more on this, check out [terraform variable definitions][tf-vars]. ```bash ./create_backend_vars.sh - + # Should generate a file `backend_config.tfvars` in the current directory. # Your file should look something like this: # @@ -116,21 +116,21 @@ Sometimes a developer will need to run Terraform locally to perform manual opera # secret_key = "some-secret-key" # region = "us-gov-west-1" ``` - + You can now run `terraform init -backend-config backend_config.tfvars` and load the remote state stored in S3 into your local Terraform config. 1. Run `terraform init` if using a local backend, or `terraform init -backend-config backend_config.tfvars` with the remote backend. -1. Run `terraform destroy -var-file variables.tfvars` to clear the current deployment (if there is one). +1. Run `terraform destroy -var-file variables.tf` to clear the current deployment (if there is one). - If the current deployment isn't destroyed, `terraform apply` will fail later because the unique service instance names are already taken. - Be cautious and weary of your target environment when destroying infrastructure. -1. Run `terraform plan -out tfapply -var-file variables.tfvars` to create a new execution plan. +1. Run `terraform plan -out tfapply -var-file variables.tfvars` to create a new execution plan. When prompted for the `cf_app_name`, you should provide the value `tanf-` where `` is: `dev`, `staging`, `prod`. 1. Run `terraform apply "tfapply"` to create the new infrastructure. A similar test deployment can also be executed from the `/scripts/deploy-infrastructure-dev.sh` script, albeit without the `destroy` step. ### Terraform State S3 Bucket -These instructions describe the creation of a new S3 bucket to hold Terraform's state. _This need only be done once per environment_ (note that currently development and staging environments share a single S3 bucket that exists in the development space). This is the only true manual steps that needs to be taken upon the initial application deployment in new environments. This should only need to be done at the beginning of a deployed app's lifetime. +These instructions describe the creation of a new S3 bucket to hold Terraform's state. _This need only be done once per environment_ (note that currently development and staging environments share a single S3 bucket that exists in the development space). This is the only true manual steps that needs to be taken upon the initial application deployment in new environments. This should only need to be done at the beginning of a deployed app's lifetime. 1. **Create S3 Bucket for Terraform State** @@ -139,7 +139,7 @@ These instructions describe the creation of a new S3 bucket to hold Terraform's ``` 1. **Create service key** - + Now we need a new service key with which to authenticate to our Cloud.gov S3 bucket from CircleCI. ```bash @@ -175,7 +175,7 @@ Below, we will use an example change that has been done on cloud.gov UI. Assume If we try to run plan or deploy at this point, then it will fail since the state doesn't have new "es-dev" elastic search service, so it assumes this is a new deployment and tries to deploy the new instance, which will fail since the name is already taken. -2. grab the id of remote change (in this case elastic service) by running ```cf``` commands. +2. grab the id of remote change (in this case elastic service) by running ```cf``` commands. for the case of our example, we can run ```cf services```, and then run ```cf service es-dev --guid ``` which will show guid of newly created elasticsearch service instance, which is required for updating state with ES instance. 3. run this command to update state: ```terraform import cloudfoundry_service_instance.elasticsearch ``` @@ -183,13 +183,13 @@ If we try to run plan or deploy at this point, then it will fail since the state You should change ```cloudfoundry_service_instance.elasticsearch``` to your instance/service you added and trying to update the state file with. #### Security - - The Terraform State S3 instance is set to be encrypted (see `main.tf#backend`). Amazon S3 [protects data at rest][s3] using 256-bit Advanced Encryption Standard. + + The Terraform State S3 instance is set to be encrypted (see `main.tf#backend`). Amazon S3 [protects data at rest][s3] using 256-bit Advanced Encryption Standard. > **Rotating credentials:** - > + > > The S3 service creates unique IAM credentials for each application binding or service key. To rotate credentials associated with an application binding, unbind and rebind the service instance to the application. To rotate credentials associated with a service key, delete and recreate the service key. - + diff --git a/terraform/create_backend_vars.sh b/terraform/create_backend_vars.sh index 72e6d107a..11de0340c 100755 --- a/terraform/create_backend_vars.sh +++ b/terraform/create_backend_vars.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [[ $# -eq 0 ]] ; then + echo 'You need to pass the env you are configuring: 'dev', 'staging', 'production'.' + exit 1 +fi + +if [[ "$1" != "dev" && "$1" != "staging" && "$1" != "production" ]] ; then + echo 'The first argument to this script must be one of: 'dev', 'staging', or 'production'.' + exit 1 +fi + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) if [ -z "$S3_CREDENTIALS" ]; then echo "Unable to get service-keys, you may need to login to Cloud.gov first" @@ -8,15 +18,14 @@ if [ -z "$S3_CREDENTIALS" ]; then fi # Requires installation of jq - https://stedolan.github.io/jq/download/ -ACCESS_KEY=$(echo "${S3_CREDENTIALS}" | jq -r '.access_key_id') -SECRET_KEY=$(echo "${S3_CREDENTIALS}" | jq -r '.secret_access_key') -REGION=$(echo "${S3_CREDENTIALS}" | jq -r '.region') -BUCKET=$(echo "${S3_CREDENTIALS}" | jq -r '.bucket') +ACCESS_KEY=$(echo "${S3_CREDENTIALS}" | jq -r '.credentials.access_key_id') +SECRET_KEY=$(echo "${S3_CREDENTIALS}" | jq -r '.credentials.secret_access_key') +REGION=$(echo "${S3_CREDENTIALS}" | jq -r '.credentials.region') +BUCKET=$(echo "${S3_CREDENTIALS}" | jq -r '.credentials.bucket') { echo "access_key = \"$ACCESS_KEY\"" echo "secret_key = \"$SECRET_KEY\"" echo "region = \"$REGION\"" echo "bucket = \"$BUCKET\"" - echo "prefix = \"dev\"" -} >> ./dev/backend_config.tfvars +} > ./$1/backend_config.tfvars diff --git a/terraform/create_tf_vars.sh b/terraform/create_tf_vars.sh index 042fcbd71..55beafb71 100755 --- a/terraform/create_tf_vars.sh +++ b/terraform/create_tf_vars.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [[ $# -eq 0 ]] ; then + echo 'You need to pass the env you are configuring: 'dev', 'staging', 'production'.' + exit 1 +fi + +if [[ "$1" != "dev" && "$1" != "staging" && "$1" != "production" ]] ; then + echo 'The first argument to this script must be one of: 'dev', 'staging', or 'production'.' + exit 1 +fi + KEYS_JSON=$(cf service-key tanf-keys deployer | grep -A4 "{") if [ -z "$KEYS_JSON" ]; then echo "Unable to get service-keys, you may need to login to Cloud.gov first" @@ -8,8 +18,8 @@ if [ -z "$KEYS_JSON" ]; then fi # Requires installation of jq - https://stedolan.github.io/jq/download/ -CF_USERNAME_DEV=$(echo "$KEYS_JSON" | jq -r '.username') -CF_PASSWORD_DEV=$(echo "$KEYS_JSON" | jq -r '.password') +CF_USERNAME_DEV=$(echo "$KEYS_JSON" | jq -r '.credentials.username') +CF_PASSWORD_DEV=$(echo "$KEYS_JSON" | jq -r '.credentials.password') CF_SPACE="tanf-dev" @@ -17,4 +27,4 @@ CF_SPACE="tanf-dev" echo "cf_password = \"$CF_PASSWORD_DEV\"" echo "cf_user = \"$CF_USERNAME_DEV\"" echo "cf_space_name = \"$CF_SPACE\"" -} >> ./dev/variables.tfvars +} > ./$1/variables.tfvars diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 98e9d422b..0b81b8114 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -51,8 +51,13 @@ resource "cloudfoundry_service_instance" "database" { name = "tdp-db-dev" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.rds.service_plans["micro-psql"] - json_params = "{\"version\": \"12\"}" + json_params = "{\"version\": \"15\"}" recursive_delete = true + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } } ### diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 7b8b6850a..c9ecf505e 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -51,8 +51,13 @@ resource "cloudfoundry_service_instance" "database" { name = "tdp-db-prod" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.rds.service_plans["medium-psql"] - json_params = "{\"version\": \"12\"}" + json_params = "{\"version\": \"15\"}" recursive_delete = true + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } } ### diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 39d3a65e9..0c4cc2576 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -51,8 +51,13 @@ resource "cloudfoundry_service_instance" "database" { name = "tdp-db-staging" space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.rds.service_plans["micro-psql"] - json_params = "{\"version\": \"12\"}" + json_params = "{\"version\": \"15\"}" recursive_delete = true + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } } ### From ed8f67e8acb7dff5c1bd9f6d45013f6a7571a960 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:43:31 -0400 Subject: [PATCH 094/105] dynamic url to email template --- .../email/templates/upcoming-submission-deadline.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 72ac24b57..55519c07a 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -4,13 +4,13 @@

Hello {{ first_name }},

This is a friendly reminder that your data files are due in 5 days.

-

Please sign in to the TANF Data Portal +

Please sign in to the TANF Data Portal to upload and submit data files for Fiscal Year {{ fiscal_year }} - {{ fiscal_quarter}} by {{ submission_deadline }}.

-

Submit your data files

+

Submit your data files

Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions. From 57ca5f621fe888de06d80743188279f71c4d2006 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:46:05 -0400 Subject: [PATCH 095/105] no-reply language change --- .../email/templates/upcoming-submission-deadline.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 55519c07a..59f0c5235 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -15,7 +15,7 @@ Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions.

-TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team by replying to this email and we will follow up with you directly.

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team if you have any questions or concerns, and we will follow up with you directly.

Thank you,

The TDP Team

From 3c36b8172928e44864ec51c87c5001bd1ca0a8aa Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:47:32 -0400 Subject: [PATCH 096/105] 3025 As an STT user, I need an accurate error report when I space-fill `COUNTY_FIPS_CODE` (#3045) * 3025 Corrected the None case and added tests * added import for from DataFileSummaryFactory * linting * correct the failing test * fixed linting * corrected validator message --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- .../tdpservice/parsers/test/test_parse.py | 79 ++++++++++++++++++- tdrs-backend/tdpservice/parsers/transforms.py | 2 + 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0c3cf0113..f762e6f91 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -12,7 +12,9 @@ from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6, Tribal_TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M4, SSP_M5, SSP_M6, SSP_M7 from tdpservice.search_indexes import documents +from tdpservice.parsers.test.factories import DataFileSummaryFactory, ParsingFileFactory from tdpservice.data_files.models import DataFile +from tdpservice.parsers import util from .. import schema_defs, aggregates from elasticsearch.helpers.errors import BulkIndexError import logging @@ -24,7 +26,41 @@ settings.GENERATE_TRAILER_ERRORS = True -# TODO: the name of this test doesn't make perfect sense anymore since it will always have errors and no records now. +@pytest.fixture +def test_datafile(stt_user, stt): + """Fixture for small_correct_file.""" + return util.create_test_datafile('small_correct_file.txt', stt_user, stt) + + +@pytest.fixture +def test_header_datafile(stt_user, stt): + """Fixture for header test.""" + return util.create_test_datafile('tanf_section1_header_test.txt', stt_user, stt) + + +@pytest.fixture +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory.build() + + +@pytest.fixture +def t2_invalid_dob_file(): + """Fixture for T2 file with an invalid DOB.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t2_invalid_dob_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A25 TAN1ED\n' + b'T22020101111111111212Q897$9 3WTTTTTY@W222122222222101221211001472201140000000000000000000000000' + b'0000000000000000000000000000000000000000000000000000000000291\n' + b'TRAILER0000001 ') + ) + return parsing_file + +# TODO: the name of this test doesn't make perfect sense anymore since it will always have errors now. +# TODO: parametrize and merge with test_zero_filled_fips_code_file @pytest.mark.django_db def test_parse_small_correct_file(small_correct_file, dfs): """Test parsing of small_correct_file.""" @@ -1695,6 +1731,47 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): for e in parser_errors: assert e.error_message in error_msgs + +@pytest.fixture +def test_file_zero_filled_fips_code(): + """Fixture for T1 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20241A01000TAN2ED\n' + b'T1202401 2132333 0140951112 43312 03 0 0 2 554145' + + b' 0 0 0 0 0 0 0 0 0 0222222 0 02229 22 \n' + + b'T2202401 21323333219550117WT@TB9BT92122222222223 1329911 34' + + b' 32 699 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' + + b' 0 0 0 0 0 01623 0 0 0\n' + + b'T2202401 21323333219561102WTT@WBP992122221222222 2329911 28' + + b' 32 699 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' + + b' 0 0 0 0 0 01432 0 0 0\n' + + b'T3202401 2132333120070906WT@@#ZY@W212222122 63981 0 012' + + b'0050201WTTYT#TT0212222122 63981 0 0 \n' + + b'TRAILER 4 ') + ) + return parsing_file + + +@pytest.mark.django_db() +def test_zero_filled_fips_code_file(test_file_zero_filled_fips_code, dfs): + """Test parsing a file with zero filled FIPS code.""" + # TODO: this test can be merged as parametrized test with "test_parse_small_correct_file" + dfs.datafile = test_file_zero_filled_fips_code + test_file_zero_filled_fips_code.year = 2024 + test_file_zero_filled_fips_code.quarter = 'Q2' + test_file_zero_filled_fips_code.save() + + parse.parse_datafile(test_file_zero_filled_fips_code, dfs) + + parser_errors = ParserError.objects.filter(file=test_file_zero_filled_fips_code) + assert 'T1 Item 2 (County FIPS Code): field is required but a value was not' + \ + ' provided.' in [i.error_message for i in parser_errors] + + @pytest.mark.parametrize("file, batch_size, model, record_type, num_errors", [ ('tanf_s1_exact_dup_file', 10000, TANF_T1, "T1", 3), ('tanf_s1_exact_dup_file', 1, TANF_T1, "T1", 3), # This forces an in memory and database deletion of records. diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index cd51e2012..6a3717817 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -40,5 +40,7 @@ def ssp_ssn_decryption_func(value, **kwargs): def zero_pad(digits): """Zero pad a string.""" def transform(value, **kwargs): + if value is None: + return None return value.lstrip().zfill(digits) return transform From 2618dd612f5651037669e125e591e270bda81fd9 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:48:50 -0400 Subject: [PATCH 097/105] turn down log sensitivity in dev --- tdrs-backend/tdpservice/email/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index aaf46c0e4..089ea4e8d 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -157,4 +157,7 @@ def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): 'object_id': loc.id, 'object_repr': loc.name, } - log(f"{loc.name} has no recipients for data submission deadline reminder.", logger_context=logger_context) + log( + f"{loc.name} has no recipients for data submission deadline reminder.", + logger_context=logger_context if not settings.DEBUG else None + ) From 05dbc4669a62e3aaabc161691df971120bddd935 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 12:14:31 -0400 Subject: [PATCH 098/105] email reply language --- .../email/templates/upcoming-submission-deadline.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 59f0c5235..191e099cd 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -15,7 +15,7 @@ Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions.

-TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team if you have any questions or concerns, and we will follow up with you directly.

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Please reach out to the TDP support team at TANFData@acf.hhs.gov if you have any questions or need assistance.

Thank you,

The TDP Team

From d532be0e1934703a5c1faaf7a63ce9364c10473b Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 13:06:37 -0400 Subject: [PATCH 099/105] fix tests --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index f762e6f91..d82124894 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1674,10 +1674,12 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (Educational Level): 00 is not in range [1, 16]. or M2 Item 37 (Educational Level): " + - "00 is not in range [98, 99].", - "M2 Item 38 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", - "M2 Item 39 (Cooperated with Child Support): 0 is not in [1, 2, 9]."} + error_msgs = { + "Item 37 (Educational Level) 00 is not in range [1, 16]. or " + "Item 37 (Educational Level) 00 is not in range [98, 99].", + "M2 Item 38 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", + "M2 Item 39 (Cooperated with Child Support): 0 is not in [1, 2, 9]." + } for e in parser_errors: assert e.error_message in error_msgs @@ -1698,10 +1700,10 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + error_msgs = {"Item 68 (Educational Level) 00 is not in range [1, 16]. or Item 68 (Educational Level) " + "00 is not in range [98, 99].", "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", - "M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + "Item 68 (Educational Level) 00 is not in range [1, 16]. or Item 68 (Educational Level) " + "00 is not in range [98, 99].", "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9]."} From d519b033f71dbbe114e2682e8ab2ce47d2301217 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 14:09:27 -0400 Subject: [PATCH 100/105] fix tests --- .../tdpservice/data_files/test/test_api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 9ae1a408e..cea6547f4 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,10 +100,11 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if Cash Amount :873 validator1 passed then Cash and " - "Cash Equivalents: Number of Months T1 Item -1 (Cash " - "and Cash Equivalents: Number of Months): 0 is not " - "larger than 0.") + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( + "if Cash Amount :873 validator1 passed then Cash " + "and Cash Equivalents: Number of Months Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + ) @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -134,9 +135,10 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not " - "larger than 0.") + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( + "if CASH_AMOUNT :873 validator1 passed then NBR_MONTHS Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + ) @staticmethod def assert_data_file_exists(data_file_data, version, user): From 5431881edcfb875ff9fbd655dd00b32cd27338ad Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 14:19:13 -0400 Subject: [PATCH 101/105] test --- tdrs-backend/tdpservice/data_files/test/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index cea6547f4..58ed6363c 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -136,8 +136,8 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if CASH_AMOUNT :873 validator1 passed then NBR_MONTHS Item 21B " - "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + "Every T1 record should have at least one corresponding T2 or T3 " + "record with the same RPT_MONTH_YEAR and CASE_NUMBER." ) @staticmethod From 66542e84faf4b8e4d7b1bc495b628e09d89afd7b Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:01:34 -0400 Subject: [PATCH 102/105] 2950 updated search index (#2964) * 2950 updated model names * 2950 added changes for creating the filter * revert unused changes back * linting * removed unused file * 2950 removed unnecessary change * Fix linting error * 2950 added Territory to TANF * 2950 added Territory to TANF corrected linting --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- .../search_indexes/admin/filters.py | 16 ++- ...029_tanf_tribal_ssp_alter_verbose_names.py | 97 +++++++++++++++++++ .../tdpservice/search_indexes/models/ssp.py | 35 +++++++ .../tdpservice/search_indexes/models/tanf.py | 35 +++++++ .../search_indexes/models/tribal.py | 35 +++++++ 5 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index 517d27e48..36e1e66da 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -61,9 +61,23 @@ def _get_lookup_choices(self, request): else: type_query = Query(type=STT.EntityType.STATE) | Query(type=STT.EntityType.TERRITORY) queryset = queryset.filter(type_query) - return (queryset.distinct().order_by('name').values_list('name', flat=True)) + def lookups(self, request, model_admin): + """Available options in dropdown.""" + listing_record_type = [i for i in str(request.path).lower().split('/') if i not in ['/', '']][-1] + options = [] + if 'tribal' in listing_record_type: + objects = STT.objects.filter(type=STT.EntityType.TRIBE) + elif 'ssp' in listing_record_type: + objects = STT.objects.filter(ssp=True) + else: + objects = STT.objects.filter(Query(type=STT.EntityType.STATE) | Query(type=STT.EntityType.TERRITORY)) + for obj in objects: + options.append((obj.stt_code, _(obj.name))) + + return options + class FiscalPeriodFilter(SimpleListFilter): """Simple filter class to filter records based on datafile fiscal year.""" diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py b/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py new file mode 100644 index 000000000..768c91d1c --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py @@ -0,0 +1,97 @@ +# Generated by Django 3.2.15 on 2024-05-15 16:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0028_education_level_to_string'), + ] + + operations = [ + migrations.AlterModelOptions( + name='ssp_m1', + options={'verbose_name': 'SSP M1'}, + ), + migrations.AlterModelOptions( + name='ssp_m2', + options={'verbose_name': 'SSP M2'}, + ), + migrations.AlterModelOptions( + name='ssp_m3', + options={'verbose_name': 'SSP M3'}, + ), + migrations.AlterModelOptions( + name='ssp_m4', + options={'verbose_name': 'SSP M4'}, + ), + migrations.AlterModelOptions( + name='ssp_m5', + options={'verbose_name': 'SSP M5'}, + ), + migrations.AlterModelOptions( + name='ssp_m6', + options={'verbose_name': 'SSP M6'}, + ), + migrations.AlterModelOptions( + name='ssp_m7', + options={'verbose_name': 'SSP M7'}, + ), + migrations.AlterModelOptions( + name='tanf_t1', + options={'verbose_name': 'TANF T1'}, + ), + migrations.AlterModelOptions( + name='tanf_t2', + options={'verbose_name': 'TANF T2'}, + ), + migrations.AlterModelOptions( + name='tanf_t3', + options={'verbose_name': 'TANF T3'}, + ), + migrations.AlterModelOptions( + name='tanf_t4', + options={'verbose_name': 'TANF T4'}, + ), + migrations.AlterModelOptions( + name='tanf_t5', + options={'verbose_name': 'TANF T5'}, + ), + migrations.AlterModelOptions( + name='tanf_t6', + options={'verbose_name': 'TANF T6'}, + ), + migrations.AlterModelOptions( + name='tanf_t7', + options={'verbose_name': 'TANF T7'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t1', + options={'verbose_name': 'Tribal TANF T1'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t2', + options={'verbose_name': 'Tribal TANF T2'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t3', + options={'verbose_name': 'Tribal TANF T3'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t4', + options={'verbose_name': 'Tribal TANF T4'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t5', + options={'verbose_name': 'Tribal TANF T5'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t6', + options={'verbose_name': 'Tribal TANF T6'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t7', + options={'verbose_name': 'Tribal TANF T7'}, + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index bb5840323..78c035db2 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -12,6 +12,11 @@ class SSP_M1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -78,6 +83,11 @@ class SSP_M2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -165,6 +175,11 @@ class SSP_M3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -206,6 +221,11 @@ class SSP_M4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -240,6 +260,11 @@ class SSP_M5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -286,6 +311,11 @@ class SSP_M6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -318,6 +348,11 @@ class SSP_M7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index bac3f008d..c57ab89ed 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -12,6 +12,11 @@ class TANF_T1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -81,6 +86,11 @@ class TANF_T2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -170,6 +180,11 @@ class TANF_T3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -211,6 +226,11 @@ class TANF_T4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -243,6 +263,11 @@ class TANF_T5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -292,6 +317,11 @@ class TANF_T6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -329,6 +359,11 @@ class TANF_T7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, diff --git a/tdrs-backend/tdpservice/search_indexes/models/tribal.py b/tdrs-backend/tdpservice/search_indexes/models/tribal.py index fd5ee805d..d42818336 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tribal.py @@ -14,6 +14,11 @@ class Tribal_TANF_T1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -83,6 +88,11 @@ class Tribal_TANF_T2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -155,6 +165,11 @@ class Tribal_TANF_T3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -195,6 +210,11 @@ class Tribal_TANF_T4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -226,6 +246,11 @@ class Tribal_TANF_T5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -274,6 +299,11 @@ class Tribal_TANF_T6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -310,6 +340,11 @@ class Tribal_TANF_T7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, From ff50f9e2bbde8216a7519800a66f301f606cb8d9 Mon Sep 17 00:00:00 2001 From: robgendron <163159602+robgendron@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:28:36 -0400 Subject: [PATCH 103/105] Create sprint-101-summary.md (#3052) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-101-summary.md | 87 ++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 docs/Sprint-Review/sprint-101-summary.md diff --git a/docs/Sprint-Review/sprint-101-summary.md b/docs/Sprint-Review/sprint-101-summary.md new file mode 100644 index 000000000..1071e6298 --- /dev/null +++ b/docs/Sprint-Review/sprint-101-summary.md @@ -0,0 +1,87 @@ +# sprint-101-summary + +6/5/2024 - 6/18/2024 + +**Dev:** + +_**Prioritized DAC and Notifications Work**_ + +* As sys admin, I want to be able to reparse datafile sets #2978 +* As a software engineer, I want to be able to test django-admin-508 #3008 +* As tech lead, I need the STT filter for search\_indexes to be updated #2950 +* As a data analyst I want to be notified of approaching data deadlines #2473 +* add `SENDGRID_API_KEY` to deploy.backend.sh #2677 +* Implement (small) data lifecycle (backup/archive ES) #3004 +* As a developer I want to test django-508 repo #2980\ + + +**DevOps:** + +_**Successful deployments across environments and pipeline stability investments**_ + +* Application health monitoring #831 + +**Design:** + +_**Close out error guide work, coordinate with dev on a plan for Cat 3 problems introduced by Cat 2 work, support spec-writing for upcoming work, and continued error audit dev ticket refinement.**_ + +* Error Report Guide #2847 is going through final edits + * Walk-on Dear Colleague letter link update to this PR (or spin up a separate ticket if deployment of the letter to OFA's website doesn't align to this) +* Deliver spec for #3014 (Blanked-out values in Submission History) +* \#3021 Updated KC Release Notes & Update Indicator FAQ - stretch goal for this sprint +* Write follow-on / spec tickets from #2909 findings - stretch/ongoing lift +* Category 3 error messages clean-up #2792 - stretch/ongoing lift + +## Tickets + +### Completed/Merged + +* [#2980 As a developer I want to test django-508 repo](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2980) +* [#2892 Correct misleading error message for unaligned reporting year/q against header year/q](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2892) +* [#2909 \[Research Spike\] OOtB OFA Kibana Experience & DIGIT Data Access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2909) +* [#2991 As tech lead, I need the sftp file transfer feature to be deprecated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2991) +* [#2847 \[Design Deliverable\] Error Report Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2847) +* [#3024 2897 follow-on for a11y-related enhancement ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3024) +* [#2897 As a data analyst I want finalized language and guidance resources in Submission History & Error Reports ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2897) + +### Submitted (QASP Review, OCIO Review) + +* [#2133 \[Dev\] Enhancement for Request Access form (Tribe discoverability) ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2133) +* [#3023 as STT approved user, I need my IP address whitelisted so i can access TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3023) +* [#3000 \[Design Deliverable\] TDP Poster for summer 2024 conferences](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3000) +* [#2795 As tech lead, I need TDP to detect duplicate records within a file and not store them in the db. ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2795) +* [#2693 \[Error Audit\] Category 2 error messages clean-up ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2693) +* [#2801 Friendly name cleanup ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2801) +* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883) +* [#3021 \[Design Deliverable\] Updated KC Release Notes & Update Indicator FAQ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3021) +* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954) + +### Ready to Merge + +* + +### Closed (Not Merged) + +* [#2491 Create root-level docker-compose configuration file(s)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2491) +* [#1690 As a system admin, I need a way to be redirected to frontend from DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1690) +* [#2351 As a user I want to be notified when the files are being scanned or uploaded when I push upload button](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2351) +* [#2591 Allow `manage.py` commands to be run by circleci](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2591) + +### Moved to Next Sprint + +**In Progress** + +* [#3004 Implement (small) data lifecycle (backup/archive ES)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3004) +* [#831 \[Spike\] As a Tech Lead, I want to get alerts when there is a backend or frontend error that affects an STT user ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/831) +* [#2978 As sys admin, I want to be able to reparse datafile sets](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2978) + +#### Blocked + +* + +**Raft Review** + +* [#2950 As tech lead, I need the STT filter for search\_indexes to be updated ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2950) +* [#3008 As a software engineer, I want to be able to test django-admin-508](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3008) +* [#3016 Spike - Cat2 Validator Improvement](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3016) +* [#2473 As a data analyst I want to be notified of approaching data deadlines](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2473) From e29f07cb5eb335f6419afeab0c113864deed4451 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 17 Jul 2024 09:01:27 -0400 Subject: [PATCH 104/105] remove extraneous item names --- tdrs-backend/tdpservice/data_files/test/test_api.py | 7 +++---- tdrs-backend/tdpservice/parsers/test/test_validators.py | 2 +- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 58ed6363c..55dba626e 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -101,8 +101,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if Cash Amount :873 validator1 passed then Cash " - "and Cash Equivalents: Number of Months Item 21B " + "if Cash Amount :873 validator1 passed then Item 21B " "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." ) @@ -136,8 +135,8 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "Every T1 record should have at least one corresponding T2 or T3 " - "record with the same RPT_MONTH_YEAR and CASE_NUMBER." + "if CASH_AMOUNT :873 validator1 passed then Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." ) @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index c39ff545b..d729efc6e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -108,7 +108,7 @@ def test_if_validators(): name='Field2', friendly_name='field 2'), ] )) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 Item 2 (field 2) 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Item 2 (field 2) 2 does not match 1.', ['Field1', 'Field2']) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 5e617c8c7..53726ee4c 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -189,7 +189,7 @@ def if_then_validator_func(value, row_schema): ending_error = "validator2 passed" error_message = (f"if {condition_field_name} " + (center_error) + - f" then {result_field_name} " + ending_error) + f" then " + ending_error) else: error_message = None From e9c02c8ff7fe174da5b3e65c6aaf0848b413ce9c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 17 Jul 2024 09:22:30 -0400 Subject: [PATCH 105/105] lint --- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 53726ee4c..e8320055a 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -189,7 +189,7 @@ def if_then_validator_func(value, row_schema): ending_error = "validator2 passed" error_message = (f"if {condition_field_name} " + (center_error) + - f" then " + ending_error) + " then " + ending_error) else: error_message = None