From 71c8e28520ee904d3a2e6e04af62f2c41fe57e7f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 5 Oct 2023 15:09:05 -0600 Subject: [PATCH] - Added m4 and m5 - Added cat 2 validators --- .../tdpservice/parsers/schema_defs/ssp/m4.py | 50 +++++++++++ .../tdpservice/parsers/schema_defs/ssp/m5.py | 83 +++++++++++++++++++ .../tdpservice/search_indexes/models/ssp.py | 82 ++++++++++++++++++ 3 files changed, 215 insertions(+) create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py new file mode 100644 index 000000000..7c65d08f0 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -0,0 +1,50 @@ +"""Schema for SSP M1 record type.""" + + +from ...util import SchemaManager +from ...fields import Field +from ...row_schema import RowSchema +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M4 + +m1 = SchemaManager( + schemas=[ + RowSchema( + model=SSP_M4, + preparsing_validators=[ + validators.hasLength(66), + ], + postparsing_validators=[], + fields=[ + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(),]), + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + required=True, validators=[validators.isAlphaNumeric()]), + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + required=True, validators=[validators.isInStringRange(0, 999)]), + Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24, + required=True, validators=[validators.isInStringRange(0, 99)]), + Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + required=True, validators=[validators.isInStringRange(0, 99999)]), + Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, + required=True, validators=[validators.isInLimits(1, 2)]), + Field(item="7", name='CLOSURE_REASON', type='string', startIndex=30, endIndex=32, + required=True, validators=[validators.or_validators(validators.isInStringRange(1, 19), + validators.matches('19'))]), + Field(item="7", name='REC_SUB_HOUSING', type='number', startIndex=32, endIndex=33, + required=True, validators=[validators.isInLimits(1, 3)]), + Field(item="7", name='REC_MED_ASSIST', type='number', startIndex=33, endIndex=34, + required=True, validators=[validators.isInLimits(1, 2)]), + Field(item="7", name='REC_FOOD_STAMPS', type='number', startIndex=34, endIndex=35, + required=True, validators=[validators.isInLimits(1, 2)]), + Field(item="7", name='REC_SUB_CC', type='number', startIndex=35, endIndex=36, + required=True, validators=[validators.isInLimits(1, 3)]), + Field(item="-1", name='BLANK', type='string', startIndex=36, endIndex=66, required=False, + validators=[]), + ] + ) + ] + ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py new file mode 100644 index 000000000..d3dc1e79d --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -0,0 +1,83 @@ +"""Schema for SSP M1 record type.""" + + +from ...util import SchemaManager +from ...transforms import ssp_ssn_decryption_func +from ...fields import TransformField, Field +from ...row_schema import RowSchema +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M5 + + +m2 = SchemaManager( + schemas=[ + RowSchema( + model=SSP_M5, + preparsing_validators=[ + validators.hasLength(150), + ], + postparsing_validators=[], + fields=[ + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(),]), + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + required=True, validators=[validators.isAlphaNumeric()]), + Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + required=True, validators=[]), + Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid(),]), + TransformField(transform_func=ssp_ssn_decryption_func, item="29", name='SSN', type='string', + startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()], + is_encrypted=False), + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, + validators=[validators.isInLimits(0, 2)]), + Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="30C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="30D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="30F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="31", name='GENDER', type='number', startIndex=43, endIndex=44, + required=True, validators=[validators.isInLimits(0, 9)]), + Field(item="28", name='REC_OASDI_INSURANCE', type='number', startIndex=44, endIndex=45, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="28", name='REC_FEDERAL_DISABILITY', type='number', startIndex=45, endIndex=46, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="28", name='REC_AID_TOTALLY_DISABLED', type='number', startIndex=46, endIndex=47, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="28", name='REC_AID_AGED_BLIND', type='number', startIndex=47, endIndex=48, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="28", name='REC_SSI', type='number', startIndex=48, endIndex=49, + required=True, validators=[validators.isInLimits(1, 2)]), + Field(item="28", name='MARITAL_STATUS', type='number', startIndex=49, endIndex=50, + required=True, validators=[validators.isInLimits(0, 5)]), + Field(item="28", name='RELATIONSHIP_HOH', type='string', startIndex=50, endIndex=52, + required=True, validators=[validators.isInStringRange(1, 10)]), + Field(item="28", name='PARENT_MINOR_CHILD', type='number', startIndex=52, endIndex=53, + required=True, validators=[validators.isInLimits(0, 2)]), + Field(item="28", name='NEEDS_OF_PREGNANT_WOMAN', type='number', startIndex=53, endIndex=54, + required=True, validators=[validators.isInLimits(0, 9)]), + Field(item="28", name='EDUCATION_LEVEL', type='string', startIndex=54, endIndex=56, + required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16), + validators.isInStringRange(98, 99))]), + Field(item="28", name='CITIZENSHIP_STATUS', type='number', startIndex=56, endIndex=57, + required=True, validators=[validators.or_validators(validators.isInLimits(0, 3), + validators.matches(9))]), + Field(item="28", name='EMPLOYMENT_STATUS', type='number', startIndex=57, endIndex=58, + required=True, validators=[validators.isInLimits(0, 3)]), + Field(item="28", name='AMOUNT_EARNED_INCOME', type='string', startIndex=58, endIndex=62, + required=True, validators=[validators.isInStringRange(0, 9999)]), + Field(item="28", name='AMOUNT_UNEARNED_INCOME', type='string', startIndex=62, endIndex=66, + required=True, validators=[validators.isInStringRange(0, 9999)]), + ], + ) + ] + ) diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index f2a325eb3..fbb0ed856 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -210,3 +210,85 @@ class SSP_M3(models.Model): CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) + +class SSP_M4(models.Model): + """ + Parsed record representing an SSP M1 data submission. + + Mapped to an elastic search index. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datafile = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m1_parent' + ) + + error = GenericRelation(ParserError) + RecordType = models.CharField(max_length=156, null=True, blank=False) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False) + COUNTY_FIPS_CODE = models.CharField( + max_length=3, + null=True, + blank=False + ) + STRATUM = models.CharField(max_length=2, null=True, blank=False) + ZIP_CODE = models.CharField(max_length=5, null=True, blank=False) + DISPOSITION = models.IntegerField(null=True, blank=False) + CLOSURE_REASON = models.CharField(max_length=2, null=True, blank=False) + REC_SUB_HOUSING = models.IntegerField(null=True, blank=False) + REC_MED_ASSIST = models.IntegerField(null=True, blank=False) + REC_FOOD_STAMPS = models.IntegerField(null=True, blank=False) + REC_SUB_CC = models.IntegerField(null=True, blank=False) + +class SSP_M5(models.Model): + """ + Parsed record representing an SSP M1 data submission. + + Mapped to an elastic search index. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datafile = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m2_parent' + ) + + error = GenericRelation(ParserError) + RecordType = models.CharField(max_length=156, null=True, blank=False) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False) + + FAMILY_AFFILIATION = models.IntegerField(null=True, blank=False) + DATE_OF_BIRTH = models.CharField(max_length=8, null=True, blank=False) + SSN = models.CharField(max_length=9, null=True, blank=False) + RACE_HISPANIC = models.IntegerField(null=True, blank=False) + RACE_AMER_INDIAN = models.IntegerField(null=True, blank=False) + RACE_ASIAN = models.IntegerField(null=True, blank=False) + RACE_BLACK = models.IntegerField(null=True, blank=False) + RACE_HAWAIIAN = models.IntegerField(null=True, blank=False) + RACE_WHITE = models.IntegerField(null=True, blank=False) + GENDER = models.IntegerField(null=True, blank=False) + REC_OASDI_INSURANCE = models.IntegerField(null=True, blank=False) + REC_FEDERAL_DISABILITY = models.IntegerField(null=True, blank=False) + REC_AID_TOTALLY_DISABLED = models.IntegerField(null=True, blank=False) + REC_AID_AGED_BLIND = models.IntegerField(null=True, blank=False) + REC_SSI = models.IntegerField(null=True, blank=False) + MARITAL_STATUS = models.IntegerField(null=True, blank=False) + RELATIONSHIP_HOH = models.CharField(max_length=2, null=True, blank=False) + PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) + NEEDS_OF_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) + CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) + EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) + AMOUNT_EARNED_INCOME = models.CharField(max_length=4, null=True, blank=False) + AMOUNT_UNEARNED_INCOME = models.CharField(max_length=4, null=True, blank=False)