Skip to content

Commit

Permalink
- Added m4 and m5
Browse files Browse the repository at this point in the history 
- Added cat 2 validators
  • Loading branch information
@elipe17
elipe17 committed Oct 5, 2023
1 parent 8de88cd commit 71c8e28
Show file tree
Hide file tree
Showing 3 changed files with 215 additions and 0 deletions.
50 changes: 50 additions & 0 deletions tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""Schema for SSP M1 record type."""


from ...util import SchemaManager
from ...fields import Field
from ...row_schema import RowSchema
from ... import validators
from tdpservice.search_indexes.models.ssp import SSP_M4

m1 = SchemaManager(
schemas=[
RowSchema(
model=SSP_M4,
preparsing_validators=[
validators.hasLength(66),
],
postparsing_validators=[],
fields=[
Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
required=True, validators=[]),
Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8,
required=True, validators=[validators.dateYearIsLargerThan(1998),
validators.dateMonthIsValid(),]),
Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19,
required=True, validators=[validators.isAlphaNumeric()]),
Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22,
required=True, validators=[validators.isInStringRange(0, 999)]),
Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24,
required=True, validators=[validators.isInStringRange(0, 99)]),
Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29,
required=True, validators=[validators.isInStringRange(0, 99999)]),
Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30,
required=True, validators=[validators.isInLimits(1, 2)]),
Field(item="7", name='CLOSURE_REASON', type='string', startIndex=30, endIndex=32,
required=True, validators=[validators.or_validators(validators.isInStringRange(1, 19),
validators.matches('19'))]),
Field(item="7", name='REC_SUB_HOUSING', type='number', startIndex=32, endIndex=33,
required=True, validators=[validators.isInLimits(1, 3)]),
Field(item="7", name='REC_MED_ASSIST', type='number', startIndex=33, endIndex=34,
required=True, validators=[validators.isInLimits(1, 2)]),
Field(item="7", name='REC_FOOD_STAMPS', type='number', startIndex=34, endIndex=35,
required=True, validators=[validators.isInLimits(1, 2)]),
Field(item="7", name='REC_SUB_CC', type='number', startIndex=35, endIndex=36,
required=True, validators=[validators.isInLimits(1, 3)]),
Field(item="-1", name='BLANK', type='string', startIndex=36, endIndex=66, required=False,
validators=[]),
]
)
]
)
83 changes: 83 additions & 0 deletions tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Schema for SSP M1 record type."""


from ...util import SchemaManager
from ...transforms import ssp_ssn_decryption_func
from ...fields import TransformField, Field
from ...row_schema import RowSchema
from ... import validators
from tdpservice.search_indexes.models.ssp import SSP_M5


m2 = SchemaManager(
schemas=[
RowSchema(
model=SSP_M5,
preparsing_validators=[
validators.hasLength(150),
],
postparsing_validators=[],
fields=[
Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
required=True, validators=[]),
Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8,
required=True, validators=[validators.dateYearIsLargerThan(1998),
validators.dateMonthIsValid(),]),
Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19,
required=True, validators=[validators.isAlphaNumeric()]),
Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20,
required=True, validators=[]),
Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28,
required=True, validators=[validators.dateYearIsLargerThan(1998),
validators.dateMonthIsValid(),]),
TransformField(transform_func=ssp_ssn_decryption_func, item="29", name='SSN', type='string',
startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()],
is_encrypted=False),
Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True,
validators=[validators.isInLimits(0, 2)]),
Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="30C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="30D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="30F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="31", name='GENDER', type='number', startIndex=43, endIndex=44,
required=True, validators=[validators.isInLimits(0, 9)]),
Field(item="28", name='REC_OASDI_INSURANCE', type='number', startIndex=44, endIndex=45,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="28", name='REC_FEDERAL_DISABILITY', type='number', startIndex=45, endIndex=46,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="28", name='REC_AID_TOTALLY_DISABLED', type='number', startIndex=46, endIndex=47,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="28", name='REC_AID_AGED_BLIND', type='number', startIndex=47, endIndex=48,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="28", name='REC_SSI', type='number', startIndex=48, endIndex=49,
required=True, validators=[validators.isInLimits(1, 2)]),
Field(item="28", name='MARITAL_STATUS', type='number', startIndex=49, endIndex=50,
required=True, validators=[validators.isInLimits(0, 5)]),
Field(item="28", name='RELATIONSHIP_HOH', type='string', startIndex=50, endIndex=52,
required=True, validators=[validators.isInStringRange(1, 10)]),
Field(item="28", name='PARENT_MINOR_CHILD', type='number', startIndex=52, endIndex=53,
required=True, validators=[validators.isInLimits(0, 2)]),
Field(item="28", name='NEEDS_OF_PREGNANT_WOMAN', type='number', startIndex=53, endIndex=54,
required=True, validators=[validators.isInLimits(0, 9)]),
Field(item="28", name='EDUCATION_LEVEL', type='string', startIndex=54, endIndex=56,
required=True, validators=[validators.or_validators(validators.isInStringRange(0, 16),
validators.isInStringRange(98, 99))]),
Field(item="28", name='CITIZENSHIP_STATUS', type='number', startIndex=56, endIndex=57,
required=True, validators=[validators.or_validators(validators.isInLimits(0, 3),
validators.matches(9))]),
Field(item="28", name='EMPLOYMENT_STATUS', type='number', startIndex=57, endIndex=58,
required=True, validators=[validators.isInLimits(0, 3)]),
Field(item="28", name='AMOUNT_EARNED_INCOME', type='string', startIndex=58, endIndex=62,
required=True, validators=[validators.isInStringRange(0, 9999)]),
Field(item="28", name='AMOUNT_UNEARNED_INCOME', type='string', startIndex=62, endIndex=66,
required=True, validators=[validators.isInStringRange(0, 9999)]),
],
)
]
)
82 changes: 82 additions & 0 deletions tdrs-backend/tdpservice/search_indexes/models/ssp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,3 +210,85 @@ class SSP_M3(models.Model):
CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False)
UNEARNED_SSI = models.IntegerField(null=True, blank=False)
OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False)

class SSP_M4(models.Model):
"""
Parsed record representing an SSP M1 data submission.
Mapped to an elastic search index.
"""

id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
datafile = models.ForeignKey(
DataFile,
blank=True,
help_text='The parent file from which this record was created.',
null=True,
on_delete=models.CASCADE,
related_name='m1_parent'
)

error = GenericRelation(ParserError)
RecordType = models.CharField(max_length=156, null=True, blank=False)
RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)
COUNTY_FIPS_CODE = models.CharField(
max_length=3,
null=True,
blank=False
)
STRATUM = models.CharField(max_length=2, null=True, blank=False)
ZIP_CODE = models.CharField(max_length=5, null=True, blank=False)
DISPOSITION = models.IntegerField(null=True, blank=False)
CLOSURE_REASON = models.CharField(max_length=2, null=True, blank=False)
REC_SUB_HOUSING = models.IntegerField(null=True, blank=False)
REC_MED_ASSIST = models.IntegerField(null=True, blank=False)
REC_FOOD_STAMPS = models.IntegerField(null=True, blank=False)
REC_SUB_CC = models.IntegerField(null=True, blank=False)

class SSP_M5(models.Model):
"""
Parsed record representing an SSP M1 data submission.
Mapped to an elastic search index.
"""

id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
datafile = models.ForeignKey(
DataFile,
blank=True,
help_text='The parent file from which this record was created.',
null=True,
on_delete=models.CASCADE,
related_name='m2_parent'
)

error = GenericRelation(ParserError)
RecordType = models.CharField(max_length=156, null=True, blank=False)
RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)

FAMILY_AFFILIATION = models.IntegerField(null=True, blank=False)
DATE_OF_BIRTH = models.CharField(max_length=8, null=True, blank=False)
SSN = models.CharField(max_length=9, null=True, blank=False)
RACE_HISPANIC = models.IntegerField(null=True, blank=False)
RACE_AMER_INDIAN = models.IntegerField(null=True, blank=False)
RACE_ASIAN = models.IntegerField(null=True, blank=False)
RACE_BLACK = models.IntegerField(null=True, blank=False)
RACE_HAWAIIAN = models.IntegerField(null=True, blank=False)
RACE_WHITE = models.IntegerField(null=True, blank=False)
GENDER = models.IntegerField(null=True, blank=False)
REC_OASDI_INSURANCE = models.IntegerField(null=True, blank=False)
REC_FEDERAL_DISABILITY = models.IntegerField(null=True, blank=False)
REC_AID_TOTALLY_DISABLED = models.IntegerField(null=True, blank=False)
REC_AID_AGED_BLIND = models.IntegerField(null=True, blank=False)
REC_SSI = models.IntegerField(null=True, blank=False)
MARITAL_STATUS = models.IntegerField(null=True, blank=False)
RELATIONSHIP_HOH = models.CharField(max_length=2, null=True, blank=False)
PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False)
NEEDS_OF_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False)
EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False)
CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False)
EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False)
AMOUNT_EARNED_INCOME = models.CharField(max_length=4, null=True, blank=False)
AMOUNT_UNEARNED_INCOME = models.CharField(max_length=4, null=True, blank=False)

0 comments on commit 71c8e28

Please sign in to comment.