Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/2709 ssp section 1 validation #2723

Merged
merged 31 commits into from
Nov 8, 2023
Merged

Feature/2709 ssp section 1 validation #2723

Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
54e773c
cat 2 validation
jtimpe Sep 22, 2023
eb00573
ssp s1 cat 3 validators
jtimpe Oct 3, 2023
069b6dd
Merge branch 'develop' into feature/2709-ssp-section-1-validation
jtimpe Oct 3, 2023
b7ca40e
Merge branch 'develop' into feature/2709-ssp-section-1-validation
jtimpe Oct 6, 2023
741cf32
debug ssp section errors
jtimpe Oct 10, 2023
bddfc4a
change string validators to number validators
jtimpe Oct 10, 2023
def741d
add df to case agg query
jtimpe Oct 10, 2023
598bb7f
update ssp s1 test
jtimpe Oct 10, 2023
5da2adc
update ssp tests
jtimpe Oct 10, 2023
e9bfc88
fix lint
jtimpe Oct 10, 2023
1d06484
Merge branch 'develop' into feature/2709-ssp-section-1-validation
jtimpe Oct 10, 2023
987ed59
update test
jtimpe Oct 10, 2023
678e3ba
rm prints
jtimpe Oct 11, 2023
f8200c8
Merge branch 'develop' into feature/2709-ssp-section-1-validation
jtimpe Oct 17, 2023
badd053
update validator
jtimpe Oct 25, 2023
98ed06d
Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
jtimpe Oct 25, 2023
a8b3cab
fix the validator i broke and update tests
jtimpe Oct 25, 2023
e9978ab
udpate validator (again)
jtimpe Oct 25, 2023
1df3e53
lint
jtimpe Oct 25, 2023
c4b935e
Update tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
jtimpe Oct 25, 2023
bd3499b
lint
jtimpe Oct 25, 2023
8889ed5
rm unneeded cat3 validator
jtimpe Oct 25, 2023
8ec5252
update docstring
jtimpe Oct 25, 2023
98b0f3c
rm unused validator
jtimpe Oct 25, 2023
a8f850a
update m2 validator
jtimpe Oct 25, 2023
3213c76
fix education level validation for m3
jtimpe Oct 25, 2023
b0a2692
add required=False to allow blanks
jtimpe Oct 26, 2023
c6a1fb2
fix other education level m3 validator
jtimpe Oct 26, 2023
96730a2
rm schema comments
jtimpe Oct 26, 2023
a321529
Merge branch 'develop' into feature/2709-ssp-section-1-validation
andrew-jameson Nov 7, 2023
bdf0463
fix tests
jtimpe Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 194 additions & 42 deletions tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,92 +14,244 @@
preparsing_validators=[
validators.hasLength(150),
],
postparsing_validators=[],
postparsing_validators=[
validators.if_then_validator(
condition_field='CASH_AMOUNT', condition_function=validators.isLargerThan(0),
result_field='NBR_MONTHS', result_function=validators.isLargerThan(0),
),
validators.if_then_validator(
condition_field='CC_AMOUNT', condition_function=validators.isLargerThan(0),
result_field='CHILDREN_COVERED', result_function=validators.isLargerThan(0),
),
validators.if_then_validator(
condition_field='CC_AMOUNT', condition_function=validators.isLargerThan(0),
result_field='CC_NBR_MONTHS', result_function=validators.isLargerThan(0),
),
validators.if_then_validator(
condition_field='TRANSP_AMOUNT', condition_function=validators.isLargerThan(0),
result_field='TRANSP_NBR_MONTHS', result_function=validators.isLargerThan(0),
),
validators.if_then_validator(
jtimpe marked this conversation as resolved.
Show resolved Hide resolved
condition_field='TRANSITION_SERVICES_AMOUNT', condition_function=validators.isLargerThan(0),
result_field='TRANSITION_NBR_MONTHS', result_function=validators.isLargerThan(0),
),
# validators.if_then_validator(
# condition_field='OTHER_AMOUNT', condition_function=validators.isLargerThan(0),
# result_field='OTHER_NBR_MONTHS', result_function=validators.isLargerThan(0),
# ),
validators.if_then_validator(
condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
result_field='WORK_REQ_SANCTION', result_function=validators.oneOf((1, 2)),
),
# validators.if_then_validator(
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove inline comments

# condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
# result_field='FAMILY_SANC_ADULT', result_function=validators.oneOf((1, 2)),
# ),
validators.if_then_validator(
condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
result_field='SANC_TEEN_PARENT', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
result_field='NON_COOPERATION_CSE', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
result_field='FAILURE_TO_COMPLY', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='SANC_REDUCTION_AMT', condition_function=validators.isLargerThan(0),
result_field='OTHER_SANCTION', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0),
result_field='FAMILY_CAP', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0),
result_field='REDUCTIONS_ON_RECEIPTS', result_function=validators.oneOf((1, 2)),
),
validators.if_then_validator(
condition_field='OTHER_TOTAL_REDUCTIONS', condition_function=validators.isLargerThan(0),
result_field='OTHER_NON_SANCTION', result_function=validators.oneOf((1, 2)),
),
validators.sumIsLarger([
"AMT_FOOD_STAMP_ASSISTANCE",
"AMT_SUB_CC",
"CASH_AMOUNT",
"CC_AMOUNT",
"CC_NBR_MONTHS"
], 0)
],
fields=[
Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
required=True, validators=[]),
Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8,
required=True, validators=[]),
required=True, validators=[
validators.dateYearIsLargerThan(1998),
validators.dateMonthIsValid(),
]),
Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19,
required=True, validators=[]),
required=True, validators=[
validators.isAlphaNumeric()
]),
Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22,
required=True, validators=[]),
required=True, validators=[
validators.isNumber(),
]),
Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24,
required=True, validators=[]),
required=True, validators=[
validators.isInStringRange(0, 99),
]),
Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29,
required=True, validators=[]),
required=True, validators=[
validators.isNumber(),
]),
Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(1, 99),
]),
Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(1, 3),
]),
Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(1, 3),
]),
Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(1, 2), # diff than t1 (1-3)
ADPennington marked this conversation as resolved.
Show resolved Hide resolved
]),
Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(1, 2),
]),
Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(0, 2),
]),
Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might be the discrepancy you were talking with Alex about, if so then disregard. For TANF any field that was multiple chars long was converted to string even if it was an int to preserve the leading zeros per Alex's request. Do we want to keep that consistency here? Likewise, any field that is only one char could be an int if it was string so long as the fields values are all ints.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

great question, glad you brought it up! i'd definitely like @ADPennington to weigh in on this - we discussed this a bit and my takeaway was that for some fields it may be useful to preserve the leading zeros, but for many fields it's more useful to have an int value in the database when querying. this kind of goes against an earlier decision that we'd like to keep the resulting data structure strictly consistent with the legacy system. for me it was easiest to not make any changes to the underlying model, and if we'd like to change anything we can (hopefully) have a smaller migration

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jtimpe @elipe17 yeah, think we lean towards ints for amounts and such and special fields like stratum we try to store as str to preserve leading zeros. there may be inconsistencies here and there but it may be hard to catch until parsing is complete. we can revisit. but so far during testing im seeing consistency.

required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(0, 2),
]),
Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81,
required=True, validators=[]),
required=True, validators=[
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jtimpe a few of these arent allowing spaces

ssps1

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The space fill ticket hasnt merged yet. It is waiting for Andrews develop/cloudgov.py PR to merge first. That way no deployments are kicked off.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once it does, all the SSP tickets will be updated with it.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ohhhh thank you! totally slipped my mind. @elipe17 @jtimpe

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ADPennington i went ahead and added required=False to all the relevant fields - i don't believe that column was in the spreadsheet when i first submitted this, sorry about that! as @elipe17 mentioned, though, #2695 still has to merge so that the validator logic executes correctly

validators.isLargerThanOrEqualTo(0),
]),
Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jtimpe tried space-filling here and see this in the report:

OTHER_NBR_MONTHS is required but a value was not provided.

]),
Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(0, 9), # diff than t1 (0-2)
]),
Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109,
required=True, validators=[]),
required=True, validators=[
validators.isLargerThanOrEqualTo(0),
]),
Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112,
required=True, validators=[]),
required=True, validators=[
validators.oneOf([1, 2]),
]),
Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113,
required=True, validators=[]),
required=True, validators=[
validators.isInLimits(0, 9),
# validators.isAlphaNumeric(), # diff than t1 (blank or 9, alpha)
]),
Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150,
required=False, validators=[]),
]
Expand Down
Loading
Loading