diff --git a/docs/Sprint-Review/sprint-83-summary.md b/docs/Sprint-Review/sprint-83-summary.md new file mode 100644 index 000000000..cf47de6cf --- /dev/null +++ b/docs/Sprint-Review/sprint-83-summary.md @@ -0,0 +1,49 @@ + +# Sprint 83 Summary + +09/30/23 - 10/11/23 + +Velocity: Dev (18) + +## Sprint Goal +* Complete parsing engine development for TANF Section (04) and begin SSP (01), close out subsmission history and metadata workflows (1613/12/10). +* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users +* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline. + + +## Tickets +### Completed/Merged +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111) + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) +* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709) +* [#2663 Investigate OWASP NightlyScan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663) + +### Blocked +* N/A + +### Raft Review +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) +* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) + +### Demo +* Internal: + * 1111, 1610, 1612 +* External: + * 1111, 1610, 1612 + diff --git a/docs/Sprint-Review/sprint-84-summary.md b/docs/Sprint-Review/sprint-84-summary.md new file mode 100644 index 000000000..a22dfd912 --- /dev/null +++ b/docs/Sprint-Review/sprint-84-summary.md @@ -0,0 +1,61 @@ +# Sprint 84 Summary +10/10/23 - 10/24/23 + +Velocity: Dev (10) + +### Sprint Goal +* Dev: + * Continue parsing engine development + * Complete SSP Sec (01) and SSP Sec (02) + * Resolve deployment blocker + * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown +* DevOps: + * 2429 - Singular Clam AV + * 2722 - Singular deployment workflow +* UX: Resume regional staff research, synthesize in-app messaging research, continue supporting onboarding/utilization +* Prod: Find path forward on Sendgrid + +## Tickets +### Completed/Merged +* [#2411 As system admin, I want to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) +* [#2429 Singular ClamAV Scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) + + + +### Ready to Merge +* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2725 file input render issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2725) + + +### Submitted (QASP Review, OCIO Review) +* [#2701 FETCH_STTS Infinite Request](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2701) +* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) +* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119) +* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592) +* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599) +* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683) +* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722) + + +### Blocked +* N/A + +### Raft Review +* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118) +* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120) +* [#2116 Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663) + +### Demo +* N/A + + + diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index f50152891..ec372396a 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -51,6 +51,8 @@ set_cf_envs() "FRONTEND_BASE_URL" "LOGGING_LEVEL" "REDIS_URI" + "JWT_KEY" + "STAGING_JWT_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" @@ -62,9 +64,13 @@ set_cf_envs() cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue + elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then + sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') + cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + else + cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi - - cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + echo "Setting var : $var_name" $cf_cmd done @@ -128,7 +134,7 @@ update_backend() bind_backend_to_services() { echo "Binding services to app: $CGAPPNAME_BACKEND" - if [ "$CFAPPNAME_BACKEND" = "tdp-backend-develop" ]; then + if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then # TODO: this is technical debt, we should either make staging mimic tanf-dev # or make unique services for all apps but we have a services limit # Introducing technical debt for release 3.0.0 specifically. diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index fa4c73101..1487cf498 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -60,7 +60,8 @@ def parse_value(self, line): class TransformField(Field): """Represents a field that requires some transformation before serializing.""" - def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True, validators=[], **kwargs): + def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True, + validators=[], **kwargs): super().__init__(item, name, type, startIndex, endIndex, required, validators) self.transform_func = transform_func self.kwargs = kwargs diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index fd0209a50..8e217e78f 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -114,7 +114,9 @@ def run_field_validators(self, instance, generate_error): else: value = getattr(instance, field.name, None) - if field.required and not value_is_empty(value, field.endIndex-field.startIndex): + is_empty = value_is_empty(value, field.endIndex-field.startIndex) + should_validate = not field.required and not is_empty + if (field.required and not is_empty) or should_validate: for validator in field.validators: validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 546910386..f31bc892c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -92,7 +92,7 @@ validators.isNumber(), ]), Field(item="5", name='STRATUM', type='string', startIndex=22, endIndex=24, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 99), ]), Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, @@ -128,7 +128,7 @@ validators.isInLimits(1, 2), ]), Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 2), ]), Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, @@ -136,7 +136,7 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, - required=True, validators=[ + required=False, validators=[ validators.isInLimits(0, 3), ]), Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, @@ -180,19 +180,19 @@ validators.isLargerThanOrEqualTo(0), ]), Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, - required=True, validators=[ + required=False, validators=[ validators.isLargerThanOrEqualTo(0), ]), Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, @@ -204,7 +204,7 @@ validators.oneOf([1, 2]), ]), Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2]), ]), Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, @@ -244,7 +244,7 @@ validators.oneOf([1, 2]), ]), Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='string', startIndex=113, endIndex=114, - required=True, validators=[ + required=False, validators=[ validators.or_validators(validators.matches('9'), validators.isEmpty()), validators.isAlphaNumeric(), ]), @@ -254,7 +254,7 @@ 6, 7, 8, 9]) ]), Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, - required=True, validators=[ + required=False, validators=[ validators.oneOf([1, 2]), ]), Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 74f0b9d52..1f2088b38 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -109,17 +109,17 @@ TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[validators.validateSSN()], is_encrypted=False), - Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, + Field(item="34A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=False, validators=[validators.isInLimits(0, 2)]), - Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, + Field(item="34B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=False, validators=[validators.isInLimits(0, 2)]), - Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, + Field(item="34C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=False, validators=[validators.isInLimits(0, 2)]), - Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, + Field(item="34D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=False, validators=[validators.isInLimits(0, 2)]), - Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, + Field(item="34E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=False, validators=[validators.isInLimits(0, 2)]), - Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, + Field(item="34F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=False, validators=[validators.isInLimits(0, 2)]), Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[validators.isLargerThanOrEqualTo(0),]), @@ -129,66 +129,66 @@ validators=[validators.oneOf([1, 2])]), Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[validators.or_validators(validators.oneOf(["1", "2"]), validators.isBlank())]), - Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, + Field(item="36D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=False, validators=[validators.isLargerThanOrEqualTo(0),]), Field(item="36E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[validators.oneOf([1, 2]),]), - Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, + Field(item="37", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=False, validators=[validators.isInLimits(0, 5),]), Field(item="38", name='RELATIONSHIP_HOH', type='string', startIndex=51, endIndex=53, required=True, validators=[validators.isInStringRange(1, 10),]), - Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=False, validators=[validators.isInLimits(0, 3),]), - Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=False, validators=[validators.isInLimits(0, 9),]), - Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=False, validators=[validators.or_validators(validators.isInStringRange(0, 16), validators.isInStringRange(98, 99) )]), - Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, + Field(item="42", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=False, validators=[validators.oneOf([0, 1, 2, 9])]), - Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, - validators=[validators.oneOf([0, 1, 2, 9]),]), - Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + required=False, validators=[validators.oneOf([0, 1, 2, 9]),]), + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=False, validators=[validators.isInStringRange(0, 999),]), - Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='number', startIndex=64, endIndex=65, - required=True, validators=[validators.isInLimits(0, 9),]), - Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=True, + required=False, validators=[validators.isInLimits(0, 9),]), + Field(item="47", name='EMPLOYMENT_STATUS', type='number', startIndex=65, endIndex=66, required=False, validators=[validators.isInLimits(0, 3),]), Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[validators.or_validators(validators.isInStringRange(0, 9), validators.oneOf(('11', '12')) )]), - Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=False, validators=[validators.oneOf(['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99'])]), - Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[ + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=False, validators=[ validators.isInStringRange(0, 99), ]), - Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=False, validators=[validators.isInStringRange(0, 99),]), - Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, required=False, validators=[validators.isInStringRange(0, 99),]), @@ -227,16 +227,16 @@ Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, required=False, validators=[validators.isInStringRange(0, 99),]), Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, - required=False, validators=[validators.isInStringRange(0, 9999),]), + required=True, validators=[validators.isInStringRange(0, 9999),]), Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=False, validators=[validators.isInStringRange(0, 9999),]), Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, - required=False, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, + required=True, validators=[validators.isInStringRange(0, 9999),]), + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=True, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=True, validators=[validators.isInStringRange(0, 9999),]), - Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=True, validators=[validators.isInStringRange(0, 9999),]), ], )] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index d22d848df..66daa1e59 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -84,17 +84,17 @@ endIndex=37, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ validators.isInLimits(0, 9) @@ -108,11 +108,11 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=46, endIndex=48, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 10) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 2, 3]) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, @@ -123,7 +123,7 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 9]) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, @@ -213,17 +213,17 @@ endIndex=78, required=True, validators=[validators.validateSSN()], is_encrypted=False), Field(item="70A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="70F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, - required=True, validators=[validators.validateRace()]), + required=False, validators=[validators.validateRace()]), Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ validators.isInLimits(0, 9) @@ -237,11 +237,11 @@ validators.oneOf([1, 2]) ]), Field(item="73", name='RELATIONSHIP_HOH', type='string', startIndex=87, endIndex=89, - required=True, validators=[ + required=False, validators=[ validators.isInStringRange(0, 10) ]), Field(item="74", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 2, 3]) ]), Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, @@ -252,7 +252,7 @@ ) ]), Field(item="76", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, - required=True, validators=[ + required=False, validators=[ validators.oneOf([0, 1, 2, 9]) ]), Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt new file mode 100644 index 000000000..c9f313404 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section1_blanks.txt @@ -0,0 +1,5 @@ +HEADER20204A06 TAN1EN +T120201011111111112230 4033611102131 0000 00000000000008730010000000000000000 00002 222200000000222901 +T2202010111111111121219740114WTTTTTY@W 2221 1 01 01 0000 0000000000000291 +T320201011111111112120190127WTTTT90W0 222 98 00000000 +TRAILER0002643 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0a9b53ea4..abdde5bc6 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -154,12 +154,13 @@ def test_parse_big_file(test_big_file, dfs): parser_errors = ParserError.objects.filter(file=test_big_file) - error_message = 'MONTHS_FED_TIME_LIMIT is required but a value was not provided.' - row_18_error = parser_errors.get(row_number=18, error_message=error_message) - assert row_18_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_18_error.error_message == error_message - assert row_18_error.content_type.model == 'tanf_t2' - assert row_18_error.object_id is not None + error_message = "14 is not in ['01', '02', '05', '07', '09', '15', '16', '17', '18', '19', '99']. " + \ + "or 14 is not blank." + row_118_error = parser_errors.get(row_number=118, error_message=error_message) + assert row_118_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE + assert row_118_error.error_message == error_message + assert row_118_error.content_type.model == 'tanf_t2' + assert row_118_error.object_id is not None assert TANF_T1.objects.count() == expected_t1_record_count assert TANF_T2.objects.count() == expected_t2_record_count @@ -205,7 +206,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): dfs.save() assert dfs.get_status() == DataFileSummary.Status.REJECTED - parser_errors = ParserError.objects.filter(file=bad_file_missing_header) + parser_errors = ParserError.objects.filter(file=bad_file_missing_header).order_by('created_at') assert parser_errors.count() == 2 @@ -217,7 +218,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.content_type is None assert err.object_id is None assert errors == { - 'header': [parser_errors[1], parser_errors[0]] + 'header': list(parser_errors) } @@ -834,6 +835,32 @@ def test_parse_tanf_section3_file(tanf_section3_file): assert second.NUM_CLOSED_CASES == 3881 assert third.NUM_CLOSED_CASES == 5453 +@pytest.fixture +def tanf_section1_file_with_blanks(stt_user, stt): + """Fixture for ADS.E2J.FTP3.TS06.""" + return util.create_test_datafile('tanf_section1_blanks.txt', stt_user, stt) + +@pytest.mark.django_db() +def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks): + """Test section 1 fields that are allowed to have blanks.""" + parse.parse_datafile(tanf_section1_file_with_blanks) + + parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) + + assert parser_errors.count() == 23 + + # Should only be cat3 validator errors + for error in parser_errors: + assert error.error_type == ParserErrorCategoryChoices.VALUE_CONSISTENCY + + t1 = TANF_T1.objects.first() + t2 = TANF_T2.objects.first() + t3 = TANF_T3.objects.first() + + assert t1.FAMILY_SANC_ADULT is None + assert t2.MARITAL_STATUS is None + assert t3.CITIZENSHIP_STATUS is None + @pytest.fixture def tanf_section4_file(stt_user, stt): """Fixture for ADS.E2J.FTP4.TS06.""" diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index b2817174e..03997597a 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -267,7 +267,7 @@ def test_field_validators_blank_and_not_required_returns_valid(first): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=False, validators=[ passing_validator(), failing_validator() ]), diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d60731ab5..d40446e5b 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -2,12 +2,23 @@ from .models import ParserErrorCategoryChoices from datetime import date +import logging + +logger = logging.getLogger(__name__) # higher order validator func def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - return lambda value: (True, None) if value is not None and validator_func(value) else (False, error_func(value)) + def validator(value): + try: + if validator_func(value): + return (True, None) + return (False, error_func(value)) + except Exception as e: + logger.debug(f"Caught exception in validator. Exception: {e}") + return (False, error_func(value)) + return validator def or_validators(*args, **kwargs): @@ -58,7 +69,8 @@ def sumIsEqual(condition_field, sum_fields=[]): def sumIsEqualFunc(value): sum = 0 for field in sum_fields: - sum += value[field] if type(value) is dict else getattr(value, field) + val = value[field] if type(value) is dict else getattr(value, field) + sum += 0 if val is None else val condition_val = value[condition_field] if type(value) is dict else getattr(value, condition_field) return (True, None) if sum == condition_val else (False, @@ -71,7 +83,8 @@ def sumIsLarger(fields, val): def sumIsLargerFunc(value): sum = 0 for field in fields: - sum += value[field] if type(value) is dict else getattr(value, field) + temp_val = value[field] if type(value) is dict else getattr(value, field) + sum += 0 if temp_val is None else temp_val return (True, None) if sum > val else (False, f"The sum of {fields} is not larger than {val}.") @@ -315,7 +328,7 @@ def validate(instance): MONTHS_FED_TIME_LIMIT = instance['MONTHS_FED_TIME_LIMIT'] if type(instance) is dict else \ getattr(instance, 'MONTHS_FED_TIME_LIMIT') if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if int(MONTHS_FED_TIME_LIMIT) < 1: + if MONTHS_FED_TIME_LIMIT is None or int(MONTHS_FED_TIME_LIMIT) < 1: return (False, 'If FAMILY_AFFILIATION == 2 and MONTHS_FED_TIME_LIMIT== 1 or 2, then MONTHS_FED_TIME_LIMIT > 1.' ) diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 6f7c7342b..b7def9383 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -70,7 +70,11 @@ class CloudGov(Common): # env_based_db_name = f'tdp_db_{cloudgov_space_suffix}_{cloudgov_name}' - db_name = database_creds['db_name'] if (cloudgov_space_suffix in ["prod", "staging"]) else env_based_db_name + logger.debug("css: " + cloudgov_space_suffix) + if (cloudgov_space_suffix in ["prod", "staging"]): + db_name = database_creds['db_name'] + else: + db_name = env_based_db_name DATABASES = { 'default': { diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf index 1ab4677bb..4ed6804f9 100644 --- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf +++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf @@ -21,9 +21,9 @@ http { log_format compression '$remote_addr - $remote_user [$time_local] ' '"proxy_host and upstream_addr": $proxy_host $upstream_addr, ' ' "request": $request, ' - '"body_bytes_sent" : $body_bytes_sent, ' + '"body_bytes_sent" : $body_bytes_sent, ' '"request_body": $request_body, ' - '"http_x_forwarded_for": $http_x_forwarded_for, ' + '"http_x_forwarded_for": $http_x_forwarded_for, ' '"host": $host, ' ' "status": $status, ' '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, ' @@ -47,7 +47,7 @@ http { } client_max_body_size 100m; - + # Block all requests except ones listed in whitelist; disabled for local # First have to correct the source IP address using real_ip_header, otherwise # the IP address will be the internal IP address of the router @@ -63,7 +63,7 @@ http { set $CSP "default-src 'self';"; set $CSP "${CSP}script-src 'self';"; set $CSP "${CSP}script-src-elem 'self';"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}img-src 'self' data:;"; set $CSP "${CSP}font-src 'self';"; set $CSP "${CSP}connect-src 'self' ${CONNECT_SRC};"; diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index 2243c720b..c4d306340 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -82,7 +82,7 @@ http { set $CSP "${CSP}prefetch-src 'none';"; set $CSP "${CSP}form-action *;"; set $CSP "${CSP}script-src-elem 'self' http://localhost:* http://www.w3.org;"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}style-src-elem 'self' 'unsafe-inline';"; set $CSP "${CSP}style-src-attr 'self';"; set $CSP "${CSP}worker-src 'none';"; @@ -104,7 +104,7 @@ http { access_log /dev/stdout compression; #access_log stderr compression; - + # Content caching # saves cached fies in /tmp # cache zone name = tdp_cache @@ -126,7 +126,7 @@ http { set $CSP "default-src 'self';"; set $CSP "${CSP}script-src 'self';"; set $CSP "${CSP}script-src-elem 'self';"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}img-src 'self' data:;"; set $CSP "${CSP}font-src 'self';"; set $CSP "${CSP}manifest-src 'self';";