diff --git a/docs/Sprint-Review/sprint-81-summary.md b/docs/Sprint-Review/sprint-81-summary.md new file mode 100644 index 000000000..439a9f9c7 --- /dev/null +++ b/docs/Sprint-Review/sprint-81-summary.md @@ -0,0 +1,55 @@ + +# Sprint 81 Summary + +08/30/23 - 09/12/23 + +Velocity: Dev (13) + +## Sprint Goal +* Continue parsing engine development for TANF Sections (02 and 04) and close out subsmission history and metadata workflows (1613/12/10). +* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >90% of total users +* DevOps to investigate singular ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline. + + +## Tickets +### Completed/Merged +* [#2626 improve parsing logging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2626) +* [#1109 TANF (02) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1109) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress + +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) +* [#2695 space-filled values update (TANF (01))](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2411 As system admin, I awnt to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) + + +### Blocked +* N/A + + +### Raft Review +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) + + +### Demo +* Internal: + * 1109 - TANF Sec (02) + * 2626 - Parsing logging enhancements + + + diff --git a/docs/Technical-Documentation/CloudFoundry-DB-Connection.md b/docs/Technical-Documentation/CloudFoundry-DB-Connection.md index 3a72e28e7..c7ca0efe8 100644 --- a/docs/Technical-Documentation/CloudFoundry-DB-Connection.md +++ b/docs/Technical-Documentation/CloudFoundry-DB-Connection.md @@ -23,3 +23,36 @@ From [this github](https://github.com/cloud-gov/cf-service-connect) which has so > `cf connect-to-service tdp-backend- tdp-db-dev` + +# How to DROP existing DB and Recreate a fresh DB + +### Connecting to DB service +First step is to connect to the instance DB (see above). + +#### Optional: DB backup +Before deleting the DB and recreating a fresh DB, you might want to create a backup from the existing data in case you decide to revert the DB changes back. + +For creating a DB backup, please see: `/tdpservice/scheduling/BACKUP_README.md` + +#### Drop and Recreate + +e.g: +>`cf connect-to-service tdp-backend-qasp tdp-db-dev` + +After connection to the DB is made (the step above will make a psql connection), then the following Postgres commands have to run: + +1. List the DBs: `\l` +2. Potgres does not _DROP_ a database when you are connected to the same DB. As such, you will have to connect to a different DB using command: +>`\c {a_database}` + + A good candiadate is: +>`\c postgres` +3. find the associated DB name with instance. E.g: `tdp_db_dev_qasp` +4. use the following command to delete the DB: +>`DROP DATABASE {DB_NAME}` +5. use the following command to create the DB: +>`CREATE DATABASE {DB_NAME}` + +After the DB is created, since the database is cinoketely empty, we will need to redeploy the app again to create tables (or alternatively we can restore a good backup), and then we should run populate stt command to add STT data to the empty DB + +>`./manage.py populatestts` diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index e8e4a3121..409d239b8 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -251,6 +251,50 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False) ) ])] + +def get_schema_manager_options(program_type): + """Return the allowed schema options.""" + match program_type: + case 'TAN': + return { + 'A': { + 'T1': schema_defs.tanf.t1, + 'T2': schema_defs.tanf.t2, + 'T3': schema_defs.tanf.t3, + }, + 'C': { + 'T4': schema_defs.tanf.t4, + 'T5': schema_defs.tanf.t5, + }, + 'G': { + 'T6': schema_defs.tanf.t6, + }, + 'S': { + 'T7': schema_defs.tanf.t7, + }, + } + case 'SSP': + return { + 'A': { + 'M1': schema_defs.ssp.m1, + 'M2': schema_defs.ssp.m2, + 'M3': schema_defs.ssp.m3, + }, + 'C': { + # 'M4': schema_options.m4, + # 'M5': schema_options.m5, + }, + 'G': { + # 'M6': schema_options.m6, + }, + 'S': { + # 'M7': schema_options.m7, + }, + } + # case tribal? + return None + + def get_schema_manager(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py index e82db47b6..7c1997236 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py @@ -4,6 +4,7 @@ from .t4 import t4 from .t5 import t5 from .t6 import t6 +from .t7 import t7 t1 = t1 t2 = t2 @@ -11,3 +12,4 @@ t4 = t4 t5 = t5 t6 = t6 +t7 = t7 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py new file mode 100644 index 000000000..2fcb4e0fd --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -0,0 +1,65 @@ +"""Schema for TANF T7 Row.""" + +from ...util import SchemaManager +from ...fields import Field, TransformField +from ...row_schema import RowSchema +from ...transforms import calendar_quarter_to_rpt_month_year +from ... import validators +from tdpservice.search_indexes.models.tanf import TANF_T7 + +schemas = [] + +validator_index = 7 +section_ind_index = 7 +stratum_index = 8 +families_index = 10 +for i in range(1, 31): + month_index = (i - 1) % 3 + sub_item_labels = ['A', 'B', 'C'] + families_value_item_number = f"6{sub_item_labels[month_index]}" + + schemas.append( + RowSchema( + model=TANF_T7, + quiet_preparser_errors=i > 1, + preparsing_validators=[ + validators.notEmpty(0, 7), + validators.notEmpty(validator_index, validator_index + 24), + ], + postparsing_validators=[], + fields=[ + Field(item="0", name="RecordType", type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.quarterIsValid()]), + TransformField( + transform_func=calendar_quarter_to_rpt_month_year(month_index), + item="3A", + name='RPT_MONTH_YEAR', + type='number', + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid() + ] + ), + Field(item="4", name='TDRS_SECTION_IND', type='string', startIndex=section_ind_index, + endIndex=section_ind_index + 1, required=True, validators=[validators.oneOf(['1', '2'])]), + Field(item="5", name='STRATUM', type='string', startIndex=stratum_index, + endIndex=stratum_index + 2, required=True, validators=[validators.isInStringRange(1, 99)]), + Field(item=families_value_item_number, name='FAMILIES_MONTH', type='number', startIndex=families_index, + endIndex=families_index + 7, required=True, validators=[validators.isInLimits(0, 9999999)]), + ] + ) + ) + + index_offset = 0 if i % 3 != 0 else 24 + validator_index += index_offset + section_ind_index += index_offset + stratum_index += index_offset + families_index += 7 if i % 3 != 0 else 10 + +t7 = SchemaManager(schemas=schemas) diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 new file mode 100644 index 000000000..5c344cf42 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 @@ -0,0 +1,3 @@ +HEADER20204S06 TAN1 N +T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 8eb309b60..b12d3c5ad 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -335,3 +335,17 @@ class Meta: NUM_BIRTHS = 1 NUM_OUTWEDLOCK_BIRTHS = 1 NUM_CLOSED_CASES = 1 + +class TanfT7Factory(factory.django.DjangoModelFactory): + """Generate TANF T7 record for testing.""" + + class Meta: + """Hardcoded meta data for TANF_T7.""" + + model = "search_indexes.TANF_T7" + + CALENDAR_QUARTER = 20204 + RPT_MONTH_YEAR = 202011 + TDRS_SECTION_IND = '1' + STRATUM = '01' + FAMILIES_MONTH = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index fd794280b..9c785f79f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -4,7 +4,7 @@ import pytest from .. import parse from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary -from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6 +from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory from tdpservice.data_files.models import DataFile @@ -833,3 +833,32 @@ def test_parse_tanf_section3_file(tanf_section3_file): assert first.NUM_CLOSED_CASES == 3884 assert second.NUM_CLOSED_CASES == 3881 assert third.NUM_CLOSED_CASES == 5453 + +@pytest.fixture +def tanf_section4_file(stt_user, stt): + """Fixture for ADS.E2J.FTP4.TS06.""" + return util.create_test_datafile('ADS.E2J.FTP4.TS06', stt_user, stt, "Stratum Data") + +@pytest.mark.django_db() +def test_parse_tanf_section4_file(tanf_section4_file): + """Test parsing TANF Section 4 submission.""" + parse.parse_datafile(tanf_section4_file) + + assert TANF_T7.objects.all().count() == 18 + + parser_errors = ParserError.objects.filter(file=tanf_section4_file) + assert parser_errors.count() == 0 + + t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH') + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202011 + assert sixth.RPT_MONTH_YEAR == 202012 + + assert first.TDRS_SECTION_IND == '2' + assert sixth.TDRS_SECTION_IND == '2' + + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index accc36269..073b7b8d8 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -129,7 +129,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'S': { 'section': DataFile.Section.STRATUM_DATA, 'models': { - # 'T7': schema_defs.tanf.t7, + 'T7': schema_defs.tanf.t7, } } }, diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index b1e5f8d5c..f9fab7f6f 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -23,6 +23,10 @@ def parse(data_file_id): dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) errors = parse_datafile(data_file) dfs.status = dfs.get_status() - dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) + + if "Case Data" in data_file.section: + dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) + dfs.save() + logger.info(f"Parsing finished for file -> {repr(data_file)} with status {dfs.status} and {len(errors)} errors.") diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 88bce804f..af3429695 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -109,12 +109,17 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ - 'record', - 'rpt_month_year', + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'TDRS_SECTION_IND', + 'STRATUM', + 'FAMILIES_MONTH', 'datafile', ] list_filter = [ + 'CALENDAR_QUARTER', CreationDateFilter, - 'rpt_month_year', + 'RPT_MONTH_YEAR', ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index aba080ff1..c613c50a2 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -343,11 +343,10 @@ class Django: model = TANF_T7 fields = [ - 'record', - 'rpt_month_year', - 'fips_code', - 'calendar_quarter', - 'tdrs_section_ind', - 'stratum', - 'families', + "RecordType", + "CALENDAR_QUARTER", + "RPT_MONTH_YEAR", + "TDRS_SECTION_IND", + "STRATUM", + "FAMILIES_MONTH", ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py b/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py new file mode 100644 index 000000000..46390d523 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.15 on 2023-09-20 18:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0017_auto_20230914_1720'), + ] + + operations = [ + migrations.RemoveField( + model_name='tanf_t7', + name='calendar_quarter', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='families', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='fips_code', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='record', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='rpt_month_year', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='stratum', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='tdrs_section_ind', + ), + migrations.AddField( + model_name='tanf_t7', + name='CALENDAR_QUARTER', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='FAMILIES_MONTH', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='RPT_MONTH_YEAR', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='RecordType', + field=models.CharField(max_length=156, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='STRATUM', + field=models.CharField(max_length=2, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='TDRS_SECTION_IND', + field=models.CharField(max_length=1, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index dd61c84c9..9e31fffe2 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -342,15 +342,13 @@ class TANF_T7(models.Model): related_name='t7_parent' ) - record = models.CharField(max_length=156, null=False, blank=False) - rpt_month_year = models.IntegerField(null=False, blank=False) - fips_code = models.CharField(max_length=100, null=False, blank=False) - - calendar_quarter = models.IntegerField(null=False, blank=False) - tdrs_section_ind = models.CharField( + RecordType = models.CharField(max_length=156, null=True, blank=False) + CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + TDRS_SECTION_IND = models.CharField( max_length=1, - null=False, + null=True, blank=False ) - stratum = models.CharField(max_length=2, null=False, blank=False) - families = models.IntegerField(null=False, blank=False) + STRATUM = models.CharField(max_length=2, null=True, blank=False) + FAMILIES_MONTH = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index cd762eb56..e8e9eb81f 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -2,7 +2,6 @@ import pytest from faker import Faker -from django.db.utils import IntegrityError from tdpservice.search_indexes import models from tdpservice.search_indexes import documents from tdpservice.parsers.util import create_test_datafile @@ -346,26 +345,27 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile): submission = models.tanf.TANF_T7() submission.datafile = test_datafile - submission.record = record_num - submission.rpt_month_year = 1 - submission.fips_code = '2' - submission.calendar_quarter = 1 - submission.tdrs_section_ind = '1' - submission.stratum = '1' - submission.families = 1 + submission.RecordType = record_num + submission.CALENDAR_YEAR = 2020 + submission.CALENDAR_QUARTER = 1 + submission.TDRS_SECTION_IND = '1' + submission.STRATUM = '01' + submission.FAMILIES_MONTH_1 = 47655 + submission.FAMILIES_MONTH_2 = 81982 + submission.FAMILIES_MONTH_3 = 9999999 submission.save() # No checks her because t7 records can't be parsed currently. - # assert submission.id is not None + assert submission.id is not None - # search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( - # 'match', - # record=record_num - # ) - # response = search.execute() + search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( + 'match', + RecordType=record_num + ) + response = search.execute() - # assert response.hits.total.value == 1 + assert response.hits.total.value == 1 @pytest.mark.django_db @@ -373,17 +373,9 @@ def test_does_not_create_index_if_model_creation_fails(): """Index creation shouldn't happen if saving a model errors.""" record_num = fake.uuid4() - with pytest.raises(IntegrityError): - submission = models.tanf.TANF_T7.objects.create( - record=record_num - # leave out a bunch of required fields - ) - - assert submission.id is None - search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( 'match', - record=record_num + RecordType=record_num ) response = search.execute()