From c94b8f7302733767e6cd3b69adaf92e0aa4dede7 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 28 Sep 2023 10:57:06 -0700 Subject: [PATCH 1/3] Create sprint-81-summary.md (#2715) --- docs/Sprint-Review/sprint-81-summary.md | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/Sprint-Review/sprint-81-summary.md diff --git a/docs/Sprint-Review/sprint-81-summary.md b/docs/Sprint-Review/sprint-81-summary.md new file mode 100644 index 000000000..439a9f9c7 --- /dev/null +++ b/docs/Sprint-Review/sprint-81-summary.md @@ -0,0 +1,55 @@ + +# Sprint 81 Summary + +08/30/23 - 09/12/23 + +Velocity: Dev (13) + +## Sprint Goal +* Continue parsing engine development for TANF Sections (02 and 04) and close out subsmission history and metadata workflows (1613/12/10). +* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >90% of total users +* DevOps to investigate singular ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline. + + +## Tickets +### Completed/Merged +* [#2626 improve parsing logging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2626) +* [#1109 TANF (02) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1109) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress + +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) +* [#2695 space-filled values update (TANF (01))](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2411 As system admin, I awnt to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) + + +### Blocked +* N/A + + +### Raft Review +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) + + +### Demo +* Internal: + * 1109 - TANF Sec (02) + * 2626 - Parsing logging enhancements + + + From 8289c015dc9aa9992cfc23cb731645fbbaf8fc69 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:46:59 -0400 Subject: [PATCH 2/3] DB-drop-and-reset-write-down (#2710) * added md file for dropping DB * added section for db_backup & merged into README * Update CloudFoundry-DB-Connection.md --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../CloudFoundry-DB-Connection.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/Technical-Documentation/CloudFoundry-DB-Connection.md b/docs/Technical-Documentation/CloudFoundry-DB-Connection.md index 3a72e28e7..c7ca0efe8 100644 --- a/docs/Technical-Documentation/CloudFoundry-DB-Connection.md +++ b/docs/Technical-Documentation/CloudFoundry-DB-Connection.md @@ -23,3 +23,36 @@ From [this github](https://github.com/cloud-gov/cf-service-connect) which has so > `cf connect-to-service tdp-backend- tdp-db-dev` + +# How to DROP existing DB and Recreate a fresh DB + +### Connecting to DB service +First step is to connect to the instance DB (see above). + +#### Optional: DB backup +Before deleting the DB and recreating a fresh DB, you might want to create a backup from the existing data in case you decide to revert the DB changes back. + +For creating a DB backup, please see: `/tdpservice/scheduling/BACKUP_README.md` + +#### Drop and Recreate + +e.g: +>`cf connect-to-service tdp-backend-qasp tdp-db-dev` + +After connection to the DB is made (the step above will make a psql connection), then the following Postgres commands have to run: + +1. List the DBs: `\l` +2. Potgres does not _DROP_ a database when you are connected to the same DB. As such, you will have to connect to a different DB using command: +>`\c {a_database}` + + A good candiadate is: +>`\c postgres` +3. find the associated DB name with instance. E.g: `tdp_db_dev_qasp` +4. use the following command to delete the DB: +>`DROP DATABASE {DB_NAME}` +5. use the following command to create the DB: +>`CREATE DATABASE {DB_NAME}` + +After the DB is created, since the database is cinoketely empty, we will need to redeploy the app again to create tables (or alternatively we can restore a good backup), and then we should run populate stt command to add STT data to the empty DB + +>`./manage.py populatestts` From fb6cc5f847ad4c2a9e670e3a2ad361af1c9f3ef8 Mon Sep 17 00:00:00 2001 From: jtimpe <111305129+jtimpe@users.noreply.github.com> Date: Fri, 29 Sep 2023 11:56:31 -0400 Subject: [PATCH 3/3] Feature/1111 tanf section 4 (#2657) * - ADding t2 schema * - small fixes to naming - added t2.py - Updating var names to match field names * - Added new doc for T2 - Updated model fo rT2 - Added active parse function * - Added T3 schema defs * - Fixing most lint errors * - Updated T3 to multirow record * - Added unit test and data files - updated field names * - Updating var names to conform to DB max var name length * - Updating based on unit test failure * - adding datafile * - Adding unit tests for t2 and t3 * - Fixed field types - Updated test file * - Removing many migrations to consolodate into one migration * Revert "- Removing many migrations to consolodate into one migration" This reverts commit 1455ae4d334f72e250405803d61d26c0a894e886. * - Resolved test issues after merge * - Resolve lint errors * - Merged 1113.2 * - Removed unnecessary file * - Rename model fields - Combined migrations * - fixed spelling in test * - Fixed lint errors * rm commented validators * - Updated schema newlines to be consistent * - Updated field names - Updated tests - Updated migration * - consolodating migrations * - Updated readme and backup script * - Updated parse logic to batch create records * - Fixed lint errors * - Batching record serialization * - Updated parse logic * - Adding big s1 datafile * - fixing lint errors * - Removing test to see if its causing weird failure * - Updating based on comments - Removing big file since circi cant handle it * - fixing error in update method * - fixing error in update * - Removed extraneous seek * - updating ignore to ignore patch files * - Made evaluate_trailer pure/immutable * Revert "- Removing test to see if its causing weird failure" This reverts commit 64b78b737c97fb9bfa70217ff70ccffea8d85429. * - Use custom transactions while parsing - Handle transaction rollback on parse failure/error - Update tests to use transaction logic - Created new decorator to rollback database when assertion error is hit - Making elastic search log level configurable - Added test for inter parse rollback * - updated store logic to one liner - updated decorator to catch all exceptions - updated tests * - removed function - renamed test * - refactored multirecord schema to be a schema manager instead - updated parse functions to use the new layout - updated bulk create to manage batch size * - Name update for clarity * - Fix lint errors * - Changing batch size to avoid crash in circi * - Updated based on review - Updated tests to reflect line number * - Updated based on review/OH - Added extra transaction safety * - Updated view to show latest datafiles - Added admin filter to show newest or all datafile records - Updated indices to allow easier elastic queries * - Updated based on review comments * - Updated based on feedback for preparse error handling - updated tests * - Updated search indices to have parent FK * - Fix lint errors * - Updated submission tests - Moved create_datafile to util * - fix lint errors * - removing frontend filtering * - addding datafile to admin model * Revert "- addding datafile to admin model" This reverts commit 35a6f24c36c3a4c00ddcfc40f20833530b0199f4. * - Fixed issue where datafile FK wasnt populating - Regenerated migration * - Readding datafile back to admin view now that the error is resolved * - adding datafile back * Revert "- Readding datafile back to admin view now that the error is resolved" This reverts commit 2807425059fd1b5b355edfb16d30d170cf869d7b. * - Removed unnecessary fields - Updated dependencies - Updated filter * - Updated document to include required fields * - Moved datafile reference to avoid confusion * - Updating based on comments. - Added base class to keep things a little more DRY * - Refactored parsing to avoid custom transactions by leveraging the FK on the records. Rollbacks are a lot slower now, but hopefully that will happen much less versus successful parses. * - small optimization for the rollback * - Fix lint errors * - Removing commented code * - Installing build essentials * Revert "- Installing build essentials" This reverts commit 782a5f7d687e60965a9100520f027b9678dbac35. * - adding setup tools and wheel * Revert "- adding setup tools and wheel" This reverts commit f529728811fba242132b7c42f9e9e09d6037fa70. * - Updating dependencies to get around pep issue * - Pin factoryboy - fix lint error * - Updating manifest * - Added EncryptedField class - Updated schema's - Updated datafiles since all are encrypted * - Fix lint errors * - Added decryption for ssp * - Making the encrypted check stronger * - Added section two models, schemas, and an initial test * - Allowing fields to be nullable * - add missing field * - Consolodating migrations * - Consolodate migration - Add tests * - Added encrypted field * - Refactored fields and rowschema into seperate files - Updated encrypted field to take decryption function * - fix lint errors * - Fixed file spacing * - fix imports - fix lint errors * - Fix import error * - Fix failing test * - Revert erroneous change * - Updating item numbers * - Fixed item numbers * - Updating to accomodate item number as string * - Removed erroneous update that was causing error - Fixed whitespace in datafiles * - White space error * - Resolved error * - Fixing test * - fix lint errors * - Added field validators for T4/T5 * - Added cat3 validators * - Resolve lint and unit test errors * - small optimization to validator * - Added tests for cat3 validators for T5 record * - Fix lint errors * - updated fields to correct types * - update race/ethnicity * - updated tests post merge - added check in make_validator to just return false if the value is None * - Updated based on review feedback * - Fix lint error * - Updating to the correct types * - Resolve conflicts * - Moving field updates to seperate migration to see if they take place * Revert "- Moving field updates to seperate migration to see if they take place" This reverts commit a3214f23e0bd9cc4cc37ad40e63ae0c98b4ea64b. * - Revert migration 15 to original state - Generate new migration for field alterations * - Merged base branch and updated tests/factories * - Fixed test * - Fixed field names to correspond to model * - Remove duplicate function - Update ssn validator - Removed asserts with large number of parse errors * t7 model, index changes * add t7 to parser * update model mapping test * add s3-s10 * - Remove print statemtnt * - Resolved filter issue that made it seem like records werent being created - Added filter to remaining records * - fix lint errors * - correct form merge conflict * - Added specific validators to avoid duplication - Updated naming of validators * TANF Section 3 Parsing/Validation (#2649) * - Updated to support parsing section 3 data * - Added validators and tests for T6 record * - resolve lint errors * - Quick rename * - Update fields in test * - Fixed conflict * - Fix error from merge conflict * - Updated to create RPT_MONTH_YEAR from CALENDAR_QUARTER - Updated tests * - Added new validators for field change * - Remove debug code * - Genericize TransformField - Update all schemas to use TransformField - Move transforms to seperate file - Fix lint errors * - put kwargs in correct spot * - Very minor change to avoid setting field unnecessarily on all TransformFields * - Updated validator to call out name of field versus the value - Updated tests * - Fix lint * - Calling super to avoid duplicate code * - Added validators for transformed field * - Updating based on merge * - Fixed remaining merge conflicts * - Fix lint errors * - update error messages based on validator updates * - Corrected validator * - Resolved bug causing file to think it was encrytped after multiple submissions and changing the encryption header in between submissions * - UPdated migration - Updated test * - Added transform fields - UPdated migrations * - Stratified T7 to one record per month per section indicator and stratum. * - Updated tests * - Fix test * - Add T7 factory * - Fix lint error * - Remove test class until cat three validators exist * - fix lint errors * - Fixing colliding migration * - Generating schemas instead of defining * - Fix lint errors * - Updated based on review feedback * rm 17/18 * remake migration 18 * fix merge error * fix merge error * fix merge error * labels are stuck * rm * dont compute aggregates for section 3/4 * move save * Update tdrs-backend/tdpservice/parsers/test/factories.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> * enhance t7 modeladmin * fix rpt_month_year off by 1 * lint * differentiate item 6a/6b/67 in t7 records --------- Co-authored-by: elipe17 Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Eric Lipe <125676261+elipe17@users.noreply.github.com> --- tdrs-backend/tdpservice/parsers/parse.py | 44 ++++++++++++ .../parsers/schema_defs/tanf/__init__.py | 2 + .../tdpservice/parsers/schema_defs/tanf/t7.py | 65 +++++++++++++++++ .../parsers/test/data/ADS.E2J.FTP4.TS06 | 3 + .../tdpservice/parsers/test/factories.py | 14 ++++ .../tdpservice/parsers/test/test_parse.py | 31 +++++++- tdrs-backend/tdpservice/parsers/util.py | 2 +- .../tdpservice/scheduling/parser_task.py | 6 +- .../tdpservice/search_indexes/admin/tanf.py | 11 ++- .../search_indexes/documents/tanf.py | 13 ++-- .../migrations/0018_auto_20230920_1846.py | 71 +++++++++++++++++++ .../tdpservice/search_indexes/models/tanf.py | 16 ++--- .../search_indexes/test/test_model_mapping.py | 40 +++++------ 13 files changed, 272 insertions(+), 46 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py create mode 100644 tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index e8e4a3121..409d239b8 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -251,6 +251,50 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False) ) ])] + +def get_schema_manager_options(program_type): + """Return the allowed schema options.""" + match program_type: + case 'TAN': + return { + 'A': { + 'T1': schema_defs.tanf.t1, + 'T2': schema_defs.tanf.t2, + 'T3': schema_defs.tanf.t3, + }, + 'C': { + 'T4': schema_defs.tanf.t4, + 'T5': schema_defs.tanf.t5, + }, + 'G': { + 'T6': schema_defs.tanf.t6, + }, + 'S': { + 'T7': schema_defs.tanf.t7, + }, + } + case 'SSP': + return { + 'A': { + 'M1': schema_defs.ssp.m1, + 'M2': schema_defs.ssp.m2, + 'M3': schema_defs.ssp.m3, + }, + 'C': { + # 'M4': schema_options.m4, + # 'M5': schema_options.m5, + }, + 'G': { + # 'M6': schema_options.m6, + }, + 'S': { + # 'M7': schema_options.m7, + }, + } + # case tribal? + return None + + def get_schema_manager(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py index e82db47b6..7c1997236 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py @@ -4,6 +4,7 @@ from .t4 import t4 from .t5 import t5 from .t6 import t6 +from .t7 import t7 t1 = t1 t2 = t2 @@ -11,3 +12,4 @@ t4 = t4 t5 = t5 t6 = t6 +t7 = t7 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py new file mode 100644 index 000000000..2fcb4e0fd --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -0,0 +1,65 @@ +"""Schema for TANF T7 Row.""" + +from ...util import SchemaManager +from ...fields import Field, TransformField +from ...row_schema import RowSchema +from ...transforms import calendar_quarter_to_rpt_month_year +from ... import validators +from tdpservice.search_indexes.models.tanf import TANF_T7 + +schemas = [] + +validator_index = 7 +section_ind_index = 7 +stratum_index = 8 +families_index = 10 +for i in range(1, 31): + month_index = (i - 1) % 3 + sub_item_labels = ['A', 'B', 'C'] + families_value_item_number = f"6{sub_item_labels[month_index]}" + + schemas.append( + RowSchema( + model=TANF_T7, + quiet_preparser_errors=i > 1, + preparsing_validators=[ + validators.notEmpty(0, 7), + validators.notEmpty(validator_index, validator_index + 24), + ], + postparsing_validators=[], + fields=[ + Field(item="0", name="RecordType", type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.quarterIsValid()]), + TransformField( + transform_func=calendar_quarter_to_rpt_month_year(month_index), + item="3A", + name='RPT_MONTH_YEAR', + type='number', + startIndex=2, + endIndex=7, + required=True, + validators=[ + validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid() + ] + ), + Field(item="4", name='TDRS_SECTION_IND', type='string', startIndex=section_ind_index, + endIndex=section_ind_index + 1, required=True, validators=[validators.oneOf(['1', '2'])]), + Field(item="5", name='STRATUM', type='string', startIndex=stratum_index, + endIndex=stratum_index + 2, required=True, validators=[validators.isInStringRange(1, 99)]), + Field(item=families_value_item_number, name='FAMILIES_MONTH', type='number', startIndex=families_index, + endIndex=families_index + 7, required=True, validators=[validators.isInLimits(0, 9999999)]), + ] + ) + ) + + index_offset = 0 if i % 3 != 0 else 24 + validator_index += index_offset + section_ind_index += index_offset + stratum_index += index_offset + families_index += 7 if i % 3 != 0 else 10 + +t7 = SchemaManager(schemas=schemas) diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 new file mode 100644 index 000000000..5c344cf42 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 @@ -0,0 +1,3 @@ +HEADER20204S06 TAN1 N +T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 8eb309b60..b12d3c5ad 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -335,3 +335,17 @@ class Meta: NUM_BIRTHS = 1 NUM_OUTWEDLOCK_BIRTHS = 1 NUM_CLOSED_CASES = 1 + +class TanfT7Factory(factory.django.DjangoModelFactory): + """Generate TANF T7 record for testing.""" + + class Meta: + """Hardcoded meta data for TANF_T7.""" + + model = "search_indexes.TANF_T7" + + CALENDAR_QUARTER = 20204 + RPT_MONTH_YEAR = 202011 + TDRS_SECTION_IND = '1' + STRATUM = '01' + FAMILIES_MONTH = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index fd794280b..9c785f79f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -4,7 +4,7 @@ import pytest from .. import parse from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary -from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6 +from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory from tdpservice.data_files.models import DataFile @@ -833,3 +833,32 @@ def test_parse_tanf_section3_file(tanf_section3_file): assert first.NUM_CLOSED_CASES == 3884 assert second.NUM_CLOSED_CASES == 3881 assert third.NUM_CLOSED_CASES == 5453 + +@pytest.fixture +def tanf_section4_file(stt_user, stt): + """Fixture for ADS.E2J.FTP4.TS06.""" + return util.create_test_datafile('ADS.E2J.FTP4.TS06', stt_user, stt, "Stratum Data") + +@pytest.mark.django_db() +def test_parse_tanf_section4_file(tanf_section4_file): + """Test parsing TANF Section 4 submission.""" + parse.parse_datafile(tanf_section4_file) + + assert TANF_T7.objects.all().count() == 18 + + parser_errors = ParserError.objects.filter(file=tanf_section4_file) + assert parser_errors.count() == 0 + + t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH') + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202011 + assert sixth.RPT_MONTH_YEAR == 202012 + + assert first.TDRS_SECTION_IND == '2' + assert sixth.TDRS_SECTION_IND == '2' + + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index accc36269..073b7b8d8 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -129,7 +129,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'S': { 'section': DataFile.Section.STRATUM_DATA, 'models': { - # 'T7': schema_defs.tanf.t7, + 'T7': schema_defs.tanf.t7, } } }, diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index b1e5f8d5c..f9fab7f6f 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -23,6 +23,10 @@ def parse(data_file_id): dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) errors = parse_datafile(data_file) dfs.status = dfs.get_status() - dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) + + if "Case Data" in data_file.section: + dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) + dfs.save() + logger.info(f"Parsing finished for file -> {repr(data_file)} with status {dfs.status} and {len(errors)} errors.") diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 88bce804f..af3429695 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -109,12 +109,17 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ - 'record', - 'rpt_month_year', + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'TDRS_SECTION_IND', + 'STRATUM', + 'FAMILIES_MONTH', 'datafile', ] list_filter = [ + 'CALENDAR_QUARTER', CreationDateFilter, - 'rpt_month_year', + 'RPT_MONTH_YEAR', ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index aba080ff1..c613c50a2 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -343,11 +343,10 @@ class Django: model = TANF_T7 fields = [ - 'record', - 'rpt_month_year', - 'fips_code', - 'calendar_quarter', - 'tdrs_section_ind', - 'stratum', - 'families', + "RecordType", + "CALENDAR_QUARTER", + "RPT_MONTH_YEAR", + "TDRS_SECTION_IND", + "STRATUM", + "FAMILIES_MONTH", ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py b/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py new file mode 100644 index 000000000..46390d523 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.15 on 2023-09-20 18:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0017_auto_20230914_1720'), + ] + + operations = [ + migrations.RemoveField( + model_name='tanf_t7', + name='calendar_quarter', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='families', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='fips_code', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='record', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='rpt_month_year', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='stratum', + ), + migrations.RemoveField( + model_name='tanf_t7', + name='tdrs_section_ind', + ), + migrations.AddField( + model_name='tanf_t7', + name='CALENDAR_QUARTER', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='FAMILIES_MONTH', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='RPT_MONTH_YEAR', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='RecordType', + field=models.CharField(max_length=156, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='STRATUM', + field=models.CharField(max_length=2, null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='TDRS_SECTION_IND', + field=models.CharField(max_length=1, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index dd61c84c9..9e31fffe2 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -342,15 +342,13 @@ class TANF_T7(models.Model): related_name='t7_parent' ) - record = models.CharField(max_length=156, null=False, blank=False) - rpt_month_year = models.IntegerField(null=False, blank=False) - fips_code = models.CharField(max_length=100, null=False, blank=False) - - calendar_quarter = models.IntegerField(null=False, blank=False) - tdrs_section_ind = models.CharField( + RecordType = models.CharField(max_length=156, null=True, blank=False) + CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + TDRS_SECTION_IND = models.CharField( max_length=1, - null=False, + null=True, blank=False ) - stratum = models.CharField(max_length=2, null=False, blank=False) - families = models.IntegerField(null=False, blank=False) + STRATUM = models.CharField(max_length=2, null=True, blank=False) + FAMILIES_MONTH = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index cd762eb56..e8e9eb81f 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -2,7 +2,6 @@ import pytest from faker import Faker -from django.db.utils import IntegrityError from tdpservice.search_indexes import models from tdpservice.search_indexes import documents from tdpservice.parsers.util import create_test_datafile @@ -346,26 +345,27 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile): submission = models.tanf.TANF_T7() submission.datafile = test_datafile - submission.record = record_num - submission.rpt_month_year = 1 - submission.fips_code = '2' - submission.calendar_quarter = 1 - submission.tdrs_section_ind = '1' - submission.stratum = '1' - submission.families = 1 + submission.RecordType = record_num + submission.CALENDAR_YEAR = 2020 + submission.CALENDAR_QUARTER = 1 + submission.TDRS_SECTION_IND = '1' + submission.STRATUM = '01' + submission.FAMILIES_MONTH_1 = 47655 + submission.FAMILIES_MONTH_2 = 81982 + submission.FAMILIES_MONTH_3 = 9999999 submission.save() # No checks her because t7 records can't be parsed currently. - # assert submission.id is not None + assert submission.id is not None - # search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( - # 'match', - # record=record_num - # ) - # response = search.execute() + search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( + 'match', + RecordType=record_num + ) + response = search.execute() - # assert response.hits.total.value == 1 + assert response.hits.total.value == 1 @pytest.mark.django_db @@ -373,17 +373,9 @@ def test_does_not_create_index_if_model_creation_fails(): """Index creation shouldn't happen if saving a model errors.""" record_num = fake.uuid4() - with pytest.raises(IntegrityError): - submission = models.tanf.TANF_T7.objects.create( - record=record_num - # leave out a bunch of required fields - ) - - assert submission.id is None - search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( 'match', - record=record_num + RecordType=record_num ) response = search.execute()