From ae7c3f86449f42961df717055566cab7b434e60c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 16 Oct 2023 10:27:36 -0400 Subject: [PATCH 1/6] ssp m6 boilerplate --- .../parsers/schema_defs/ssp/__init__.py | 2 + .../tdpservice/parsers/schema_defs/ssp/m6.py | 174 ++++++++++++++++++ .../parsers/test/data/ADS.E2J.NDM3.MS24 | 3 + .../tdpservice/parsers/test/test_parse.py | 37 +++- tdrs-backend/tdpservice/parsers/util.py | 9 +- tdrs-backend/tdpservice/parsers/validators.py | 4 + .../search_indexes/documents/ssp.py | 41 ++++- .../tdpservice/search_indexes/models/ssp.py | 37 ++++ .../search_indexes/test/test_model_mapping.py | 39 ++++ 9 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py create mode 100644 tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py index f4d66c258..b3542bf07 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py @@ -1,7 +1,9 @@ from .m1 import m1 from .m2 import m2 from .m3 import m3 +from .m6 import m6 m1 = m1 m2 = m2 m3 = m3 +m6 = m6 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py new file mode 100644 index 000000000..36d477ef0 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -0,0 +1,174 @@ +"""Schema for HEADER row of all submission types.""" + + +from ...util import SchemaManager +from ...transforms import calendar_quarter_to_rpt_month_year +from ...fields import Field, TransformField +from ...row_schema import RowSchema +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M6 + + + +## item numbering different between ssp and tanf? + +s1 = RowSchema( + model=SSP_M6, + preparsing_validators=[ + validators.hasLength(379), + ], + postparsing_validators=[ + validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), + validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + ], + fields=[ + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.quarterIsValid()]), + TransformField(calendar_quarter_to_rpt_month_year(0), item="4", name='RPT_MONTH_YEAR', type='number', + startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid()]), + Field(item="4A", name='NUM_APPLICATIONS', type='number', startIndex=7, endIndex=15, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="5A", name='NUM_APPROVED', type='number', startIndex=31, endIndex=39, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="6A", name='NUM_DENIED', type='number', startIndex=55, endIndex=63, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="7A", name='ASSISTANCE', type='number', startIndex=79, endIndex=91, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="8A", name='NUM_FAMILIES', type='number', startIndex=115, endIndex=123, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="9A", name='NUM_2_PARENTS', type='number', startIndex=139, endIndex=147, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="10A", name='NUM_1_PARENTS', type='number', startIndex=163, endIndex=171, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="11A", name='NUM_NO_PARENTS', type='number', startIndex=187, endIndex=195, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="12A", name='NUM_RECIPIENTS', type='number', startIndex=211, endIndex=219, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="13A", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=235, endIndex=243, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="14A", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=259, endIndex=267, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="15A", name='NUM_NONCUSTODIALS', type='number', startIndex=283, endIndex=291, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="16A", name='NUM_BIRTHS', type='number', startIndex=307, endIndex=315, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="17A", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=331, endIndex=339, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="18A", name='NUM_CLOSED_CASES', type='number', startIndex=355, endIndex=363, + required=True, validators=[validators.isInLimits(0, 99999999)]), + ], +) + +s2 = RowSchema( + model=SSP_M6, + preparsing_validators=[ + validators.hasLength(379), + ], + postparsing_validators=[ + validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), + validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + ], + fields=[ + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[]), + TransformField(calendar_quarter_to_rpt_month_year(1), item="4", name='RPT_MONTH_YEAR', type='number', + startIndex=2, endIndex=7, required=True, validators=[]), + Field(item="4B", name='NUM_APPLICATIONS', type='number', startIndex=15, endIndex=23, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="5B", name='NUM_APPROVED', type='number', startIndex=39, endIndex=47, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="6B", name='NUM_DENIED', type='number', startIndex=63, endIndex=71, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="7B", name='ASSISTANCE', type='number', startIndex=91, endIndex=103, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="8B", name='NUM_FAMILIES', type='number', startIndex=123, endIndex=131, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="9B", name='NUM_2_PARENTS', type='number', startIndex=147, endIndex=155, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="10B", name='NUM_1_PARENTS', type='number', startIndex=171, endIndex=179, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="11B", name='NUM_NO_PARENTS', type='number', startIndex=195, endIndex=203, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="12B", name='NUM_RECIPIENTS', type='number', startIndex=219, endIndex=227, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="13B", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=243, endIndex=251, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="14B", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=267, endIndex=275, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="15B", name='NUM_NONCUSTODIALS', type='number', startIndex=291, endIndex=299, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="16B", name='NUM_BIRTHS', type='number', startIndex=315, endIndex=323, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="17B", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=339, endIndex=347, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="18B", name='NUM_CLOSED_CASES', type='number', startIndex=363, endIndex=371, + required=True, validators=[validators.isInLimits(0, 99999999)]), + ], +) + +s3 = RowSchema( + model=SSP_M6, + preparsing_validators=[ + validators.hasLength(379), + ], + postparsing_validators=[ + validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), + validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + ], + fields=[ + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, + required=True, validators=[]), + Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[]), + TransformField(calendar_quarter_to_rpt_month_year(2), item="4", name='RPT_MONTH_YEAR', type='number', + startIndex=2, endIndex=7, required=True, validators=[]), + Field(item="4C", name='NUM_APPLICATIONS', type='number', startIndex=23, endIndex=31, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="5C", name='NUM_APPROVED', type='number', startIndex=47, endIndex=55, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="6C", name='NUM_DENIED', type='number', startIndex=71, endIndex=79, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="7C", name='ASSISTANCE', type='number', startIndex=103, endIndex=115, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="8C", name='NUM_FAMILIES', type='number', startIndex=131, endIndex=139, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="9C", name='NUM_2_PARENTS', type='number', startIndex=155, endIndex=163, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="10C", name='NUM_1_PARENTS', type='number', startIndex=179, endIndex=187, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="11C", name='NUM_NO_PARENTS', type='number', startIndex=203, endIndex=211, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="12C", name='NUM_RECIPIENTS', type='number', startIndex=227, endIndex=235, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="13C", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=251, endIndex=259, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="14C", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=275, endIndex=283, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="15C", name='NUM_NONCUSTODIALS', type='number', startIndex=299, endIndex=307, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="16C", name='NUM_BIRTHS', type='number', startIndex=323, endIndex=331, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="17C", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=347, endIndex=355, + required=True, validators=[validators.isInLimits(0, 99999999)]), + Field(item="18C", name='NUM_CLOSED_CASES', type='number', startIndex=371, endIndex=379, + required=True, validators=[validators.isInLimits(0, 99999999)]), + ], +) + + +m6 = SchemaManager( + schemas=[ + s1, + s2, + s3 + ] + ) diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 new file mode 100644 index 000000000..aeff2cbf5 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 @@ -0,0 +1,3 @@ +HEADER20184G24 SSP1 N +M620184000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 +TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 5a940a71b..b9f6c473c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -5,7 +5,7 @@ from .. import parse from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 -from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 +from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M6 from .factories import DataFileSummaryFactory from tdpservice.data_files.models import DataFile from .. import schema_defs, util @@ -861,3 +861,38 @@ def test_parse_tanf_section4_file(tanf_section4_file): assert first.FAMILIES_MONTH == 274 assert sixth.FAMILIES_MONTH == 499 + + +@pytest.fixture +def ssp_section3_file(stt_user, stt): + """Fixture for ADS.E2J.FTP3.TS06.""" + return util.create_test_datafile('ADS.E2J.NDM3.MS24', stt_user, stt, "SSP Aggregate Data") + + +@pytest.mark.django_db() +def test_parse_ssp_section3_file(ssp_section3_file): + """Test parsing TANF Section 3 submission.""" + parse.parse_datafile(ssp_section3_file) + + assert SSP_M6.objects.all().count() == 3 + + parser_errors = ParserError.objects.filter(file=ssp_section3_file) + assert parser_errors.count() == 0 + + m6_objs = SSP_M6.objects.all().order_by('NUM_APPROVED') + + first = m6_objs.first() + second = m6_objs[1] + third = m6_objs[2] + + assert first.RPT_MONTH_YEAR == 202012 + assert second.RPT_MONTH_YEAR == 202011 + assert third.RPT_MONTH_YEAR == 202010 + + assert first.NUM_APPROVED == 3924 + assert second.NUM_APPROVED == 3977 + assert third.NUM_APPROVED == 4301 + + assert first.NUM_CLOSED_CASES == 3884 + assert second.NUM_CLOSED_CASES == 3881 + assert third.NUM_CLOSED_CASES == 5453 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 6ed48e44e..181d6244c 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -168,7 +168,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'G': { 'section': DataFile.Section.SSP_AGGREGATE_DATA, 'models': { - # 'S6': schema_defs.ssp.m6, + 'M6': schema_defs.ssp.m6, } }, 'S': { @@ -228,7 +228,12 @@ def get_program_model(str_prog, str_section, str_model): def get_section_reference(str_prog, str_section): """Return the named section reference for a given program and section.""" - return get_schema_options(program=str_prog, section=str_section, query='section') + print('****************************========================') + print(str_prog) + print(str_section) + res = get_schema_options(program=str_prog, section=str_section, query='section') + print(res) + return res def get_text_from_df(df): """Return the short-hand text for program, section for a given datafile.""" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index a8722794d..a56418b89 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -351,6 +351,10 @@ def validate_header_section_matches_submission(datafile, section, generate_error """Validate header section matches submission section.""" is_valid = datafile.section == section + print('**********************') + print(section) + print(datafile.section) + error = None if not is_valid: error = generate_error( diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index c3c431529..b1536bfa6 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -2,7 +2,7 @@ from django_elasticsearch_dsl import Document from django_elasticsearch_dsl.registries import registry -from ..models.ssp import SSP_M1, SSP_M2, SSP_M3 +from ..models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M6 from .document_base import DocumentBase @registry.register_document @@ -198,3 +198,42 @@ class Django: 'UNEARNED_SSI', 'OTHER_UNEARNED_INCOME', ] + + +@registry.register_document +class SSP_M6DataSubmissionDocument(DocumentBase, Document): + """Elastic search model mapping for a parsed SSP M6 data file.""" + + class Index: + """ElasticSearch index generation settings.""" + + name = 'ssp_m6_submissions' + settings = { + 'number_of_shards': 1, + 'number_of_replicas': 0, + } + + class Django: + """Django model reference and field mapping.""" + + model = SSP_M6 + fields = [ + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'NUM_APPLICATIONS', + 'NUM_APPROVED', + 'NUM_DENIED', + 'ASSISTANCE', + 'NUM_FAMILIES', + 'NUM_2_PARENTS', + 'NUM_1_PARENTS', + 'NUM_NO_PARENTS', + 'NUM_RECIPIENTS', + 'NUM_ADULT_RECIPIENTS', + 'NUM_CHILD_RECIPIENTS', + 'NUM_NONCUSTODIALS', + 'NUM_BIRTHS', + 'NUM_OUTWEDLOCK_BIRTHS', + 'NUM_CLOSED_CASES' + ] \ No newline at end of file diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index f2a325eb3..0bd08a67f 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -210,3 +210,40 @@ class SSP_M3(models.Model): CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) + + +class SSP_M6(models.Model): + """ + Parsed record representing an M6 data submission. + + Mapped to an elastic search index. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datafile = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m6_parent' + ) + + RecordType = models.CharField(max_length=156, null=True, blank=False) + CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) + RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) + NUM_APPLICATIONS = models.IntegerField(null=True, blank=True) + NUM_APPROVED = models.IntegerField(null=True, blank=True) + NUM_DENIED = models.IntegerField(null=True, blank=True) + ASSISTANCE = models.IntegerField(null=True, blank=True) + NUM_FAMILIES = models.IntegerField(null=True, blank=True) + NUM_2_PARENTS = models.IntegerField(null=True, blank=True) + NUM_1_PARENTS = models.IntegerField(null=True, blank=True) + NUM_NO_PARENTS = models.IntegerField(null=True, blank=True) + NUM_RECIPIENTS = models.IntegerField(null=True, blank=True) + NUM_ADULT_RECIPIENTS = models.IntegerField(null=True, blank=True) + NUM_CHILD_RECIPIENTS = models.IntegerField(null=True, blank=True) + NUM_NONCUSTODIALS = models.IntegerField(null=True, blank=True) + NUM_BIRTHS = models.IntegerField(null=True, blank=True) + NUM_OUTWEDLOCK_BIRTHS = models.IntegerField(null=True, blank=True) + NUM_CLOSED_CASES = models.IntegerField(null=True, blank=True) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 4d81eaac9..c533afba6 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -578,3 +578,42 @@ def test_can_create_and_index_ssp_m3_submission(): response = search.execute() assert response.hits.total.value == 1 + + +@pytest.mark.django_db +def test_can_create_and_index_ssp_m6_submission(): + """SSP M6 submissions can be created and mapped.""" + record_num = fake.uuid4() + + submission = models.tanf.SSP_M6() + submission.datafile = test_datafile + submission.RecordType = record_num + submission.CALENDAR_QUARTER = 1 + submission.RPT_MONTH_YEAR = 1 + submission.NUM_APPLICATIONS = 1 + submission.NUM_APPROVED = 1 + submission.NUM_DENIED = 1 + submission.ASSISTANCE = 1 + submission.NUM_FAMILIES = 1 + submission.NUM_2_PARENTS = 1 + submission.NUM_1_PARENTS = 1 + submission.NUM_NO_PARENTS = 1 + submission.NUM_RECIPIENTS = 1 + submission.NUM_ADULT_RECIPIENTS = 1 + submission.NUM_CHILD_RECIPIENTS = 1 + submission.NUM_NONCUSTODIALS = 1 + submission.NUM_BIRTHS = 1 + submission.NUM_OUTWEDLOCK_BIRTHS = 1 + submission.NUM_CLOSED_CASES = 1 + + submission.save() + + assert submission.id is not None + + search = documents.tanf.SSP_M6DataSubmissionDocument.search().query( + 'match', + RecordType=record_num + ) + response = search.execute() + + assert response.hits.total.value == 1 From 5129fa1eaf2f49ae79dab03420bd2b07be81ed1f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 18 Oct 2023 12:44:16 -0400 Subject: [PATCH 2/6] update ssp m6 field names --- .../tdpservice/parsers/schema_defs/ssp/m6.py | 161 ++++++++---------- .../search_indexes/admin/__init__.py | 1 + .../tdpservice/search_indexes/admin/ssp.py | 18 ++ .../search_indexes/documents/ssp.py | 20 +-- .../search_indexes/migrations/0019_ssp_m6.py | 36 ++++ .../tdpservice/search_indexes/models/ssp.py | 18 +- 6 files changed, 137 insertions(+), 117 deletions(-) create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0019_ssp_m6.py diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 36d477ef0..3a00f9fb9 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -8,19 +8,16 @@ from ... import validators from tdpservice.search_indexes.models.ssp import SSP_M6 - - -## item numbering different between ssp and tanf? - s1 = RowSchema( model=SSP_M6, preparsing_validators=[ - validators.hasLength(379), + validators.hasLength(259), ], postparsing_validators=[ - validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), - validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), - validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + validators.sumIsEqual( + "SSPMOE_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual( + "NUM_RECIPIENTS", ["ADULT_RECIPIENTS", "CHILD_RECIPIENTS"]), ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, @@ -28,38 +25,28 @@ Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.quarterIsValid()]), - TransformField(calendar_quarter_to_rpt_month_year(0), item="4", name='RPT_MONTH_YEAR', type='number', + TransformField(calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.dateMonthIsValid()]), - Field(item="4A", name='NUM_APPLICATIONS', type='number', startIndex=7, endIndex=15, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="5A", name='NUM_APPROVED', type='number', startIndex=31, endIndex=39, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="6A", name='NUM_DENIED', type='number', startIndex=55, endIndex=63, + Field(item="3A", name='SSPMOE_FAMILIES', type='number', startIndex=7, endIndex=15, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="7A", name='ASSISTANCE', type='number', startIndex=79, endIndex=91, - required=True, validators=[validators.isInLimits(0, 999999999999)]), - Field(item="8A", name='NUM_FAMILIES', type='number', startIndex=115, endIndex=123, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="9A", name='NUM_2_PARENTS', type='number', startIndex=139, endIndex=147, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="10A", name='NUM_1_PARENTS', type='number', startIndex=163, endIndex=171, + Field(item="4A", name='NUM_2_PARENTS', type='number', startIndex=31, endIndex=39, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="11A", name='NUM_NO_PARENTS', type='number', startIndex=187, endIndex=195, + Field(item="5A", name='NUM_1_PARENTS', type='number', startIndex=55, endIndex=63, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="12A", name='NUM_RECIPIENTS', type='number', startIndex=211, endIndex=219, + Field(item="6A", name='NUM_NO_PARENTS', type='number', startIndex=79, endIndex=87, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="13A", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=235, endIndex=243, + Field(item="7A", name='NUM_RECIPIENTS', type='number', startIndex=103, endIndex=111, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="14A", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=259, endIndex=267, + Field(item="8A", name='ADULT_RECIPIENTS', type='number', startIndex=127, endIndex=135, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="15A", name='NUM_NONCUSTODIALS', type='number', startIndex=283, endIndex=291, + Field(item="9A", name='CHILD_RECIPIENTS', type='number', startIndex=151, endIndex=159, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="16A", name='NUM_BIRTHS', type='number', startIndex=307, endIndex=315, + Field(item="10A", name='NONCUSTODIALS', type='number', startIndex=175, endIndex=183, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="17A", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=331, endIndex=339, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="18A", name='NUM_CLOSED_CASES', type='number', startIndex=355, endIndex=363, + Field(item="11A", name='AMT_ASSISTANCE', type='number', startIndex=199, endIndex=211, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="12A", name='CLOSED_CASES', type='number', startIndex=235, endIndex=243, required=True, validators=[validators.isInLimits(0, 99999999)]), ], ) @@ -67,49 +54,42 @@ s2 = RowSchema( model=SSP_M6, preparsing_validators=[ - validators.hasLength(379), + validators.hasLength(259), ], postparsing_validators=[ - validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), - validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), - validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + validators.sumIsEqual( + "SSPMOE_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual( + "NUM_RECIPIENTS", ["ADULT_RECIPIENTS", "CHILD_RECIPIENTS"]), ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, - required=True, validators=[]), - TransformField(calendar_quarter_to_rpt_month_year(1), item="4", name='RPT_MONTH_YEAR', type='number', - startIndex=2, endIndex=7, required=True, validators=[]), - Field(item="4B", name='NUM_APPLICATIONS', type='number', startIndex=15, endIndex=23, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="5B", name='NUM_APPROVED', type='number', startIndex=39, endIndex=47, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="6B", name='NUM_DENIED', type='number', startIndex=63, endIndex=71, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="7B", name='ASSISTANCE', type='number', startIndex=91, endIndex=103, - required=True, validators=[validators.isInLimits(0, 999999999999)]), - Field(item="8B", name='NUM_FAMILIES', type='number', startIndex=123, endIndex=131, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="9B", name='NUM_2_PARENTS', type='number', startIndex=147, endIndex=155, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="10B", name='NUM_1_PARENTS', type='number', startIndex=171, endIndex=179, + Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.quarterIsValid()]), + TransformField(calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', type='number', + startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid()]), + Field(item="3B", name='SSPMOE_FAMILIES', type='number', startIndex=15, endIndex=23, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="11B", name='NUM_NO_PARENTS', type='number', startIndex=195, endIndex=203, + Field(item="4B", name='NUM_2_PARENTS', type='number', startIndex=39, endIndex=47, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="12B", name='NUM_RECIPIENTS', type='number', startIndex=219, endIndex=227, + Field(item="5B", name='NUM_1_PARENTS', type='number', startIndex=63, endIndex=71, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="13B", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=243, endIndex=251, + Field(item="6B", name='NUM_NO_PARENTS', type='number', startIndex=87, endIndex=95, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="14B", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=267, endIndex=275, + Field(item="7B", name='NUM_RECIPIENTS', type='number', startIndex=111, endIndex=119, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="15B", name='NUM_NONCUSTODIALS', type='number', startIndex=291, endIndex=299, + Field(item="8B", name='ADULT_RECIPIENTS', type='number', startIndex=135, endIndex=143, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="16B", name='NUM_BIRTHS', type='number', startIndex=315, endIndex=323, + Field(item="9B", name='CHILD_RECIPIENTS', type='number', startIndex=159, endIndex=167, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="17B", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=339, endIndex=347, + Field(item="10B", name='NONCUSTODIALS', type='number', startIndex=183, endIndex=191, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="18B", name='NUM_CLOSED_CASES', type='number', startIndex=363, endIndex=371, + Field(item="11B", name='AMT_ASSISTANCE', type='number', startIndex=211, endIndex=223, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="12B", name='CLOSED_CASES', type='number', startIndex=243, endIndex=251, required=True, validators=[validators.isInLimits(0, 99999999)]), ], ) @@ -117,58 +97,51 @@ s3 = RowSchema( model=SSP_M6, preparsing_validators=[ - validators.hasLength(379), + validators.hasLength(259), ], postparsing_validators=[ - validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), - validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), - validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]), + validators.sumIsEqual( + "SSPMOE_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]), + validators.sumIsEqual( + "NUM_RECIPIENTS", ["ADULT_RECIPIENTS", "CHILD_RECIPIENTS"]), ], fields=[ Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, - required=True, validators=[]), - TransformField(calendar_quarter_to_rpt_month_year(2), item="4", name='RPT_MONTH_YEAR', type='number', - startIndex=2, endIndex=7, required=True, validators=[]), - Field(item="4C", name='NUM_APPLICATIONS', type='number', startIndex=23, endIndex=31, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="5C", name='NUM_APPROVED', type='number', startIndex=47, endIndex=55, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="6C", name='NUM_DENIED', type='number', startIndex=71, endIndex=79, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="7C", name='ASSISTANCE', type='number', startIndex=103, endIndex=115, - required=True, validators=[validators.isInLimits(0, 999999999999)]), - Field(item="8C", name='NUM_FAMILIES', type='number', startIndex=131, endIndex=139, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="9C", name='NUM_2_PARENTS', type='number', startIndex=155, endIndex=163, - required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="10C", name='NUM_1_PARENTS', type='number', startIndex=179, endIndex=187, + Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, + required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.quarterIsValid()]), + TransformField(calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', type='number', + startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), + validators.dateMonthIsValid()]), + Field(item="3C", name='SSPMOE_FAMILIES', type='number', startIndex=23, endIndex=31, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="11C", name='NUM_NO_PARENTS', type='number', startIndex=203, endIndex=211, + Field(item="4C", name='NUM_2_PARENTS', type='number', startIndex=47, endIndex=55, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="12C", name='NUM_RECIPIENTS', type='number', startIndex=227, endIndex=235, + Field(item="5C", name='NUM_1_PARENTS', type='number', startIndex=71, endIndex=79, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="13C", name='NUM_ADULT_RECIPIENTS', type='number', startIndex=251, endIndex=259, + Field(item="6C", name='NUM_NO_PARENTS', type='number', startIndex=95, endIndex=103, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="14C", name='NUM_CHILD_RECIPIENTS', type='number', startIndex=275, endIndex=283, + Field(item="7C", name='NUM_RECIPIENTS', type='number', startIndex=119, endIndex=127, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="15C", name='NUM_NONCUSTODIALS', type='number', startIndex=299, endIndex=307, + Field(item="8C", name='ADULT_RECIPIENTS', type='number', startIndex=143, endIndex=151, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="16C", name='NUM_BIRTHS', type='number', startIndex=323, endIndex=331, + Field(item="9C", name='CHILD_RECIPIENTS', type='number', startIndex=167, endIndex=175, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="17C", name='NUM_OUTWEDLOCK_BIRTHS', type='number', startIndex=347, endIndex=355, + Field(item="10C", name='NONCUSTODIALS', type='number', startIndex=191, endIndex=199, required=True, validators=[validators.isInLimits(0, 99999999)]), - Field(item="18C", name='NUM_CLOSED_CASES', type='number', startIndex=371, endIndex=379, + Field(item="11C", name='AMT_ASSISTANCE', type='number', startIndex=223, endIndex=235, + required=True, validators=[validators.isInLimits(0, 999999999999)]), + Field(item="12C", name='CLOSED_CASES', type='number', startIndex=251, endIndex=259, required=True, validators=[validators.isInLimits(0, 99999999)]), ], ) m6 = SchemaManager( - schemas=[ - s1, - s2, - s3 - ] - ) + schemas=[ + s1, + s2, + s3 + ] +) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py index 92131fd35..19ad02979 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/__init__.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/__init__.py @@ -13,3 +13,4 @@ admin.site.register(models.ssp.SSP_M1, ssp.SSP_M1Admin) admin.site.register(models.ssp.SSP_M2, ssp.SSP_M2Admin) admin.site.register(models.ssp.SSP_M3, ssp.SSP_M3Admin) +admin.site.register(models.ssp.SSP_M6, ssp.SSP_M6Admin) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 17ee5a4c8..d8e2088af 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -2,6 +2,7 @@ from django.contrib import admin from .filter import CreationDateFilter + class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" @@ -53,3 +54,20 @@ class SSP_M3Admin(admin.ModelAdmin): CreationDateFilter, 'RPT_MONTH_YEAR', ] + + +class SSP_M6Admin(admin.ModelAdmin): + """ModelAdmin class for parsed M6 data files.""" + + list_display = [ + 'RecordType', + 'CALENDAR_QUARTER', + 'RPT_MONTH_YEAR', + 'datafile', + ] + + list_filter = [ + 'CALENDAR_QUARTER', + CreationDateFilter, + 'RPT_MONTH_YEAR' + ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index b1536bfa6..d047acd0d 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -5,6 +5,7 @@ from ..models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M6 from .document_base import DocumentBase + @registry.register_document class SSP_M1DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed SSP M1 data file.""" @@ -221,19 +222,14 @@ class Django: 'RecordType', 'CALENDAR_QUARTER', 'RPT_MONTH_YEAR', - 'NUM_APPLICATIONS', - 'NUM_APPROVED', - 'NUM_DENIED', - 'ASSISTANCE', - 'NUM_FAMILIES', + 'SSPMOE_FAMILIES', 'NUM_2_PARENTS', 'NUM_1_PARENTS', 'NUM_NO_PARENTS', 'NUM_RECIPIENTS', - 'NUM_ADULT_RECIPIENTS', - 'NUM_CHILD_RECIPIENTS', - 'NUM_NONCUSTODIALS', - 'NUM_BIRTHS', - 'NUM_OUTWEDLOCK_BIRTHS', - 'NUM_CLOSED_CASES' - ] \ No newline at end of file + 'ADULT_RECIPIENTS', + 'CHILD_RECIPIENTS', + 'NONCUSTODIALS', + 'AMT_ASSISTANCE', + 'CLOSED_CASES', + ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0019_ssp_m6.py b/tdrs-backend/tdpservice/search_indexes/migrations/0019_ssp_m6.py new file mode 100644 index 000000000..e41a275b3 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0019_ssp_m6.py @@ -0,0 +1,36 @@ +# Generated by Django 3.2.15 on 2023-10-18 16:19 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('search_indexes', '0018_auto_20230920_1846'), + ] + + operations = [ + migrations.CreateModel( + name='SSP_M6', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('RecordType', models.CharField(max_length=156, null=True)), + ('CALENDAR_QUARTER', models.IntegerField(blank=True, null=True)), + ('RPT_MONTH_YEAR', models.IntegerField(null=True)), + ('SSPMOE_FAMILIES', models.IntegerField(blank=True, null=True)), + ('NUM_2_PARENTS', models.IntegerField(blank=True, null=True)), + ('NUM_1_PARENTS', models.IntegerField(blank=True, null=True)), + ('NUM_NO_PARENTS', models.IntegerField(blank=True, null=True)), + ('NUM_RECIPIENTS', models.IntegerField(blank=True, null=True)), + ('ADULT_RECIPIENTS', models.IntegerField(blank=True, null=True)), + ('CHILD_RECIPIENTS', models.IntegerField(blank=True, null=True)), + ('NONCUSTODIALS', models.IntegerField(blank=True, null=True)), + ('AMT_ASSISTANCE', models.IntegerField(blank=True, null=True)), + ('CLOSED_CASES', models.IntegerField(blank=True, null=True)), + ('datafile', models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m6_parent', to='data_files.datafile')), + ], + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index 0bd08a67f..05c599c39 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -232,18 +232,14 @@ class SSP_M6(models.Model): RecordType = models.CharField(max_length=156, null=True, blank=False) CALENDAR_QUARTER = models.IntegerField(null=True, blank=True) RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) - NUM_APPLICATIONS = models.IntegerField(null=True, blank=True) - NUM_APPROVED = models.IntegerField(null=True, blank=True) - NUM_DENIED = models.IntegerField(null=True, blank=True) - ASSISTANCE = models.IntegerField(null=True, blank=True) - NUM_FAMILIES = models.IntegerField(null=True, blank=True) + + SSPMOE_FAMILIES = models.IntegerField(null=True, blank=True) NUM_2_PARENTS = models.IntegerField(null=True, blank=True) NUM_1_PARENTS = models.IntegerField(null=True, blank=True) NUM_NO_PARENTS = models.IntegerField(null=True, blank=True) NUM_RECIPIENTS = models.IntegerField(null=True, blank=True) - NUM_ADULT_RECIPIENTS = models.IntegerField(null=True, blank=True) - NUM_CHILD_RECIPIENTS = models.IntegerField(null=True, blank=True) - NUM_NONCUSTODIALS = models.IntegerField(null=True, blank=True) - NUM_BIRTHS = models.IntegerField(null=True, blank=True) - NUM_OUTWEDLOCK_BIRTHS = models.IntegerField(null=True, blank=True) - NUM_CLOSED_CASES = models.IntegerField(null=True, blank=True) + ADULT_RECIPIENTS = models.IntegerField(null=True, blank=True) + CHILD_RECIPIENTS = models.IntegerField(null=True, blank=True) + NONCUSTODIALS = models.IntegerField(null=True, blank=True) + AMT_ASSISTANCE = models.IntegerField(null=True, blank=True) + CLOSED_CASES = models.IntegerField(null=True, blank=True) From d4f855b08a007274e04bd061bddc2907649ee986 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 18 Oct 2023 12:52:28 -0400 Subject: [PATCH 3/6] update tests --- .../tdpservice/parsers/schema_defs/ssp/m6.py | 4 +- .../parsers/test/data/ADS.E2J.NDM3.MS24 | 2 +- .../tdpservice/parsers/test/test_parse.py | 136 ++++++++++++------ 3 files changed, 93 insertions(+), 49 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 3a00f9fb9..6bea60b11 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -68,7 +68,7 @@ Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.quarterIsValid()]), - TransformField(calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', type='number', + TransformField(calendar_quarter_to_rpt_month_year(1), item="2B", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.dateMonthIsValid()]), Field(item="3B", name='SSPMOE_FAMILIES', type='number', startIndex=15, endIndex=23, @@ -111,7 +111,7 @@ Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.quarterIsValid()]), - TransformField(calendar_quarter_to_rpt_month_year(0), item="2B", name='RPT_MONTH_YEAR', type='number', + TransformField(calendar_quarter_to_rpt_month_year(2), item="2B", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998), validators.dateMonthIsValid()]), Field(item="3C", name='SSPMOE_FAMILIES', type='number', startIndex=23, endIndex=31, diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 index aeff2cbf5..bbe8ee4f0 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 @@ -1,3 +1,3 @@ HEADER20184G24 SSP1 N M620184000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 -TRAILER0000001 +TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b9f6c473c..24b0a416e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -21,11 +21,13 @@ def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" return util.create_test_datafile('small_correct_file.txt', stt_user, stt) + @pytest.fixture def dfs(): """Fixture for DataFileSummary.""" return DataFileSummaryFactory.create() + @pytest.mark.django_db def test_parse_small_correct_file(test_datafile, dfs): """Test parsing of small_correct_file.""" @@ -34,13 +36,17 @@ def test_parse_small_correct_file(test_datafile, dfs): parse.parse_datafile(test_datafile) dfs.status = dfs.get_status() - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ - {'accepted_without_errors': 1, 'accepted_with_errors': 0, 'month': 'Oct'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} - ]} + {'accepted_without_errors': 1, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Dec'} + ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 @@ -58,6 +64,7 @@ def test_parse_small_correct_file(test_datafile, dfs): assert t1.SANC_REDUCTION_AMT == 0 assert t1.FAMILY_NEW_CHILD == 2 + @pytest.mark.django_db def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" @@ -71,7 +78,8 @@ def test_parse_section_mismatch(test_datafile, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 1, 'months': [ {'accepted_without_errors': 'N/A', @@ -83,7 +91,7 @@ def test_parse_section_mismatch(test_datafile, dfs): {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} - ]} + ]} assert parser_errors.count() == 1 err = parser_errors.first() @@ -129,6 +137,7 @@ def test_big_file(stt_user, stt): """Fixture for ADS.E2J.FTP1.TS06.""" return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) + @pytest.mark.django_db @pytest.mark.skip(reason="long runtime") # big_files def test_parse_big_file(test_big_file, dfs): @@ -143,18 +152,23 @@ def test_parse_big_file(test_big_file, dfs): parse.parse_datafile(test_big_file) dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ - {'accepted_without_errors': 171, 'accepted_with_errors': 99, 'month': 'Oct'}, - {'accepted_without_errors': 169, 'accepted_with_errors': 104, 'month': 'Nov'}, - {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'} - ]} + {'accepted_without_errors': 171, + 'accepted_with_errors': 99, 'month': 'Oct'}, + {'accepted_without_errors': 169, + 'accepted_with_errors': 104, 'month': 'Nov'}, + {'accepted_without_errors': 166, + 'accepted_with_errors': 106, 'month': 'Dec'} + ]} parser_errors = ParserError.objects.filter(file=test_big_file) error_message = 'MONTHS_FED_TIME_LIMIT is required but a value was not provided.' - row_18_error = parser_errors.get(row_number=18, error_message=error_message) + row_18_error = parser_errors.get( + row_number=18, error_message=error_message) assert row_18_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert row_18_error.error_message == error_message assert row_18_error.content_type.model == 'tanf_t2' @@ -196,6 +210,7 @@ def bad_file_missing_header(stt_user, stt): """Fixture for bad_missing_header.""" return util.create_test_datafile('bad_missing_header.txt', stt_user, stt) + @pytest.mark.django_db def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" @@ -275,6 +290,7 @@ def bad_trailer_file(stt_user, stt): """Fixture for bad_trailer_1.""" return util.create_test_datafile('bad_trailer_1.txt', stt_user, stt) + @pytest.mark.django_db def test_parse_bad_trailer_file(bad_trailer_file, dfs): """Test parsing bad_trailer_1.""" @@ -379,7 +395,7 @@ def test_parse_empty_file(empty_file, dfs): {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} - ]} + ]} parser_errors = ParserError.objects.filter(file=empty_file).order_by('id') @@ -421,15 +437,20 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 1, 'months': [ - {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} - ]} - - parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) + {'accepted_without_errors': 5, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Dec'} + ]} + + parser_errors = ParserError.objects.filter( + file=small_ssp_section1_datafile) assert parser_errors.count() == 1 err = parser_errors.first() @@ -477,11 +498,13 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile): assert SSP_M2.objects.count() == expected_m2_record_count assert SSP_M3.objects.count() == expected_m3_record_count + @pytest.fixture def small_tanf_section1_datafile(stt_user, stt): """Fixture for small_tanf_section1.""" return util.create_test_datafile('small_tanf_section1.txt', stt_user, stt) + @pytest.mark.django_db def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" @@ -492,13 +515,17 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ - {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} - ]} + {'accepted_without_errors': 5, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Dec'} + ]} assert TANF_T2.objects.count() == 5 @@ -526,6 +553,7 @@ def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): assert TANF_T2.objects.count() == 5 assert TANF_T3.objects.count() == 6 + @pytest.mark.django_db() def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" @@ -548,6 +576,7 @@ def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): assert t3_6.GENDER == 2 assert t3_6.EDUCATION_LEVEL == '98' + @pytest.fixture def super_big_s1_file(stt_user, stt): """Fixture for ADS.E2J.NDM1.TS53_fake.""" @@ -563,11 +592,13 @@ def test_parse_super_big_s1_file(super_big_s1_file): assert TANF_T2.objects.count() == 112794 assert TANF_T3.objects.count() == 172595 + @pytest.fixture def super_big_s1_rollback_file(stt_user, stt): """Fixture for ADS.E2J.NDM1.TS53_fake.rollback.""" return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt) + @pytest.mark.django_db() @pytest.mark.skip(reason="cuz") # big_files def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file): @@ -609,7 +640,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED - parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) + parser_errors = ParserError.objects.filter( + file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 [print(parser_error) for parser_error in parser_errors] @@ -651,7 +683,8 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" errors = parse.parse_datafile(bad_ssp_s1__row_missing_required_field) - parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) + parser_errors = ParserError.objects.filter( + file=bad_ssp_s1__row_missing_required_field) assert parser_errors.count() == 5 row_2_error = parser_errors.get(row_number=2) @@ -692,16 +725,19 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel 'trailer': [trailer_error], } + @pytest.mark.django_db def test_dfs_set_case_aggregates(test_datafile, dfs): """Test that the case aggregates are set correctly.""" test_datafile.section = 'Active Case Data' test_datafile.save() - parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried + # this still needs to execute to create db objects to be queried + parse.parse_datafile(test_datafile) dfs.file = test_datafile dfs.save() dfs.status = dfs.get_status() - dfs.case_aggregates = util.case_aggregates_by_month(test_datafile, dfs.status) + dfs.case_aggregates = util.case_aggregates_by_month( + test_datafile, dfs.status) dfs.save() for month in dfs.case_aggregates['months']: @@ -709,6 +745,7 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): assert month['accepted_without_errors'] == 1 assert month['accepted_with_errors'] == 0 + @pytest.mark.django_db def test_get_schema_options(dfs): """Test use-cases for translating strings to named object references.""" @@ -733,10 +770,10 @@ def test_get_schema_options(dfs): # get model models = util.get_program_models('TAN', 'A') assert models == { - 'T1': schema_defs.tanf.t1, - 'T2': schema_defs.tanf.t2, - 'T3': schema_defs.tanf.t3, - } + 'T1': schema_defs.tanf.t1, + 'T2': schema_defs.tanf.t2, + 'T3': schema_defs.tanf.t3, + } model = util.get_program_model('TAN', 'A', 'T1') assert model == schema_defs.tanf.t1 @@ -753,11 +790,13 @@ def test_get_schema_options(dfs): # get section str # get ref section + @pytest.fixture def small_tanf_section2_file(stt_user, stt): """Fixture for tanf section2 datafile.""" return util.create_test_datafile('small_tanf_section2.txt', stt_user, stt, 'Closed Case Data') + @pytest.mark.django_db() def test_parse_small_tanf_section2_file(small_tanf_section2_file): """Test parsing a small TANF Section 2 submission.""" @@ -779,11 +818,13 @@ def test_parse_small_tanf_section2_file(small_tanf_section2_file): assert t5.GENDER == 2 assert t5.AMOUNT_UNEARNED_INCOME == '0000' + @pytest.fixture def tanf_section2_file(stt_user, stt): """Fixture for ADS.E2J.FTP2.TS06.""" return util.create_test_datafile('ADS.E2J.FTP2.TS06', stt_user, stt, 'Closed Case Data') + @pytest.mark.django_db() def test_parse_tanf_section2_file(tanf_section2_file): """Test parsing TANF Section 2 submission.""" @@ -800,11 +841,13 @@ def test_parse_tanf_section2_file(tanf_section2_file): assert err.content_type.model == "tanf_t5" assert err.object_id is not None + @pytest.fixture def tanf_section3_file(stt_user, stt): """Fixture for ADS.E2J.FTP3.TS06.""" return util.create_test_datafile('ADS.E2J.FTP3.TS06', stt_user, stt, "Aggregate Data") + @pytest.mark.django_db() def test_parse_tanf_section3_file(tanf_section3_file): """Test parsing TANF Section 3 submission.""" @@ -833,11 +876,13 @@ def test_parse_tanf_section3_file(tanf_section3_file): assert second.NUM_CLOSED_CASES == 3881 assert third.NUM_CLOSED_CASES == 5453 + @pytest.fixture def tanf_section4_file(stt_user, stt): """Fixture for ADS.E2J.FTP4.TS06.""" return util.create_test_datafile('ADS.E2J.FTP4.TS06', stt_user, stt, "Stratum Data") + @pytest.mark.django_db() def test_parse_tanf_section4_file(tanf_section4_file): """Test parsing TANF Section 4 submission.""" @@ -874,25 +919,24 @@ def test_parse_ssp_section3_file(ssp_section3_file): """Test parsing TANF Section 3 submission.""" parse.parse_datafile(ssp_section3_file) - assert SSP_M6.objects.all().count() == 3 + m6_objs = SSP_M6.objects.all().order_by('RPT_MONTH_YEAR') + assert m6_objs.count() == 3 parser_errors = ParserError.objects.filter(file=ssp_section3_file) assert parser_errors.count() == 0 - m6_objs = SSP_M6.objects.all().order_by('NUM_APPROVED') - first = m6_objs.first() second = m6_objs[1] third = m6_objs[2] - assert first.RPT_MONTH_YEAR == 202012 - assert second.RPT_MONTH_YEAR == 202011 - assert third.RPT_MONTH_YEAR == 202010 + assert first.RPT_MONTH_YEAR == 201810 + assert second.RPT_MONTH_YEAR == 201811 + assert third.RPT_MONTH_YEAR == 201812 - assert first.NUM_APPROVED == 3924 - assert second.NUM_APPROVED == 3977 - assert third.NUM_APPROVED == 4301 + assert first.SSPMOE_FAMILIES == 15869 + assert second.SSPMOE_FAMILIES == 16008 + assert third.SSPMOE_FAMILIES == 15956 - assert first.NUM_CLOSED_CASES == 3884 - assert second.NUM_CLOSED_CASES == 3881 - assert third.NUM_CLOSED_CASES == 5453 + assert first.NUM_RECIPIENTS == 51355 + assert second.NUM_RECIPIENTS == 51696 + assert third.NUM_RECIPIENTS == 51348 From b186ae569c31317c7e794e13040fe3df5a4a40f0 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 18 Oct 2023 12:58:03 -0400 Subject: [PATCH 4/6] rm logging --- tdrs-backend/tdpservice/parsers/util.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 181d6244c..4073ae17e 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -228,12 +228,7 @@ def get_program_model(str_prog, str_section, str_model): def get_section_reference(str_prog, str_section): """Return the named section reference for a given program and section.""" - print('****************************========================') - print(str_prog) - print(str_section) - res = get_schema_options(program=str_prog, section=str_section, query='section') - print(res) - return res + return get_schema_options(program=str_prog, section=str_section, query='section') def get_text_from_df(df): """Return the short-hand text for program, section for a given datafile.""" From 0ddd03db1f982b29a35ee4400a39e1b494ed0ec8 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 18 Oct 2023 12:58:20 -0400 Subject: [PATCH 5/6] rm logging --- tdrs-backend/tdpservice/parsers/validators.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index a56418b89..a8722794d 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -351,10 +351,6 @@ def validate_header_section_matches_submission(datafile, section, generate_error """Validate header section matches submission section.""" is_valid = datafile.section == section - print('**********************') - print(section) - print(datafile.section) - error = None if not is_valid: error = generate_error( From dc99ebdbe5a139b1ab03df203d27934f72913233 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 18 Oct 2023 14:47:43 -0400 Subject: [PATCH 6/6] fix tests --- .../tdpservice/search_indexes/test/test_model_mapping.py | 6 +++--- tdrs-backend/tdpservice/users/test/test_permissions.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index c533afba6..15581d304 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -581,11 +581,11 @@ def test_can_create_and_index_ssp_m3_submission(): @pytest.mark.django_db -def test_can_create_and_index_ssp_m6_submission(): +def test_can_create_and_index_ssp_m6_submission(test_datafile): """SSP M6 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.SSP_M6() + submission = models.ssp.SSP_M6() submission.datafile = test_datafile submission.RecordType = record_num submission.CALENDAR_QUARTER = 1 @@ -610,7 +610,7 @@ def test_can_create_and_index_ssp_m6_submission(): assert submission.id is not None - search = documents.tanf.SSP_M6DataSubmissionDocument.search().query( + search = documents.ssp.SSP_M6DataSubmissionDocument.search().query( 'match', RecordType=record_num ) diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 2f25347aa..d942fd0c8 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -123,6 +123,9 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'search_indexes.add_ssp_m3', 'search_indexes.view_ssp_m3', 'search_indexes.change_ssp_m3', + 'search_indexes.add_ssp_m6', + 'search_indexes.view_ssp_m6', + 'search_indexes.change_ssp_m6', } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions