SSP Section 4 Parsing & Validation

tdrs-backend/tdpservice/parsers/parse.py

@@ -251,50 +251,6 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False)
             )
         ])]
 
-
-def get_schema_manager_options(program_type):
-    """Return the allowed schema options."""
-    match program_type:
-        case 'TAN':
-            return {
-                'A': {
-                    'T1': schema_defs.tanf.t1,
-                    'T2': schema_defs.tanf.t2,
-                    'T3': schema_defs.tanf.t3,
-                },
-                'C': {
-                    'T4': schema_defs.tanf.t4,
-                    'T5': schema_defs.tanf.t5,
-                },
-                'G': {
-                    'T6': schema_defs.tanf.t6,
-                },
-                'S': {
-                    'T7': schema_defs.tanf.t7,
-                },
-            }
-        case 'SSP':
-            return {
-                'A': {
-                    'M1': schema_defs.ssp.m1,
-                    'M2': schema_defs.ssp.m2,
-                    'M3': schema_defs.ssp.m3,
-                },
-                'C': {
-                    # 'M4': schema_options.m4,
-                    # 'M5': schema_options.m5,
-                },
-                'G': {
-                    # 'M6': schema_options.m6,
-                },
-                'S': {
-                    # 'M7': schema_options.m7,
-                },
-            }
-        # case tribal?
-    return None
-
-
 def get_schema_manager(line, section, program_type):
     """Return the appropriate schema for the line."""
     line_type = line[0:2]

tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py

@@ -1,7 +1,9 @@
 from .m1 import m1
 from .m2 import m2
 from .m3 import m3
+from .m7 import m7
 
 m1 = m1
 m2 = m2
 m3 = m3
+m7 = m7

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py

@@ -0,0 +1,65 @@
+"""Schema for TANF T7 Row."""
+
+from ...util import SchemaManager
+from ...fields import Field, TransformField
+from ...row_schema import RowSchema
+from ...transforms import calendar_quarter_to_rpt_month_year
+from ... import validators
+from tdpservice.search_indexes.models.ssp import SSP_M7
+
+schemas = []
+
+validator_index = 7
+section_ind_index = 7
+stratum_index = 8
+families_index = 10
+
+sub_item_labels = ['5A', '5B', '5C']
+families_item_numbers = [sub_item_labels[i % 3] for i in range(30)]
+
+for i in range(1, 31):
+    schemas.append(
+      RowSchema(
+          model=SSP_M7,
+          quiet_preparser_errors=i > 1,
+          preparsing_validators=[
+              validators.notEmpty(0, 7),
+              validators.notEmpty(validator_index, validator_index + 24),
+          ],
+          postparsing_validators=[],
+          fields=[
+              Field(item="0", name="RecordType", type='string', startIndex=0, endIndex=2,
+                    required=True, validators=[]),
+              Field(item="2", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7,
+                    required=True, validators=[validators.dateYearIsLargerThan(1998),
+                                               validators.quarterIsValid()]),
+              TransformField(
+                  transform_func=calendar_quarter_to_rpt_month_year((i - 1) % 3),
+                  item="2A",
+                  name='RPT_MONTH_YEAR',
+                  type='number',
+                  startIndex=2,
+                  endIndex=7,
+                  required=True,
+                  validators=[
+                      validators.dateYearIsLargerThan(1998),
+                      validators.dateMonthIsValid()
+                  ]
+              ),
+              Field(item="3", name='TDRS_SECTION_IND', type='string', startIndex=section_ind_index,
+                    endIndex=section_ind_index + 1, required=True, validators=[validators.oneOf(['1', '2'])]),
+              Field(item="4", name='STRATUM', type='string', startIndex=stratum_index,
+                    endIndex=stratum_index + 2, required=True, validators=[validators.isInStringRange(0, 99)]),
+              Field(item=families_item_numbers[i - 1], name='FAMILIES_MONTH', type='number', startIndex=families_index,
+                    endIndex=families_index + 7, required=True, validators=[validators.isInLimits(0, 9999999)]),
+          ]
+      )
+    )
+
+    index_offset = 0 if i % 3 != 0 else 24
+    validator_index += index_offset
+    section_ind_index += index_offset
+    stratum_index += index_offset
+    families_index += 7 if i % 3 != 0 else 10
+
+m7 = SchemaManager(schemas=schemas)

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24

@@ -0,0 +1,3 @@
+HEADER20184S24   SSP1 N
+M720184101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
+TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -5,7 +5,7 @@
 from .. import parse
 from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary
 from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7
-from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3
+from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M7
 from .factories import DataFileSummaryFactory
 from tdpservice.data_files.models import DataFile
 from .. import schema_defs, util
@@ -888,3 +888,21 @@ def test_parse_tanf_section4_file(tanf_section4_file):
 
     assert first.FAMILIES_MONTH == 274
     assert sixth.FAMILIES_MONTH == 499
+
+@pytest.fixture
+def ssp_section4_file(stt_user, stt):
+    """Fixture for ADS.E2J.NDM4.MS24."""
+    return util.create_test_datafile('ADS.E2J.NDM4.MS24', stt_user, stt, "SSP Stratum Data")
+
+@pytest.mark.django_db()
+def test_parse_ssp_section4_file(ssp_section4_file):
+    """Test parsing SSP Section 4 submission."""
+    parse.parse_datafile(ssp_section4_file)
+
+    m7_objs = SSP_M7.objects.all().order_by('FAMILIES_MONTH')
+
+    assert m7_objs.count() == 12
+
+    first = m7_objs.first()
+    assert first.RPT_MONTH_YEAR == 201810
+    assert first.FAMILIES_MONTH == 748

tdrs-backend/tdpservice/parsers/util.py

@@ -161,20 +161,20 @@ def get_schema_options(program, section, query=None, model=None, model_name=None
             'C': {
                 'section': DataFile.Section.SSP_CLOSED_CASE_DATA,
                 'models': {
-                    # 'S4': schema_defs.ssp.m4,
-                    # 'S5': schema_defs.ssp.m5,
+                    # 'M4': schema_defs.ssp.m4,
+                    # 'M5': schema_defs.ssp.m5,
                 }
             },
             'G': {
                 'section': DataFile.Section.SSP_AGGREGATE_DATA,
                 'models': {
-                    # 'S6': schema_defs.ssp.m6,
+                    # 'M6': schema_defs.ssp.m6,
                 }
             },
             'S': {
                 'section': DataFile.Section.SSP_STRATUM_DATA,
                 'models': {
-                    # 'S7': schema_defs.ssp.m7,
+                    'M7': schema_defs.ssp.m7,
                 }
             }
         },

tdrs-backend/tdpservice/search_indexes/admin/__init__.py

@@ -13,3 +13,4 @@
 admin.site.register(models.ssp.SSP_M1, ssp.SSP_M1Admin)
 admin.site.register(models.ssp.SSP_M2, ssp.SSP_M2Admin)
 admin.site.register(models.ssp.SSP_M3, ssp.SSP_M3Admin)
+admin.site.register(models.ssp.SSP_M7, ssp.SSP_M7Admin)

tdrs-backend/tdpservice/search_indexes/admin/ssp.py

@@ -53,3 +53,22 @@ class SSP_M3Admin(admin.ModelAdmin):
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
+
+class SSP_M7Admin(admin.ModelAdmin):
+    """ModelAdmin class for parsed M7 data files."""
+
+    list_display = [
+        'RecordType',
+        'CALENDAR_QUARTER',
+        'RPT_MONTH_YEAR',
+        'TDRS_SECTION_IND',
+        'STRATUM',
+        'FAMILIES_MONTH',
+        'datafile',
+    ]
+
+    list_filter = [
+        'CALENDAR_QUARTER',
+        CreationDateFilter,
+        'RPT_MONTH_YEAR',
+    ]

tdrs-backend/tdpservice/search_indexes/documents/ssp.py

@@ -2,7 +2,7 @@
 
 from django_elasticsearch_dsl import Document
 from django_elasticsearch_dsl.registries import registry
-from ..models.ssp import SSP_M1, SSP_M2, SSP_M3
+from ..models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M7
 from .document_base import DocumentBase
 
 @registry.register_document
@@ -198,3 +198,29 @@ class Django:
             'UNEARNED_SSI',
             'OTHER_UNEARNED_INCOME',
         ]
+
+@registry.register_document
+class SSP_M7DataSubmissionDocument(DocumentBase, Document):
+    """Elastic search model mapping for a parsed SSP M7 data file."""
+
+    class Index:
+        """ElasticSearch index generation settings."""
+
+        name = 'ssp_m7_submissions'
+        settings = {
+            'number_of_shards': 1,
+            'number_of_replicas': 0,
+        }
+
+    class Django:
+        """Django model reference and field mapping."""
+
+        model = SSP_M7
+        fields = [
+            "RecordType",
+            "CALENDAR_QUARTER",
+            "RPT_MONTH_YEAR",
+            "TDRS_SECTION_IND",
+            "STRATUM",
+            "FAMILIES_MONTH",
+        ]

tdrs-backend/tdpservice/search_indexes/migrations/0019_ssp_m7.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.15 on 2023-10-19 14:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0012_datafile_s3_versioning_id'),
+        ('search_indexes', '0018_auto_20230920_1846'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SSP_M7',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('RecordType', models.CharField(max_length=156, null=True)),
+                ('CALENDAR_QUARTER', models.IntegerField(blank=True, null=True)),
+                ('RPT_MONTH_YEAR', models.IntegerField(null=True)),
+                ('TDRS_SECTION_IND', models.CharField(max_length=1, null=True)),
+                ('STRATUM', models.CharField(max_length=2, null=True)),
+                ('FAMILIES_MONTH', models.IntegerField(null=True)),
+                ('datafile', models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m7_parent', to='data_files.datafile')),
+            ],
+        ),
+    ]

tdrs-backend/tdpservice/search_indexes/models/ssp.py

@@ -2,9 +2,7 @@
 
 import uuid
 from django.db import models
-from django.contrib.contenttypes.fields import GenericRelation
 from tdpservice.data_files.models import DataFile
-from tdpservice.parsers.models import ParserError
 
 
 class SSP_M1(models.Model):
@@ -24,7 +22,6 @@ class SSP_M1(models.Model):
         related_name='m1_parent'
     )
 
-    error = GenericRelation(ParserError)
     RecordType = models.CharField(max_length=156, null=True, blank=False)
     RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
     CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)
@@ -36,9 +33,7 @@ class SSP_M1(models.Model):
     )
     STRATUM = models.CharField(max_length=2, null=True, blank=False)
     ZIP_CODE = models.CharField(max_length=5, null=True, blank=False)
-    # FUNDING_STREAM = models.IntegerField(null=True, blank=False)
     DISPOSITION = models.IntegerField(null=True, blank=False)
-    # NEW_APPLICANT = models.IntegerField(null=True, blank=False)
     NBR_FAMILY_MEMBERS = models.IntegerField(null=True, blank=False)
     FAMILY_TYPE = models.IntegerField(null=True, blank=False)
     TANF_ASST_IN_6MONTHS = models.IntegerField(null=True, blank=False)
@@ -74,8 +69,6 @@ class SSP_M1(models.Model):
     REDUCTIONS_ON_RECEIPTS = models.IntegerField(null=True, blank=False)
     OTHER_NON_SANCTION = models.IntegerField(null=True, blank=False)
     WAIVER_EVAL_CONTROL_GRPS = models.IntegerField(null=True, blank=False)
-    # FAMILY_EXEMPT_TIME_LIMITS = models.IntegerField(null=True, blank=False)
-    # FAMILY_NEW_CHILD = models.IntegerField(null=True, blank=False)
 
 
 class SSP_M2(models.Model):
@@ -95,7 +88,6 @@ class SSP_M2(models.Model):
         related_name='m2_parent'
     )
 
-    error = GenericRelation(ParserError)
     RecordType = models.CharField(max_length=156, null=True, blank=False)
     RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
     CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)
@@ -124,9 +116,6 @@ class SSP_M2(models.Model):
     EDUCATION_LEVEL = models.IntegerField(null=True, blank=False)
     CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False)
     COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False)
-    # months_fed_time_limit = models.FloatField(null=True, blank=False)
-    # months_state_time_limit = models.FloatField(null=True, blank=False)
-    # current_month_state_exempt = models.IntegerField(null=True, blank=False)
     EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False)
     WORK_ELIGIBLE_INDICATOR = models.IntegerField(null=True, blank=False)
     WORK_PART_STATUS = models.IntegerField(null=True, blank=False)
@@ -186,7 +175,6 @@ class SSP_M3(models.Model):
         related_name='m3_parent'
     )
 
-    error = GenericRelation(ParserError)
     RecordType = models.CharField(max_length=156, null=True, blank=False)
     RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
     CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)
@@ -210,3 +198,31 @@ class SSP_M3(models.Model):
     CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False)
     UNEARNED_SSI = models.IntegerField(null=True, blank=False)
     OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False)
+
+class SSP_M7(models.Model):
+    """
+    Parsed record representing an SSP M3 data submission.
+
+    Mapped to an elastic search index.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    datafile = models.ForeignKey(
+        DataFile,
+        blank=True,
+        help_text='The parent file from which this record was created.',
+        null=True,
+        on_delete=models.CASCADE,
+        related_name='m7_parent'
+    )
+
+    RecordType = models.CharField(max_length=156, null=True, blank=False)
+    CALENDAR_QUARTER = models.IntegerField(null=True, blank=True)
+    RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
+    TDRS_SECTION_IND = models.CharField(
+        max_length=1,
+        null=True,
+        blank=False
+    )
+    STRATUM = models.CharField(max_length=2, null=True, blank=False)
+    FAMILIES_MONTH = models.IntegerField(null=True, blank=False)

tdrs-backend/tdpservice/search_indexes/models/tanf.py

@@ -2,9 +2,7 @@
 
 import uuid
 from django.db import models
-from django.contrib.contenttypes.fields import GenericRelation
 from tdpservice.data_files.models import DataFile
-from tdpservice.parsers.models import ParserError
 
 
 class TANF_T1(models.Model):
@@ -24,7 +22,6 @@ class TANF_T1(models.Model):
         related_name='t1_parent'
     )
 
-    error = GenericRelation(ParserError)
     RecordType = models.CharField(max_length=156, null=True, blank=False)
     RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
     CASE_NUMBER = models.CharField(max_length=11, null=True, blank=False)