Feature/1111 tanf section 4

tdrs-backend/tdpservice/parsers/parse.py

@@ -272,7 +272,7 @@ def get_schema_manager_options(program_type):
                     'T6': schema_defs.tanf.t6,
                 },
                 'S': {
-                    # 'T7': schema_options.t7,
+                    'T7': schema_defs.tanf.t7,
                 },
             }
         case 'SSP':

tdrs-backend/tdpservice/parsers/schema_defs/tanf/__init__.py

@@ -4,10 +4,12 @@
 from .t4 import t4
 from .t5 import t5
 from .t6 import t6
+from .t7 import t7
 
 t1 = t1
 t2 = t2
 t3 = t3
 t4 = t4
 t5 = t5
 t6 = t6
+t7 = t7

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py

@@ -0,0 +1,51 @@
+"""Schema for TANF T7 Row."""
+
+from ...util import SchemaManager
+from ...fields import Field, TransformField
+from ...row_schema import RowSchema
+from ...transforms import calendar_quarter_to_rpt_month_year
+from ... import validators
+from tdpservice.search_indexes.models.tanf import TANF_T7
+
+schemas = []
+
+validator_index = 7
+section_ind_index = 7
+stratum_index = 8
+families_index = 10
+for i in range(1, 31):
+    schemas.append(
+      RowSchema(
+          model=TANF_T7,
+          quiet_preparser_errors=i > 1,
+          preparsing_validators=[
+              validators.notEmpty(0, 7),
+              validators.notEmpty(validator_index, validator_index + 24),
+          ],
+          postparsing_validators=[],
+          fields=[
+              Field(item="0", name="RecordType", type='string', startIndex=0, endIndex=2,
+                    required=True, validators=[]),
+              Field(item="3", name='CALENDAR_QUARTER', type='number', startIndex=2, endIndex=7,
+                    required=True, validators=[validators.dateYearIsLargerThan(1998),
+                                               validators.quarterIsValid()]),
+              TransformField(calendar_quarter_to_rpt_month_year(i % 3), item="3A", name='RPT_MONTH_YEAR', type='number',
+                             startIndex=2, endIndex=7, required=True, validators=[validators.dateYearIsLargerThan(1998),
+                                                                                  validators.dateMonthIsValid()]),
+              Field(item="4", name='TDRS_SECTION_IND', type='string', startIndex=section_ind_index,
+                    endIndex=section_ind_index + 1, required=True, validators=[validators.oneOf(['1', '2'])]),
+              Field(item="5", name='STRATUM', type='string', startIndex=stratum_index,
+                    endIndex=stratum_index + 2, required=True, validators=[validators.isInStringRange(1, 99)]),
+              Field(item="6A", name='FAMILIES_MONTH', type='number', startIndex=families_index,
+                    endIndex=families_index + 7, required=True, validators=[validators.isInLimits(0, 9999999)]),
+          ]
+      )
+    )
+
+    index_offset = 0 if i % 3 != 0 else 24
+    validator_index += index_offset
+    section_ind_index += index_offset
+    stratum_index += index_offset
+    families_index += 7 if i % 3 != 0 else 10
+
+t7 = SchemaManager(schemas=schemas)

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06

@@ -0,0 +1,3 @@
+HEADER20204S06   TAN1 N
+T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/factories.py

@@ -278,3 +278,17 @@ class Meta:
     NUM_BIRTHS = 1
     NUM_OUTWEDLOCK_BIRTHS = 1
     NUM_CLOSED_CASES = 1
+
+class TanfT7Factory(factory.django.DjangoModelFactory):
+    """Generate TANF T6 record for testing."""
+
+    class Meta:
+        """Hardcoded meta data for TANF_T7."""
+
+        model = "search_indexes.TANF_T7"
+
+    CALENDAR_QUARTER = 20204
+    RPT_MONTH_YEAR = 202011
+    TDRS_SECTION_IND = '1'
+    STRATUM = '01'
+    FAMILIES_MONTH = 1

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -5,7 +5,7 @@
 from ..util import create_test_datafile
 from .. import parse
 from ..models import ParserError, ParserErrorCategoryChoices
-from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6
+from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7
 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3
 import logging
 
@@ -656,3 +656,32 @@ def test_parse_tanf_section3_file(tanf_section3_file):
     assert first.NUM_CLOSED_CASES == 3884
     assert second.NUM_CLOSED_CASES == 3881
     assert third.NUM_CLOSED_CASES == 5453
+
+@pytest.fixture
+def tanf_section4_file(stt_user, stt):
+    """Fixture for ADS.E2J.FTP4.TS06."""
+    return create_test_datafile('ADS.E2J.FTP4.TS06', stt_user, stt, "Stratum Data")
+
+@pytest.mark.django_db()
+def test_parse_tanf_section4_file(tanf_section4_file):
+    """Test parsing TANF Section 4 submission."""
+    parse.parse_datafile(tanf_section4_file)
+
+    assert TANF_T7.objects.all().count() == 18
+
+    parser_errors = ParserError.objects.filter(file=tanf_section4_file)
+    assert parser_errors.count() == 0
+
+    t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH')
+
+    first = t7_objs.first()
+    sixth = t7_objs[5]
+
+    assert first.RPT_MONTH_YEAR == 202012
+    assert sixth.RPT_MONTH_YEAR == 202010
+
+    assert first.TDRS_SECTION_IND == '2'
+    assert sixth.TDRS_SECTION_IND == '2'
+
+    assert first.FAMILIES_MONTH == 274
+    assert sixth.FAMILIES_MONTH == 499

tdrs-backend/tdpservice/search_indexes/admin/tanf.py

@@ -109,12 +109,14 @@ class TANF_T7Admin(admin.ModelAdmin):
     """ModelAdmin class for parsed T7 data files."""
 
     list_display = [
-        'record',
-        'rpt_month_year',
+        'RecordType',
+        'CALENDAR_QUARTER',
+        'RPT_MONTH_YEAR',
         'datafile',
     ]
 
     list_filter = [
+        'CALENDAR_QUARTER',
         CreationDateFilter,
-        'rpt_month_year',
+        'RPT_MONTH_YEAR',
     ]

tdrs-backend/tdpservice/search_indexes/documents/tanf.py

@@ -343,11 +343,10 @@ class Django:
 
         model = TANF_T7
         fields = [
-            'record',
-            'rpt_month_year',
-            'fips_code',
-            'calendar_quarter',
-            'tdrs_section_ind',
-            'stratum',
-            'families',
+            "RecordType",
+            "CALENDAR_QUARTER",
+            "RPT_MONTH_YEAR",
+            "TDRS_SECTION_IND",
+            "STRATUM",
+            "FAMILIES_MONTH",
         ]

tdrs-backend/tdpservice/search_indexes/migrations/0018_auto_20230920_1846.py

@@ -0,0 +1,71 @@
+# Generated by Django 3.2.15 on 2023-09-20 18:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('search_indexes', '0017_auto_20230914_1720'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='calendar_quarter',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='families',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='fips_code',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='record',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='rpt_month_year',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='stratum',
+        ),
+        migrations.RemoveField(
+            model_name='tanf_t7',
+            name='tdrs_section_ind',
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='CALENDAR_QUARTER',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='FAMILIES_MONTH',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='RPT_MONTH_YEAR',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='RecordType',
+            field=models.CharField(max_length=156, null=True),
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='STRATUM',
+            field=models.CharField(max_length=2, null=True),
+        ),
+        migrations.AddField(
+            model_name='tanf_t7',
+            name='TDRS_SECTION_IND',
+            field=models.CharField(max_length=1, null=True),
+        ),
+    ]

tdrs-backend/tdpservice/search_indexes/models/tanf.py

@@ -342,15 +342,13 @@ class TANF_T7(models.Model):
         related_name='t7_parent'
     )
 
-    record = models.CharField(max_length=156, null=False, blank=False)
-    rpt_month_year = models.IntegerField(null=False, blank=False)
-    fips_code = models.CharField(max_length=100, null=False, blank=False)
-
-    calendar_quarter = models.IntegerField(null=False, blank=False)
-    tdrs_section_ind = models.CharField(
+    RecordType = models.CharField(max_length=156, null=True, blank=False)
+    CALENDAR_QUARTER = models.IntegerField(null=True, blank=True)
+    RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False)
+    TDRS_SECTION_IND = models.CharField(
         max_length=1,
-        null=False,
+        null=True,
         blank=False
     )
-    stratum = models.CharField(max_length=2, null=False, blank=False)
-    families = models.IntegerField(null=False, blank=False)
+    STRATUM = models.CharField(max_length=2, null=True, blank=False)
+    FAMILIES_MONTH = models.IntegerField(null=True, blank=False)

tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py

@@ -2,7 +2,6 @@
 
 import pytest
 from faker import Faker
-from django.db.utils import IntegrityError
 from tdpservice.search_indexes import models
 from tdpservice.search_indexes import documents
 from tdpservice.parsers.util import create_test_datafile
@@ -346,44 +345,37 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile):
 
     submission = models.tanf.TANF_T7()
     submission.datafile = test_datafile
-    submission.record = record_num
-    submission.rpt_month_year = 1
-    submission.fips_code = '2'
-    submission.calendar_quarter = 1
-    submission.tdrs_section_ind = '1'
-    submission.stratum = '1'
-    submission.families = 1
+    submission.RecordType = record_num
+    submission.CALENDAR_YEAR = 2020
+    submission.CALENDAR_QUARTER = 1
+    submission.TDRS_SECTION_IND = '1'
+    submission.STRATUM = '01'
+    submission.FAMILIES_MONTH_1 = 47655
+    submission.FAMILIES_MONTH_2 = 81982
+    submission.FAMILIES_MONTH_3 = 9999999
 
     submission.save()
 
     # No checks her because t7 records can't be parsed currently.
-    # assert submission.id is not None
+    assert submission.id is not None
 
-    # search = documents.tanf.TANF_T7DataSubmissionDocument.search().query(
-    #     'match',
-    #     record=record_num
-    # )
-    # response = search.execute()
+    search = documents.tanf.TANF_T7DataSubmissionDocument.search().query(
+        'match',
+        RecordType=record_num
+    )
+    response = search.execute()
 
-    # assert response.hits.total.value == 1
+    assert response.hits.total.value == 1
 
 
 @pytest.mark.django_db
 def test_does_not_create_index_if_model_creation_fails():
     """Index creation shouldn't happen if saving a model errors."""
     record_num = fake.uuid4()
 
-    with pytest.raises(IntegrityError):
-        submission = models.tanf.TANF_T7.objects.create(
-            record=record_num
-            # leave out a bunch of required fields
-        )
-
-        assert submission.id is None
-
     search = documents.tanf.TANF_T7DataSubmissionDocument.search().query(
         'match',
-        record=record_num
+        RecordType=record_num
     )
 
     response = search.execute()