Merge branch 'develop' into 2839-nginx-dns

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py

@@ -13,6 +13,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -49,7 +50,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),
@@ -177,6 +178,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -213,7 +215,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),
@@ -341,6 +343,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -377,7 +380,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py

@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
@@ -49,7 +50,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py

@@ -7,13 +7,13 @@
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument
 
-
 s1 = RowSchema(
     record_type="T6",
     document=TANF_T6DataSubmissionDocument(),
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -56,7 +56,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid(),
             ],
         ),
@@ -234,6 +234,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -448,6 +449,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py

@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
@@ -49,7 +50,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),

tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py

@@ -7,13 +7,13 @@
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument
 
-
 s1 = RowSchema(
     record_type="T6",
     document=Tribal_TANF_T6DataSubmissionDocument(),
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),
@@ -44,7 +44,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid(),
             ],
         ),
@@ -222,6 +222,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),
@@ -424,6 +425,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),

tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py

@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
@@ -49,7 +50,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06

@@ -1,3 +1,3 @@
-HEADER20204G06   TAN1 D
-T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
+HEADER20214G06   TAN1 D
+T620214000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142

@@ -1,3 +1,3 @@
-HEADER20194G00142TAN1ED
-T620194       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
+HEADER20214G00142TAN1ED
+T620214       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06

@@ -1,3 +1,3 @@
-HEADER20204S06   TAN1 D
-T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S06   TAN1 D
+T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24

@@ -1,3 +1,3 @@
-HEADER20184G24   SSP1 D
-M620184000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
+HEADER20214G24   SSP1 D
+M620214000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24

@@ -1,3 +1,3 @@
-HEADER20184S24   SSP1 D
-M720184101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
+HEADER20214S24   SSP1 D
+M720214101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt

@@ -1,3 +1,3 @@
-HEADER20204S06   TAN1 D
-T720204700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S06   TAN1 D
+T720214700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt

@@ -1,3 +1,3 @@
-HEADER20204S00142TAN1ED
-T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S00142TAN1ED
+T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt

@@ -0,0 +1,3 @@
+HEADER20204S00142TAN1ED
+T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+TRAILER0000001         

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -1027,7 +1027,7 @@ def tanf_section3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section3_file(tanf_section3_file, dfs):
     """Test parsing TANF Section 3 submission."""
-    tanf_section3_file.year = 2021
+    tanf_section3_file.year = 2022
     tanf_section3_file.quarter = 'Q1'
 
     dfs.datafile = tanf_section3_file
@@ -1056,9 +1056,9 @@ def test_parse_tanf_section3_file(tanf_section3_file, dfs):
     second = t6_objs[1]
     third = t6_objs[2]
 
-    assert first.RPT_MONTH_YEAR == 202012
-    assert second.RPT_MONTH_YEAR == 202011
-    assert third.RPT_MONTH_YEAR == 202010
+    assert first.RPT_MONTH_YEAR == 202112
+    assert second.RPT_MONTH_YEAR == 202111
+    assert third.RPT_MONTH_YEAR == 202110
 
     assert first.NUM_APPROVED == 3924
     assert second.NUM_APPROVED == 3977
@@ -1109,7 +1109,7 @@ def tanf_section4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section4_file(tanf_section4_file, dfs):
     """Test parsing TANF Section 4 submission."""
-    tanf_section4_file.year = 2021
+    tanf_section4_file.year = 2022
     tanf_section4_file.quarter = 'Q1'
 
     dfs.datafile = tanf_section4_file
@@ -1137,8 +1137,8 @@ def test_parse_tanf_section4_file(tanf_section4_file, dfs):
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202012
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202112
 
     assert first.TDRS_SECTION_IND == '2'
     assert sixth.TDRS_SECTION_IND == '2'
@@ -1196,7 +1196,7 @@ def ssp_section4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     """Test parsing SSP Section 4 submission."""
-    ssp_section4_file.year = 2019
+    ssp_section4_file.year = 2022
     ssp_section4_file.quarter = 'Q1'
 
     dfs.datafile = ssp_section4_file
@@ -1217,7 +1217,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     assert m7_objs.count() == 12
 
     first = m7_objs.first()
-    assert first.RPT_MONTH_YEAR == 201810
+    assert first.RPT_MONTH_YEAR == 202110
     assert first.FAMILIES_MONTH == 748
 
 @pytest.fixture
@@ -1304,7 +1304,7 @@ def ssp_section3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     """Test parsing TANF Section 3 submission."""
-    ssp_section3_file.year = 2019
+    ssp_section3_file.year = 2022
     ssp_section3_file.quarter = 'Q1'
 
     dfs.datafile = ssp_section3_file
@@ -1332,9 +1332,9 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     second = m6_objs[1]
     third = m6_objs[2]
 
-    assert first.RPT_MONTH_YEAR == 201810
-    assert second.RPT_MONTH_YEAR == 201811
-    assert third.RPT_MONTH_YEAR == 201812
+    assert first.RPT_MONTH_YEAR == 202110
+    assert second.RPT_MONTH_YEAR == 202111
+    assert third.RPT_MONTH_YEAR == 202112
 
     assert first.SSPMOE_FAMILIES == 15869
     assert second.SSPMOE_FAMILIES == 16008
@@ -1487,7 +1487,7 @@ def tribal_section_3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_3_file(tribal_section_3_file, dfs):
     """Test parsing Tribal TANF Section 3 submission."""
-    tribal_section_3_file.year = 2020
+    tribal_section_3_file.year = 2022
     tribal_section_3_file.quarter = 'Q1'
 
     dfs.datafile = tribal_section_3_file
@@ -1523,7 +1523,7 @@ def tribal_section_4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_file(tribal_section_4_file, dfs):
     """Test parsing Tribal TANF Section 4 submission."""
-    tribal_section_4_file.year = 2021
+    tribal_section_4_file.year = 2022
     tribal_section_4_file.quarter = 'Q1'
 
     dfs.datafile = tribal_section_4_file
@@ -1546,8 +1546,8 @@ def test_parse_tribal_section_4_file(tribal_section_4_file, dfs):
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202012
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202112
 
     assert first.TDRS_SECTION_IND == '2'
     assert sixth.TDRS_SECTION_IND == '2'
@@ -1620,6 +1620,8 @@ def tanf_section_4_file_with_errors(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs):
     """Test parsing TANF Section 4 submission."""
+    tanf_section_4_file_with_errors.year = 2022
+    tanf_section_4_file_with_errors.quarter = 'Q1'
     dfs.datafile = tanf_section_4_file_with_errors
 
     parse.parse_datafile(tanf_section_4_file_with_errors, dfs)
@@ -1638,15 +1640,16 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d
     assert TANF_T7.objects.all().count() == 18
 
     parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors)
+
     assert parser_errors.count() == 6
 
     t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH')
 
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202010
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202110
 
     assert first.TDRS_SECTION_IND == '1'
     assert sixth.TDRS_SECTION_IND == '1'
@@ -1701,3 +1704,25 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file
 
     assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE
     assert error.error_message == "HEADER update indicator: U does not match D."
+
+@pytest.fixture
+def tribal_section_4_bad_quarter(stt_user, stt):
+    """Fixture for tribal_section_4_bad_quarter."""
+    return util.create_test_datafile('tribal_section_4_fake_bad_quarter.txt', stt_user, stt, "Tribal Stratum Data")
+
+@pytest.mark.django_db()
+def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
+    """Test handling invalid quarter value that raises a ValueError exception."""
+    tribal_section_4_bad_quarter.year = 2021
+    tribal_section_4_bad_quarter.quarter = 'Q1'
+    dfs.datafile = tribal_section_4_bad_quarter
+
+    parse.parse_datafile(tribal_section_4_bad_quarter, dfs)
+    parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter).order_by('id')
+
+    assert parser_errors.count() == 3
+
+    parser_errors.first().error_message == "T7: 2020  is invalid. Calendar Quarter must be a numeric" + \
+        "representing the Calendar Year and Quarter formatted as YYYYQ"
+
+    Tribal_TANF_T7.objects.count() == 0