resolving merge conflict problems

tdrs-backend/tdpservice/parsers/models.py

@@ -91,6 +91,7 @@ class Status(models.TextChoices):
     def get_status(self):
         """Set and return the status field based on errors and models associated with datafile."""
         errors = ParserError.objects.filter(file=self.datafile)
+        [print(error) for error in errors]
 
         # excluding row-level pre-checks and trailer pre-checks.
         precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\

tdrs-backend/tdpservice/parsers/parse.py

@@ -35,7 +35,8 @@ def parse_datafile(datafile):
 
     section_is_valid, section_error = validators.validate_header_section_matches_submission(
         datafile,
-        util.get_section_reference(program_type, section)
+        util.get_section_reference(program_type, section),
+        util.make_generate_parser_error(datafile, 1)
     )
 
     if not section_is_valid:

tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt

@@ -1,12 +1,12 @@
 HEADER20204A06   TAN1EN
 T12020101111111111223003403361110213120000300000000000008730010000000000000000000000000000000000222222000000002229012                                       
-T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011400000000000000000000000000000000000000000000000000000000000000000000000000000000000291
+T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011500000000000000000000000000000000000000000000000000000000000000000000000000000000000291
 T320201011111111112120190127WTTTT90W022212222204398100000000                                                                                                
 T12020101111111111524503401311110233110374300000000000005450320000000000000000000000000000000000222222000000002229021                                       
 T2202010111111111152219730113WTTTT@#Z@2221222122211012210110630023080700000000000000000000000000000000000000000000000000000000000000000000000551019700000000
 T320201011111111115120160401WTTTT@BTB22212212204398100000000                                                                                                
 T12020101111111114023001401101120213110336300000000000002910410000000000000000000000000000000000222222000000002229012                                       
-T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011500000000000000000000000000000000000000000000000000000000000000000000000000000000000000
 T320201011111111140120170423WTTTT@@T#22212222204398100000000                                                                                                
 T12020101111111114721801401711120212110374300000000000003820060000000000000000000000000000000000222222000000002229012                                       
 T2202010111111111471219800223WTTTT@TTW2222212222221012212110065423010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -32,7 +32,7 @@ def test_parse_small_correct_file(test_datafile, dfs):
     dfs.datafile = test_datafile
     dfs.save()
 
-    errors = parse.parse_datafile(test_datafile)
+    parse.parse_datafile(test_datafile)
     dfs.status = dfs.get_status()
     dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status)
     assert dfs.case_aggregates == {'rejected': 0,
@@ -42,7 +42,6 @@ def test_parse_small_correct_file(test_datafile, dfs):
                                        {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'}
                                     ]}
 
-
     assert dfs.get_status() == DataFileSummary.Status.ACCEPTED
 
     assert TANF_T1.objects.count() == 1
@@ -132,7 +131,7 @@ def test_big_file(stt_user, stt):
     return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt)
 
 @pytest.mark.django_db
-@pytest.mark.big_files
+@pytest.mark.skip(reason="long runtime")  # big_files
 def test_parse_big_file(test_big_file, dfs):
     """Test parsing of ADS.E2J.FTP1.TS06."""
     expected_t1_record_count = 815
@@ -142,7 +141,7 @@ def test_parse_big_file(test_big_file, dfs):
     dfs.datafile = test_big_file
     dfs.save()
 
-    errors = parse.parse_datafile(test_big_file)
+    parse.parse_datafile(test_big_file)
     dfs.status = dfs.get_status()
     assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS
     dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status)
@@ -153,7 +152,6 @@ def test_parse_big_file(test_big_file, dfs):
                                        {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'}
                                     ]}
 
-
     parser_errors = ParserError.objects.filter(file=test_big_file)
 
     error_message = 'MONTHS_FED_TIME_LIMIT is required but a value was not provided.'
@@ -384,7 +382,6 @@ def test_parse_empty_file(empty_file, dfs):
                                         'month': 'Dec'}
                                     ]}
 
-
     parser_errors = ParserError.objects.filter(file=empty_file).order_by('id')
 
     assert parser_errors.count() == 2
@@ -492,7 +489,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs):
     dfs.datafile = small_tanf_section1_datafile
     dfs.save()
 
-    errors = parse.parse_datafile(small_tanf_section1_datafile)
+    parse.parse_datafile(small_tanf_section1_datafile)
 
     dfs.status = dfs.get_status()
     assert dfs.status == DataFileSummary.Status.ACCEPTED
@@ -573,7 +570,7 @@ def super_big_s1_rollback_file(stt_user, stt):
     return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt)
 
 @pytest.mark.django_db()
-@pytest.mark.big_files
+@pytest.mark.skip(reason="cuz")  # big_files
 def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file):
     """Test parsing of super_big_s1_rollback_file.
 
@@ -609,7 +606,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field
     dfs.datafile = bad_tanf_s1__row_missing_required_field
     dfs.save()
 
-    errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field)
+    parse.parse_datafile(bad_tanf_s1__row_missing_required_field)
 
     assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS
 
@@ -806,7 +803,7 @@ def test_parse_tanf_section2_file(tanf_section2_file):
 @pytest.fixture
 def tanf_section3_file(stt_user, stt):
     """Fixture for ADS.E2J.FTP3.TS06."""
-    return create_test_datafile('ADS.E2J.FTP3.TS06', stt_user, stt, "Aggregate Data")
+    return util.create_test_datafile('ADS.E2J.FTP3.TS06', stt_user, stt, "Aggregate Data")
 
 @pytest.mark.django_db()
 def test_parse_tanf_section3_file(tanf_section3_file):

tdrs-backend/tdpservice/parsers/util.py

@@ -5,9 +5,7 @@
 from tdpservice.data_files.models import DataFile
 from datetime import datetime
 from pathlib import Path
-from .fields import EncryptedField
 from .fields import TransformField
-from datetime import datetime
 import logging
 
 logger = logging.getLogger(__name__)
@@ -65,6 +63,7 @@ def generate(schema, error_category, error_message, record=None, field=None):
 
     return generate
 
+
 class SchemaManager:
     """Manages one or more RowSchema's and runs all parsers and validators."""
 

tdrs-backend/tdpservice/parsers/validators.py

@@ -1,8 +1,6 @@
 """Generic parser validator functions for use in schema definitions."""
 
-from .util import generate_parser_error
 from .models import ParserErrorCategoryChoices
-from tdpservice.data_files.models import DataFile
 from datetime import date
 
 # higher order validator func
@@ -348,61 +346,14 @@ def validate(instance):
             return (True, None)
     return lambda instance: validate(instance)
 
-def validate_single_header_trailer(datafile):
-    """Validate that a raw datafile has one trailer and one footer."""
-    line_number = 0
-    headers = 0
-    trailers = 0
-    is_valid = True
-    error_message = None
-
-    for rawline in datafile.file:
-        line = rawline.decode()
-        line_number += 1
-
-        if line.startswith('HEADER'):
-            headers += 1
-        elif line.startswith('TRAILER'):
-            trailers += 1
-
-        if headers > 1:
-            is_valid = False
-            error_message = 'Multiple headers found.'
-            break
-
-        if trailers > 1:
-            is_valid = False
-            error_message = 'Multiple trailers found.'
-            break
-
-    if headers == 0:
-        is_valid = False
-        error_message = 'No headers found.'
 
-    error = None
-    if not is_valid:
-        error = generate_parser_error(
-            datafile=datafile,
-            line_number=line_number,
-            schema=None,
-            error_category=ParserErrorCategoryChoices.PRE_CHECK,
-            error_message=error_message,
-            record=None,
-            field=None
-        )
-
-    return is_valid, error
-
-
-def validate_header_section_matches_submission(datafile, section):
+def validate_header_section_matches_submission(datafile, section, generate_error):
     """Validate header section matches submission section."""
     is_valid = datafile.section == section
 
     error = None
     if not is_valid:
-        error = generate_parser_error(
-            datafile=datafile,
-            line_number=1,
+        error = generate_error(
             schema=None,
             error_category=ParserErrorCategoryChoices.PRE_CHECK,
             error_message=f"Data does not match the expected layout for {datafile.section}.",