Merge branch 'develop' into 2411-metadata-parsed-datafiles-3

tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py

@@ -14,14 +14,5 @@ class Migration(migrations.Migration):
             model_name='parsererror',
             name='error_type',
             field=models.TextField(choices=[('1', 'File pre-check'), ('2', 'Record value invalid'), ('3', 'Record value consistency'), ('4', 'Case consistency'), ('5', 'Section consistency'), ('6', 'Historical consistency')], max_length=128),
-        ),
-        migrations.CreateModel(
-            name='DataFileSummary',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)),
-                ('case_aggregates', models.JSONField(null=True)),
-                ('datafile', models.ForeignKey(on_delete=models.deletion.CASCADE, to='data_files.datafile')),
-            ],
-        ),
+        )
     ]

tdrs-backend/tdpservice/parsers/migrations/0007_datafilesummary.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.15 on 2023-09-20 15:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0012_datafile_s3_versioning_id'),
+        ('parsers', '0006_auto_20230810_1500'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataFileSummary',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)),
+                ('case_aggregates', models.JSONField(null=True)),
+                ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_files.datafile')),
+            ],
+        ),
+    ]

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -612,6 +612,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field
 
     parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field)
     assert parser_errors.count() == 4
+    [print(parser_error) for parser_error in parser_errors]
 
     error_message = 'RPT_MONTH_YEAR is required but a value was not provided.'
     row_2_error = parser_errors.get(row_number=2, error_message=error_message)
@@ -632,7 +633,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field
     assert row_4_error.content_type.model == 'tanf_t3'
     assert row_4_error.object_id is not None
 
-    error_message = 'Record Type is missing from record.'
+    error_message = 'Unknown Record_Type was found.'
     row_5_error = parser_errors.get(row_number=5, error_message=error_message)
     assert row_5_error.error_type == ParserErrorCategoryChoices.PRE_CHECK
     assert row_5_error.error_message == error_message

tdrs-backend/tdpservice/parsers/util.py

@@ -123,7 +123,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None
             'G': {
                 'section': DataFile.Section.AGGREGATE_DATA,
                 'models': {
-                    # 'T6': schema_defs.tanf.t6,
+                    'T6': schema_defs.tanf.t6,
                 }
             },
             'S': {

tdrs-backend/tdpservice/scheduling/parser_task.py

@@ -19,11 +19,10 @@ def parse(data_file_id):
     # for undetermined amount of time.
     data_file = DataFile.objects.get(id=data_file_id)
 
-    logger.info(f"DataFile parsing started for file {data_file.filename}")
-
+    logger.info(f"DataFile parsing started for file -> {repr(data_file)}")
     dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING)
     errors = parse_datafile(data_file)
     dfs.status = dfs.get_status()
     dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status)
     dfs.save()
-    logger.info(f"DataFile parsing finished with status {dfs.status} and {len(errors)} errors: {errors}")
+    logger.info(f"Parsing finished for file -> {repr(data_file)} with status {dfs.status} and {len(errors)} errors.")