lint

tdrs-backend/tdpservice/data_files/models.py

@@ -81,12 +81,17 @@ class Meta:
 
 class ReparseFileMeta(models.Model):
     """Meta data model representing a single file parse within a reparse execution."""
+
     data_file = models.ForeignKey('data_files.DataFile', on_delete=models.CASCADE, related_name='reparse_file_metas')
-    reparse_meta = models.ForeignKey('search_indexes.ReparseMeta', on_delete=models.CASCADE, related_name='reparse_file_metas')
+    reparse_meta = models.ForeignKey(
+        'search_indexes.ReparseMeta',
+        on_delete=models.CASCADE,
+        related_name='reparse_file_metas'
+    )
 
     finished = models.BooleanField(default=False)
     success = models.BooleanField(default=False)
-    started_at = models.DateTimeField(auto_now_add=False, null=True)  # set at beg of parse run
+    started_at = models.DateTimeField(auto_now_add=False, null=True)
     finished_at = models.DateTimeField(auto_now_add=False, null=True)
 
     num_records_created = models.PositiveIntegerField(default=0)

tdrs-backend/tdpservice/parsers/parse.py

@@ -13,7 +13,6 @@
 from tdpservice.parsers.schema_defs.utils import get_section_reference, get_program_model
 from tdpservice.parsers.case_consistency_validator import CaseConsistencyValidator
 from tdpservice.parsers.util import log_parser_exception
-from tdpservice.search_indexes.models.reparse_meta import ReparseMeta
 
 logger = logging.getLogger(__name__)
 
@@ -474,7 +473,6 @@ def parse_datafile_lines(datafile, dfs, program_type, section, is_encrypted, cas
                  f"validated {case_consistency_validator.total_cases_validated} of them.")
     dfs.save()
 
-
     return errors
 
 

tdrs-backend/tdpservice/search_indexes/admin/reparse_meta.py

@@ -9,10 +9,12 @@ class ReparseMetaAdmin(ReadOnlyAdminMixin):
     inlines = [DataFileInline]
 
     def reparse_is_finished(self, instance):
+        """Overload instance property for ui checkboxes."""
         return instance.is_finished
     reparse_is_finished.boolean = True
 
     def reparse_is_success(self, instance):
+        """Overload instance property for ui checkboxes."""
         return instance.is_success
     reparse_is_success.boolean = True
 

tdrs-backend/tdpservice/search_indexes/models/reparse_meta.py

@@ -1,9 +1,7 @@
 """Meta data model for tracking reparsed files."""
 
-from django.db import models, transaction
-from django.db.utils import DatabaseError
+from django.db import models
 from django.db.models import Max
-from tdpservice.search_indexes.util import count_all_records
 import logging
 
 logger = logging.getLogger(__name__)
@@ -12,6 +10,7 @@
 class ReparseMeta(models.Model):
     """
     Meta data model representing a single execution of `clean_and_reparse`.
+
     Because this model is intended to be queried in a distributed and parrallel fashion, all queries should rely on
     database level locking to ensure race conditions aren't introduced. See `increment_files_reparsed` for an example.
     """
@@ -39,44 +38,52 @@ class Meta:
 
     @property
     def is_finished(self):
+        """Return True if all associated ReparseFileMeta objects are finished."""
         return all([r.finished for r in self.reparse_file_metas.all()])
 
     @property
     def is_success(self):
+        """Return True if all associated ReparseFileMeta objects are successful."""
         return all([r.success for r in self.reparse_file_metas.all()])
 
     @property
     def finished_at(self):
+        """Return the finished_at timestamp of the last ReparseFileMeta object."""
         last_parse = self.reparse_file_metas.order_by('-finished_at').first()
         return last_parse.finished_at if last_parse else None
 
     @property
     def num_files(self):
+        """Return the number of associated ReparseFileMeta objects."""
         return self.reparse_file_metas.count()
 
     @property
     def num_files_completed(self):
+        """Return the number of completed ReparseFileMeta objects."""
         return self.reparse_file_metas.filter(finished=True).count()
 
     @property
     def num_files_succeeded(self):
+        """Return the number of successful ReparseFileMeta objects."""
         return self.reparse_file_metas.filter(finished=True, success=True).count()
 
     @property
     def num_files_failed(self):
+        """Return the number of failed ReparseFileMeta objects."""
         return self.reparse_file_metas.filter(finished=True, success=False).count()
 
     @property
     def num_records_created(self):
+        """Return the sum of records created for all associated ReparseFileMeta objects."""
         return sum([r.num_records_created for r in self.reparse_file_metas.all()])
 
-
     # remove unused statics or change to utils funcs in own app and/or make new cleanup ticket for future
 
     @staticmethod
     def file_counts_match(meta_model):
         """
         Check whether the file counts match.
+
         This function assumes the meta_model has been passed in a distributed/thread safe way. If the database row
         containing this model has not been locked the caller will experience race issues.
         """
@@ -90,6 +97,7 @@ def file_counts_match(meta_model):
     def assert_all_files_done(meta_model):
         """
         Check if all files have been parsed with or without exceptions.
+
         This function assumes the meta_model has been passed in a distributed/thread safe way. If the database row
         containing this model has not been locked the caller will experience race issues.
         """