Reparse Memory Management

tdrs-backend/tdpservice/parsers/validators/category3.py

@@ -386,7 +386,7 @@ def validate(record, row_schema):
                 "Caught exception in validator: validate__WORK_ELIGIBLE_INDICATOR__HOH__AGE. " +
                 f"With field values: {vals}."
             )
-            logger.error(f'Exception: {e}')
+            logger.debug(f'Exception: {e}')
             # Per conversation with Alex on 03/26/2024, returning the true case during exception handling to avoid
             # confusing the STTs.
             return true_case

tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py

@@ -2,6 +2,7 @@
 
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
+from django.core.paginator import Paginator
 from django.db.utils import DatabaseError
 from elasticsearch.exceptions import ElasticsearchException
 from tdpservice.data_files.models import DataFile
@@ -61,6 +62,7 @@ def __handle_elastic(self, new_indices, log_context):
         """Create new Elastic indices and delete old ones."""
         if new_indices:
             try:
+                logger.info("Creating new elastic indexes.")
                 call_command('tdp_search_index', '--create', '-f', '--use-alias')
                 log("Index creation complete.",
                     logger_context=log_context,
@@ -82,7 +84,10 @@ def __delete_summaries(self, file_ids, log_context):
         """Raw delete all DataFileSummary objects."""
         try:
             qset = DataFileSummary.objects.filter(datafile_id__in=file_ids)
+            count = qset.count()
+            logger.info(f"Deleting {count} datafile summary objects.")
             qset._raw_delete(qset.db)
+            logger.info("Successfully deleted datafile summary objects.")
         except DatabaseError as e:
             log('Encountered a DatabaseError while deleting DataFileSummary from Postgres. The database '
                 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!',
@@ -102,11 +107,16 @@ def __delete_records(self, file_ids, new_indices, log_context):
         for doc in DOCUMENTS:
             try:
                 model = doc.Django.model
-                qset = model.objects.filter(datafile_id__in=file_ids)
-                total_deleted += qset.count()
+                qset = model.objects.filter(datafile_id__in=file_ids).order_by('id')
+                count = qset.count()
+                total_deleted += count
+                logger.info(f"Deleting {count} records of type: {model}.")
                 if not new_indices:
                     # If we aren't creating new indices, then we don't want duplicate data in the existing indices.
-                    doc().update(qset, refresh=True, action='delete')
+                    paginator = Paginator(qset, settings.BULK_CREATE_BATCH_SIZE)
+                    document = doc()
+                    for page in paginator:
+                        document.update(page.object_list, action='delete')
                 qset._raw_delete(qset.db)
             except ElasticsearchException as e:
                 log(f'Elastic document delete failed for type {model}. The database and Elastic are INCONSISTENT! '
@@ -132,7 +142,10 @@ def __delete_errors(self, file_ids, log_context):
         """Raw delete all ParserErrors for each file ID."""
         try:
             qset = ParserError.objects.filter(file_id__in=file_ids)
+            count = qset.count()
+            logger.info(f"Deleting {count} parser errors.")
             qset._raw_delete(qset.db)
+            logger.info("Successfully deleted parser errors.")
         except DatabaseError as e:
             log('Encountered a DatabaseError while deleting ParserErrors from Postgres. The database '
                 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!',