3106 reparse django action (#3167)

* initial commit - added filter * 3076 added test * linting * move import to top * lint * 3076 correction * 3076 Fixed today filter * linting * 3076 applied a11y review changes * 3076 fix failing test * 3106 first draft * clean up * clean up * clean admin * clean admin * parsing command * delete unwanted imports * added modal form * added styling to confirmation page * changes * 3106 middle page works * 3106 can add files for reparsing * 3106 Correct linting * bypassing status check to get this deployed for testing * injected working js * remove some of the comments * revert changes * linting and corrections * lint * redirect to meta model * 3106 replying comments * move imports to the top * Update admin.py: renamed reparse command * 3106 removed unused function * 3106 fixed reparsing command issues * resolve merge conflict errors * move admin action to celery task * rm print * rm prints * lint * rm unused * unused imports * script styling * changed log message * change log for action --------- Co-authored-by: andrew-jameson <[email protected]> Co-authored-by: Andrew <[email protected]> Co-authored-by: Jan Timpe <[email protected]> Co-authored-by: Alex P. <[email protected]>
raft-tech · Oct 18, 2024 · 702f72f · 702f72f
1 parent 7811e8d
commit 702f72f
Show file tree

Hide file tree

Showing 4 changed files with 128 additions and 25 deletions.
diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py
@@ -7,6 +7,10 @@
 from django.conf import settings
 from django.utils.html import format_html
 from datetime import datetime, timedelta, timezone
+from django.shortcuts import redirect
+from django.utils.translation import ngettext
+from django.contrib import messages
+from tdpservice.data_files.tasks import reparse_files
 
 DOMAIN = settings.FRONTEND_BASE_URL
 
@@ -22,11 +26,44 @@ def has_change_permission(self, request, obj=None):
         """Read only permissions."""
         return False
 
-
 @admin.register(DataFile)
 class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
     """Admin class for DataFile models."""
 
+    class Media:
+        """Media class for DataFileAdmin."""
+
+        js = ('admin/js/admin/admin_datafile_model.js',)
+
+    actions = ['reparse']
+
+    def reparse(self, request, queryset):
+        """Reparse the selected data files."""
+        files = queryset.values_list("id", flat=True)
+        reparse_files.delay(list(files))
+
+        self.message_user(
+            request,
+            ngettext(
+                "%d file successfully submitted for reparsing.",
+                "%d files successfully submitted for reparsing.",
+                files.count(),
+            )
+            % files.count(),
+            messages.SUCCESS,
+        )
+        return redirect("/admin/search_indexes/reparsemeta/")
+
+    def get_actions(self, request):
+        """Return the actions."""
+        actions = super().get_actions(request)
+        if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists():
+            actions.pop("reparse", None)
+        else:
+            if "reparse" not in actions:
+                actions["reparse"] = (self.reparse, "reparse", "Reparse selected data files)")
+        return actions
+
     def status(self, obj):
         """Return the status of the data file summary."""
         return DataFileSummary.objects.get(datafile=obj).status

diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js
@@ -0,0 +1,30 @@
+$(window).on('load', function() {
+    console.log('loaded');
+    var submitBtn=document.querySelector('button[type=submit]');    // add the first listener
+    var theForm = submitBtn.parentNode.parentNode;
+
+    for (var i = 0; i < theForm.childNodes.length; i++) {
+        if (theForm.childNodes[i].className == "actions") {
+          form_header = theForm.childNodes[i];
+
+          for (var i = 0; i < form_header.childNodes.length; i++) {
+              if (form_header.childNodes[i].className == "action-counter") {
+                number_of_files = form_header.childNodes[i];
+                break;
+              }
+            }
+
+          break;
+        }
+    }
+    submitBtn.addEventListener('click', function(e) {
+        e.preventDefault();
+        if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) {
+            console.log('submitting');
+            theForm.submit();
+        } else {
+            console.log('not submitting');
+        };
+    });
+
+});
diff --git a/tdrs-backend/tdpservice/data_files/tasks.py b/tdrs-backend/tdpservice/data_files/tasks.py
@@ -4,6 +4,7 @@
 from datetime import timedelta
 from django.utils import timezone
 from django.contrib.auth.models import Group
+from django.core.management import call_command
 from django.db.models import Q, Count
 from tdpservice.users.models import AccountApprovalStatusChoices, User
 from tdpservice.data_files.models import DataFile
@@ -46,3 +47,10 @@ def notify_stuck_files():
         ).values_list('username', flat=True).distinct()
 
         send_stuck_file_email(stuck_files, recipients)
+
+
+@shared_task
+def reparse_files(file_ids):
+    """Call the clean_and_reparse management command with a list of file ids."""
+    file_ids_str = ",".join(map(str, file_ids))
+    call_command("clean_and_reparse", f"-f {file_ids_str}")
diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py
@@ -33,6 +33,7 @@ def add_arguments(self, parser):
         parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.")
         parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, "
                             "fiscal_year/quarter aren't necessary.")
+        parser.add_argument("-f", "--files", nargs='+', type=str, help="Re-parse specific datafiles by datafile id")
 
     def _get_log_context(self, system_user):
         """Return logger context."""
@@ -263,32 +264,20 @@ def _handle_input(self, testing, continue_msg):
                 print('Cancelled.')
                 exit(0)
 
-    def handle(self, *args, **options):
-        """Delete and reparse datafiles matching a query."""
-        fiscal_year = options.get('fiscal_year', None)
-        fiscal_quarter = options.get('fiscal_quarter', None)
-        reparse_all = options.get('all', False)
-        new_indices = reparse_all is True
-
-        # Option that can only be specified by calling `handle` directly and passing it.
-        testing = options.get('testing', False)
-        ##
-
-        args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all
-
-        if not args_passed:
-            logger.warn("No arguments supplied.")
-            self.print_help("manage.py", "clean_and_parse")
-            return
-
+    def get_files_to_reparse(case, fiscal_year, fiscal_quarter, selected_files, reparse_all):
+        """Get the files to reparse."""
         backup_file_name = "/tmp/reparsing_backup"
         files = DataFile.objects.all()
         continue_msg = "You have selected to reparse datafiles for FY {fy} and {q}. The reparsed files "
+        if selected_files:
+            files = files.filter(id__in=selected_files)
+            backup_file_name += "_selected_files"
+            continue_msg = continue_msg.format(fy=f"selected files: {str(selected_files)}", q="Q1-4")
         if reparse_all:
             backup_file_name += "_FY_All_Q1-4"
             continue_msg = continue_msg.format(fy="All", q="Q1-4")
         else:
-            if not fiscal_year and not fiscal_quarter:
+            if not fiscal_year and not fiscal_quarter and not selected_files:
                 print(
                     'Options --fiscal_year and --fiscal_quarter not set. '
                     'Provide either option to continue, or --all to wipe all submissions.'
@@ -306,6 +295,38 @@ def handle(self, *args, **options):
                 files = files.filter(quarter=fiscal_quarter)
                 backup_file_name += f"_FY_All_{fiscal_quarter}"
                 continue_msg = continue_msg.format(fy="All", q=fiscal_quarter)
+        return files, backup_file_name, continue_msg
+
+    def handle(self, *args, **options):
+        """Delete and reparse datafiles matching a query."""
+        fiscal_year = options.get('fiscal_year', None)
+        fiscal_quarter = options.get('fiscal_quarter', None)
+        reparse_all = options.get('all', False)
+        print(f'************** reparse all {reparse_all}')
+        selected_files = options.get('files', None)
+        selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None
+        print(f'************** selected files {selected_files}')
+        new_indices = reparse_all is True
+
+        # Option that can only be specified by calling `handle` directly and passing it.
+        testing = options.get('testing', False)
+        ##
+
+        args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files
+
+        if not args_passed:
+            logger.warning("No arguments supplied.")
+            self.print_help("manage.py", "clean_and_parse")
+            return
+
+        # Set up the backup file name and continue message
+        files, backup_file_name, continue_msg = self.get_files_to_reparse(
+            fiscal_year,
+            fiscal_quarter,
+            selected_files,
+            reparse_all)
+
+        # end of the if statement
 
         fmt_str = "be" if new_indices else "NOT be"
         continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str)
@@ -314,7 +335,8 @@ def handle(self, *args, **options):
         fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})"
         continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str)
 
-        self._handle_input(testing, continue_msg)
+        if not selected_files:
+            self._handle_input(testing, continue_msg)
 
         system_user, created = User.objects.get_or_create(username='system')
         if created:
@@ -323,10 +345,16 @@ def handle(self, *args, **options):
 
         all_fy = "All"
         all_q = "Q1-4"
-        log(f"Starting clean and reparse command for FY {fiscal_year if fiscal_year else all_fy} and "
-            f"{fiscal_quarter if fiscal_quarter else all_q}",
-            logger_context=log_context,
-            level='info')
+
+        if not selected_files:
+            log(f"Starting clean and reparse command for FY {fiscal_year if fiscal_year else all_fy} and "
+                f"{fiscal_quarter if fiscal_quarter else all_q}",
+                logger_context=log_context,
+                level='info')
+        else:
+            log(f"Starting clean and reparse action for files: {str(selected_files)}",
+                logger_context=log_context,
+                level='info')
 
         if num_files == 0:
             log(f"No files available for the selected Fiscal Year: {fiscal_year if fiscal_year else all_fy} and "