From 702f72fe8405f33278aefbbd939b61a73d2e7b1f Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:32:19 -0400 Subject: [PATCH] 3106 reparse django action (#3167) * initial commit - added filter * 3076 added test * linting * move import to top * lint * 3076 correction * 3076 Fixed today filter * linting * 3076 applied a11y review changes * 3076 fix failing test * 3106 first draft * clean up * clean up * clean admin * clean admin * parsing command * delete unwanted imports * added modal form * added styling to confirmation page * changes * 3106 middle page works * 3106 can add files for reparsing * 3106 Correct linting * bypassing status check to get this deployed for testing * injected working js * remove some of the comments * revert changes * linting and corrections * lint * redirect to meta model * 3106 replying comments * move imports to the top * Update admin.py: renamed reparse command * 3106 removed unused function * 3106 fixed reparsing command issues * resolve merge conflict errors * move admin action to celery task * rm print * rm prints * lint * rm unused * unused imports * script styling * changed log message * change log for action --------- Co-authored-by: andrew-jameson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: Jan Timpe Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- .../tdpservice/data_files/admin/admin.py | 39 +++++++++- .../admin/js/admin/admin_datafile_model.js | 30 ++++++++ tdrs-backend/tdpservice/data_files/tasks.py | 8 ++ .../management/commands/clean_and_reparse.py | 76 +++++++++++++------ 4 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index eae060c36..2056f68d9 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -7,6 +7,10 @@ from django.conf import settings from django.utils.html import format_html from datetime import datetime, timedelta, timezone +from django.shortcuts import redirect +from django.utils.translation import ngettext +from django.contrib import messages +from tdpservice.data_files.tasks import reparse_files DOMAIN = settings.FRONTEND_BASE_URL @@ -22,11 +26,44 @@ def has_change_permission(self, request, obj=None): """Read only permissions.""" return False - @admin.register(DataFile) class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" + class Media: + """Media class for DataFileAdmin.""" + + js = ('admin/js/admin/admin_datafile_model.js',) + + actions = ['reparse'] + + def reparse(self, request, queryset): + """Reparse the selected data files.""" + files = queryset.values_list("id", flat=True) + reparse_files.delay(list(files)) + + self.message_user( + request, + ngettext( + "%d file successfully submitted for reparsing.", + "%d files successfully submitted for reparsing.", + files.count(), + ) + % files.count(), + messages.SUCCESS, + ) + return redirect("/admin/search_indexes/reparsemeta/") + + def get_actions(self, request): + """Return the actions.""" + actions = super().get_actions(request) + if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists(): + actions.pop("reparse", None) + else: + if "reparse" not in actions: + actions["reparse"] = (self.reparse, "reparse", "Reparse selected data files)") + return actions + def status(self, obj): """Return the status of the data file summary.""" return DataFileSummary.objects.get(datafile=obj).status diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js new file mode 100644 index 000000000..31d52ae20 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js @@ -0,0 +1,30 @@ +$(window).on('load', function() { + console.log('loaded'); + var submitBtn=document.querySelector('button[type=submit]'); // add the first listener + var theForm = submitBtn.parentNode.parentNode; + + for (var i = 0; i < theForm.childNodes.length; i++) { + if (theForm.childNodes[i].className == "actions") { + form_header = theForm.childNodes[i]; + + for (var i = 0; i < form_header.childNodes.length; i++) { + if (form_header.childNodes[i].className == "action-counter") { + number_of_files = form_header.childNodes[i]; + break; + } + } + + break; + } + } + submitBtn.addEventListener('click', function(e) { + e.preventDefault(); + if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) { + console.log('submitting'); + theForm.submit(); + } else { + console.log('not submitting'); + }; + }); + +}); diff --git a/tdrs-backend/tdpservice/data_files/tasks.py b/tdrs-backend/tdpservice/data_files/tasks.py index 0ea5446af..dc480c5a0 100644 --- a/tdrs-backend/tdpservice/data_files/tasks.py +++ b/tdrs-backend/tdpservice/data_files/tasks.py @@ -4,6 +4,7 @@ from datetime import timedelta from django.utils import timezone from django.contrib.auth.models import Group +from django.core.management import call_command from django.db.models import Q, Count from tdpservice.users.models import AccountApprovalStatusChoices, User from tdpservice.data_files.models import DataFile @@ -46,3 +47,10 @@ def notify_stuck_files(): ).values_list('username', flat=True).distinct() send_stuck_file_email(stuck_files, recipients) + + +@shared_task +def reparse_files(file_ids): + """Call the clean_and_reparse management command with a list of file ids.""" + file_ids_str = ",".join(map(str, file_ids)) + call_command("clean_and_reparse", f"-f {file_ids_str}") diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index 48d4cf3fe..0187a5abe 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -33,6 +33,7 @@ def add_arguments(self, parser): parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.") parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, " "fiscal_year/quarter aren't necessary.") + parser.add_argument("-f", "--files", nargs='+', type=str, help="Re-parse specific datafiles by datafile id") def _get_log_context(self, system_user): """Return logger context.""" @@ -263,32 +264,20 @@ def _handle_input(self, testing, continue_msg): print('Cancelled.') exit(0) - def handle(self, *args, **options): - """Delete and reparse datafiles matching a query.""" - fiscal_year = options.get('fiscal_year', None) - fiscal_quarter = options.get('fiscal_quarter', None) - reparse_all = options.get('all', False) - new_indices = reparse_all is True - - # Option that can only be specified by calling `handle` directly and passing it. - testing = options.get('testing', False) - ## - - args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all - - if not args_passed: - logger.warn("No arguments supplied.") - self.print_help("manage.py", "clean_and_parse") - return - + def get_files_to_reparse(case, fiscal_year, fiscal_quarter, selected_files, reparse_all): + """Get the files to reparse.""" backup_file_name = "/tmp/reparsing_backup" files = DataFile.objects.all() continue_msg = "You have selected to reparse datafiles for FY {fy} and {q}. The reparsed files " + if selected_files: + files = files.filter(id__in=selected_files) + backup_file_name += "_selected_files" + continue_msg = continue_msg.format(fy=f"selected files: {str(selected_files)}", q="Q1-4") if reparse_all: backup_file_name += "_FY_All_Q1-4" continue_msg = continue_msg.format(fy="All", q="Q1-4") else: - if not fiscal_year and not fiscal_quarter: + if not fiscal_year and not fiscal_quarter and not selected_files: print( 'Options --fiscal_year and --fiscal_quarter not set. ' 'Provide either option to continue, or --all to wipe all submissions.' @@ -306,6 +295,38 @@ def handle(self, *args, **options): files = files.filter(quarter=fiscal_quarter) backup_file_name += f"_FY_All_{fiscal_quarter}" continue_msg = continue_msg.format(fy="All", q=fiscal_quarter) + return files, backup_file_name, continue_msg + + def handle(self, *args, **options): + """Delete and reparse datafiles matching a query.""" + fiscal_year = options.get('fiscal_year', None) + fiscal_quarter = options.get('fiscal_quarter', None) + reparse_all = options.get('all', False) + print(f'************** reparse all {reparse_all}') + selected_files = options.get('files', None) + selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None + print(f'************** selected files {selected_files}') + new_indices = reparse_all is True + + # Option that can only be specified by calling `handle` directly and passing it. + testing = options.get('testing', False) + ## + + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files + + if not args_passed: + logger.warning("No arguments supplied.") + self.print_help("manage.py", "clean_and_parse") + return + + # Set up the backup file name and continue message + files, backup_file_name, continue_msg = self.get_files_to_reparse( + fiscal_year, + fiscal_quarter, + selected_files, + reparse_all) + + # end of the if statement fmt_str = "be" if new_indices else "NOT be" continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str) @@ -314,7 +335,8 @@ def handle(self, *args, **options): fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})" continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str) - self._handle_input(testing, continue_msg) + if not selected_files: + self._handle_input(testing, continue_msg) system_user, created = User.objects.get_or_create(username='system') if created: @@ -323,10 +345,16 @@ def handle(self, *args, **options): all_fy = "All" all_q = "Q1-4" - log(f"Starting clean and reparse command for FY {fiscal_year if fiscal_year else all_fy} and " - f"{fiscal_quarter if fiscal_quarter else all_q}", - logger_context=log_context, - level='info') + + if not selected_files: + log(f"Starting clean and reparse command for FY {fiscal_year if fiscal_year else all_fy} and " + f"{fiscal_quarter if fiscal_quarter else all_q}", + logger_context=log_context, + level='info') + else: + log(f"Starting clean and reparse action for files: {str(selected_files)}", + logger_context=log_context, + level='info') if num_files == 0: log(f"No files available for the selected Fiscal Year: {fiscal_year if fiscal_year else all_fy} and "