From 0d2980bef86e8447c6e8e9c12afb48b3fe3a039a Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Fri, 6 Oct 2023 19:43:34 +0000 Subject: [PATCH 1/2] feat: removes non-functional retirement partner report scripts FIXES: APER-2936 --- setup.cfg | 2 - tubular/edx_api.py | 26 - .../delete_expired_partner_gdpr_reports.py | 133 --- tubular/scripts/retirement_partner_report.py | 404 --------- tubular/tests/retirement_helpers.py | 13 - tubular/tests/test_delete_expired_reports.py | 246 ------ tubular/tests/test_edx_api.py | 74 -- .../tests/test_retirement_partner_report.py | 809 ------------------ 8 files changed, 1707 deletions(-) delete mode 100755 tubular/scripts/delete_expired_partner_gdpr_reports.py delete mode 100755 tubular/scripts/retirement_partner_report.py delete mode 100644 tubular/tests/test_delete_expired_reports.py delete mode 100644 tubular/tests/test_retirement_partner_report.py diff --git a/setup.cfg b/setup.cfg index ebec2525..8b042fef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ console_scripts = create_private_to_public_pr.py = tubular.scripts.create_private_to_public_pr:create_private_to_public_pr create_tag.py = tubular.scripts.create_tag:create_tag delete-asg.py = tubular.scripts.delete_asg:delete_asg - delete_expired_partner_gdpr_reports.py = tubular.scripts.delete_expired_partner_gdpr_reports:delete_expired_reports drupal_backup_database.py = tubular.scripts.drupal_backup_database:backup_database drupal_clear_varnish.py = tubular.scripts.drupal_clear_varnish:clear_varnish_cache drupal_deploy.py = tubular.scripts.drupal_deploy:deploy @@ -53,7 +52,6 @@ console_scripts = retire_one_learner.py = tubular.scripts.retire_one_learner:retire_learner retirement_archive_and_cleanup.py = tubular.scripts.retirement_archive_and_cleanup:archive_and_cleanup retirement_bulk_status_update.py = tubular.scripts.retirement_bulk_status_update:update_statuses - retirement_partner_report.py = tubular.scripts.retirement_partner_report:generate_report retrieve_latest_base_ami.py = tubular.scripts.retrieve_latest_base_ami:retrieve_latest_base_ami retrieve_base_ami.py = tubular.scripts.retrieve_base_ami:retrieve_base_ami rollback_asg.py = tubular.scripts.rollback_asg:rollback diff --git a/tubular/edx_api.py b/tubular/edx_api.py index 8ff9f612..efab883e 100644 --- a/tubular/edx_api.py +++ b/tubular/edx_api.py @@ -318,32 +318,6 @@ def retirement_lms_retire(self, learner): api_url = self.get_api_url('api/user/v1/accounts/retire') return self._request('POST', api_url, json=data) - @_retry_lms_api() - def retirement_partner_queue(self, learner): - """ - Calls LMS to add the given user to the retirement reporting queue - """ - data = {'username': learner['original_username']} - api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report') - return self._request('PUT', api_url, json=data) - - @_retry_lms_api() - def retirement_partner_report(self): - """ - Retrieves the list of users to create partner reports for and set their status to - processing - """ - api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report') - return self._request('POST', api_url) - - @_retry_lms_api() - def retirement_partner_cleanup(self, usernames): - """ - Removes the given users from the partner reporting queue - """ - api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report_cleanup') - return self._request('POST', api_url, json=usernames) - @_retry_lms_api() def retirement_retire_proctoring_data(self, learner): """ diff --git a/tubular/scripts/delete_expired_partner_gdpr_reports.py b/tubular/scripts/delete_expired_partner_gdpr_reports.py deleted file mode 100755 index 79ee56c2..00000000 --- a/tubular/scripts/delete_expired_partner_gdpr_reports.py +++ /dev/null @@ -1,133 +0,0 @@ -#! /usr/bin/env python3 -""" -Command-line script to delete GDPR partner reports on Google Drive that were created over N days ago. -""" - - -from datetime import datetime, timedelta -from functools import partial -from os import path -import io -import json -import logging -import sys - -import click -import yaml -from pytz import UTC - -# Add top-level module path to sys.path before importing tubular code. -sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) - -from tubular.google_api import DriveApi # pylint: disable=wrong-import-position -from tubular.scripts.helpers import _log, _fail, _fail_exception # pylint: disable=wrong-import-position -from tubular.scripts.retirement_partner_report import REPORTING_FILENAME_PREFIX # pylint: disable=wrong-import-position - -SCRIPT_SHORTNAME = 'delete_expired_reports' -LOG = partial(_log, SCRIPT_SHORTNAME) -FAIL = partial(_fail, SCRIPT_SHORTNAME) -FAIL_EXCEPTION = partial(_fail_exception, SCRIPT_SHORTNAME) - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -# Return codes for various fail cases -ERR_NO_CONFIG = -1 -ERR_BAD_CONFIG = -2 -ERR_NO_SECRETS = -3 -ERR_BAD_SECRETS = -4 -ERR_DELETING_REPORTS = -5 -ERR_BAD_AGE = -6 - - -def _config_or_exit(config_file, google_secrets_file): - """ - Returns the config values from the given file, allows overriding of passed in values. - """ - try: - with io.open(config_file, 'r') as config: - config = yaml.safe_load(config) - - # Check required value - if 'drive_partners_folder' not in config or not config['drive_partners_folder']: - FAIL(ERR_BAD_CONFIG, 'No drive_partners_folder in config, or it is empty!') - - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_BAD_CONFIG, 'Failed to read config file {}'.format(config_file), exc) - - try: - # Just load and parse the file to make sure it's legit JSON before doing - # all of the work to delete old reports. - with open(google_secrets_file, 'r') as secrets_f: - json.load(secrets_f) - - config['google_secrets_file'] = google_secrets_file - return config - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_BAD_SECRETS, 'Failed to read secrets file {}'.format(google_secrets_file), exc) - - -@click.command("delete_expired_reports") -@click.option( - '--config_file', - help='YAML file that contains retirement-related configuration for this environment.' -) -@click.option( - '--google_secrets_file', - help='JSON file with Google service account credentials for deletion purposes.' -) -@click.option( - '--age_in_days', - type=int, - help='Days ago from the current time - before which all GDPR partner reports will be deleted.' -) -@click.option( - '--as_user_account', - is_flag=True, - help=( - 'Whether or not the given secrets file is an OAuth2 JSON token, ' - 'which means that the authentication is done using a ' - 'user account, and NOT a service account.' - ), - show_default=True, -) -def delete_expired_reports( - config_file, google_secrets_file, age_in_days, as_user_account -): - """ - Performs the partner report deletion as needed. - """ - LOG('Starting partner report deletion using config file "{}", Google config "{}", and {} days back'.format( - config_file, google_secrets_file, age_in_days - )) - - if not config_file: - FAIL(ERR_NO_CONFIG, 'No config file passed in.') - - if not google_secrets_file: - FAIL(ERR_NO_SECRETS, 'No secrets file passed in.') - - if age_in_days <= 0: - FAIL(ERR_BAD_AGE, 'age_in_days must be a positive integer.') - - config = _config_or_exit(config_file, google_secrets_file) - - try: - delete_before_dt = datetime.now(UTC) - timedelta(days=age_in_days) - drive = DriveApi( - config['google_secrets_file'], as_user_account=as_user_account - ) - LOG('DriveApi configured') - drive.delete_files_older_than( - config['drive_partners_folder'], - delete_before_dt, - mimetype='text/csv', - prefix="{}_{}".format(REPORTING_FILENAME_PREFIX, config['partner_report_platform_name']) - ) - LOG('Partner report deletion complete') - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_DELETING_REPORTS, 'Unexpected error occurred!', exc) - - -if __name__ == '__main__': - # pylint: disable=unexpected-keyword-arg, no-value-for-parameter - delete_expired_reports(auto_envvar_prefix='RETIREMENT') diff --git a/tubular/scripts/retirement_partner_report.py b/tubular/scripts/retirement_partner_report.py deleted file mode 100755 index 3c1e6fd7..00000000 --- a/tubular/scripts/retirement_partner_report.py +++ /dev/null @@ -1,404 +0,0 @@ -#! /usr/bin/env python3 -# coding=utf-8 - -""" -Command-line script to drive the partner reporting part of the retirement process -""" - -from collections import defaultdict, OrderedDict -from datetime import date -from functools import partial -import logging -import os -import sys -import unicodedata -import unicodecsv as csv - -import click -from six import text_type - -# Add top-level module path to sys.path before importing tubular code. -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from tubular.google_api import DriveApi # pylint: disable=wrong-import-position -# pylint: disable=wrong-import-position -from tubular.scripts.helpers import ( - _config_with_drive_or_exit, - _fail, - _fail_exception, - _log, - _setup_lms_api_or_exit -) - -# Return codes for various fail cases -ERR_SETUP_FAILED = -1 -ERR_FETCHING_LEARNERS = -2 -ERR_NO_CONFIG = -3 -ERR_NO_SECRETS = -4 -ERR_NO_OUTPUT_DIR = -5 -ERR_BAD_CONFIG = -6 -ERR_BAD_SECRETS = -7 -ERR_UNKNOWN_ORG = -8 -ERR_REPORTING = -9 -ERR_DRIVE_UPLOAD = -10 -ERR_CLEANUP = -11 -ERR_DRIVE_LISTING = -12 - -SCRIPT_SHORTNAME = 'Partner report' -LOG = partial(_log, SCRIPT_SHORTNAME) -FAIL = partial(_fail, SCRIPT_SHORTNAME) -FAIL_EXCEPTION = partial(_fail_exception, SCRIPT_SHORTNAME) -CONFIG_WITH_DRIVE_OR_EXIT = partial(_config_with_drive_or_exit, FAIL_EXCEPTION, ERR_BAD_CONFIG, ERR_BAD_SECRETS) -SETUP_LMS_OR_EXIT = partial(_setup_lms_api_or_exit, FAIL, ERR_SETUP_FAILED) - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - -# Prefix which starts all generated report filenames. -REPORTING_FILENAME_PREFIX = 'user_retirement' - -# We'll store the access token here once retrieved -AUTH_HEADER = {} - -# This text template will be the comment body for all new CSV uploads. The -# following format variables need to be provided: -# tags: space delimited list of google user tags, e.g. "+user1@gmail.com +user2@gmail.com" -NOTIFICATION_MESSAGE_TEMPLATE = """ -Hello from edX. Dear {tags}, a new report listing the learners enrolled in your institution’s courses on edx.org that have requested deletion of their edX account and associated personal data within the last week has been published to Google Drive. Please access your folder to see the latest report. -""".strip() - -LEARNER_CREATED_KEY = 'created' # This key is currently required to exist in the learner -LEARNER_ORIGINAL_USERNAME_KEY = 'original_username' # This key is currently required to exist in the learner -ORGS_KEY = 'orgs' -ORGS_CONFIG_KEY = 'orgs_config' -ORGS_CONFIG_ORG_KEY = 'org' -ORGS_CONFIG_FIELD_HEADINGS_KEY = 'field_headings' -ORGS_CONFIG_LEARNERS_KEY = 'learners' - -# Default field headings for the CSV file -DEFAULT_FIELD_HEADINGS = ['user_id', 'original_username', 'original_email', 'original_name', 'deletion_completed'] - - -def _check_all_learner_orgs_or_exit(config, learners): - """ - Checks all learners and their orgs, ensuring that each org has a mapping to a partner Drive folder. - If any orgs are missing a mapping, fails after printing the mismatched orgs. - """ - # Loop through all learner orgs, checking for their mappings. - mismatched_orgs = set() - for learner in learners: - # Check the orgs with standard fields - if ORGS_KEY in learner: - for org in learner[ORGS_KEY]: - if org not in config['org_partner_mapping']: - mismatched_orgs.add(org) - - # Check the orgs with custom configurations (orgs with custom fields) - if ORGS_CONFIG_KEY in learner: - for org_config in learner[ORGS_CONFIG_KEY]: - org_name = org_config[ORGS_CONFIG_ORG_KEY] - if org_name not in config['org_partner_mapping']: - mismatched_orgs.add(org_name) - if mismatched_orgs: - FAIL( - ERR_UNKNOWN_ORG, - 'Partners for organizations {} do not exist in configuration.'.format(text_type(mismatched_orgs)) - ) - - -def _get_orgs_and_learners_or_exit(config): - """ - Contacts LMS to get the list of learners to report on and the orgs they belong to. - Reformats them into dicts with keys of the orgs and lists of learners as the value - and returns a tuple of that dict plus a list of all of the learner usernames. - """ - try: - LOG('Retrieving all learners on which to report from the LMS.') - learners = config['LMS'].retirement_partner_report() - LOG('Retrieved {} learners from the LMS.'.format(len(learners))) - - _check_all_learner_orgs_or_exit(config, learners) - - orgs = defaultdict() - usernames = [] - - # Organize the learners, create separate dicts per partner, making sure each partner is in the mapping. - # Learners can appear in more than one dict. It is assumed that each org has 1 and only 1 set of field headings. - for learner in learners: - usernames.append({'original_username': learner[LEARNER_ORIGINAL_USERNAME_KEY]}) - - # Use the datetime upon which the record was 'created' in the partner reporting queue - # as the approximate time upon which user retirement was completed ('deletion_completed') - # for the record's user. - learner['deletion_completed'] = learner[LEARNER_CREATED_KEY] - - # Create a list of orgs who should be notified about this user - if ORGS_KEY in learner: - for org_name in learner[ORGS_KEY]: - reporting_org_names = config['org_partner_mapping'][org_name] - _add_reporting_org(orgs, reporting_org_names, DEFAULT_FIELD_HEADINGS, learner) - - # Check for orgs with custom fields - if ORGS_CONFIG_KEY in learner: - for org_config in learner[ORGS_CONFIG_KEY]: - org_name = org_config[ORGS_CONFIG_ORG_KEY] - org_headings = org_config[ORGS_CONFIG_FIELD_HEADINGS_KEY] - reporting_org_names = config['org_partner_mapping'][org_name] - _add_reporting_org(orgs, reporting_org_names, org_headings, learner) - - return orgs, usernames - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_FETCHING_LEARNERS, 'Unexpected exception occurred!', exc) - - -def _add_reporting_org(orgs, org_names, org_headings, learner): - """ - Add the learner to the org - """ - for org_name in org_names: - # Create the org, if necessary - orgs[org_name] = orgs.get( - org_name, - { - ORGS_CONFIG_FIELD_HEADINGS_KEY: org_headings, - ORGS_CONFIG_LEARNERS_KEY: [] - } - ) - - # Add the learner to the list of learners in the org - orgs[org_name][ORGS_CONFIG_LEARNERS_KEY].append(learner) - - -def _generate_report_files_or_exit(config, report_data, output_dir): - """ - Spins through the partners, creating a single CSV file for each - """ - # We'll store all of the partner to file links here so we can be sure all files generated successfully - # before trying to push to Google, minimizing the cases where we might have to overwrite files - # already up there. - partner_filenames = {} - - for partner_name in report_data: - try: - partner = report_data[partner_name] - partner_headings = partner[ORGS_CONFIG_FIELD_HEADINGS_KEY] - partner_learners = partner[ORGS_CONFIG_LEARNERS_KEY] - outfile = _generate_report_file_or_exit(config, output_dir, partner_name, partner_headings, - partner_learners) - partner_filenames[partner_name] = outfile - LOG('Report complete for partner {}'.format(partner_name)) - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_REPORTING, 'Error reporting retirement for partner {}'.format(partner_name), exc) - - return partner_filenames - - -def _generate_report_file_or_exit(config, output_dir, partner, field_headings, field_values): - """ - Create a CSV file for the partner - """ - LOG('Starting report for partner {}: {} learners to add. Field headings are {}'.format( - partner, - len(field_values), - field_headings - )) - - outfile = os.path.join(output_dir, '{}_{}_{}_{}.csv'.format( - REPORTING_FILENAME_PREFIX, config['partner_report_platform_name'], partner, date.today().isoformat() - )) - - # If there is already a file for this date, assume it is bad and replace it - try: - os.remove(outfile) - except OSError: - pass - - with open(outfile, 'wb') as f: - writer = csv.DictWriter(f, field_headings, dialect=csv.excel, extrasaction='ignore') - writer.writeheader() - writer.writerows(field_values) - - return outfile - - -def _config_drive_folder_map_or_exit(config): - """ - Lists folders under our top level parent for this environment and returns - a dict of {partner name: folder id}. Partner names should match the values - in config['org_partner_mapping'] - """ - drive = DriveApi(config['google_secrets_file']) - - try: - LOG('Attempting to find all partner sub-directories on Drive.') - folders = drive.walk_files( - config['drive_partners_folder'], - mimetype='application/vnd.google-apps.folder', - recurse=False - ) - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_DRIVE_LISTING, 'Finding partner directories on Drive failed.', exc) - - if not folders: - FAIL(ERR_DRIVE_LISTING, 'Finding partner directories on Drive failed. Check your permissions.') - - # As in _config_or_exit we force normalize the unicode here to make sure the keys - # match. Otherwise the name we get back from Google won't match what's in the YAML config. - config['partner_folder_mapping'] = OrderedDict() - for folder in folders: - folder['name'] = unicodedata.normalize('NFKC', text_type(folder['name'])) - config['partner_folder_mapping'][folder['name']] = folder['id'] - - -def _push_files_to_google(config, partner_filenames): - """ - Copy the file to Google drive for this partner - - Returns: - List of file IDs for the uploaded csv files. - """ - # First make sure we have Drive folders for all partners - failed_partners = [] - for partner in partner_filenames: - if partner not in config['partner_folder_mapping']: - failed_partners.append(partner) - - if failed_partners: - FAIL(ERR_BAD_CONFIG, 'These partners have retiring learners, but no Drive folder: {}'.format(failed_partners)) - - file_ids = {} - drive = DriveApi(config['google_secrets_file']) - for partner in partner_filenames: - # This is populated on the fly in _config_drive_folder_map_or_exit - folder_id = config['partner_folder_mapping'][partner] - file_id = None - with open(partner_filenames[partner], 'rb') as f: - try: - drive_filename = os.path.basename(partner_filenames[partner]) - LOG('Attempting to upload {} to {} Drive folder.'.format(drive_filename, partner)) - file_id = drive.create_file_in_folder(folder_id, drive_filename, f, "text/csv") - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_DRIVE_UPLOAD, 'Drive upload failed for: {}'.format(drive_filename), exc) - file_ids[partner] = file_id - return file_ids - - -def _add_comments_to_files(config, file_ids): - """ - Add comments to the uploaded csv files, triggering email notification. - - Args: - file_ids (dict): Mapping of partner names to Drive file IDs corresponding to the newly uploaded csv files. - """ - drive = DriveApi(config['google_secrets_file']) - - partner_folders_to_permissions = drive.list_permissions_for_files( - config['partner_folder_mapping'].values(), - fields='emailAddress', - ) - - # create a mapping of partners to a list of permissions dicts: - permissions = { - partner: partner_folders_to_permissions[config['partner_folder_mapping'][partner]] - for partner in file_ids - } - - # throw out all denied addresses, and flatten the permissions dicts to just the email: - external_emails = { - partner: [ - perm['emailAddress'] - for perm in permissions[partner] - if not any( - perm['emailAddress'].lower().endswith(denied_domain.lower()) - for denied_domain in config['denied_notification_domains'] - ) - ] - for partner in permissions - } - - file_ids_and_comments = [] - for partner in file_ids: - if not external_emails[partner]: - LOG( - 'WARNING: could not find a POC for the following partner: "{}". ' - 'Double check the partner folder permissions in Google Drive.' - .format(partner) - ) - else: - tag_string = ' '.join('+' + email for email in external_emails[partner]) - comment_content = NOTIFICATION_MESSAGE_TEMPLATE.format(tags=tag_string) - file_ids_and_comments.append((file_ids[partner], comment_content)) - - try: - LOG('Adding notification comments to uploaded csv files.') - drive.create_comments_for_files(file_ids_and_comments) - except Exception as exc: # pylint: disable=broad-except - # do not fail the script here, since comment errors are non-critical - LOG('WARNING: there was an error adding Google Drive comments to the csv files: {}'.format(exc)) - - -@click.command("generate_report") -@click.option( - '--config_file', - help='YAML file that contains retirement related configuration for this environment.' -) -@click.option( - '--google_secrets_file', - help='JSON file with Google service account credentials for uploading.' -) -@click.option( - '--output_dir', - help='The local directory that the script will write the reports to.' -) -@click.option( - '--comments/--no_comments', - default=True, - help='Do or skip adding notification comments to the reports.' -) -def generate_report(config_file, google_secrets_file, output_dir, comments): - """ - Retrieves a JWT token as the retirement service learner, then performs the reporting process as that user. - - - Accepts the configuration file with all necessary credentials and URLs for a single environment - - Gets the users in the LMS reporting queue and the partners they need to be reported to - - Generates a single report per partner - - Pushes the reports to Google Drive - - On success tells LMS to remove the users who succeeded from the reporting queue - """ - LOG('Starting partner report using config file {} and Google config {}'.format(config_file, google_secrets_file)) - - try: - if not config_file: - FAIL(ERR_NO_CONFIG, 'No config file passed in.') - - if not google_secrets_file: - FAIL(ERR_NO_SECRETS, 'No secrets file passed in.') - - # The Jenkins DSL is supposed to create this path for us - if not output_dir or not os.path.exists(output_dir): - FAIL(ERR_NO_OUTPUT_DIR, 'No output_dir passed in or path does not exist.') - - config = CONFIG_WITH_DRIVE_OR_EXIT(config_file, google_secrets_file) - SETUP_LMS_OR_EXIT(config) - _config_drive_folder_map_or_exit(config) - report_data, all_usernames = _get_orgs_and_learners_or_exit(config) - # If no usernames were returned, then no reports need to be generated. - if all_usernames: - partner_filenames = _generate_report_files_or_exit(config, report_data, output_dir) - - # All files generated successfully, now push them to Google - report_file_ids = _push_files_to_google(config, partner_filenames) - - if comments: - # All files uploaded successfully, now add comments to them to trigger notifications - _add_comments_to_files(config, report_file_ids) - - # Success, tell LMS to remove these users from the queue - config['LMS'].retirement_partner_cleanup(all_usernames) - LOG('All reports completed and uploaded to Google.') - except Exception as exc: # pylint: disable=broad-except - FAIL_EXCEPTION(ERR_CLEANUP, 'Unexpected error occurred! Users may be stuck in the processing state!', exc) - - -if __name__ == '__main__': - # pylint: disable=unexpected-keyword-arg, no-value-for-parameter - generate_report(auto_envvar_prefix='RETIREMENT') diff --git a/tubular/tests/retirement_helpers.py b/tubular/tests/retirement_helpers.py index ba502c80..01ae2324 100644 --- a/tubular/tests/retirement_helpers.py +++ b/tubular/tests/retirement_helpers.py @@ -42,18 +42,9 @@ TEST_DENIED_NOTIFICATION_DOMAINS = { '@edx.org', - '@partner-reporting-automation.iam.gserviceaccount.com', } -def flatten_partner_list(partner_list): - """ - Flattens a list of lists into a list. - [["Org1X"], ["Org2X"], ["Org3X", "Org4X"]] => ["Org1X", "Org2X", "Org3X", "Org4X"] - """ - return [partner for sublist in partner_list for partner in sublist] - - def fake_config_file(f, orgs=None, fetch_ecom_segment_id=False): """ Create a config file for a single test. Combined with CliRunner.isolated_filesystem() to @@ -72,9 +63,6 @@ def fake_config_file(f, orgs=None, fetch_ecom_segment_id=False): 'segment': 'https://segment.invalid/graphql', }, 'retirement_pipeline': TEST_RETIREMENT_PIPELINE, - 'partner_report_platform_name': TEST_PLATFORM_NAME, - 'org_partner_mapping': orgs, - 'drive_partners_folder': 'FakeDriveID', 'denied_notification_domains': TEST_DENIED_NOTIFICATION_DOMAINS, 'sailthru_key': 'fake_sailthru_key', 'sailthru_secret': 'fake_sailthru_secret', @@ -153,7 +141,6 @@ def fake_google_secrets_file(f): secrets = { "type": "service_account", - "project_id": "partner-reporting-automation", "private_key_id": "foo", "private_key": fake_private_key, "client_email": "bogus@serviceacct.invalid", diff --git a/tubular/tests/test_delete_expired_reports.py b/tubular/tests/test_delete_expired_reports.py deleted file mode 100644 index 765f16ff..00000000 --- a/tubular/tests/test_delete_expired_reports.py +++ /dev/null @@ -1,246 +0,0 @@ -# coding=utf-8 -""" -Test the retire_one_learner.py script -""" - - -import os - -from click.testing import CliRunner -from mock import patch - -from tubular.scripts.delete_expired_partner_gdpr_reports import ( - ERR_NO_CONFIG, - ERR_BAD_CONFIG, - ERR_NO_SECRETS, - ERR_BAD_SECRETS, - ERR_DELETING_REPORTS, - ERR_BAD_AGE, - delete_expired_reports -) -from tubular.scripts.retirement_partner_report import REPORTING_FILENAME_PREFIX -from tubular.tests.retirement_helpers import TEST_PLATFORM_NAME, fake_config_file, fake_google_secrets_file - -TEST_CONFIG_FILENAME = 'test_config.yml' -TEST_GOOGLE_SECRETS_FILENAME = 'test_google_secrets.json' - - -def _call_script(age_in_days=1, expect_success=True): - """ - Call the report deletion script with a generic, temporary config file. - Returns the CliRunner.invoke results - """ - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_FILENAME, 'w') as config_f: - fake_config_file(config_f) - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: - fake_google_secrets_file(secrets_f) - - result = runner.invoke( - delete_expired_reports, - args=[ - '--config_file', - TEST_CONFIG_FILENAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--age_in_days', - age_in_days - ] - ) - - print(result) - print(result.output) - - if expect_success: - assert result.exit_code == 0 - - return result - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.delete_files') -def test_successful_report_deletion(*args): - mock_delete_files = args[0] - mock_walk_files = args[1] - mock_driveapi = args[2] - - test_created_date = '2018-07-13T22:21:45.600275+00:00' - file_prefix = '{}_{}'.format(REPORTING_FILENAME_PREFIX, TEST_PLATFORM_NAME) - - mock_walk_files.return_value = [ - { - 'id': 'folder1', - 'name': '{}.csv'.format(file_prefix), - 'createdTime': test_created_date, - }, - { - 'id': 'folder2', - 'name': '{}_foo.csv'.format(file_prefix), - 'createdTime': test_created_date, - }, - { - 'id': 'folder3', - 'name': '{}___bar.csv'.format(file_prefix), - 'createdTime': test_created_date, - }, - ] - mock_delete_files.return_value = None - mock_driveapi.return_value = None - - result = _call_script() - - # Make sure the files were listed - assert mock_walk_files.call_count == 1 - - # Make sure we tried to delete the files - assert mock_delete_files.call_count == 1 - - assert 'Partner report deletion complete' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.delete_files') -def test_deletion_report_no_matching_files(*args): - mock_delete_files = args[0] - mock_walk_files = args[1] - mock_driveapi = args[2] - - test_created_date = '2018-07-13T22:21:45.600275+00:00' - mock_walk_files.return_value = [ - { - 'id': 'folder1', - 'name': 'not_this.csv', - 'createdTime': test_created_date, - }, - { - 'id': 'folder2', - 'name': 'or_this.csv', - 'createdTime': test_created_date, - }, - { - 'id': 'folder3', - 'name': 'foo.csv', - 'createdTime': test_created_date, - }, - ] - mock_delete_files.return_value = None - mock_driveapi.return_value = None - - result = _call_script() - - # Make sure the files were listed - assert mock_walk_files.call_count == 1 - - # Make sure we did *not* try to delete the files - nothing to delete. - assert mock_delete_files.call_count == 0 - - assert 'Partner report deletion complete' in result.output - - -def test_no_config(): - runner = CliRunner() - result = runner.invoke(delete_expired_reports) - print(result.output) - assert result.exit_code == ERR_NO_CONFIG - assert 'No config file' in result.output - - -def test_no_secrets(): - runner = CliRunner() - result = runner.invoke(delete_expired_reports, args=['--config_file', 'does_not_exist.yml']) - print(result.output) - assert result.exit_code == ERR_NO_SECRETS - assert 'No secrets file' in result.output - - -def test_bad_config(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_FILENAME, 'w') as config_f: - config_f.write(']this is bad yaml') - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - config_f.write('{this is bad json but we should not get to parsing it') - - result = runner.invoke( - delete_expired_reports, - args=[ - '--config_file', - TEST_CONFIG_FILENAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--age_in_days', 1 - ] - ) - print(result.output) - assert result.exit_code == ERR_BAD_CONFIG - assert 'Failed to read' in result.output - - -def test_bad_secrets(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_FILENAME, 'w') as config_f: - fake_config_file(config_f) - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - config_f.write('{this is bad json') - - tmp_output_dir = 'test_output_dir' - os.mkdir(tmp_output_dir) - - result = runner.invoke( - delete_expired_reports, - args=[ - '--config_file', - TEST_CONFIG_FILENAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--age_in_days', 1 - ] - ) - print(result.output) - assert result.exit_code == ERR_BAD_SECRETS - assert 'Failed to read' in result.output - - -def test_bad_age_in_days(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_FILENAME, 'w') as config_f: - fake_config_file(config_f) - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - fake_google_secrets_file(config_f) - - result = runner.invoke( - delete_expired_reports, - args=[ - '--config_file', - TEST_CONFIG_FILENAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--age_in_days', -1000 - ] - ) - print(result.output) - assert result.exit_code == ERR_BAD_AGE - assert 'must be a positive integer' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.delete_files_older_than') -def test_deletion_error(*args): - mock_delete_old_reports = args[0] - mock_drive_init = args[1] - - mock_delete_old_reports.side_effect = Exception() - mock_drive_init.return_value = None - - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_DELETING_REPORTS - assert 'Unexpected error occurred' in result.output diff --git a/tubular/tests/test_edx_api.py b/tubular/tests/test_edx_api.py index 8db6e27e..73d99718 100644 --- a/tubular/tests/test_edx_api.py +++ b/tubular/tests/test_edx_api.py @@ -161,11 +161,6 @@ def test_update_leaner_retirement_state(self, mock_method): 'mock_method': 'retirement_lms_retire', 'method': 'POST', }, - { - 'api_url': 'api/user/v1/accounts/retirement_partner_report/', - 'mock_method': 'retirement_partner_queue', - 'method': 'PUT', - }, ) @unpack @patch.multiple( @@ -177,7 +172,6 @@ def test_update_leaner_retirement_state(self, mock_method): retirement_retire_notes=DEFAULT, retirement_lms_retire_misc=DEFAULT, retirement_lms_retire=DEFAULT, - retirement_partner_queue=DEFAULT, ) def test_learner_retirement(self, api_url, mock_method, method, **kwargs): json_data = { @@ -191,38 +185,6 @@ def test_learner_retirement(self, api_url, mock_method, method, **kwargs): getattr(self.lms_api, mock_method)(get_fake_user_retirement(original_username=FAKE_ORIGINAL_USERNAME)) kwargs[mock_method].assert_called_once_with(get_fake_user_retirement(original_username=FAKE_ORIGINAL_USERNAME)) - @patch.object(edx_api.LmsApi, 'retirement_partner_report') - def test_retirement_partner_report(self, mock_method): - responses.add( - POST, - urljoin(self.lms_base_url, 'api/user/v1/accounts/retirement_partner_report/') - ) - self.lms_api.retirement_partner_report( - learner=get_fake_user_retirement( - original_username=FAKE_ORIGINAL_USERNAME - ) - ) - mock_method.assert_called_once_with( - learner=get_fake_user_retirement( - original_username=FAKE_ORIGINAL_USERNAME - ) - ) - - @patch.object(edx_api.LmsApi, 'retirement_partner_cleanup') - def test_retirement_partner_cleanup(self, mock_method): - json_data = FAKE_USERNAMES - responses.add( - POST, - urljoin(self.lms_base_url, 'api/user/v1/accounts/retirement_partner_report_cleanup/'), - match=[matchers.json_params_matcher(json_data)] - ) - self.lms_api.retirement_partner_cleanup( - usernames=FAKE_USERNAMES - ) - mock_method.assert_called_once_with( - usernames=FAKE_USERNAMES - ) - @patch.object(edx_api.LmsApi, 'retirement_retire_proctoring_data') def test_retirement_retire_proctoring_data(self, mock_method): learner = get_fake_user_retirement() @@ -305,25 +267,6 @@ def test_retrieve_learner_queue_backoff( self.lms_api.learners_to_retire( TEST_RETIREMENT_QUEUE_STATES, cool_off_days=365) - @data(104) - @responses.activate - @patch('tubular.edx_api._backoff_handler') - @patch.object(edx_api.LmsApi, 'retirement_partner_cleanup') - def test_retirement_partner_cleanup_backoff_on_connection_error( - self, - svr_status_code, - mock_backoff_handler, - mock_retirement_partner_cleanup - ): - mock_backoff_handler.side_effect = BackoffTriedException - response = requests.Response() - response.status_code = svr_status_code - mock_retirement_partner_cleanup.retirement_partner_cleanup.side_effect = ConnectionError( - response=response - ) - with self.assertRaises(BackoffTriedException): - self.lms_api.retirement_partner_cleanup([{'original_username': 'test'}]) - class TestEcommerceApi(OAuth2Mixin, unittest.TestCase): """ @@ -343,23 +286,6 @@ def setUp(self): 'the_client_secret' ) - @patch.object(edx_api.EcommerceApi, 'retire_learner') - def test_retirement_partner_report(self, mock_method): - json_data = { - 'username': FAKE_ORIGINAL_USERNAME, - } - responses.add( - POST, - urljoin(self.lms_base_url, 'api/v2/user/retire/'), - match=[matchers.json_params_matcher(json_data)] - ) - self.ecommerce_api.retire_learner( - learner=get_fake_user_retirement(original_username=FAKE_ORIGINAL_USERNAME) - ) - mock_method.assert_called_once_with( - learner=get_fake_user_retirement(original_username=FAKE_ORIGINAL_USERNAME) - ) - @patch.object(edx_api.EcommerceApi, 'retire_learner') def get_tracking_key(self, mock_method): original_username = { diff --git a/tubular/tests/test_retirement_partner_report.py b/tubular/tests/test_retirement_partner_report.py deleted file mode 100644 index d6c13741..00000000 --- a/tubular/tests/test_retirement_partner_report.py +++ /dev/null @@ -1,809 +0,0 @@ -# coding=utf-8 -""" -Test the retire_one_learner.py script -""" - - -import csv -import os -import unicodedata -from datetime import date -import time - -from click.testing import CliRunner -from mock import DEFAULT, patch -from six import PY2, itervalues - -from tubular.scripts.retirement_partner_report import ( - DEFAULT_FIELD_HEADINGS, - ERR_BAD_CONFIG, - ERR_BAD_SECRETS, - ERR_CLEANUP, - ERR_FETCHING_LEARNERS, - ERR_NO_CONFIG, - ERR_NO_SECRETS, - ERR_NO_OUTPUT_DIR, - ERR_REPORTING, - ERR_SETUP_FAILED, - ERR_UNKNOWN_ORG, - ERR_DRIVE_LISTING, - LEARNER_CREATED_KEY, - LEARNER_ORIGINAL_USERNAME_KEY, - ORGS_CONFIG_FIELD_HEADINGS_KEY, - ORGS_CONFIG_KEY, - ORGS_CONFIG_LEARNERS_KEY, - ORGS_CONFIG_ORG_KEY, - ORGS_KEY, - REPORTING_FILENAME_PREFIX, - SETUP_LMS_OR_EXIT, - generate_report, - _generate_report_files_or_exit, # pylint: disable=protected-access - _get_orgs_and_learners_or_exit, # pylint: disable=protected-access -) - -from tubular.tests.retirement_helpers import fake_config_file, fake_google_secrets_file, flatten_partner_list, FAKE_ORGS, TEST_PLATFORM_NAME - -TEST_CONFIG_YML_NAME = 'test_config.yml' -TEST_GOOGLE_SECRETS_FILENAME = 'test_google_secrets.json' -DELETION_TIME = time.strftime("%Y-%m-%dT%H:%M:%S") -UNICODE_NAME_CONSTANT = '阿碧' -USER_ID = '12345' -TEST_ORGS_CONFIG = [ - { - ORGS_CONFIG_ORG_KEY: 'orgCustom', - ORGS_CONFIG_FIELD_HEADINGS_KEY: ['heading_1', 'heading_2', 'heading_3'] - }, - { - ORGS_CONFIG_ORG_KEY: 'otherCustomOrg', - ORGS_CONFIG_FIELD_HEADINGS_KEY: ['unique_id'] - } -] -DEFAULT_FIELD_VALUES = { - 'user_id': USER_ID, - LEARNER_ORIGINAL_USERNAME_KEY: 'username', - 'original_email': 'invalid', - 'original_name': UNICODE_NAME_CONSTANT, - 'deletion_completed': DELETION_TIME -} - - -def _call_script(expect_success=True, expected_num_rows=10, config_orgs=None, expected_fields=None): - """ - Call the retired learner script with the given username and a generic, temporary config file. - Returns the CliRunner.invoke results - """ - if expected_fields is None: - expected_fields = DEFAULT_FIELD_VALUES - if config_orgs is None: - config_orgs = FAKE_ORGS - - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_YML_NAME, 'w') as config_f: - fake_config_file(config_f, config_orgs) - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: - fake_google_secrets_file(secrets_f) - - tmp_output_dir = 'test_output_dir' - os.mkdir(tmp_output_dir) - - result = runner.invoke( - generate_report, - args=[ - '--config_file', - TEST_CONFIG_YML_NAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--output_dir', - tmp_output_dir - ] - ) - - print(result) - print(result.output) - - if expect_success: - assert result.exit_code == 0 - - if config_orgs is None: - # These are the orgs - config_org_vals = flatten_partner_list(FAKE_ORGS.values()) - else: - config_org_vals = flatten_partner_list(config_orgs.values()) - - # Normalize the unicode as the script does - if PY2: - config_org_vals = [org.decode('utf-8') for org in config_org_vals] - - config_org_vals = [unicodedata.normalize('NFKC', org) for org in config_org_vals] - - for org in config_org_vals: - outfile = os.path.join(tmp_output_dir, '{}_{}_{}_{}.csv'.format( - REPORTING_FILENAME_PREFIX, TEST_PLATFORM_NAME, org, date.today().isoformat() - )) - - with open(outfile, 'r') as csvfile: - reader = csv.DictReader(csvfile) - rows = [] - for row in reader: - for field_key in expected_fields: - field_value = expected_fields[field_key] - assert field_value in row[field_key] - rows.append(row) - - # Confirm the number of rows - assert len(rows) == expected_num_rows - return result - - -def _fake_retirement_report_user(seed_val, user_orgs=None, user_orgs_config=None): - """ - Creates unique user to populate a fake report with. - - seed_val is a number or other unique value for this user, will be formatted into - user values to make sure they're distinct. - - user_orgs, if given, should be a list of orgs that will be associated with the user. - - user_orgs_config, if given, should be a list of dicts mapping orgs to their customized - field headings. These orgs will also be associated with the user. - """ - user_info = { - 'user_id': USER_ID, - LEARNER_ORIGINAL_USERNAME_KEY: 'username_{}'.format(seed_val), - 'original_email': 'user_{}@foo.invalid'.format(seed_val), - 'original_name': '{} {}'.format(UNICODE_NAME_CONSTANT, seed_val), - LEARNER_CREATED_KEY: DELETION_TIME, - } - - if user_orgs is not None: - user_info[ORGS_KEY] = user_orgs - - if user_orgs_config is not None: - user_info[ORGS_CONFIG_KEY] = user_orgs_config - - return user_info - - -def _fake_retirement_report(num_users=10, user_orgs=None, user_orgs_config=None): - """ - Fake the output of a retirement report with unique users - """ - return [_fake_retirement_report_user(i, user_orgs, user_orgs_config) for i in range(num_users)] - - -@patch('tubular.edx_api.LmsApi.retirement_partner_report') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -def test_report_generation_multiple_partners(*args, **kwargs): - mock_get_access_token = args[0] - mock_retirement_report = args[1] - - org_1_users = [_fake_retirement_report_user(i, user_orgs=['org1']) for i in range(1,3)] - org_2_users = [_fake_retirement_report_user(i, user_orgs=['org2']) for i in range(3,5)] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_retirement_report.return_value = org_1_users + org_2_users - - config = { - 'client_id': 'bogus id', - 'client_secret': 'supersecret', - 'base_urls': { - 'lms': 'https://stage-edx-edxapp.edx.invalid/', - }, - 'org_partner_mapping': { - 'org1': ['Org1X'], - 'org2': ['Org2X', 'Org2Xb'], - } - } - SETUP_LMS_OR_EXIT(config) - orgs, usernames = _get_orgs_and_learners_or_exit(config) - - assert usernames == [{'original_username': 'username_{}'.format(username)} for username in range(1,5)] - - def _get_learner_usernames(org_data): - return [learner['original_username'] for learner in org_data['learners']] - - assert _get_learner_usernames(orgs['Org1X']) == ['username_1', 'username_2'] - - # Org2X and Org2Xb should have the same learners in their report data - assert _get_learner_usernames(orgs['Org2X']) == _get_learner_usernames(orgs['Org2Xb']) == ['username_3', 'username_4'] - - # Org2X and Org2Xb report data should match - assert orgs['Org2X'] == orgs['Org2Xb'] - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.create_file_in_folder') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.list_permissions_for_files') -@patch('tubular.google_api.DriveApi.create_comments_for_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT, - retirement_partner_cleanup=DEFAULT -) -def test_successful_report(*args, **kwargs): - mock_get_access_token = args[0] - mock_create_comments = args[1] - mock_list_permissions = args[2] - mock_walk_files = args[3] - mock_create_files = args[4] - mock_driveapi = args[5] - mock_retirement_report = kwargs['retirement_partner_report'] - mock_retirement_cleanup = kwargs['retirement_partner_cleanup'] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_create_comments.return_value = None - fake_partners = list(itervalues(FAKE_ORGS)) - # Generate the list_permissions return value. - # The first few have POCs. - mock_list_permissions.return_value = { - 'folder' + partner: [ - {'emailAddress': 'some.contact@example.com'}, # The POC. - {'emailAddress': 'another.contact@edx.org'}, - ] - for partner in flatten_partner_list(fake_partners[:2]) - } - # The last one does not have any POCs. - mock_list_permissions.return_value.update({ - 'folder' + partner: [ - {'emailAddress': 'another.contact@edx.org'}, - ] - for partner in fake_partners[2] - }) - mock_walk_files.return_value = [{'name': partner, 'id': 'folder' + partner} for partner in flatten_partner_list(FAKE_ORGS.values())] - mock_create_files.side_effect = ['foo', 'bar', 'baz', 'qux'] - mock_driveapi.return_value = None - mock_retirement_report.return_value = _fake_retirement_report(user_orgs=list(FAKE_ORGS.keys())) - - result = _call_script() - - # Make sure we're getting the LMS token - mock_get_access_token.assert_called_once() - - # Make sure that we get the report - mock_retirement_report.assert_called_once() - - # Make sure we tried to upload the files - assert mock_create_files.call_count == 4 - - # Make sure we tried to add comments to the files - assert mock_create_comments.call_count == 1 - # First [0] returns all positional args, second [0] gets the first positional arg. - create_comments_file_ids, create_comments_messages = zip(*mock_create_comments.call_args[0][0]) - assert set(create_comments_file_ids).issubset(set(['foo', 'bar', 'baz', 'qux'])) - assert len(create_comments_file_ids) == 2 # only two comments created, the third didn't have a POC. - assert all('+some.contact@example.com' in msg for msg in create_comments_messages) - assert all('+another.contact@edx.org' not in msg for msg in create_comments_messages) - assert 'WARNING: could not find a POC' in result.output - - # Make sure we tried to remove the users from the queue - mock_retirement_cleanup.assert_called_with( - [{'original_username': user[LEARNER_ORIGINAL_USERNAME_KEY]} for user in mock_retirement_report.return_value] - ) - - assert 'All reports completed and uploaded to Google.' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.create_file_in_folder') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.list_permissions_for_files') -@patch('tubular.google_api.DriveApi.create_comments_for_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT, - retirement_partner_cleanup=DEFAULT -) -def test_successful_report_org_config(*args, **kwargs): - mock_get_access_token = args[0] - mock_create_comments = args[1] - mock_list_permissions = args[2] - mock_walk_files = args[3] - mock_create_files = args[4] - mock_driveapi = args[5] - mock_retirement_report = kwargs['retirement_partner_report'] - mock_retirement_cleanup = kwargs['retirement_partner_cleanup'] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_create_comments.return_value = None - fake_custom_orgs = { - 'orgCustom': ['firstBlah'] - } - fake_partners = list(itervalues(fake_custom_orgs)) - mock_list_permissions.return_value = { - 'folder' + partner: [ - {'emailAddress': 'some.contact@example.com'}, # The POC. - {'emailAddress': 'another.contact@edx.org'}, - ] - for partner in flatten_partner_list(fake_partners[:2]) - } - mock_walk_files.return_value = [{'name': partner, 'id': 'folder' + partner} for partner in - flatten_partner_list(fake_custom_orgs.values())] - mock_create_files.side_effect = ['foo', 'bar', 'baz'] - mock_driveapi.return_value = None - expected_num_users = 1 - - orgs_config = [ - { - ORGS_CONFIG_ORG_KEY: 'orgCustom', - ORGS_CONFIG_FIELD_HEADINGS_KEY: ['heading_1', 'heading_2', 'heading_3'] - } - ] - - # Input from the LMS - report_data = [ - { - 'heading_1': 'h1val', - 'heading_2': 'h2val', - 'heading_3': 'h3val', - LEARNER_ORIGINAL_USERNAME_KEY: 'blah', - LEARNER_CREATED_KEY: DELETION_TIME, - ORGS_CONFIG_KEY: orgs_config - } - ] - - # Resulting csv file content - expected_fields = { - 'heading_1': 'h1val', - 'heading_2': 'h2val', - 'heading_3': 'h3val', - } - - mock_retirement_report.return_value = report_data - - result = _call_script(expected_num_rows=expected_num_users, config_orgs=fake_custom_orgs, - expected_fields=expected_fields) - - # Make sure we're getting the LMS token - mock_get_access_token.assert_called_once() - - # Make sure that we get the report - mock_retirement_report.assert_called_once() - - # Make sure we tried to remove the users from the queue - mock_retirement_cleanup.assert_called_with( - [{'original_username': user[LEARNER_ORIGINAL_USERNAME_KEY]} for user in mock_retirement_report.return_value] - ) - - assert 'All reports completed and uploaded to Google.' in result.output - - -def test_no_config(): - runner = CliRunner() - result = runner.invoke(generate_report) - print(result.output) - assert result.exit_code == ERR_NO_CONFIG - assert 'No config file' in result.output - - -def test_no_secrets(): - runner = CliRunner() - result = runner.invoke(generate_report, args=['--config_file', 'does_not_exist.yml']) - print(result.output) - assert result.exit_code == ERR_NO_SECRETS - assert 'No secrets file' in result.output - - -def test_no_output_dir(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_YML_NAME, 'w') as config_f: - config_f.write('irrelevant') - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - config_f.write('irrelevant') - - result = runner.invoke( - generate_report, - args=[ - '--config_file', - TEST_CONFIG_YML_NAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME - ] - ) - print(result.output) - assert result.exit_code == ERR_NO_OUTPUT_DIR - assert 'No output_dir' in result.output - - -def test_bad_config(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_YML_NAME, 'w') as config_f: - config_f.write(']this is bad yaml') - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - config_f.write('{this is bad json but we should not get to parsing it') - - tmp_output_dir = 'test_output_dir' - os.mkdir(tmp_output_dir) - - result = runner.invoke( - generate_report, - args=[ - '--config_file', - TEST_CONFIG_YML_NAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--output_dir', - tmp_output_dir - ] - ) - print(result.output) - assert result.exit_code == ERR_BAD_CONFIG - assert 'Failed to read' in result.output - - -def test_bad_secrets(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_YML_NAME, 'w') as config_f: - fake_config_file(config_f) - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - config_f.write('{this is bad json') - - tmp_output_dir = 'test_output_dir' - os.mkdir(tmp_output_dir) - - result = runner.invoke( - generate_report, - args=[ - '--config_file', - TEST_CONFIG_YML_NAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--output_dir', - tmp_output_dir - ] - ) - print(result.output) - assert result.exit_code == ERR_BAD_SECRETS - assert 'Failed to read' in result.output - - -def test_bad_output_dir(): - runner = CliRunner() - with runner.isolated_filesystem(): - with open(TEST_CONFIG_YML_NAME, 'w') as config_f: - fake_config_file(config_f) - - with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: - fake_google_secrets_file(config_f) - - result = runner.invoke( - generate_report, - args=[ - '--config_file', - TEST_CONFIG_YML_NAME, - '--google_secrets_file', - TEST_GOOGLE_SECRETS_FILENAME, - '--output_dir', - 'does_not_exist/at_all' - ] - ) - print(result.output) - assert result.exit_code == ERR_NO_OUTPUT_DIR - assert 'or path does not exist' in result.output - - -@patch('tubular.edx_api.BaseApiClient.get_access_token') -def test_setup_failed(*args): - mock_get_access_token = args[0] - mock_get_access_token.side_effect = Exception('boom') - - result = _call_script(expect_success=False) - mock_get_access_token.assert_called_once() - assert result.exit_code == ERR_SETUP_FAILED - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT) -def test_fetching_learners_failed(*args, **kwargs): - mock_get_access_token = args[0] - mock_walk_files = args[1] - mock_drive_init = args[2] - mock_retirement_report = kwargs['retirement_partner_report'] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_walk_files.return_value = [{'name': 'dummy_file_name', 'id': 'dummy_file_id'}] - mock_drive_init.return_value = None - mock_retirement_report.side_effect = Exception('failed to get learners') - - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_FETCHING_LEARNERS - assert 'failed to get learners' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -def test_listing_folders_failed(*args): - mock_get_access_token = args[0] - mock_walk_files = args[1] - mock_drive_init = args[2] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_walk_files.side_effect = [[], Exception()] - mock_drive_init.return_value = None - - # call it once; this time walk_files will return an empty list. - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_DRIVE_LISTING - assert 'Finding partner directories on Drive failed' in result.output - - # call it a second time; this time walk_files will throw an exception. - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_DRIVE_LISTING - assert 'Finding partner directories on Drive failed' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT) -def test_unknown_org(*args, **kwargs): - mock_get_access_token = args[0] - mock_drive_init = args[2] - mock_retirement_report = kwargs['retirement_partner_report'] - - mock_drive_init.return_value = None - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - - orgs = ['orgA', 'orgB'] - - mock_retirement_report.return_value = [_fake_retirement_report_user(i, orgs, TEST_ORGS_CONFIG) for i in range(10)] - - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_UNKNOWN_ORG - assert 'orgA' in result.output - assert 'orgB' in result.output - assert 'orgCustom' in result.output - assert 'otherCustomOrg' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT) -def test_unknown_org_custom(*args, **kwargs): - mock_get_access_token = args[0] - mock_drive_init = args[2] - mock_retirement_report = kwargs['retirement_partner_report'] - - mock_drive_init.return_value = None - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - - custom_orgs_config = [ - { - ORGS_CONFIG_ORG_KEY: 'singleCustomOrg', - ORGS_CONFIG_FIELD_HEADINGS_KEY: ['first_heading', 'second_heading'] - } - ] - - mock_retirement_report.return_value = [_fake_retirement_report_user(i, None, custom_orgs_config) for i in range(2)] - - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_UNKNOWN_ORG - assert 'organizations {\'singleCustomOrg\'} do not exist' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch('unicodecsv.DictWriter') -@patch('tubular.edx_api.LmsApi.retirement_partner_report') -def test_reporting_error(*args): - mock_retirement_report = args[0] - mock_dictwriter = args[1] - mock_get_access_token = args[2] - mock_drive_init = args[4] - - error_msg = 'Fake unable to write csv' - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_dictwriter.side_effect = Exception(error_msg) - mock_drive_init.return_value = None - mock_retirement_report.return_value = _fake_retirement_report(user_orgs=list(FAKE_ORGS.keys())) - - result = _call_script(expect_success=False) - - assert result.exit_code == ERR_REPORTING - assert error_msg in result.output - -@patch('tubular.google_api.DriveApi.list_permissions_for_files') -@patch('tubular.google_api.DriveApi.create_comments_for_files') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.create_file_in_folder') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT, - retirement_partner_cleanup=DEFAULT -) -def test_cleanup_error(*args, **kwargs): - mock_get_access_token = args[0] - mock_create_files = args[1] - mock_driveapi = args[2] - mock_walk_files = args[3] - mock_create_comments = args[4] - mock_list_permissions = args[5] - mock_retirement_report = kwargs['retirement_partner_report'] - mock_retirement_cleanup = kwargs['retirement_partner_cleanup'] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_create_files.return_value = True - mock_driveapi.return_value = None - mock_walk_files.return_value = [{'name': partner, 'id': 'folder' + partner} for partner in flatten_partner_list(FAKE_ORGS.values())] - fake_partners = list(itervalues(FAKE_ORGS)) - # Generate the list_permissions return value. - mock_list_permissions.return_value = { - 'folder' + partner: [ - {'emailAddress': 'some.contact@example.com'}, # The POC. - {'emailAddress': 'another.contact@edx.org'}, - {'emailAddress': 'third@edx.org'} - ] - for partner in flatten_partner_list(fake_partners) - } - mock_create_comments.return_value = None - - - mock_retirement_report.return_value = _fake_retirement_report(user_orgs=list(FAKE_ORGS.keys())) - mock_retirement_cleanup.side_effect = Exception('Mock cleanup exception') - - result = _call_script(expect_success=False) - - mock_retirement_cleanup.assert_called_with( - [{'original_username': user[LEARNER_ORIGINAL_USERNAME_KEY]} for user in mock_retirement_report.return_value] - ) - - assert result.exit_code == ERR_CLEANUP - assert 'Users may be stuck in the processing state!' in result.output - - -@patch('tubular.google_api.DriveApi.__init__') -@patch('tubular.google_api.DriveApi.create_file_in_folder') -@patch('tubular.google_api.DriveApi.walk_files') -@patch('tubular.google_api.DriveApi.list_permissions_for_files') -@patch('tubular.google_api.DriveApi.create_comments_for_files') -@patch('tubular.edx_api.BaseApiClient.get_access_token') -@patch.multiple( - 'tubular.edx_api.LmsApi', - retirement_partner_report=DEFAULT, - retirement_partner_cleanup=DEFAULT -) -def test_google_unicode_folder_names(*args, **kwargs): - mock_get_access_token = args[0] - mock_create_comments = args[1] - mock_list_permissions = args[2] - mock_walk_files = args[3] - mock_create_files = args[4] - mock_driveapi = args[5] - mock_retirement_report = kwargs['retirement_partner_report'] - mock_retirement_cleanup = kwargs['retirement_partner_cleanup'] - - mock_get_access_token.return_value = ('THIS_IS_A_JWT', None) - mock_list_permissions.return_value = { - 'folder' + partner: [ - {'emailAddress': 'some.contact@example.com'}, - {'emailAddress': 'another.contact@edx.org'}, - ] - for partner in [ - unicodedata.normalize('NFKC', u'TéstX'), - unicodedata.normalize('NFKC', u'TéstX2'), - unicodedata.normalize('NFKC', u'TéstX3'), - ] - } - mock_walk_files.return_value = [ - {'name': partner, 'id': 'folder' + partner} - for partner in [ - unicodedata.normalize('NFKC', u'TéstX'), - unicodedata.normalize('NFKC', u'TéstX2'), - unicodedata.normalize('NFKC', u'TéstX3'), - ] - ] - mock_create_files.side_effect = ['foo', 'bar', 'baz'] - mock_driveapi.return_value = None - mock_retirement_report.return_value = _fake_retirement_report(user_orgs=list(FAKE_ORGS.keys())) - - config_orgs = { - 'org1': [unicodedata.normalize('NFKC', u'TéstX')], - 'org2': [unicodedata.normalize('NFD', u'TéstX2')], - 'org3': [unicodedata.normalize('NFKD', u'TéstX3')], - } - - result = _call_script(config_orgs=config_orgs) - - # Make sure we're getting the LMS token - mock_get_access_token.assert_called_once() - - # Make sure that we get the report - mock_retirement_report.assert_called_once() - - # Make sure we tried to upload the files - assert mock_create_files.call_count == 3 - - # Make sure we tried to add comments to the files - assert mock_create_comments.call_count == 1 - # First [0] returns all positional args, second [0] gets the first positional arg. - create_comments_file_ids, create_comments_messages = zip(*mock_create_comments.call_args[0][0]) - assert set(create_comments_file_ids) == set(['foo', 'bar', 'baz']) - assert all('+some.contact@example.com' in msg for msg in create_comments_messages) - assert all('+another.contact@edx.org' not in msg for msg in create_comments_messages) - - # Make sure we tried to remove the users from the queue - mock_retirement_cleanup.assert_called_with( - [{'original_username': user[LEARNER_ORIGINAL_USERNAME_KEY]} for user in mock_retirement_report.return_value] - ) - - assert 'All reports completed and uploaded to Google.' in result.output - - -def test_file_content_custom_headings(): - runner = CliRunner() - with runner.isolated_filesystem(): - config = {'partner_report_platform_name': 'fake_platform_name'} - tmp_output_dir = 'test_output_dir' - os.mkdir(tmp_output_dir) - - # Custom headings and values - ch1 = 'special_id' - ch1v = '134456765432' - ch2 = 'alternate_heading_for_email' - ch2v = 'zxcvbvcxz@blah.com' - custom_field_headings = [ch1, ch2] - - org_name = 'my_delightful_org' - username = 'unique_user' - learner_data = [ - { - ch1: ch1v, - ch2: ch2v, - LEARNER_ORIGINAL_USERNAME_KEY: username, - LEARNER_CREATED_KEY: DELETION_TIME, - } - ] - report_data = { - org_name: { - ORGS_CONFIG_FIELD_HEADINGS_KEY: custom_field_headings, - ORGS_CONFIG_LEARNERS_KEY: learner_data - } - } - - partner_filenames = _generate_report_files_or_exit(config, report_data, tmp_output_dir) - - assert len(partner_filenames) == 1 - filename = partner_filenames[org_name] - with open(filename) as f: - file_content = f.read() - - # Custom field headings - for ch in custom_field_headings: - # Verify custom field headings are present - assert ch in file_content - # Verify custom field values are present - assert ch1v in file_content - assert ch2v in file_content - - # Default field headings - for h in DEFAULT_FIELD_HEADINGS: - # Verify default field headings are not present - assert h not in file_content - # Verify default field values are not present - assert username not in file_content - assert DELETION_TIME not in file_content From a8a1950a82ec1f5da1d9a3d45737f65dac22db33 Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Fri, 6 Oct 2023 20:00:23 +0000 Subject: [PATCH 2/2] feat: removes non-functional retirement partner report scripts FIXES: APER-2936 --- tubular/tests/retirement_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubular/tests/retirement_helpers.py b/tubular/tests/retirement_helpers.py index 01ae2324..e8748c60 100644 --- a/tubular/tests/retirement_helpers.py +++ b/tubular/tests/retirement_helpers.py @@ -63,6 +63,8 @@ def fake_config_file(f, orgs=None, fetch_ecom_segment_id=False): 'segment': 'https://segment.invalid/graphql', }, 'retirement_pipeline': TEST_RETIREMENT_PIPELINE, + 'org_partner_mapping': orgs, + 'drive_partners_folder': 'FakeDriveID', 'denied_notification_domains': TEST_DENIED_NOTIFICATION_DOMAINS, 'sailthru_key': 'fake_sailthru_key', 'sailthru_secret': 'fake_sailthru_secret',