From dd4b84ae203f7b137f38a9a0a951821e555ba518 Mon Sep 17 00:00:00 2001 From: James Tanner Date: Tue, 10 Oct 2023 09:32:19 -0400 Subject: [PATCH] Checkin. No-Issue Signed-off-by: James Tanner --- galaxy_ng/app/api/v1/serializers.py | 6 ++ galaxy_ng/app/api/v1/tasks.py | 127 +++++++++++++++++++++------- 2 files changed, 101 insertions(+), 32 deletions(-) diff --git a/galaxy_ng/app/api/v1/serializers.py b/galaxy_ng/app/api/v1/serializers.py index 1cfc8cce3d..96b3f4e19c 100644 --- a/galaxy_ng/app/api/v1/serializers.py +++ b/galaxy_ng/app/api/v1/serializers.py @@ -187,6 +187,8 @@ class LegacyRoleSerializer(serializers.ModelSerializer): # import_branch. github_branch = serializers.SerializerMethodField() + imported = serializers.SerializerMethodField() + commit = serializers.SerializerMethodField() commit_message = serializers.SerializerMethodField() @@ -202,6 +204,7 @@ class Meta: 'upstream_id', 'created', 'modified', + 'imported', 'github_user', 'username', 'github_repo', @@ -238,6 +241,9 @@ def get_created(self, obj): def get_modified(self, obj): return obj.pulp_created + def get_imported(self, obj): + return obj.full_metadata.get('imported') + def get_github_user(self, obj): """ Return the github_user. diff --git a/galaxy_ng/app/api/v1/tasks.py b/galaxy_ng/app/api/v1/tasks.py index 9da2efec8e..84ee7e4f04 100644 --- a/galaxy_ng/app/api/v1/tasks.py +++ b/galaxy_ng/app/api/v1/tasks.py @@ -58,6 +58,17 @@ def legacy_role_import( """ logger.debug('START LEGACY ROLE IMPORT') + print(f'REQUEST_USERNAME:{request_username}') + print(f'GITHUB_USER:{github_user}') + print(f'GITHUB_REPO:{github_repo}') + print(f'GITHUB_REFERENCE:{github_reference}') + print(f'ALTERNATE_ROLE_NAME:{alternate_role_name}') + + clone_url = None + real_role = None + real_github_user = github_user + real_github_repo = github_repo + # prevent empty strings? if not github_reference: github_reference = None @@ -76,13 +87,37 @@ def legacy_role_import( if not v3_namespace: raise Exception(f'No v3 namespace exists for {github_user}') + # some roles have their github_user set differently from their namespace name ... + candidates = LegacyRole.objects.filter(namespace__name=github_user, name=github_repo) + if candidates.count() > 0: + real_role = candidates.first() + logger.info(f'Using {real_role} as basis for import task') + rr_github_user = real_role.full_metadata.get('github_user') + rr_github_repo = real_role.full_metadata.get('github_repo') + if rr_github_user and rr_github_repo: + real_github_user = rr_github_user + real_github_repo = rr_github_repo + logger.info(f'using github_user and github_repo from {real_role}') + clone_url = f'https://github.com/{rr_github_user}/{rr_github_repo}' + elif rr_github_user: + real_github_user = rr_github_user + logger.info(f'using github_user from {real_role}') + clone_url = f'https://github.com/{rr_github_user}/{github_repo}' + elif rr_github_repo: + real_github_repo = rr_github_repo + logger.info(f'using github_repo from {real_role}') + clone_url = f'https://github.com/{github_user}/{github_repo}' + with tempfile.TemporaryDirectory() as tmp_path: # galaxy-importer requires importing legacy roles from the role's parent directory. os.chdir(tmp_path) # galaxy-importer wants the role's directory to be the name of the role. checkout_path = os.path.join(tmp_path, github_repo) - clone_url = f'https://github.com/{github_user}/{github_repo}' + if clone_url is None: + clone_url = f'https://github.com/{github_user}/{github_repo}' + + logger.info(f'clone from {clone_url}') # pygit didn't have an obvious way to prevent interactive clones ... pid = subprocess.run( @@ -95,32 +130,42 @@ def legacy_role_import( # bind the checkout to a pygit object gitrepo = Repo(checkout_path) - github_commit = None - github_commit_date = None - + # the github_reference could be a branch OR a tag name ... if github_reference is not None: - gitrepo.checkout(github_reference) - github_commit = get_tag_commit_hash( - clone_url, - github_reference, - checkout_path=checkout_path - ) - github_commit_date = get_tag_commit_date( - clone_url, - github_reference, - checkout_path=checkout_path - ) + + branch_names = [x.name for x in gitrepo.branches] + tag_names = [x.name for x in gitrepo.tags] + + cmd = None + if github_reference in branch_names: + cmd = f'git checkout origin/{github_reference}' + elif github_reference in tag_names: + cmd = f'git checkout tags/{github_reference} -b local_${github_reference}' + else: + raise Exception(f'{github_reference} is not a valid branch or tag name') + + logger.info(cmd) + pid = subprocess.run(cmd, cwd=checkout_path, shell=True) + assert pid.returncode == 0 + + last_commit = [x for x in gitrepo.iter_commits()][0] + else: - # FIXME - figure out the old logic. - if gitrepo.tags: - tag = gitrepo.tags[-1] - github_reference = tag.name - github_commit = tag.commit.hexsha - github_commit_date = datetime.datetime.fromtimestamp(tag.commit.committed_date) - github_commit_date = github_commit_date.isoformat() + # use the default branch ... + github_reference = gitrepo.active_branch.name + + # use latest commit on HEAD + last_commit = gitrepo.head.commit - logger.debug(f'GITHUB_REFERENCE: {github_reference}') - logger.debug(f'GITHUB_COMMIT: {github_commit}') + # relevant data for this new role version ... + github_commit = last_commit.hexsha + github_commit_message = last_commit.message + github_commit_date = last_commit.committed_datetime.isoformat() + + logger.info(f'GITHUB_REFERENCE: {github_reference}') + logger.info(f'GITHUB_COMMIT: {github_commit}') + logger.info(f'GITHUB_COMMIT_MESSAGE: {github_commit_message}') + logger.info(f'GITHUB_COMMIT_DATE: {github_commit_date}') # Parse legacy role with galaxy-importer. importer_config = Config() @@ -128,9 +173,11 @@ def legacy_role_import( galaxy_info = result["metadata"]["galaxy_info"] logger.debug(f"TAGS: {galaxy_info['galaxy_tags']}") + # munge the role name via an order of precedence role_name = result["name"] or alternate_role_name or \ github_repo.replace("ansible-role-", "") + ''' # check if this namespace/name/version has already been imported old = LegacyRole.objects.filter(namespace=namespace, name=role_name).first() if old is not None: @@ -143,15 +190,18 @@ def legacy_role_import( + ' has already been imported' ) raise Exception(msg) + ''' new_full_metadata = { 'imported': datetime.datetime.now().isoformat(), 'clone_url': clone_url, 'tags': galaxy_info["galaxy_tags"], 'commit': github_commit, - 'github_user': github_user, - 'github_repo': github_repo, + 'github_user': real_github_user, + 'github_repo': real_github_repo, 'github_reference': github_reference, + 'commit': github_commit, + 'commit_message': github_commit_message, 'issue_tracker_url': galaxy_info["issue_tracker_url"] or clone_url + "/issues", 'dependencies': result["metadata"]["dependencies"], 'versions': [], @@ -165,17 +215,21 @@ def legacy_role_import( } # Make the object - this_role, _ = LegacyRole.objects.get_or_create( - namespace=namespace, - name=role_name - ) + if real_role: + this_role = real_role + else: + this_role, _ = LegacyRole.objects.get_or_create( + namespace=namespace, + name=role_name + ) # Combine old versions with new ... old_metadata = copy.deepcopy(this_role.full_metadata) new_full_metadata['versions'] = old_metadata.get('versions', []) ts = datetime.datetime.now().isoformat() - new_full_metadata['versions'].append({ + + new_version = { 'name': github_reference, 'version': github_reference, 'release_date': ts, @@ -186,7 +240,16 @@ def legacy_role_import( 'url': None, 'commit_date': github_commit_date, 'commit_sha': github_commit - }) + } + versions_by_sha = dict((x['commit_sha'], x) for x in new_full_metadata.get('versions', [])) + if github_commit not in versions_by_sha: + new_full_metadata['versions'].append(new_version) + else: + msg = ( + f'{namespace.name}.{role_name} {github_reference} commit:{github_commit}' + + ' has already been imported' + ) + raise Exception(msg) # Save the new metadata this_role.full_metadata = new_full_metadata