fix role imports with mismatched github_user

MANIFEST.in

@@ -17,3 +17,4 @@ exclude docs_requirements.txt
 exclude .readthedocs.yaml
 include django-automated-logging-LICENSE.txt
 include galaxy_ng/automated_logging/templates/dal/admin/view.html
+recursive-include galaxy_ng *.py

dev/oci_unit_test

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -x
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: Argument 1 must be set and a valid oci profile name (such as 'community')."
+    exit 1
+fi
+
+profile=$1
+shift
+echo "USING PROFILE ${profile}"
+
+# find the oci-env checkout
+OCI_ENV_PATH=$(dirname $(pip show oci-env | egrep ^Location | awk '{print $2}'))
+echo "FOUND OCI_ENV_PATH: ${OCI_ENV_PATH}"
+
+export COMPOSE_INTERACTIVE_NO_CLI=1
+env_path=dev/oci_env_integration/oci_env_configs/$profile.compose.env
+
+oci-env -e ${env_path} compose exec pulp /bin/bash \
+    -c "python3.11 -m pip install git+https://github.com/pulp/pulp-smash.git"
+
+oci-env -e ${env_path} compose exec pulp /bin/bash \
+    -c "python3.11 -m pip install -r /src/galaxy_ng/unittest_requirements.txt"
+
+oci-env -e ${env_path} compose exec pulp /bin/bash \
+    -c "sudo -u postgres psql -c 'ALTER USER pulp CREATEDB;'"
+
+oci-env -e ${env_path} compose exec pulp /bin/bash -c \
+    "
+    source /opt/oci_env/base/container_scripts/configure_pulp_smash.sh
+    cd /src/galaxy_ng
+    sudo -u pulp -E PULP_DATABASES__default__USER=postgres pytest --capture=no -v -r sx --color=yes --pyargs galaxy_ng.tests.unit.app.api.v1.test_tasks::test_legacy_role_import_with_tag_name
+    "

galaxy_ng/app/api/v1/serializers.py

@@ -187,6 +187,8 @@ class LegacyRoleSerializer(serializers.ModelSerializer):
     # import_branch.
     github_branch = serializers.SerializerMethodField()
 
+    imported = serializers.SerializerMethodField()
+
     commit = serializers.SerializerMethodField()
     commit_message = serializers.SerializerMethodField()
 
@@ -202,6 +204,7 @@ class Meta:
             'upstream_id',
             'created',
             'modified',
+            'imported',
             'github_user',
             'username',
             'github_repo',
@@ -238,6 +241,9 @@ def get_created(self, obj):
     def get_modified(self, obj):
         return obj.pulp_created
 
+    def get_imported(self, obj):
+        return obj.full_metadata.get('imported')
+
     def get_github_user(self, obj):
         """
         Return the github_user.
@@ -446,13 +452,15 @@ class LegacyImportSerializer(serializers.Serializer):
     github_user = serializers.CharField()
     github_repo = serializers.CharField()
     alternate_role_name = serializers.CharField(required=False)
+    github_reference = serializers.CharField(required=False)
 
     class Meta:
         model = None
         fields = [
             'github_user',
             'github_repo',
-            'alternate_role_name'
+            'alternate_role_name',
+            'github_reference',
         ]
 
 

galaxy_ng/app/api/v1/tasks.py

@@ -12,8 +12,6 @@
 
 from galaxy_ng.app.models.auth import User
 from galaxy_ng.app.utils.galaxy import upstream_role_iterator
-from galaxy_ng.app.utils.git import get_tag_commit_hash
-from galaxy_ng.app.utils.git import get_tag_commit_date
 from galaxy_ng.app.utils.legacy import process_namespace
 
 from galaxy_ng.app.api.v1.models import LegacyNamespace
@@ -26,6 +24,58 @@
 logger = logging.getLogger(__name__)
 
 
+def find_real_role(github_user, github_repo):
+    """
+    Given the github_user and github_repo attributes, find a matching
+    role with those matching values and then return the necessary
+    properties from the role needed to to do an import.
+
+    :param github_user:
+        The github_user as passed in to the CLI for imports.
+    :param github_repo:
+        The github_repo as passed in to the CLI for imports.
+    """
+
+    # if we find a role, this will point at it
+    real_role = None
+
+    # figure out the actual namespace name
+    real_namespace_name = github_user
+
+    # figure out the actual github user
+    real_github_user = github_user
+
+    # figure out the actual github repo
+    real_github_repo = github_repo
+
+    # the role role can influence the clone url
+    clone_url = None
+
+    # some roles have their github_user set differently from their namespace name ...
+    candidates = LegacyRole.objects.filter(
+        full_metadata__github_user=github_user,
+        full_metadata__github_repo=github_repo
+    )
+    if candidates.count() > 0:
+        real_role = candidates.first()
+        logger.info(f'Using {real_role} as basis for import task')
+        rr_github_user = real_role.full_metadata.get('github_user')
+        rr_github_repo = real_role.full_metadata.get('github_repo')
+        real_namespace_name = real_role.namespace.name
+        if rr_github_user and rr_github_repo:
+            real_github_user = rr_github_user
+            real_github_repo = rr_github_repo
+            clone_url = f'https://github.com/{rr_github_user}/{rr_github_repo}'
+        elif rr_github_user:
+            real_github_user = rr_github_user
+            clone_url = f'https://github.com/{rr_github_user}/{github_repo}'
+        elif rr_github_repo:
+            real_github_repo = rr_github_repo
+            clone_url = f'https://github.com/{github_user}/{github_repo}'
+
+    return real_role, real_namespace_name, real_github_user, real_github_repo, clone_url
+
+
 def legacy_role_import(
     request_username=None,
     github_user=None,
@@ -56,24 +106,41 @@ def legacy_role_import(
     there will not be any duplicate key errors. However,
     the "commit" field should always reflect the latest import.
     """
-    logger.debug('START LEGACY ROLE IMPORT')
+    logger.info('START LEGACY ROLE IMPORT')
+    logger.info(f'REQUEST_USERNAME:{request_username}')
+    logger.info(f'GITHUB_USER:{github_user}')
+    logger.info(f'GITHUB_REPO:{github_repo}')
+    logger.info(f'GITHUB_REFERENCE:{github_reference}')
+    logger.info(f'ALTERNATE_ROLE_NAME:{alternate_role_name}')
+
+    clone_url = None
+    github_reference_is_tag = False
 
     # prevent empty strings?
     if not github_reference:
         github_reference = None
 
+    # some roles have their github_user set differently from their namespace name ...
+    real_role, real_namespace_name, real_github_user, real_github_repo, clone_url = \
+        find_real_role(github_user, github_repo)
+    if real_role:
+        logger.info(f'Using {real_role} as basis for import task')
+
     # this shouldn't happen but just in case ...
     if request_username and not User.objects.filter(username=request_username).exists():
+        logger.error(f'Username {request_username} does not exist in galaxy')
         raise Exception(f'Username {request_username} does not exist in galaxy')
 
     # the user should have a legacy and v3 namespace if they logged in ...
-    namespace = LegacyNamespace.objects.filter(name=github_user).first()
+    namespace = LegacyNamespace.objects.filter(name=real_namespace_name).first()
     if not namespace:
+        logger.error(f'No legacy namespace exists for {github_user}')
         raise Exception(f'No legacy namespace exists for {github_user}')
 
     # we have to have a v3 namespace because of the rbac based ownership ...
     v3_namespace = namespace.namespace
     if not v3_namespace:
+        logger.error(f'No v3 namespace exists for {github_user}')
         raise Exception(f'No v3 namespace exists for {github_user}')
 
     with tempfile.TemporaryDirectory() as tmp_path:
@@ -82,76 +149,96 @@ def legacy_role_import(
 
         # galaxy-importer wants the role's directory to be the name of the role.
         checkout_path = os.path.join(tmp_path, github_repo)
-        clone_url = f'https://github.com/{github_user}/{github_repo}'
+        if clone_url is None:
+            clone_url = f'https://github.com/{github_user}/{github_repo}'
+
+        logger.info(f'CLONING {clone_url}')
 
         # pygit didn't have an obvious way to prevent interactive clones ...
         pid = subprocess.run(
             f'GIT_TERMINAL_PROMPT=0 git clone --recurse-submodules {clone_url} {checkout_path}',
             shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
         )
         if pid.returncode != 0:
+            error = pid.stdout.decode('utf-8')
+            logger.error(f'cloning failed: {error}')
             raise Exception(f'git clone for {clone_url} failed')
 
         # bind the checkout to a pygit object
         gitrepo = Repo(checkout_path)
 
-        github_commit = None
-        github_commit_date = None
-
+        # the github_reference could be a branch OR a tag name ...
         if github_reference is not None:
-            gitrepo.checkout(github_reference)
-            github_commit = get_tag_commit_hash(
-                clone_url,
-                github_reference,
-                checkout_path=checkout_path
-            )
-            github_commit_date = get_tag_commit_date(
-                clone_url,
-                github_reference,
-                checkout_path=checkout_path
-            )
+
+            branch_names = [x.name for x in gitrepo.branches]
+            tag_names = [x.name for x in gitrepo.tags]
+
+            cmd = None
+            if github_reference in branch_names:
+                current_branch = gitrepo.active_branch.name
+                if github_reference != current_branch:
+                    cmd = f'git checkout origin/{github_reference}'
+            elif github_reference in tag_names:
+                github_reference_is_tag = True
+                cmd = f'git checkout tags/{github_reference} -b local_${github_reference}'
+            else:
+                raise Exception(f'{github_reference} is not a valid branch or tag name')
+
+            if cmd:
+                logger.info(f'switching to {github_reference} in checkout via {cmd}')
+                pid = subprocess.run(
+                    cmd,
+                    cwd=checkout_path,
+                    shell=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                )
+                if pid.returncode != 0:
+                    error = pid.stdout.decode('utf-8')
+                    logger.error('{cmd} failed: {error}')
+                    raise Exception(f'{cmd} failed')
+
+            last_commit = [x for x in gitrepo.iter_commits()][0]
+
         else:
-            # FIXME - figure out the old logic.
-            if gitrepo.tags:
-                tag = gitrepo.tags[-1]
-                github_reference = tag.name
-                github_commit = tag.commit.hexsha
-                github_commit_date = datetime.datetime.fromtimestamp(tag.commit.committed_date)
-                github_commit_date = github_commit_date.isoformat()
+            # use the default branch ...
+            github_reference = gitrepo.active_branch.name
 
-        logger.debug(f'GITHUB_REFERENCE: {github_reference}')
-        logger.debug(f'GITHUB_COMMIT: {github_commit}')
+            # use latest commit on HEAD
+            last_commit = gitrepo.head.commit
+
+        # relevant data for this new role version ...
+        github_commit = last_commit.hexsha
+        github_commit_message = last_commit.message
+        github_commit_date = last_commit.committed_datetime.isoformat()
+
+        logger.info(f'GITHUB_REFERENCE: {github_reference}')
+        logger.info(f'GITHUB_COMMIT: {github_commit}')
+        logger.info(f'GITHUB_COMMIT_MESSAGE: {github_commit_message}')
+        logger.info(f'GITHUB_COMMIT_DATE: {github_commit_date}')
 
         # Parse legacy role with galaxy-importer.
         importer_config = Config()
         result = import_legacy_role(checkout_path, namespace.name, importer_config, logger)
         galaxy_info = result["metadata"]["galaxy_info"]
-        logger.debug(f"TAGS: {galaxy_info['galaxy_tags']}")
+        logger.debug(f"FOUND TAGS: {galaxy_info['galaxy_tags']}")
 
+        # munge the role name via an order of precedence
         role_name = result["name"] or alternate_role_name or \
             github_repo.replace("ansible-role-", "")
 
-        # check if this namespace/name/version has already been imported
-        old = LegacyRole.objects.filter(namespace=namespace, name=role_name).first()
-        if old is not None:
-            old_versions = old.full_metadata.get('versions', [])
-            old_versions = [x['name'] for x in old_versions]
-            logger.debug(f'OLD VERSIONS: {old_versions}')
-            if github_reference in old_versions:
-                msg = (
-                    f'{namespace.name}.{role_name} {github_reference}'
-                    + ' has already been imported'
-                )
-                raise Exception(msg)
-
         new_full_metadata = {
             'imported': datetime.datetime.now().isoformat(),
             'clone_url': clone_url,
             'tags': galaxy_info["galaxy_tags"],
             'commit': github_commit,
-            'github_user': github_user,
-            'github_repo': github_repo,
+            'github_user': real_github_user,
+            'github_repo': real_github_repo,
             'github_reference': github_reference,
+            'commit': github_commit,
+            'commit_message': github_commit_message,
             'issue_tracker_url': galaxy_info["issue_tracker_url"] or clone_url + "/issues",
             'dependencies': result["metadata"]["dependencies"],
             'versions': [],
@@ -165,28 +252,49 @@ def legacy_role_import(
         }
 
         # Make the object
-        this_role, _ = LegacyRole.objects.get_or_create(
-            namespace=namespace,
-            name=role_name
-        )
+        if real_role:
+            this_role = real_role
+        else:
+            this_role, _ = LegacyRole.objects.get_or_create(
+                namespace=namespace,
+                name=role_name
+            )
 
         # Combine old versions with new ...
         old_metadata = copy.deepcopy(this_role.full_metadata)
 
         new_full_metadata['versions'] = old_metadata.get('versions', [])
         ts = datetime.datetime.now().isoformat()
-        new_full_metadata['versions'].append({
+
+        # only make a download url for tags?
+        download_url = None
+        if github_reference_is_tag:
+            download_url = (
+                f'https://github.com/{real_github_user}/{real_github_repo}'
+                + f'/archive/{github_reference}.tar.gz'
+            )
+
+        new_version = {
             'name': github_reference,
             'version': github_reference,
             'release_date': ts,
             'created': ts,
             'modified': ts,
             'active': None,
-            'download_url': None,
+            'download_url': download_url,
             'url': None,
             'commit_date': github_commit_date,
             'commit_sha': github_commit
-        })
+        }
+        versions_by_sha = dict((x['commit_sha'], x) for x in new_full_metadata.get('versions', []))
+        if github_commit not in versions_by_sha:
+            new_full_metadata['versions'].append(new_version)
+        else:
+            msg = (
+                f'{namespace.name}.{role_name} {github_reference} commit:{github_commit}'
+                + ' has already been imported'
+            )
+            raise Exception(msg)
 
         # Save the new metadata
         this_role.full_metadata = new_full_metadata

galaxy_ng/app/api/v1/viewsets/roles.py

@@ -190,6 +190,7 @@ def create(self, request):
                 'id': task_id,
                 'github_user': kwargs['github_user'],
                 'github_repo': kwargs['github_repo'],
+                'github_reference': kwargs.get('github_reference'),
                 'summary_fields': {
                     'role': {
                         'name': role_name