more community management commands (#1897)

* Add some logging to the sync-galaxy-namespaces command. No-Issue Signed-off-by: James Tanner <[email protected]> * Fix lint errors. No-Issue Signed-off-by: James Tanner <[email protected]> * Add command for syncing collections. No-Issue Signed-off-by: James Tanner <[email protected]> * Skip already sync'ed collections. No-Issue Signed-off-by: James Tanner <[email protected]> * Checkin. No-Issue Signed-off-by: James Tanner <[email protected]> * More stuff. No-Issue Signed-off-by: James Tanner <[email protected]> * Wrong variable. No-Issue Signed-off-by: James Tanner <[email protected]> * Allow downloading logos for a specific namespace. No-Issue Signed-off-by: James Tanner <[email protected]> * Add unit test stubs. No-Issue Signed-off-by: James Tanner <[email protected]> * Oops. No-Issue Signed-off-by: James Tanner <[email protected]> --------- Signed-off-by: James Tanner <[email protected]>
ansible · Sep 28, 2023 · cecf61a · cecf61a
1 parent 44fb926
commit cecf61a
Show file tree

Hide file tree

Showing 9 changed files with 407 additions and 37 deletions.
diff --git a/galaxy_ng/app/api/v1/models.py b/galaxy_ng/app/api/v1/models.py
@@ -111,6 +111,9 @@ class LegacyNamespace(models.Model):
     def __repr__(self):
         return f'<LegacyNamespace: {self.name}>'
 
+    def __str__(self):
+        return self.name
+
 
 class LegacyRole(models.Model):
     """
@@ -154,6 +157,9 @@ class LegacyRole(models.Model):
     def __repr__(self):
         return f'<LegacyRole: {self.namespace.name}.{self.name}>'
 
+    def __str__(self):
+        return f'{self.namespace.name}.{self.name}'
+
 
 class LegacyRoleDownloadCount(models.Model):
     legacyrole = models.OneToOneField(

diff --git a/galaxy_ng/app/management/commands/copy-owner-permissions.py b/galaxy_ng/app/management/commands/copy-owner-permissions.py
@@ -0,0 +1,53 @@
+import logging
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+from galaxy_ng.app.utils.rbac import get_owned_v3_namespaces
+from galaxy_ng.app.utils.rbac import add_user_to_v3_namespace
+from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
+
+
+logger = logging.getLogger(__name__)
+
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    """
+    Mirror ownership from one user to another
+    """
+
+    help = 'Add user <dst_username> as an owner to all the things <src_username> owns.'
+
+    def add_arguments(self, parser):
+        parser.add_argument("src_username")
+        parser.add_argument("dst_username")
+        parser.add_argument("--create", action="store_true")
+
+    def handle(self, *args, **options):
+
+        src_user = User.objects.filter(username=options['src_username']).first()
+        dst_user = User.objects.filter(username=options['dst_username']).first()
+
+        if not src_user:
+            raise Exception(
+                f'source username {options["src_username"]} was not found in the system'
+            )
+
+        if not dst_user:
+            if not options['create']:
+                raise Exception(
+                    f'dest username {options["dst_username"]} was not found in the system'
+                )
+            dst_user, _ = User.objects.get_or_create(username=options['dst_username'])
+
+        # find all the namespaces owned by the source user ...
+        namespaces = get_owned_v3_namespaces(src_user)
+        for namespace in namespaces:
+            current_owners = get_v3_namespace_owners(namespace)
+            if dst_user not in current_owners:
+                logger.info(f'add {dst_user} to {namespace}')
+                add_user_to_v3_namespace(dst_user, namespace)
+            else:
+                logger.info(f'{dst_user} alreay owns {namespace}')
diff --git a/galaxy_ng/app/management/commands/download-namespace-logos.py b/galaxy_ng/app/management/commands/download-namespace-logos.py
@@ -22,14 +22,22 @@ class Command(BaseCommand):
 
     help = 'Download namespace logos.'
 
+    def add_arguments(self, parser):
+        parser.add_argument("--namespace", help="find and sync only this namespace name")
+
     def echo(self, message, style=None):
         style = style or self.style.SUCCESS
         self.stdout.write(style(message))
 
     def handle(self, *args, **options):
 
+        kwargs = {
+            'namespace_name': options['namespace'],
+        }
+
         task = dispatch(
             download_all_logos,
+            kwargs=kwargs,
             exclusive_resources=list(AnsibleRepository.objects.all()),
         )
 
@@ -44,8 +52,12 @@ def handle(self, *args, **options):
             sys.exit(1)
 
 
-def download_all_logos():
-    for namespace in Namespace.objects.all():
+def download_all_logos(namespace_name=None):
+    if namespace_name:
+        qs = Namespace.objects.filter(name=namespace_name)
+    else:
+        qs = Namespace.objects.all()
+    for namespace in qs:
         download_logo = False
         if namespace._avatar_url:
             download_logo = True

diff --git a/galaxy_ng/app/management/commands/sync-galaxy-collections.py b/galaxy_ng/app/management/commands/sync-galaxy-collections.py
@@ -0,0 +1,181 @@
+import logging
+import yaml
+import sys
+import time
+
+import django_guid
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+from galaxy_ng.app.utils.galaxy import upstream_collection_iterator
+from galaxy_ng.app.utils.legacy import process_namespace
+
+from pulp_ansible.app.models import CollectionVersion
+from pulp_ansible.app.models import CollectionRemote
+from pulp_ansible.app.models import AnsibleRepository
+from pulp_ansible.app.tasks.collections import sync
+from pulp_ansible.app.tasks.collections import rebuild_repository_collection_versions_metadata
+
+from pulpcore.plugin.tasking import dispatch
+from pulpcore.plugin.constants import TASK_FINAL_STATES, TASK_STATES
+
+
+# Set logging_uid, this does not seem to get generated when task called via management command
+django_guid.set_guid(django_guid.utils.generate_guid())
+
+
+logger = logging.getLogger(__name__)
+
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    """
+    Iterates through every upstream namespace and syncs it.
+    """
+
+    help = 'Sync upstream namespaces+owners from [old-]galaxy.ansible.com'
+
+    def add_arguments(self, parser):
+        parser.add_argument("--baseurl", default="https://old-galaxy.ansible.com")
+        parser.add_argument("--namespace", help="find and sync only this namespace name")
+        parser.add_argument("--name", help="find and sync only this name")
+        parser.add_argument("--rebuild_only", action="store_true", help="only rebuild metadata")
+        parser.add_argument("--limit", type=int)
+
+    def echo(self, message, style=None):
+        style = style or self.style.SUCCESS
+        logger.info(style(message))
+
+    def handle(self, *args, **options):
+
+        remote = CollectionRemote.objects.filter(name='community').first()
+        repo = AnsibleRepository.objects.filter(name='published').first()
+
+        counter = 0
+        processed_namespaces = set()
+        for namespace_info, collection_info, collection_versions in upstream_collection_iterator(
+            baseurl=options['baseurl'],
+            collection_namespace=options['namespace'],
+            collection_name=options['name'],
+            limit=options['limit'],
+        ):
+            counter += 1
+            logger.info(
+                f"{counter}. {collection_info['namespace']['name']}.{collection_info['name']}"
+                + f" versions:{len(collection_versions)}"
+            )
+
+            if namespace_info['name'] not in processed_namespaces:
+                process_namespace(namespace_info['name'], namespace_info)
+                processed_namespaces.add(namespace_info['name'])
+
+            # pulp_ansible sync isn't smart enough to do this ...
+            should_sync = False
+            should_rebuild = False
+            for cvdata in collection_versions:
+                cv = CollectionVersion.objects.filter(
+                    namespace=collection_info['namespace']['name'],
+                    name=collection_info['name'],
+                    version=cvdata['version']
+                ).first()
+
+                if not cv:
+                    should_sync = True
+                    should_rebuild = True
+                elif not cv.contents or not cv.requires_ansible:
+                    should_rebuild = True
+
+            self.echo(f'sync: {should_sync}')
+            self.echo(f'rebuild: {should_rebuild}')
+
+            if should_sync and not options['rebuild_only']:
+
+                # build a single collection requirements
+                requirements = {
+                    'collections': [
+                        collection_info['namespace']['name'] + '.' + collection_info['name']
+                    ]
+                }
+                requirements_yaml = yaml.dump(requirements)
+
+                # set the remote's requirements
+                remote.requirements_file = requirements_yaml
+                remote.save()
+
+                self.echo(
+                    f"dispatching sync for {collection_info['namespace']['name']}"
+                    + f".{collection_info['name']}"
+                )
+                self.do_sync_dispatch(
+                    remote,
+                    repo,
+                    collection_info['namespace']['name'],
+                    collection_info['name']
+                )
+
+            if should_rebuild:
+                self.echo(
+                    f"dispatching rebuild for {collection_info['namespace']['name']}"
+                    + f".{collection_info['name']}"
+                )
+                self.do_rebuild_dispatch(
+                    repo,
+                    collection_info['namespace']['name'],
+                    collection_info['name']
+                )
+
+    def do_sync_dispatch(self, remote, repository, namespace, name):
+
+        # dispatch the real pulp_ansible sync code
+        task = dispatch(
+            sync,
+            kwargs={
+                'remote_pk': str(remote.pk),
+                'repository_pk': str(repository.pk),
+                'mirror': False,
+                'optimize': False,
+            },
+            exclusive_resources=[repository],
+        )
+
+        while task.state not in TASK_FINAL_STATES:
+            time.sleep(2)
+            self.echo(f"Syncing {namespace}.{name} {task.state}")
+            task.refresh_from_db()
+
+        self.echo(f"Syncing {namespace}.{name} {task.state}")
+
+        if task.state == TASK_STATES.FAILED:
+            self.echo(
+                f"Task failed with error ({namespace}.{name}): {task.error}", self.style.ERROR
+            )
+            sys.exit(1)
+
+    def do_rebuild_dispatch(self, repository, namespace, name):
+
+        repository_version = repository.latest_version()
+
+        task = dispatch(
+            rebuild_repository_collection_versions_metadata,
+            kwargs={
+                'repository_version_pk': str(repository_version.pk),
+                'namespace': namespace,
+                'name': name
+            },
+            exclusive_resources=[repository],
+        )
+
+        while task.state not in TASK_FINAL_STATES:
+            time.sleep(2)
+            self.echo(f"Rebuild {namespace}.{name} {task.state}")
+            task.refresh_from_db()
+
+        self.echo(f"Rebuild {namespace}.{name} {task.state}")
+
+        if task.state == TASK_STATES.FAILED:
+            self.echo(
+                f"Task failed with error ({namespace}.{name}): {task.error}", self.style.ERROR
+            )
+            sys.exit(1)
diff --git a/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py b/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py
@@ -22,9 +22,10 @@ class Command(BaseCommand):
     help = 'Sync upstream namespaces+owners from [old-]galaxy.ansible.com'
 
     def add_arguments(self, parser):
-        parser.add_argument("--baseurl", default="https://galaxy.ansible.com")
+        parser.add_argument("--baseurl", default="https://old-galaxy.ansible.com")
         parser.add_argument("--name", help="find and sync only this namespace name")
         parser.add_argument("--id", help="find and sync only this namespace id")
+        parser.add_argument("--force", action="store_true")
         parser.add_argument("--limit", type=int)
         parser.add_argument("--start_page", type=int)
 
@@ -37,12 +38,12 @@ def handle(self, *args, **options):
         if options.get('name'):
             ns_name, ns_info = find_namespace(baseurl=options['baseurl'], name=options['name'])
             self.echo(f'PROCESSING {ns_info["id"]}:{ns_name}')
-            process_namespace(ns_name, ns_info)
+            process_namespace(ns_name, ns_info, force=options['force'])
 
         elif options.get('id'):
             ns_name, ns_info = find_namespace(baseurl=options['baseurl'], id=options['id'])
             self.echo(f'PROCESSING {ns_info["id"]}:{ns_name}')
-            process_namespace(ns_name, ns_info)
+            process_namespace(ns_name, ns_info, force=options['force'])
 
         else:
 
@@ -60,4 +61,4 @@ def handle(self, *args, **options):
                     f'({total}|{count})'
                     + f' PROCESSING {namespace_info["id"]}:{namespace_name}'
                 )
-                process_namespace(namespace_name, namespace_info)
+                process_namespace(namespace_name, namespace_info, force=options['force'])
diff --git a/galaxy_ng/app/management/commands/sync-galaxy-roles.py b/galaxy_ng/app/management/commands/sync-galaxy-roles.py
@@ -15,7 +15,7 @@ class Command(BaseCommand):
     help = 'Sync upstream roles from [old-]galaxy.ansible.com'
 
     def add_arguments(self, parser):
-        parser.add_argument("--baseurl", default="https://galaxy.ansible.com")
+        parser.add_argument("--baseurl", default="https://old-galaxy.ansible.com")
         parser.add_argument("--github_user", help="find and sync only this namespace name")
         parser.add_argument("--role_name", help="find and sync only this role name")
         parser.add_argument("--limit", type=int)