Checkin.

No-Issue

Signed-off-by: James Tanner <[email protected]>

galaxy_ng/app/api/v1/models.py

@@ -108,6 +108,9 @@ class LegacyNamespace(models.Model):
         editable=True
     )
 
+    def __repr__(self):
+        return f'<LegacyNamespace: {self.name}>'
+
 
 class LegacyRole(models.Model):
     """
@@ -148,6 +151,9 @@ class LegacyRole(models.Model):
         default=dict
     )
 
+    def __repr__(self):
+        return f'<LegacyRole: {self.namespace.name}.{self.name}>'
+
 
 class LegacyRoleDownloadCount(models.Model):
     legacyrole = models.OneToOneField(

galaxy_ng/app/api/v1/tasks.py

@@ -17,6 +17,7 @@
 from galaxy_ng.app.utils.git import get_tag_commit_date
 from galaxy_ng.app.utils.rbac import add_user_to_v3_namespace
 from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
+from galaxy_ng.app.utils.legacy import process_namespace
 
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
@@ -218,8 +219,6 @@ def legacy_sync_from_upstream(
 
     logger.debug('SYNC INDEX EXISTING NAMESPACES')
     nsmap = {}
-    for ns in LegacyNamespace.objects.all():
-        nsmap[ns.name] = ns
 
     # allow the user to specify how many roles to sync
     if limit is not None:
@@ -236,6 +235,12 @@ def legacy_sync_from_upstream(
     }
     for ns_data, rdata, rversions in upstream_role_iterator(**iterator_kwargs):
 
+        if ns_data['name'] not in nsmap:
+            namespace, v3_namespace = process_namespace(ns_data['name'], ns_data)
+            nsmap[ns_data['name']] = (namespace, v3_namespace)
+        else:
+            namespace, v3_namespace = nsmap[ns_data['name']]
+
         ruser = rdata.get('github_user')
         rname = rdata.get('name')
 
@@ -267,6 +272,7 @@ def legacy_sync_from_upstream(
         role_type = rdata.get('role_type', 'ANS')
         role_download_count = rdata.get('download_count', 0)
 
+        '''
         if ruser not in nsmap:
             logger.debug(f'SYNC NAMESPACE GET_OR_CREATE {ruser}')
             logger.info(f'SYNC NAMESPACE GET_OR_CREATE {ruser}')
@@ -306,6 +312,7 @@ def legacy_sync_from_upstream(
             nsmap[ruser] = namespace
         else:
             namespace = nsmap[ruser]
+        '''
 
         if rkey not in rmap:
             logger.debug(f'SYNC create initial role for {rkey}')

galaxy_ng/app/management/commands/sync-galaxy-namespaces.py

@@ -27,11 +27,6 @@ def add_arguments(self, parser):
         parser.add_argument("--id", help="find and sync only this namespace id")
         parser.add_argument("--limit", type=int)
         parser.add_argument("--startpage", type=int)
-        parser.add_argument(
-            "--use-async",
-            action="store_true",
-            help="process the namespaces asynchronously"
-        )
 
     def echo(self, message, style=None):
         style = style or self.style.SUCCESS

galaxy_ng/app/management/commands/sync-galaxy-roles.py

@@ -0,0 +1,36 @@
+import django_guid
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+from galaxy_ng.app.api.v1.tasks import legacy_sync_from_upstream
+
+
+# Set logging_uid, this does not seem to get generated when task called via management command
+django_guid.set_guid(django_guid.utils.generate_guid())
+
+
+class Command(BaseCommand):
+    """
+    Iterates through every upstream namespace and syncs it.
+    """
+
+    help = 'Sync upstream namespaces from [old-]galaxy.ansible.com'
+
+    def add_arguments(self, parser):
+        parser.add_argument("--baseurl", default="https://galaxy.ansible.com")
+        parser.add_argument("--github_user", help="find and sync only this namespace name")
+        parser.add_argument("--role_name", help="find and sync only this role name")
+        parser.add_argument("--limit", type=int)
+
+    def echo(self, message, style=None):
+        style = style or self.style.SUCCESS
+        self.stdout.write(style(message))
+
+    def handle(self, *args, **options):
+
+        legacy_sync_from_upstream(
+            baseurl=options['baseurl'],
+            github_user=options['github_user'],
+            role_name=options['role_name'],
+            limit=options['limit'],
+        )

galaxy_ng/app/utils/galaxy.py

@@ -106,6 +106,20 @@ def find_namespace(baseurl=None, name=None, id=None):
     return ns_name, ns_info
 
 
+def get_namespace_owners_details(baseurl, ns_id):
+    # get the owners too
+    owners = []
+    next_owners_url = baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
+    while next_owners_url:
+        o_data = requests.get(next_owners_url).json()
+        for owner in o_data['results']:
+            owners.append(owner)
+        if not o_data.get('next'):
+            break
+        next_owners_url = baseurl + o_data['next_link']
+    return owners
+
+
 def upstream_namespace_iterator(
     baseurl=None,
     limit=None,
@@ -158,6 +172,7 @@ def upstream_namespace_iterator(
             ns_id = ndata['id']
 
             # get the owners too
+            '''
             owners = []
             next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
             while next_owners_url:
@@ -167,7 +182,8 @@ def upstream_namespace_iterator(
                 if not o_data.get('next'):
                     break
                 next_owners_url = _baseurl + o_data['next_link']
-            ndata['summary_fields']['owners'] = owners
+            '''
+            ndata['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id)
 
             # send the collection
             namespace_count += 1
@@ -251,6 +267,7 @@ def upstream_collection_iterator(
                 namespace_cache[ns_id] = namespace_data
 
                 # get the owners too
+                '''
                 owners = []
                 next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
                 while next_owners_url:
@@ -260,8 +277,8 @@ def upstream_collection_iterator(
                     if not o_data.get('next'):
                         break
                     next_owners_url = _baseurl + o_data['next_link']
-
-                namespace_cache[ns_id]['summary_fields']['owners'] = owners
+                '''
+                namespace_cache[ns_id]['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id)
 
             else:
                 namespace_data = namespace_cache[ns_id]
@@ -319,13 +336,15 @@ def upstream_role_iterator(
         if role_name:
             params.append(f'name={role_name}')
         next_url = _baseurl + '/api/v1/roles/?' + '&'.join(params)
+    else:
+        next_url = _baseurl + '/api/v1/roles/'
 
     namespace_cache = {}
 
     pagenum = 0
     role_count = 0
     while next_url:
-        logger.info(f'fetch {pagenum} {next_url}')
+        logger.info(f'fetch {pagenum} {next_url} role-count:{role_count}')
 
         page = requests.get(next_url)
 
@@ -369,6 +388,7 @@ def upstream_role_iterator(
                 namespace_cache[ns_id] = namespace_data
 
                 # get the owners too
+                '''
                 owners = []
                 next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
                 while next_owners_url:
@@ -378,8 +398,8 @@ def upstream_role_iterator(
                     if not o_data.get('next'):
                         break
                     next_owners_url = _baseurl + o_data['next_link']
-
-                namespace_cache[ns_id]['summary_fields']['owners'] = owners
+                '''
+                namespace_cache[ns_id]['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id)
 
             else:
                 namespace_data = namespace_cache[ns_id]

galaxy_ng/app/utils/legacy.py

@@ -149,3 +149,5 @@ def process_namespace(namespace_name, namespace_info):
                 owner.save()
 
         add_user_to_v3_namespace(owner, namespace)
+
+    return legacy_namespace, namespace