diff --git a/galaxy_ng/app/api/v1/models.py b/galaxy_ng/app/api/v1/models.py index 95a74e2ebc..c282d44c89 100644 --- a/galaxy_ng/app/api/v1/models.py +++ b/galaxy_ng/app/api/v1/models.py @@ -108,6 +108,9 @@ class LegacyNamespace(models.Model): editable=True ) + def __repr__(self): + return f'' + class LegacyRole(models.Model): """ @@ -148,6 +151,9 @@ class LegacyRole(models.Model): default=dict ) + def __repr__(self): + return f'' + class LegacyRoleDownloadCount(models.Model): legacyrole = models.OneToOneField( diff --git a/galaxy_ng/app/api/v1/tasks.py b/galaxy_ng/app/api/v1/tasks.py index 2e1cec17ee..75eaf674f8 100644 --- a/galaxy_ng/app/api/v1/tasks.py +++ b/galaxy_ng/app/api/v1/tasks.py @@ -17,6 +17,7 @@ from galaxy_ng.app.utils.git import get_tag_commit_date from galaxy_ng.app.utils.rbac import add_user_to_v3_namespace from galaxy_ng.app.utils.rbac import get_v3_namespace_owners +from galaxy_ng.app.utils.legacy import process_namespace from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole @@ -218,8 +219,6 @@ def legacy_sync_from_upstream( logger.debug('SYNC INDEX EXISTING NAMESPACES') nsmap = {} - for ns in LegacyNamespace.objects.all(): - nsmap[ns.name] = ns # allow the user to specify how many roles to sync if limit is not None: @@ -236,6 +235,12 @@ def legacy_sync_from_upstream( } for ns_data, rdata, rversions in upstream_role_iterator(**iterator_kwargs): + if ns_data['name'] not in nsmap: + namespace, v3_namespace = process_namespace(ns_data['name'], ns_data) + nsmap[ns_data['name']] = (namespace, v3_namespace) + else: + namespace, v3_namespace = nsmap[ns_data['name']] + ruser = rdata.get('github_user') rname = rdata.get('name') @@ -267,6 +272,7 @@ def legacy_sync_from_upstream( role_type = rdata.get('role_type', 'ANS') role_download_count = rdata.get('download_count', 0) + ''' if ruser not in nsmap: logger.debug(f'SYNC NAMESPACE GET_OR_CREATE {ruser}') logger.info(f'SYNC NAMESPACE GET_OR_CREATE {ruser}') @@ -306,6 +312,7 @@ def legacy_sync_from_upstream( nsmap[ruser] = namespace else: namespace = nsmap[ruser] + ''' if rkey not in rmap: logger.debug(f'SYNC create initial role for {rkey}') diff --git a/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py b/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py index f3f19e278c..6b4d0a0870 100644 --- a/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py +++ b/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py @@ -27,11 +27,6 @@ def add_arguments(self, parser): parser.add_argument("--id", help="find and sync only this namespace id") parser.add_argument("--limit", type=int) parser.add_argument("--startpage", type=int) - parser.add_argument( - "--use-async", - action="store_true", - help="process the namespaces asynchronously" - ) def echo(self, message, style=None): style = style or self.style.SUCCESS diff --git a/galaxy_ng/app/management/commands/sync-galaxy-roles.py b/galaxy_ng/app/management/commands/sync-galaxy-roles.py new file mode 100644 index 0000000000..12d528ec31 --- /dev/null +++ b/galaxy_ng/app/management/commands/sync-galaxy-roles.py @@ -0,0 +1,36 @@ +import django_guid +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand + +from galaxy_ng.app.api.v1.tasks import legacy_sync_from_upstream + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +class Command(BaseCommand): + """ + Iterates through every upstream namespace and syncs it. + """ + + help = 'Sync upstream namespaces from [old-]galaxy.ansible.com' + + def add_arguments(self, parser): + parser.add_argument("--baseurl", default="https://galaxy.ansible.com") + parser.add_argument("--github_user", help="find and sync only this namespace name") + parser.add_argument("--role_name", help="find and sync only this role name") + parser.add_argument("--limit", type=int) + + def echo(self, message, style=None): + style = style or self.style.SUCCESS + self.stdout.write(style(message)) + + def handle(self, *args, **options): + + legacy_sync_from_upstream( + baseurl=options['baseurl'], + github_user=options['github_user'], + role_name=options['role_name'], + limit=options['limit'], + ) diff --git a/galaxy_ng/app/utils/galaxy.py b/galaxy_ng/app/utils/galaxy.py index 210639c52e..0319e9950a 100644 --- a/galaxy_ng/app/utils/galaxy.py +++ b/galaxy_ng/app/utils/galaxy.py @@ -106,6 +106,20 @@ def find_namespace(baseurl=None, name=None, id=None): return ns_name, ns_info +def get_namespace_owners_details(baseurl, ns_id): + # get the owners too + owners = [] + next_owners_url = baseurl + f'/api/v1/namespaces/{ns_id}/owners/' + while next_owners_url: + o_data = requests.get(next_owners_url).json() + for owner in o_data['results']: + owners.append(owner) + if not o_data.get('next'): + break + next_owners_url = baseurl + o_data['next_link'] + return owners + + def upstream_namespace_iterator( baseurl=None, limit=None, @@ -158,6 +172,7 @@ def upstream_namespace_iterator( ns_id = ndata['id'] # get the owners too + ''' owners = [] next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/' while next_owners_url: @@ -167,7 +182,8 @@ def upstream_namespace_iterator( if not o_data.get('next'): break next_owners_url = _baseurl + o_data['next_link'] - ndata['summary_fields']['owners'] = owners + ''' + ndata['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id) # send the collection namespace_count += 1 @@ -251,6 +267,7 @@ def upstream_collection_iterator( namespace_cache[ns_id] = namespace_data # get the owners too + ''' owners = [] next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/' while next_owners_url: @@ -260,8 +277,8 @@ def upstream_collection_iterator( if not o_data.get('next'): break next_owners_url = _baseurl + o_data['next_link'] - - namespace_cache[ns_id]['summary_fields']['owners'] = owners + ''' + namespace_cache[ns_id]['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id) else: namespace_data = namespace_cache[ns_id] @@ -319,13 +336,15 @@ def upstream_role_iterator( if role_name: params.append(f'name={role_name}') next_url = _baseurl + '/api/v1/roles/?' + '&'.join(params) + else: + next_url = _baseurl + '/api/v1/roles/' namespace_cache = {} pagenum = 0 role_count = 0 while next_url: - logger.info(f'fetch {pagenum} {next_url}') + logger.info(f'fetch {pagenum} {next_url} role-count:{role_count}') page = requests.get(next_url) @@ -369,6 +388,7 @@ def upstream_role_iterator( namespace_cache[ns_id] = namespace_data # get the owners too + ''' owners = [] next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/' while next_owners_url: @@ -378,8 +398,8 @@ def upstream_role_iterator( if not o_data.get('next'): break next_owners_url = _baseurl + o_data['next_link'] - - namespace_cache[ns_id]['summary_fields']['owners'] = owners + ''' + namespace_cache[ns_id]['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id) else: namespace_data = namespace_cache[ns_id] diff --git a/galaxy_ng/app/utils/legacy.py b/galaxy_ng/app/utils/legacy.py index ed740f91c0..2cd76fa715 100644 --- a/galaxy_ng/app/utils/legacy.py +++ b/galaxy_ng/app/utils/legacy.py @@ -149,3 +149,5 @@ def process_namespace(namespace_name, namespace_info): owner.save() add_user_to_v3_namespace(owner, namespace) + + return legacy_namespace, namespace