Force a v3 namespace for all legacy namespaces.

No-Issue

Signed-off-by: James Tanner <[email protected]>

galaxy_ng/app/access_control/access_policy.py

@@ -23,6 +23,7 @@
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
 from galaxy_ng.app.constants import COMMUNITY_DOMAINS
+from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
 
 from galaxy_ng.app.access_control.statements import PULP_VIEWSETS
 
@@ -770,8 +771,14 @@ def is_namespace_owner(self, request, viewset, action):
         if namespace is None and github_user and user.username == github_user:
             return True
 
-        # allow owners to do things in the namespace
-        if namespace and user.username in [x.username for x in namespace.owners.all()]:
+        # v1 namespace rbac is controlled via their v3 namespace
+        v3_namespace = namespace.namespace
+        if not v3_namespace:
+            return False
+
+        # use the helper to get the list of owners
+        owners = get_v3_namespace_owners(v3_namespace)
+        if owners and user in owners:
             return True
 
         return False

galaxy_ng/app/api/v1/filtersets.py

@@ -5,11 +5,13 @@
 from galaxy_ng.app.models.auth import User
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
+from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
 
 
 class LegacyNamespaceFilter(filterset.FilterSet):
 
     keywords = filters.CharFilter(method='keywords_filter')
+    owner = filters.CharFilter(method='owner_filter')
 
     sort = filters.OrderingFilter(
         fields=(
@@ -31,6 +33,23 @@ def keywords_filter(self, queryset, name, value):
 
         return queryset
 
+    def owner_filter(self, queryset, name, value):
+        # find the owner on the linked v3 namespace
+
+        # FIXME - this is terribly slow
+        pks = []
+        for ns1 in LegacyNamespace.objects.all():
+            if not ns1.namespace:
+                continue
+            ns3 = ns1.namespace
+            owners = get_v3_namespace_owners(ns3)
+            if value in [x.username for x in owners]:
+                pks.append(ns1.id)
+
+        queryset = queryset.filter(id__in=pks)
+
+        return queryset
+
 
 class LegacyUserFilter(filterset.FilterSet):
 
@@ -61,6 +80,7 @@ class LegacyRoleFilter(filterset.FilterSet):
     tag = filters.CharFilter(method='tags_filter')
     autocomplete = filters.CharFilter(method='autocomplete_filter')
     owner__username = filters.CharFilter(method='owner__username_filter')
+    namespace = filters.CharFilter(method='namespace_filter')
 
     sort = filters.OrderingFilter(
         fields=(
@@ -76,6 +96,9 @@ class Meta:
     def github_user_filter(self, queryset, name, value):
         return queryset.filter(namespace__name=value)
 
+    def namespace_filter(self, queryset, name, value):
+        return queryset.filter(namespace__name__iexact=value)
+
     def owner__username_filter(self, queryset, name, value):
         """
         The cli uses this filter to find a role by the namespace.

galaxy_ng/app/api/v1/models.py

@@ -108,6 +108,9 @@ class LegacyNamespace(models.Model):
         editable=True
     )
 
+    def __repr__(self):
+        return f'<LegacyNamespace: {self.name}>'
+
 
 class LegacyRole(models.Model):
     """
@@ -148,6 +151,9 @@ class LegacyRole(models.Model):
         default=dict
     )
 
+    def __repr__(self):
+        return f'<LegacyRole: {self.namespace.name}.{self.name}>'
+
 
 class LegacyRoleDownloadCount(models.Model):
     legacyrole = models.OneToOneField(

galaxy_ng/app/api/v1/serializers.py

@@ -3,6 +3,8 @@
 from pulpcore.plugin.util import get_url
 
 from galaxy_ng.app.models.auth import User
+from galaxy_ng.app.models.namespace import Namespace
+from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
 from galaxy_ng.app.api.v1.models import LegacyRoleDownloadCount
@@ -49,8 +51,11 @@ def get_date_joined(self, obj):
         return obj.created
 
     def get_summary_fields(self, obj):
-        owners = obj.owners.all()
-        owners = [{'id': x.id, 'username': x.username} for x in owners]
+
+        owners = []
+        if obj.namespace:
+            owner_objects = get_v3_namespace_owners(obj.namespace)
+            owners = [{'id': x.id, 'username': x.username} for x in owner_objects]
 
         # link the v1 namespace to the v3 namespace so that users
         # don't need to query the database to figure it out.
@@ -82,6 +87,22 @@ def get_id(self, obj):
         return obj.id
 
 
+class LegacyNamespaceProviderSerializer(serializers.ModelSerializer):
+
+    pulp_href = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Namespace
+        fields = [
+            'id',
+            'name',
+            'pulp_href'
+        ]
+
+    def get_pulp_href(self, obj):
+        return get_url(obj)
+
+
 class LegacyUserSerializer(serializers.ModelSerializer):
 
     summary_fields = serializers.SerializerMethodField()

galaxy_ng/app/api/v1/tasks.py

@@ -2,6 +2,7 @@
 import datetime
 import logging
 import os
+import subprocess
 import tempfile
 
 from django.db import transaction
@@ -13,6 +14,7 @@
 from galaxy_ng.app.utils.galaxy import upstream_role_iterator
 from galaxy_ng.app.utils.git import get_tag_commit_hash
 from galaxy_ng.app.utils.git import get_tag_commit_date
+from galaxy_ng.app.utils.legacy import process_namespace
 
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
@@ -60,27 +62,39 @@ def legacy_role_import(
     if not github_reference:
         github_reference = None
 
-    if LegacyNamespace.objects.filter(name=github_user).count() == 0:
-        logger.debug(f'CREATE NEW NAMESPACE {github_user}')
-        namespace, _ = LegacyNamespace.objects.get_or_create(name=github_user)
+    # this shouldn't happen but just in case ...
+    if request_username and not User.objects.filter(username=request_username).exists():
+        raise Exception(f'Username {request_username} does not exist in galaxy')
 
-        # set the owner to this request user ...
-        user = User.objects.filter(username=request_username).first()
-        namespace.owners.add(user)
+    # the user should have a legacy and v3 namespace if they logged in ...
+    namespace = LegacyNamespace.objects.filter(name=github_user).first()
+    if not namespace:
+        raise Exception(f'No legacy namespace exists for {github_user}')
 
-    else:
-        logger.debug(f'USE EXISTING NAMESPACE {github_user}')
-        namespace = LegacyNamespace.objects.filter(name=github_user).first()
+    # we have to have a v3 namespace because of the rbac based ownership ...
+    v3_namespace = namespace.namespace
+    if not v3_namespace:
+        raise Exception(f'No v3 namespace exists for {github_user}')
 
     with tempfile.TemporaryDirectory() as tmp_path:
         # galaxy-importer requires importing legacy roles from the role's parent directory.
         os.chdir(tmp_path)
 
         # galaxy-importer wants the role's directory to be the name of the role.
         checkout_path = os.path.join(tmp_path, github_repo)
-
         clone_url = f'https://github.com/{github_user}/{github_repo}'
-        gitrepo = Repo.clone_from(clone_url, checkout_path, multi_options=["--recurse-submodules"])
+
+        # pygit didn't have an obvious way to prevent interactive clones ...
+        pid = subprocess.run(
+            f'GIT_TERMINAL_PROMPT=0 git clone --recurse-submodules {clone_url} {checkout_path}',
+            shell=True,
+        )
+        if pid.returncode != 0:
+            raise Exception(f'git clone for {clone_url} failed')
+
+        # bind the checkout to a pygit object
+        gitrepo = Repo(checkout_path)
+
         github_commit = None
         github_commit_date = None
 
@@ -126,7 +140,7 @@ def legacy_role_import(
             if github_reference in old_versions:
                 msg = (
                     f'{namespace.name}.{role_name} {github_reference}'
-                    + 'has already been imported'
+                    + ' has already been imported'
                 )
                 raise Exception(msg)
 
@@ -186,7 +200,8 @@ def legacy_sync_from_upstream(
     github_user=None,
     role_name=None,
     role_version=None,
-    limit=None
+    limit=None,
+    start_page=None,
 ):
     """
     Sync legacy roles from a remote v1 api.
@@ -214,8 +229,6 @@ def legacy_sync_from_upstream(
 
     logger.debug('SYNC INDEX EXISTING NAMESPACES')
     nsmap = {}
-    for ns in LegacyNamespace.objects.all():
-        nsmap[ns.name] = ns
 
     # allow the user to specify how many roles to sync
     if limit is not None:
@@ -228,12 +241,23 @@ def legacy_sync_from_upstream(
         'baseurl': baseurl,
         'github_user': github_user,
         'role_name': role_name,
-        'limit': limit
+        'limit': limit,
+        'start_page': start_page,
     }
     for ns_data, rdata, rversions in upstream_role_iterator(**iterator_kwargs):
 
+        # processing a namespace should make owners and set rbac as needed ...
+        if ns_data['name'] not in nsmap:
+            namespace, v3_namespace = process_namespace(ns_data['name'], ns_data)
+            nsmap[ns_data['name']] = (namespace, v3_namespace)
+        else:
+            namespace, v3_namespace = nsmap[ns_data['name']]
+
         ruser = rdata.get('github_user')
         rname = rdata.get('name')
+
+        logger.info(f'POPULATE {ruser}.{rname}')
+
         rkey = (ruser, rname)
         remote_id = rdata['id']
         role_versions = rversions[:]
@@ -260,19 +284,6 @@ def legacy_sync_from_upstream(
         role_type = rdata.get('role_type', 'ANS')
         role_download_count = rdata.get('download_count', 0)
 
-        if ruser not in nsmap:
-            logger.debug(f'SYNC NAMESPACE GET_OR_CREATE {ruser}')
-            namespace, _ = LegacyNamespace.objects.get_or_create(name=ruser)
-
-            # if the ns has owners, create them and set them
-            for owner_info in ns_data['summary_fields']['owners']:
-                user, _ = User.objects.get_or_create(username=owner_info['username'])
-                namespace.owners.add(user)
-
-            nsmap[ruser] = namespace
-        else:
-            namespace = nsmap[ruser]
-
         if rkey not in rmap:
             logger.debug(f'SYNC create initial role for {rkey}')
             this_role, _ = LegacyRole.objects.get_or_create(

galaxy_ng/app/api/v1/urls.py

@@ -10,6 +10,7 @@
     LegacyRolesSyncViewSet,
     LegacyNamespacesViewSet,
     LegacyNamespaceOwnersViewSet,
+    LegacyNamespaceProvidersViewSet,
     LegacyUsersViewSet
 )
 
@@ -86,7 +87,7 @@
     ),
     path(
         'namespaces/',
-        LegacyNamespacesViewSet.as_view({"get": "list"}),
+        LegacyNamespacesViewSet.as_view({"get": "list", "post": "create"}),
         name='legacy_namespace-list'
     ),
     path(
@@ -99,6 +100,11 @@
         LegacyNamespaceOwnersViewSet.as_view({"get": "list", "put": "update"}),
         name='legacy_namespace_owners-list'
     ),
+    path(
+        'namespaces/<int:pk>/providers/',
+        LegacyNamespaceProvidersViewSet.as_view({"get": "list", "put": "update", "post": "update"}),
+        name='legacy_namespace_providers-list'
+    ),
 
     path('', LegacyRootView.as_view(), name='legacy-root')
 ]

galaxy_ng/app/api/v1/viewsets/__init__.py

@@ -1,6 +1,7 @@
 from .namespaces import (
     LegacyNamespacesViewSet,
     LegacyNamespaceOwnersViewSet,
+    LegacyNamespaceProvidersViewSet,
 )
 
 from .users import (
@@ -23,6 +24,7 @@
 __all__ = (
     LegacyNamespacesViewSet,
     LegacyNamespaceOwnersViewSet,
+    LegacyNamespaceProvidersViewSet,
     LegacyUsersViewSet,
     LegacyRolesViewSet,
     LegacyRolesSyncViewSet,