From 6c01a6d995e363636639733c98606cdf05bc73ce Mon Sep 17 00:00:00 2001 From: James Tanner Date: Wed, 20 Sep 2023 13:34:15 -0400 Subject: [PATCH] Force a v3 namespace for all legacy namespaces. No-Issue Signed-off-by: James Tanner --- galaxy_ng/app/access_control/access_policy.py | 11 +- galaxy_ng/app/api/v1/filtersets.py | 23 + galaxy_ng/app/api/v1/models.py | 6 + galaxy_ng/app/api/v1/serializers.py | 25 +- galaxy_ng/app/api/v1/tasks.py | 69 +-- galaxy_ng/app/api/v1/urls.py | 8 +- galaxy_ng/app/api/v1/viewsets/__init__.py | 2 + galaxy_ng/app/api/v1/viewsets/namespaces.py | 73 ++- galaxy_ng/app/dynaconf_hooks.py | 8 +- .../management/commands/import-galaxy-role.py | 33 ++ .../commands/sync-galaxy-namespaces.py | 63 +++ .../management/commands/sync-galaxy-roles.py | 37 ++ galaxy_ng/app/utils/galaxy.py | 307 +++++++++++- galaxy_ng/app/utils/legacy.py | 134 +++++ galaxy_ng/app/utils/namespaces.py | 70 +++ galaxy_ng/app/utils/rbac.py | 23 +- galaxy_ng/social/__init__.py | 116 +++-- galaxy_ng/social/pipeline/user.py | 104 +++- .../tests/integration/api/test_aiindex.py | 1 + .../tests/integration/api/test_community.py | 18 + .../test_community_namespace_rbac.py | 470 ++++++++++++++++++ .../integration/community/test_v1_api.py | 21 + galaxy_ng/tests/integration/utils/__init__.py | 2 +- galaxy_ng/tests/integration/utils/legacy.py | 7 +- profiles/community/github_mock/flaskapp.py | 4 +- 25 files changed, 1538 insertions(+), 97 deletions(-) create mode 100644 galaxy_ng/app/management/commands/import-galaxy-role.py create mode 100644 galaxy_ng/app/management/commands/sync-galaxy-namespaces.py create mode 100644 galaxy_ng/app/management/commands/sync-galaxy-roles.py create mode 100644 galaxy_ng/app/utils/legacy.py create mode 100644 galaxy_ng/app/utils/namespaces.py create mode 100644 galaxy_ng/tests/integration/community/test_community_namespace_rbac.py diff --git a/galaxy_ng/app/access_control/access_policy.py b/galaxy_ng/app/access_control/access_policy.py index 3bbc847959..e9173df876 100644 --- a/galaxy_ng/app/access_control/access_policy.py +++ b/galaxy_ng/app/access_control/access_policy.py @@ -23,6 +23,7 @@ from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole from galaxy_ng.app.constants import COMMUNITY_DOMAINS +from galaxy_ng.app.utils.rbac import get_v3_namespace_owners from galaxy_ng.app.access_control.statements import PULP_VIEWSETS @@ -770,8 +771,14 @@ def is_namespace_owner(self, request, viewset, action): if namespace is None and github_user and user.username == github_user: return True - # allow owners to do things in the namespace - if namespace and user.username in [x.username for x in namespace.owners.all()]: + # v1 namespace rbac is controlled via their v3 namespace + v3_namespace = namespace.namespace + if not v3_namespace: + return False + + # use the helper to get the list of owners + owners = get_v3_namespace_owners(v3_namespace) + if owners and user in owners: return True return False diff --git a/galaxy_ng/app/api/v1/filtersets.py b/galaxy_ng/app/api/v1/filtersets.py index 76d9bc0222..694d08c4a6 100644 --- a/galaxy_ng/app/api/v1/filtersets.py +++ b/galaxy_ng/app/api/v1/filtersets.py @@ -5,11 +5,13 @@ from galaxy_ng.app.models.auth import User from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole +from galaxy_ng.app.utils.rbac import get_v3_namespace_owners class LegacyNamespaceFilter(filterset.FilterSet): keywords = filters.CharFilter(method='keywords_filter') + owner = filters.CharFilter(method='owner_filter') sort = filters.OrderingFilter( fields=( @@ -31,6 +33,23 @@ def keywords_filter(self, queryset, name, value): return queryset + def owner_filter(self, queryset, name, value): + # find the owner on the linked v3 namespace + + # FIXME - this is terribly slow + pks = [] + for ns1 in LegacyNamespace.objects.all(): + if not ns1.namespace: + continue + ns3 = ns1.namespace + owners = get_v3_namespace_owners(ns3) + if value in [x.username for x in owners]: + pks.append(ns1.id) + + queryset = queryset.filter(id__in=pks) + + return queryset + class LegacyUserFilter(filterset.FilterSet): @@ -61,6 +80,7 @@ class LegacyRoleFilter(filterset.FilterSet): tag = filters.CharFilter(method='tags_filter') autocomplete = filters.CharFilter(method='autocomplete_filter') owner__username = filters.CharFilter(method='owner__username_filter') + namespace = filters.CharFilter(method='namespace_filter') sort = filters.OrderingFilter( fields=( @@ -76,6 +96,9 @@ class Meta: def github_user_filter(self, queryset, name, value): return queryset.filter(namespace__name=value) + def namespace_filter(self, queryset, name, value): + return queryset.filter(namespace__name__iexact=value) + def owner__username_filter(self, queryset, name, value): """ The cli uses this filter to find a role by the namespace. diff --git a/galaxy_ng/app/api/v1/models.py b/galaxy_ng/app/api/v1/models.py index 95a74e2ebc..c282d44c89 100644 --- a/galaxy_ng/app/api/v1/models.py +++ b/galaxy_ng/app/api/v1/models.py @@ -108,6 +108,9 @@ class LegacyNamespace(models.Model): editable=True ) + def __repr__(self): + return f'' + class LegacyRole(models.Model): """ @@ -148,6 +151,9 @@ class LegacyRole(models.Model): default=dict ) + def __repr__(self): + return f'' + class LegacyRoleDownloadCount(models.Model): legacyrole = models.OneToOneField( diff --git a/galaxy_ng/app/api/v1/serializers.py b/galaxy_ng/app/api/v1/serializers.py index 77c95a1c71..37ac43d292 100644 --- a/galaxy_ng/app/api/v1/serializers.py +++ b/galaxy_ng/app/api/v1/serializers.py @@ -3,6 +3,8 @@ from pulpcore.plugin.util import get_url from galaxy_ng.app.models.auth import User +from galaxy_ng.app.models.namespace import Namespace +from galaxy_ng.app.utils.rbac import get_v3_namespace_owners from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole from galaxy_ng.app.api.v1.models import LegacyRoleDownloadCount @@ -49,8 +51,11 @@ def get_date_joined(self, obj): return obj.created def get_summary_fields(self, obj): - owners = obj.owners.all() - owners = [{'id': x.id, 'username': x.username} for x in owners] + + owners = [] + if obj.namespace: + owner_objects = get_v3_namespace_owners(obj.namespace) + owners = [{'id': x.id, 'username': x.username} for x in owner_objects] # link the v1 namespace to the v3 namespace so that users # don't need to query the database to figure it out. @@ -82,6 +87,22 @@ def get_id(self, obj): return obj.id +class LegacyNamespaceProviderSerializer(serializers.ModelSerializer): + + pulp_href = serializers.SerializerMethodField() + + class Meta: + model = Namespace + fields = [ + 'id', + 'name', + 'pulp_href' + ] + + def get_pulp_href(self, obj): + return get_url(obj) + + class LegacyUserSerializer(serializers.ModelSerializer): summary_fields = serializers.SerializerMethodField() diff --git a/galaxy_ng/app/api/v1/tasks.py b/galaxy_ng/app/api/v1/tasks.py index bfd989b232..03be9fde4c 100644 --- a/galaxy_ng/app/api/v1/tasks.py +++ b/galaxy_ng/app/api/v1/tasks.py @@ -2,6 +2,7 @@ import datetime import logging import os +import subprocess import tempfile from django.db import transaction @@ -13,6 +14,7 @@ from galaxy_ng.app.utils.galaxy import upstream_role_iterator from galaxy_ng.app.utils.git import get_tag_commit_hash from galaxy_ng.app.utils.git import get_tag_commit_date +from galaxy_ng.app.utils.legacy import process_namespace from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole @@ -60,17 +62,19 @@ def legacy_role_import( if not github_reference: github_reference = None - if LegacyNamespace.objects.filter(name=github_user).count() == 0: - logger.debug(f'CREATE NEW NAMESPACE {github_user}') - namespace, _ = LegacyNamespace.objects.get_or_create(name=github_user) + # this shouldn't happen but just in case ... + if request_username and not User.objects.filter(username=request_username).exists(): + raise Exception(f'Username {request_username} does not exist in galaxy') - # set the owner to this request user ... - user = User.objects.filter(username=request_username).first() - namespace.owners.add(user) + # the user should have a legacy and v3 namespace if they logged in ... + namespace = LegacyNamespace.objects.filter(name=github_user).first() + if not namespace: + raise Exception(f'No legacy namespace exists for {github_user}') - else: - logger.debug(f'USE EXISTING NAMESPACE {github_user}') - namespace = LegacyNamespace.objects.filter(name=github_user).first() + # we have to have a v3 namespace because of the rbac based ownership ... + v3_namespace = namespace.namespace + if not v3_namespace: + raise Exception(f'No v3 namespace exists for {github_user}') with tempfile.TemporaryDirectory() as tmp_path: # galaxy-importer requires importing legacy roles from the role's parent directory. @@ -78,9 +82,19 @@ def legacy_role_import( # galaxy-importer wants the role's directory to be the name of the role. checkout_path = os.path.join(tmp_path, github_repo) - clone_url = f'https://github.com/{github_user}/{github_repo}' - gitrepo = Repo.clone_from(clone_url, checkout_path, multi_options=["--recurse-submodules"]) + + # pygit didn't have an obvious way to prevent interactive clones ... + pid = subprocess.run( + f'GIT_TERMINAL_PROMPT=0 git clone --recurse-submodules {clone_url} {checkout_path}', + shell=True, + ) + if pid.returncode != 0: + raise Exception(f'git clone for {clone_url} failed') + + # bind the checkout to a pygit object + gitrepo = Repo(checkout_path) + github_commit = None github_commit_date = None @@ -126,7 +140,7 @@ def legacy_role_import( if github_reference in old_versions: msg = ( f'{namespace.name}.{role_name} {github_reference}' - + 'has already been imported' + + ' has already been imported' ) raise Exception(msg) @@ -186,7 +200,8 @@ def legacy_sync_from_upstream( github_user=None, role_name=None, role_version=None, - limit=None + limit=None, + start_page=None, ): """ Sync legacy roles from a remote v1 api. @@ -214,8 +229,6 @@ def legacy_sync_from_upstream( logger.debug('SYNC INDEX EXISTING NAMESPACES') nsmap = {} - for ns in LegacyNamespace.objects.all(): - nsmap[ns.name] = ns # allow the user to specify how many roles to sync if limit is not None: @@ -228,12 +241,23 @@ def legacy_sync_from_upstream( 'baseurl': baseurl, 'github_user': github_user, 'role_name': role_name, - 'limit': limit + 'limit': limit, + 'start_page': start_page, } for ns_data, rdata, rversions in upstream_role_iterator(**iterator_kwargs): + # processing a namespace should make owners and set rbac as needed ... + if ns_data['name'] not in nsmap: + namespace, v3_namespace = process_namespace(ns_data['name'], ns_data) + nsmap[ns_data['name']] = (namespace, v3_namespace) + else: + namespace, v3_namespace = nsmap[ns_data['name']] + ruser = rdata.get('github_user') rname = rdata.get('name') + + logger.info(f'POPULATE {ruser}.{rname}') + rkey = (ruser, rname) remote_id = rdata['id'] role_versions = rversions[:] @@ -260,19 +284,6 @@ def legacy_sync_from_upstream( role_type = rdata.get('role_type', 'ANS') role_download_count = rdata.get('download_count', 0) - if ruser not in nsmap: - logger.debug(f'SYNC NAMESPACE GET_OR_CREATE {ruser}') - namespace, _ = LegacyNamespace.objects.get_or_create(name=ruser) - - # if the ns has owners, create them and set them - for owner_info in ns_data['summary_fields']['owners']: - user, _ = User.objects.get_or_create(username=owner_info['username']) - namespace.owners.add(user) - - nsmap[ruser] = namespace - else: - namespace = nsmap[ruser] - if rkey not in rmap: logger.debug(f'SYNC create initial role for {rkey}') this_role, _ = LegacyRole.objects.get_or_create( diff --git a/galaxy_ng/app/api/v1/urls.py b/galaxy_ng/app/api/v1/urls.py index 67dd5fc380..5c010f616f 100644 --- a/galaxy_ng/app/api/v1/urls.py +++ b/galaxy_ng/app/api/v1/urls.py @@ -10,6 +10,7 @@ LegacyRolesSyncViewSet, LegacyNamespacesViewSet, LegacyNamespaceOwnersViewSet, + LegacyNamespaceProvidersViewSet, LegacyUsersViewSet ) @@ -86,7 +87,7 @@ ), path( 'namespaces/', - LegacyNamespacesViewSet.as_view({"get": "list"}), + LegacyNamespacesViewSet.as_view({"get": "list", "post": "create"}), name='legacy_namespace-list' ), path( @@ -99,6 +100,11 @@ LegacyNamespaceOwnersViewSet.as_view({"get": "list", "put": "update"}), name='legacy_namespace_owners-list' ), + path( + 'namespaces//providers/', + LegacyNamespaceProvidersViewSet.as_view({"get": "list", "put": "update", "post": "update"}), + name='legacy_namespace_providers-list' + ), path('', LegacyRootView.as_view(), name='legacy-root') ] diff --git a/galaxy_ng/app/api/v1/viewsets/__init__.py b/galaxy_ng/app/api/v1/viewsets/__init__.py index 8b3e2352df..bfdc57ce38 100644 --- a/galaxy_ng/app/api/v1/viewsets/__init__.py +++ b/galaxy_ng/app/api/v1/viewsets/__init__.py @@ -1,6 +1,7 @@ from .namespaces import ( LegacyNamespacesViewSet, LegacyNamespaceOwnersViewSet, + LegacyNamespaceProvidersViewSet, ) from .users import ( @@ -23,6 +24,7 @@ __all__ = ( LegacyNamespacesViewSet, LegacyNamespaceOwnersViewSet, + LegacyNamespaceProvidersViewSet, LegacyUsersViewSet, LegacyRolesViewSet, LegacyRolesSyncViewSet, diff --git a/galaxy_ng/app/api/v1/viewsets/namespaces.py b/galaxy_ng/app/api/v1/viewsets/namespaces.py index c40b462c0c..0f40997bc0 100644 --- a/galaxy_ng/app/api/v1/viewsets/namespaces.py +++ b/galaxy_ng/app/api/v1/viewsets/namespaces.py @@ -14,14 +14,19 @@ from rest_framework.pagination import PageNumberPagination from galaxy_ng.app.access_control.access_policy import LegacyAccessPolicy +from galaxy_ng.app.utils.rbac import get_v3_namespace_owners +from galaxy_ng.app.utils.rbac import add_user_to_v3_namespace +from galaxy_ng.app.utils.rbac import remove_user_from_v3_namespace from galaxy_ng.app.models.auth import User +from galaxy_ng.app.models import Namespace from galaxy_ng.app.api.v1.models import ( LegacyNamespace, ) from galaxy_ng.app.api.v1.serializers import ( LegacyNamespacesSerializer, LegacyNamespaceOwnerSerializer, + LegacyNamespaceProviderSerializer, LegacyUserSerializer ) @@ -83,6 +88,18 @@ class LegacyNamespacesViewSet( def destroy(self, request, pk=None): return super().destroy(self, request, pk) + def create(self, request): + + ns_name = request.data.get('name') + namespace, created = LegacyNamespace.objects.get_or_create(name=ns_name) + + return Response( + self.serializer_class( + namespace, + many=False + ).data + ) + class LegacyNamespaceOwnersViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): """ @@ -100,7 +117,11 @@ class LegacyNamespaceOwnersViewSet(viewsets.GenericViewSet, mixins.ListModelMixi authentication_classes = GALAXY_AUTHENTICATION_CLASSES def get_queryset(self): - return get_object_or_404(LegacyNamespace, pk=self.kwargs["pk"]).owners.all() + # return get_object_or_404(LegacyNamespace, pk=self.kwargs["pk"]).owners.all() + ns = get_object_or_404(LegacyNamespace, pk=self.kwargs["pk"]) + if ns.namespace: + return get_v3_namespace_owners(ns.namespace) + return [] @extend_schema( parameters=[LegacyNamespaceOwnerSerializer(many=True)], @@ -120,6 +141,52 @@ def update(self, request, pk): if len(pks) != new_owners.count(): raise exceptions.ValidationError("user doesn't exist", code="invalid") - ns.owners.set(new_owners) + # get the foreign key ... + v3_namespace = ns.namespace + + # get the list of current owners + current_owners = get_v3_namespace_owners(v3_namespace) + + # remove all owners not in the new list + for current_owner in current_owners: + if current_owner not in new_owners: + remove_user_from_v3_namespace(current_owner, v3_namespace) + + # add new owners if not in the list + for new_owner in new_owners: + if new_owner not in current_owners: + add_user_to_v3_namespace(new_owner, v3_namespace) + + return Response( + self.serializer_class( + get_v3_namespace_owners(v3_namespace), + many=True + ).data + ) + + +class LegacyNamespaceProvidersViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): + + serializer_class = LegacyNamespaceProviderSerializer + pagination_class = None + permission_classes = [LegacyAccessPolicy] + authentication_classes = GALAXY_AUTHENTICATION_CLASSES + + def get_queryset(self): + ns = get_object_or_404(LegacyNamespace, pk=self.kwargs["pk"]) + if ns.namespace: + return [ns.namespace] + return [] + + def update(self, request, pk): + '''Bind a v3 namespace to the legacy namespace''' + legacy_ns = get_object_or_404(LegacyNamespace, pk=pk) + + v3_id = request.data['id'] + v3_namespace = get_object_or_404(Namespace, id=v3_id) + + if legacy_ns.namespace != v3_namespace: + legacy_ns.namespace = v3_namespace + legacy_ns.save() - return Response(self.serializer_class(ns.owners.all(), many=True).data) + return Response({}) diff --git a/galaxy_ng/app/dynaconf_hooks.py b/galaxy_ng/app/dynaconf_hooks.py index 09505227ba..5455793cca 100755 --- a/galaxy_ng/app/dynaconf_hooks.py +++ b/galaxy_ng/app/dynaconf_hooks.py @@ -222,17 +222,15 @@ def configure_socialauth(settings: Dynaconf) -> Dict[str, Any]: "rest_framework.authentication.BasicAuthentication", ] - # Override the get_username step so that social users - # are associated to existing users instead of creating - # a whole new randomized username. + # Override the get_username and create_user steps + # to conform to our super special user validation + # requirements data['SOCIAL_AUTH_PIPELINE'] = [ 'social_core.pipeline.social_auth.social_details', 'social_core.pipeline.social_auth.social_uid', 'social_core.pipeline.social_auth.auth_allowed', 'social_core.pipeline.social_auth.social_user', - # 'social_core.pipeline.user.get_username', 'galaxy_ng.social.pipeline.user.get_username', - # 'social_core.pipeline.user.create_user', 'galaxy_ng.social.pipeline.user.create_user', 'social_core.pipeline.social_auth.associate_user', 'social_core.pipeline.social_auth.load_extra_data', diff --git a/galaxy_ng/app/management/commands/import-galaxy-role.py b/galaxy_ng/app/management/commands/import-galaxy-role.py new file mode 100644 index 0000000000..c8987d7737 --- /dev/null +++ b/galaxy_ng/app/management/commands/import-galaxy-role.py @@ -0,0 +1,33 @@ +import django_guid +from django.core.management.base import BaseCommand + +from galaxy_ng.app.api.v1.tasks import legacy_role_import + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +class Command(BaseCommand): + """ + Import a role using the same functions the API uses. + """ + + help = 'Import a role using the same functions the API uses.' + + def add_arguments(self, parser): + parser.add_argument("--github_user", required=True) + parser.add_argument("--github_repo", required=True) + parser.add_argument("--role_name", help="find and sync only this namespace name") + parser.add_argument("--branch", help="find and sync only this namespace name") + parser.add_argument("--request_username", help="set the uploader's username") + + def handle(self, *args, **options): + # This is the same function that api/v1/imports eventually calls ... + legacy_role_import( + request_username=options['request_username'], + github_user=options['github_user'], + github_repo=options['github_repo'], + github_reference=options['branch'], + alternate_role_name=options['role_name'], + ) diff --git a/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py b/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py new file mode 100644 index 0000000000..3902e73a18 --- /dev/null +++ b/galaxy_ng/app/management/commands/sync-galaxy-namespaces.py @@ -0,0 +1,63 @@ +import django_guid +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand + +from galaxy_ng.app.utils.galaxy import upstream_namespace_iterator +from galaxy_ng.app.utils.galaxy import find_namespace +from galaxy_ng.app.utils.legacy import process_namespace + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +User = get_user_model() + + +class Command(BaseCommand): + """ + Iterates through every upstream namespace and syncs it. + """ + + help = 'Sync upstream namespaces+owners from [old-]galaxy.ansible.com' + + def add_arguments(self, parser): + parser.add_argument("--baseurl", default="https://galaxy.ansible.com") + parser.add_argument("--name", help="find and sync only this namespace name") + parser.add_argument("--id", help="find and sync only this namespace id") + parser.add_argument("--limit", type=int) + parser.add_argument("--start_page", type=int) + + def echo(self, message, style=None): + style = style or self.style.SUCCESS + self.stdout.write(style(message)) + + def handle(self, *args, **options): + + if options.get('name'): + ns_name, ns_info = find_namespace(baseurl=options['baseurl'], name=options['name']) + self.echo(f'PROCESSING {ns_info["id"]}:{ns_name}') + process_namespace(ns_name, ns_info) + + elif options.get('id'): + ns_name, ns_info = find_namespace(baseurl=options['baseurl'], id=options['id']) + self.echo(f'PROCESSING {ns_info["id"]}:{ns_name}') + process_namespace(ns_name, ns_info) + + else: + + count = 0 + for total, namespace_info in upstream_namespace_iterator( + baseurl=options['baseurl'], + start_page=options['start_page'], + limit=options['limit'], + ): + + count += 1 + namespace_name = namespace_info['name'] + + self.echo( + f'({total}|{count})' + + f' PROCESSING {namespace_info["id"]}:{namespace_name}' + ) + process_namespace(namespace_name, namespace_info) diff --git a/galaxy_ng/app/management/commands/sync-galaxy-roles.py b/galaxy_ng/app/management/commands/sync-galaxy-roles.py new file mode 100644 index 0000000000..679046e670 --- /dev/null +++ b/galaxy_ng/app/management/commands/sync-galaxy-roles.py @@ -0,0 +1,37 @@ +import django_guid +from django.core.management.base import BaseCommand +from galaxy_ng.app.api.v1.tasks import legacy_sync_from_upstream + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +class Command(BaseCommand): + """ + Iterates through api/v1/roles and sync all roles found. + """ + + help = 'Sync upstream roles from [old-]galaxy.ansible.com' + + def add_arguments(self, parser): + parser.add_argument("--baseurl", default="https://galaxy.ansible.com") + parser.add_argument("--github_user", help="find and sync only this namespace name") + parser.add_argument("--role_name", help="find and sync only this role name") + parser.add_argument("--limit", type=int) + parser.add_argument("--start_page", type=int) + + def echo(self, message, style=None): + style = style or self.style.SUCCESS + self.stdout.write(style(message)) + + def handle(self, *args, **options): + + # This is the function that api/v1/sync eventually calls in a task ... + legacy_sync_from_upstream( + baseurl=options['baseurl'], + github_user=options['github_user'], + role_name=options['role_name'], + limit=options['limit'], + start_page=options['start_page'], + ) diff --git a/galaxy_ng/app/utils/galaxy.py b/galaxy_ng/app/utils/galaxy.py index 414da79f9e..84d63e6c49 100644 --- a/galaxy_ng/app/utils/galaxy.py +++ b/galaxy_ng/app/utils/galaxy.py @@ -1,11 +1,16 @@ import logging import requests +import time from urllib.parse import urlparse logger = logging.getLogger(__name__) +def generate_unverified_email(github_id): + return str(github_id) + '@GALAXY.GITHUB.UNVERIFIED.COM' + + def uuid_to_int(uuid): """Cast a uuid to a reversable int""" return int(uuid.replace("-", ""), 16) @@ -24,6 +29,22 @@ def int_to_uuid(num): return uuid +def safe_fetch(url): + rr = None + counter = 0 + while True: + counter += 1 + logger.info(f'fetch {url}') + rr = requests.get(url) + if rr.status_code < 500: + return rr + + logger.info(f'waiting 60s to refetch {url}') + time.sleep(60) + + return rr + + def paginated_results(next_url): """Iterate through a paginated query and combine the results.""" parsed = urlparse(next_url) @@ -31,22 +52,267 @@ def paginated_results(next_url): results = [] while next_url: logger.info(f'fetch {next_url}') - rr = requests.get(next_url) + rr = safe_fetch(next_url) ds = rr.json() + results.extend(ds['results']) - if ds['next_link']: + if 'next' in ds: + next_url = ds['next'] + elif ds['next_link']: next_url = _baseurl + ds['next_link'] else: next_url = None + + if next_url and not next_url.startswith(_baseurl + '/api/v1'): + next_url = _baseurl + '/api/v1' + next_url + + if next_url and not next_url.startswith(_baseurl): + next_url = _baseurl + next_url + return results +def find_namespace(baseurl=None, name=None, id=None): + + logger.info(f'baseurl1: {baseurl}') + if baseurl is None or not baseurl: + baseurl = 'https://galaxy.ansible.com' + baseurl += '/api/v1/namespaces' + logger.info(f'baseurl2: {baseurl}') + + parsed = urlparse(baseurl) + _baseurl = parsed.scheme + '://' + parsed.netloc + + ns_name = None + ns_info = None + + if name: + qurl = baseurl + f'/?name={name}' + rr = safe_fetch(qurl) + ds = rr.json() + + ns_info = ds['results'][0] + ns_name = ns_info['name'] + + elif id: + qurl = baseurl + f'/{id}/' + rr = safe_fetch(qurl) + ds = rr.json() + + ns_name = ds['name'] + ns_info = ds + + # get the owners too + ns_id = ns_info['id'] + owners = [] + next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/' + while next_owners_url: + o_data = safe_fetch(next_owners_url).json() + for owner in o_data['results']: + owners.append(owner) + if not o_data.get('next'): + break + next_owners_url = _baseurl + o_data['next_link'] + + ns_info['summary_fields']['owners'] = owners + + return ns_name, ns_info + + +def get_namespace_owners_details(baseurl, ns_id): + # get the owners too + owners = [] + next_owners_url = baseurl + f'/api/v1/namespaces/{ns_id}/owners/' + while next_owners_url: + o_data = safe_fetch(next_owners_url).json() + for owner in o_data['results']: + owners.append(owner) + if not o_data.get('next'): + break + if 'next_link' in o_data and not o_data.get('next_link'): + break + next_owners_url = baseurl + o_data['next_link'] + return owners + + +def upstream_namespace_iterator( + baseurl=None, + limit=None, + start_page=None, + require_content=True, +): + + """Abstracts the pagination of v2 collections into a generator with error handling.""" + logger.info(f'baseurl1: {baseurl}') + if baseurl is None or not baseurl: + baseurl = 'https://galaxy.ansible.com/api/v1/namespaces' + if not baseurl.rstrip().endswith('/api/v1/namespaces'): + baseurl = baseurl.rstrip() + '/api/v1/namespaces' + logger.info(f'baseurl2: {baseurl}') + + # normalize the upstream url + parsed = urlparse(baseurl) + _baseurl = parsed.scheme + '://' + parsed.netloc + + # default to baseurl or construct from parameters + next_url = baseurl + + pagenum = 0 + namespace_count = 0 + + if start_page: + pagenum = start_page + next_url = next_url + f'?page={pagenum}' + + while next_url: + logger.info(f'fetch {pagenum} {next_url}') + + page = safe_fetch(next_url) + + # Some upstream pages return ISEs for whatever reason. + if page.status_code >= 500: + if 'page=' in next_url: + next_url = next_url.replace(f'page={pagenum}', f'page={pagenum+1}') + else: + next_url = next_url.rstrip('/') + '/?page={pagenum+1}' + pagenum += 1 + continue + + ds = page.json() + total = ds['count'] + + for ndata in ds['results']: + + if not ndata['summary_fields']['content_counts'] and require_content: + continue + + ns_id = ndata['id'] + + # get the owners too + ndata['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id) + + # send the collection + namespace_count += 1 + yield total, ndata + + # break early if count reached + if limit is not None and namespace_count >= limit: + break + + # break early if count reached + if limit is not None and namespace_count >= limit: + break + + # break if no next page + if not ds.get('next_link'): + break + + pagenum += 1 + next_url = _baseurl + ds['next_link'] + + +def upstream_collection_iterator( + baseurl=None, + limit=None, + collection_namespace=None, + collection_name=None, + get_versions=True, +): + """Abstracts the pagination of v2 collections into a generator with error handling.""" + logger.info(f'baseurl1: {baseurl}') + if baseurl is None or not baseurl: + baseurl = 'https://galaxy.ansible.com/api/v2/collections' + logger.info(f'baseurl2: {baseurl}') + + # normalize the upstream url + parsed = urlparse(baseurl) + _baseurl = parsed.scheme + '://' + parsed.netloc + + # default to baseurl or construct from parameters + next_url = baseurl + + ''' + params = [] + if github_user or github_repo or role_name: + if github_user: + params.append(f'owner__username={github_user}') + if role_name: + params.append(f'name={role_name}') + next_url = _baseurl + '/api/v1/roles/?' + '&'.join(params) + ''' + + namespace_cache = {} + + pagenum = 0 + collection_count = 0 + while next_url: + logger.info(f'fetch {pagenum} {next_url}') + + page = safe_fetch(next_url) + + # Some upstream pages return ISEs for whatever reason. + if page.status_code >= 500: + if 'page=' in next_url: + next_url = next_url.replace(f'page={pagenum}', f'page={pagenum+1}') + else: + next_url = next_url.rstrip('/') + '/?page={pagenum+1}' + pagenum += 1 + continue + + ds = page.json() + + for cdata in ds['results']: + + # Get the namespace+owners + ns_id = cdata['namespace']['id'] + if ns_id not in namespace_cache: + logger.info(_baseurl + f'/api/v1/namespaces/{ns_id}/') + ns_url = _baseurl + f'/api/v1/namespaces/{ns_id}/' + namespace_data = safe_fetch(ns_url).json() + # logger.info(namespace_data) + namespace_cache[ns_id] = namespace_data + + # get the owners too + namespace_cache[ns_id]['summary_fields']['owners'] = \ + get_namespace_owners_details(_baseurl, ns_id) + + else: + namespace_data = namespace_cache[ns_id] + + # get the versions + if get_versions: + collection_versions = paginated_results(cdata['versions_url']) + else: + collection_versions = [] + + # send the collection + collection_count += 1 + yield namespace_data, cdata, collection_versions + + # break early if count reached + if limit is not None and collection_count >= limit: + break + + # break early if count reached + if limit is not None and collection_count >= limit: + break + + # break if no next page + if not ds.get('next_link'): + break + + pagenum += 1 + next_url = _baseurl + ds['next_link'] + + def upstream_role_iterator( baseurl=None, limit=None, github_user=None, github_repo=None, - role_name=None + role_name=None, + get_versions=True, + start_page=None, ): """Abstracts the pagination of v1 roles into a generator with error handling.""" logger.info(f'baseurl1: {baseurl}') @@ -67,15 +333,23 @@ def upstream_role_iterator( if role_name: params.append(f'name={role_name}') next_url = _baseurl + '/api/v1/roles/?' + '&'.join(params) + else: + next_url = _baseurl + '/api/v1/roles/' + + if start_page: + if '?' in next_url: + next_url += f'&page={start_page}' + else: + next_url = next_url.rstrip('/') + f'/?page={start_page}' namespace_cache = {} pagenum = 0 role_count = 0 while next_url: - logger.info(f'fetch {pagenum} {next_url}') + logger.info(f'fetch {pagenum} {next_url} role-count:{role_count}') - page = requests.get(next_url) + page = safe_fetch(next_url) # Some upstream pages return ISEs for whatever reason. if page.status_code >= 500: @@ -95,7 +369,8 @@ def upstream_role_iterator( role_upstream_url = _baseurl + f'/api/v1/roles/{remote_id}/' logger.info(f'fetch {role_upstream_url}') - role_page = requests.get(role_upstream_url) + # role_page = requests.get(role_upstream_url) + role_page = safe_fetch(role_upstream_url) if role_page.status_code == 404: continue @@ -110,15 +385,29 @@ def upstream_role_iterator( # Get the namespace+owners ns_id = role_data['summary_fields']['namespace']['id'] if ns_id not in namespace_cache: + logger.info(_baseurl + f'/api/v1/namespaces/{ns_id}/') ns_url = _baseurl + f'/api/v1/namespaces/{ns_id}/' - namespace_data = requests.get(ns_url).json() + + nsd_rr = safe_fetch(ns_url) + try: + namespace_data = nsd_rr.json() + except requests.exceptions.JSONDecodeError: + continue namespace_cache[ns_id] = namespace_data + + # get the owners too + namespace_cache[ns_id]['summary_fields']['owners'] = \ + get_namespace_owners_details(_baseurl, ns_id) + else: namespace_data = namespace_cache[ns_id] # Get all of the versions because they have more info than the summary - versions_url = role_upstream_url + 'versions' - role_versions = paginated_results(versions_url) + if get_versions: + versions_url = role_upstream_url + 'versions' + role_versions = paginated_results(versions_url) + else: + role_versions = [] # send the role role_count += 1 diff --git a/galaxy_ng/app/utils/legacy.py b/galaxy_ng/app/utils/legacy.py new file mode 100644 index 0000000000..082a2c2fcb --- /dev/null +++ b/galaxy_ng/app/utils/legacy.py @@ -0,0 +1,134 @@ +# import trio + +# import datetime +import re +# import sys +# import time +# import django_guid +from django.contrib.auth import get_user_model + +# from django.core.management.base import BaseCommand +# from pulp_ansible.app.models import AnsibleRepository +# from pulpcore.plugin.constants import TASK_FINAL_STATES, TASK_STATES +# from pulpcore.plugin.tasking import dispatch + +from galaxy_ng.app.api.v1.models import LegacyNamespace +from galaxy_ng.app.models import Namespace +# from galaxy_ng.app.tasks.namespaces import _create_pulp_namespace +# from galaxy_ng.app.utils.galaxy import upstream_collection_iterator +# from galaxy_ng.app.utils.galaxy import upstream_namespace_iterator +# from galaxy_ng.app.utils.galaxy import upstream_role_iterator +# from galaxy_ng.app.utils.galaxy import find_namespace +from galaxy_ng.app.utils.galaxy import generate_unverified_email +from galaxy_ng.app.utils.namespaces import generate_v3_namespace_from_attributes +from galaxy_ng.app.utils.rbac import add_user_to_v3_namespace + +# from pulpcore.plugin.files import PulpTemporaryUploadedFile +# from pulpcore.plugin.download import HttpDownloader +# from pulp_ansible.app.models import AnsibleNamespaceMetadata, AnsibleNamespace +# from pulpcore.plugin.tasking import add_and_remove, dispatch +# from pulpcore.plugin.models import RepositoryContent, Artifact, ContentArtifact + + +User = get_user_model() + + +def sanitize_avatar_url(url): + '''Remove all the non-url characters people have put in their avatar urls''' + regex = ( + r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)" + + r"(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|" + + r"(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" + ) + + for match in re.findall(regex, url): + if match and 'http' in match[0]: + return match[0] + + return None + + +def process_namespace(namespace_name, namespace_info): + '''Do all the work to sync a legacy namespace and build it's v3 counterpart''' + + # get or create a legacy namespace with an identical name first ... + legacy_namespace, legacy_namespace_created = \ + LegacyNamespace.objects.get_or_create(name=namespace_name) + + if legacy_namespace.namespace: + namespace = legacy_namespace.namespace + namespace_created = False + + else: + _owners = namespace_info['summary_fields']['owners'] + _owners = [(x['github_id'], x['username']) for x in _owners] + _matched_owners = [x for x in _owners if x[1].lower() == namespace_name.lower()] + + if _matched_owners: + _owner_id = _matched_owners[0][0] + v3_namespace_name = \ + generate_v3_namespace_from_attributes(username=namespace_name, github_id=_owner_id) + else: + v3_namespace_name = generate_v3_namespace_from_attributes(username=namespace_name) + + namespace, namespace_created = Namespace.objects.get_or_create(name=v3_namespace_name) + + # bind legacy and v3 + legacy_namespace.namespace = namespace + legacy_namespace.save() + + changed = False + + if namespace_info['avatar_url'] and not namespace.avatar_url: + avatar_url = sanitize_avatar_url(namespace_info['avatar_url']) + if avatar_url: + namespace.avatar_url = avatar_url + changed = True + + if namespace_info['company'] and not namespace.company: + if len(namespace_info['company']) >= 60: + namespace.company = namespace_info['company'][:60] + else: + namespace.company = namespace_info['company'] + changed = True + + if namespace_info['email'] and not namespace.email: + namespace.email = namespace_info['email'] + changed = True + + if namespace_info['description'] and not namespace.description: + if len(namespace_info['description']) >= 60: + namespace.description = namespace_info['description'][:60] + else: + namespace.description = namespace_info['description'] + changed = True + + if changed: + namespace.save() + + for owner_info in namespace_info['summary_fields']['owners']: + + unverified_email = generate_unverified_email(owner_info['github_id']) + + owner_created = False + owner = User.objects.filter(username=unverified_email).first() + if not owner: + owner = User.objects.filter(email=unverified_email).first() + + # should always have an email set with default of the unverified email + if owner and not owner.email: + owner.email = unverified_email + owner.save() + + if not owner: + + owner, owner_created = User.objects.get_or_create(username=owner_info['username']) + + if owner_created or not owner.email: + # the new user should have the unverified email until they actually login + owner.email = unverified_email + owner.save() + + add_user_to_v3_namespace(owner, namespace) + + return legacy_namespace, namespace diff --git a/galaxy_ng/app/utils/namespaces.py b/galaxy_ng/app/utils/namespaces.py new file mode 100644 index 0000000000..9861d95822 --- /dev/null +++ b/galaxy_ng/app/utils/namespaces.py @@ -0,0 +1,70 @@ +import re +from galaxy_importer.constants import NAME_REGEXP + + +def generate_v3_namespace_from_attributes(username=None, github_id=None): + + if validate_namespace_name(username): + return username + + transformed = transform_namespace_name(username) + if validate_namespace_name(transformed): + return transformed + + return map_v3_namespace(username) + + +def map_v3_namespace(v1_namespace): + """ + 1. remove unwanted characters + 2. convert - to _ + 3. if name starts with number or _, prepend prefix + """ + + prefix = "gh_" + no_start = tuple(x for x in "0123456789_") + + name = v1_namespace.lower() + name = name.replace("-", "_") + name = re.sub(r'[^a-z0-9_]', "", name) + if name.startswith(no_start) or len(name) <= 2: + name = prefix + name.lstrip("_") + + return name + + +def generate_available_namespace_name(Namespace, login, github_id): + # we're only here because session_login is already taken as + # a namespace name and we need a new one for the user + + # this makes the weird gh_{BLAH} name ... + # namespace_name = ns_utils.map_v3_namespace(session_login) + + # we should iterate and append 0,1,2,3,4,5,etc on the login name until we find one that is free + counter = -1 + while True: + counter += 1 + namespace_name = transform_namespace_name(login) + str(counter) + if Namespace.objects.filter(name=namespace_name).count() == 0: + return namespace_name + + +def validate_namespace_name(name): + """Similar validation to the v3 namespace serializer.""" + + # galaxy has "extra" requirements for a namespace ... + # https://github.com/ansible/galaxy-importer/blob/master/galaxy_importer/constants.py#L45 + # NAME_REGEXP = re.compile(r"^(?!.*__)[a-z]+[0-9a-z_]*$") + + if not NAME_REGEXP.match(name): + return False + if len(name) < 2: + return False + if name.startswith('_'): + return False + return True + + +def transform_namespace_name(name): + """Convert namespace name to valid v3 name.""" + return name.replace('-', '_').lower() diff --git a/galaxy_ng/app/utils/rbac.py b/galaxy_ng/app/utils/rbac.py index fe8ec92032..e50d6359e8 100644 --- a/galaxy_ng/app/utils/rbac.py +++ b/galaxy_ng/app/utils/rbac.py @@ -62,6 +62,11 @@ def add_user_to_v3_namespace(user: User, namespace: Namespace) -> None: assign_role(role_name, user, namespace) +def remove_user_from_v3_namespace(user: User, namespace: Namespace) -> None: + role_name = 'galaxy.collection_namespace_owner' + remove_role(role_name, user, namespace) + + def get_v3_namespace_owners(namespace: Namespace) -> list: """ Return a list of users that own a v3 namespace. @@ -78,5 +83,19 @@ def get_v3_namespace_owners(namespace: Namespace) -> list: include_model_permissions=False ) owners.extend(list(current_users)) - owners = sorted(set(owners)) - return owners + unique_owners = [] + for owner in owners: + if owner not in unique_owners: + unique_owners.append(owner) + return unique_owners + + +def get_owned_v3_namespaces(user: User): + + owned = [] + for namespace in Namespace.objects.all(): + owners = get_v3_namespace_owners(namespace) + if user in owners: + owned.append(namespace) + + return owned diff --git a/galaxy_ng/social/__init__.py b/galaxy_ng/social/__init__.py index a0ba716fcf..99572ce339 100644 --- a/galaxy_ng/social/__init__.py +++ b/galaxy_ng/social/__init__.py @@ -5,10 +5,11 @@ from django.db import transaction from social_core.backends.github import GithubOAuth2 -from galaxy_ng.app.models.auth import Group, User +# from galaxy_ng.app.models.auth import Group, User from galaxy_ng.app.models import Namespace from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.utils import rbac +# from galaxy_ng.app.utils import namespaces as ns_utils from galaxy_importer.constants import NAME_REGEXP @@ -43,13 +44,11 @@ def do_auth(self, access_token, *args, **kwargs): # userdata = id, login, access_token data = self.get_github_user(access_token) - # print('-' * 100) - # from pprint import pprint - # pprint(data) - # print('-' * 100) # extract the login now to prevent mutation + gid = data['id'] login = data['login'] + email = data['email'] # ensure access_token is passed in as a kwarg if data is not None and 'access_token' not in data: @@ -57,32 +56,82 @@ def do_auth(self, access_token, *args, **kwargs): kwargs.update({'response': data, 'backend': self}) - # use upstream auth method + # use upstream auth method to get or create the new user ... auth_response = self.strategy.authenticate(*args, **kwargs) - # create a legacynamespace? - legacy_namespace, legacy_created = self._ensure_legacynamespace(login) + # make a v3 namespace for the user ... + v3_namespace, v3_namespace_created = \ + self.handle_v3_namespace(auth_response, email, login, gid) - # define namespace, validate and create ... - namespace_name = self.transform_namespace_name(login) - print(f'NAMESPACE NAME: {namespace_name}') - if self.validate_namespace_name(namespace_name): + # create a legacynamespace and bind to the v3 namespace? + legacy_namespace, legacy_namespace_created = \ + self._ensure_legacynamespace(login, v3_namespace) - # Need user for group and rbac binding - user = User.objects.filter(username=login).first() + return auth_response - # Create a group to bind rbac perms. - group, _ = self._ensure_group(namespace_name, user) + def handle_v3_namespace(self, session_user, session_email, session_login, github_id): - # create a v3 namespace? - v3_namespace, v3_created = self._ensure_namespace(namespace_name, user, group) + logger.debug( + f'HANDLING V3 NAMESPACE session_user:{session_user}' + + f' session_email:{session_email} session_login:{session_login}' + ) - # bind the v3 namespace to the v1 namespace - if legacy_created and v3_created: - legacy_namespace.namespace = v3_namespace - legacy_namespace.save() + namespace_created = False - return auth_response + # first make the namespace name ... + namespace_name = self.transform_namespace_name(session_login) + + logger.debug(f'TRANSFORMED NAME: {namespace_name}') + + if not self.validate_namespace_name(namespace_name): + logger.debug(f'DID NOT VALIDATE NAMESPACE NAME: {namespace_name}') + return False, False + + # does the namespace already exist? + found_namespace = Namespace.objects.filter(name=namespace_name).first() + + logger.debug(f'FOUND NAMESPACE: {found_namespace}') + + # is it owned by this userid? + if found_namespace: + logger.debug(f'FOUND EXISTING NAMESPACE: {found_namespace}') + owners = rbac.get_v3_namespace_owners(found_namespace) + logger.debug(f'FOUND EXISTING OWNERS: {owners}') + + if session_user in owners: + return found_namespace, False + + # should always have a namespace ... + if found_namespace: + logger.debug( + f'GENERATING A NEW NAMESPACE NAME SINCE USER DOES NOT OWN {found_namespace}' + ) + namespace_name = self.generate_available_namespace_name(session_login, github_id) + logger.debug(f'FINAL NAMESPACE NAME {namespace_name}') + + # create a v3 namespace? + namespace, namespace_created = self._ensure_namespace(namespace_name, session_user) + + owned = rbac.get_owned_v3_namespaces(session_user) + logger.debug(f'NS OWNED BY {session_user}: {owned}') + + return namespace, namespace_created + + def generate_available_namespace_name(self, session_login, github_id): + # we're only here because session_login is already taken as a + # namespace name and we need a new one for the user + + # this makes the weird gh_{BLAH} name ... + # namespace_name = ns_utils.map_v3_namespace(session_login) + + # we should iterate and append 0,1,2,3,4,5,etc on the login name + # until we find one that is free + counter = -1 + while True: + counter += 1 + namespace_name = self.transform_namespace_name(session_login) + str(counter) + if Namespace.objects.filter(name=namespace_name).count() == 0: + return namespace_name def validate_namespace_name(self, name): """Similar validation to the v3 namespace serializer.""" @@ -103,6 +152,7 @@ def transform_namespace_name(self, name): """Convert namespace name to valid v3 name.""" return name.replace('-', '_').lower() + ''' def _ensure_group(self, namespace_name, user): """Create a group in the form of :""" with transaction.atomic(): @@ -111,26 +161,27 @@ def _ensure_group(self, namespace_name, user): if created: rbac.add_user_to_group(user, group) return group, created + ''' - def _ensure_namespace(self, name, user, group): + def _ensure_namespace(self, namespace_name, user): """Create an auto v3 namespace for the account""" with transaction.atomic(): - namespace, created = Namespace.objects.get_or_create(name=name) - print(f'NAMESPACE:{namespace} CREATED:{created}') + namespace, created = Namespace.objects.get_or_create(name=namespace_name) owners = rbac.get_v3_namespace_owners(namespace) if created or not owners: # Binding by user breaks the UI workflow ... - # rbac.add_user_to_v3_namespace(user, namespace) - rbac.add_group_to_v3_namespace(group, namespace) + rbac.add_user_to_v3_namespace(user, namespace) return namespace, created - def _ensure_legacynamespace(self, login): + def _ensure_legacynamespace(self, login, v3_namespace): """Create an auto legacynamespace for the account""" + ''' # userdata = id, login, access_token user = User.objects.filter(username=login).first() + ''' # make the namespace with transaction.atomic(): @@ -139,9 +190,10 @@ def _ensure_legacynamespace(self, login): name=login ) - # add the user to the owners - if created: - legacy_namespace.owners.add(user) + # bind the v3 namespace + if created or not legacy_namespace.namespace: + legacy_namespace.namespace = v3_namespace + legacy_namespace.save() return legacy_namespace, created diff --git a/galaxy_ng/social/pipeline/user.py b/galaxy_ng/social/pipeline/user.py index cdc7e97f0e..59edefb3b2 100644 --- a/galaxy_ng/social/pipeline/user.py +++ b/galaxy_ng/social/pipeline/user.py @@ -2,6 +2,7 @@ from galaxy_ng.app.models.auth import User +from galaxy_ng.app.utils.galaxy import generate_unverified_email USER_FIELDS = ['username', 'email'] @@ -14,6 +15,9 @@ def get_username(strategy, details, backend, user=None, *args, **kwargs): def create_user(strategy, details, backend, user=None, *args, **kwargs): if user: + if user.username != details.get('username'): + user.username = details.get('username') + user.save() return {'is_new': False} fields = dict( @@ -26,16 +30,102 @@ def create_user(strategy, details, backend, user=None, *args, **kwargs): # bypass the strange logic that can't find the user ... ? username = details.get('username') - if username: - found_user = User.objects.filter(username=username).first() - if found_user is not None: - return { - 'is_new': False, - 'user': found_user - } + email = details.get('email') + github_id = details.get('id') + if not github_id: + github_id = kwargs['response']['id'] + + # check for all possible emails ... + possible_emails = [generate_unverified_email(github_id), email] + for possible_email in possible_emails: + found_email = User.objects.filter(email=possible_email).first() + if found_email is not None: + break + + if found_email is not None: + + # fix the username if they've changed their login since last time + if found_email.username != username: + found_email.username = username + found_email.save() + + if found_email.email != email: + found_email.email = email + found_email.save() + + return { + 'is_new': False, + 'user': found_email + } + + found_username = User.objects.filter(username=username).first() + if found_username is not None and found_username.email: + # we have an old user who's got the username but it's not the same person logging in ... + # so change that username? The email should be unique right? + found_username.username = found_username.email + found_username.save() + + found_username = User.objects.filter(username=username).first() + if found_username is not None: + return { + 'is_new': False, + 'user': found_username + } new_user = strategy.create_user(**fields) return { 'is_new': True, 'user': new_user } + + +def user_details(strategy, details, backend, user=None, *args, **kwargs): + """Update user details using data from provider.""" + + if not user: + return + + changed = False # flag to track changes + + # Default protected user fields (username, id, pk and email) can be ignored + # by setting the SOCIAL_AUTH_NO_DEFAULT_PROTECTED_USER_FIELDS to True + if strategy.setting("NO_DEFAULT_PROTECTED_USER_FIELDS") is True: + protected = () + else: + protected = ( + "username", + "id", + "pk", + "email", + "password", + "is_active", + "is_staff", + "is_superuser", + ) + + protected = protected + tuple(strategy.setting("PROTECTED_USER_FIELDS", [])) + + # Update user model attributes with the new data sent by the current + # provider. Update on some attributes is disabled by default, for + # example username and id fields. It's also possible to disable update + # on fields defined in SOCIAL_AUTH_PROTECTED_USER_FIELDS. + field_mapping = strategy.setting("USER_FIELD_MAPPING", {}, backend) + for name, value in details.items(): + # Convert to existing user field if mapping exists + name = field_mapping.get(name, name) + if value is None or not hasattr(user, name) or name in protected: + continue + + current_value = getattr(user, name, None) + if current_value == value: + continue + + immutable_fields = tuple(strategy.setting("IMMUTABLE_USER_FIELDS", [])) + if name in immutable_fields and current_value: + continue + + changed = True + setattr(user, name, value) + + if changed: + strategy.storage.user.changed(user) diff --git a/galaxy_ng/tests/integration/api/test_aiindex.py b/galaxy_ng/tests/integration/api/test_aiindex.py index 82537c590e..41fdeae7f2 100644 --- a/galaxy_ng/tests/integration/api/test_aiindex.py +++ b/galaxy_ng/tests/integration/api/test_aiindex.py @@ -91,6 +91,7 @@ def test_legacy_namespace_add_list_remove_aiindex(ansible_config, legacy_namespa cfg = ansible_config("github_user_1") with SocialGithubClient(config=cfg) as client: + assert ( client.post( "_ui/v1/ai_deny_index/legacy_namespace/", diff --git a/galaxy_ng/tests/integration/api/test_community.py b/galaxy_ng/tests/integration/api/test_community.py index ba6eb1853c..245f4cd947 100644 --- a/galaxy_ng/tests/integration/api/test_community.py +++ b/galaxy_ng/tests/integration/api/test_community.py @@ -152,7 +152,10 @@ def test_me_social_with_v1_synced_user(ansible_config): """ Make sure social auth associates to the correct username """ username = 'geerlingguy' + real_email = 'geerlingguy@nohaxx.me' + unverified_email = '481677@GALAXY.GITHUB.UNVERIFIED.COM' cleanup_social_user(username, ansible_config) + cleanup_social_user(unverified_email, ansible_config) admin_config = ansible_config("admin") admin_client = get_client( @@ -166,6 +169,10 @@ def test_me_social_with_v1_synced_user(ansible_config): resp = admin_client('/api/v1/sync/', method='POST', args=pargs) wait_for_v1_task(resp=resp, api_client=admin_client) + # should have an unverified user ... + unverified_user = admin_client(f'/api/_ui/v1/users/?username={username}')['data'][0] + assert unverified_user['email'] == unverified_email + # set the social config ... cfg = ansible_config(username) @@ -176,7 +183,18 @@ def test_me_social_with_v1_synced_user(ansible_config): uinfo = resp.json() assert uinfo['username'] == cfg.get('username') + # should have the same ID ... + assert uinfo['id'] == unverified_user['id'] + + # should have the right email + assert uinfo['email'] == real_email + + # the unverified email should not be a user ... + bad_users = admin_client(f'/api/_ui/v1/users/?username={unverified_email}') + assert bad_users['meta']['count'] == 0, bad_users + +@pytest.mark.skip(reason='no longer creating groups for social users') @pytest.mark.deployment_community def test_social_auth_creates_group(ansible_config): diff --git a/galaxy_ng/tests/integration/community/test_community_namespace_rbac.py b/galaxy_ng/tests/integration/community/test_community_namespace_rbac.py new file mode 100644 index 0000000000..ba709094ad --- /dev/null +++ b/galaxy_ng/tests/integration/community/test_community_namespace_rbac.py @@ -0,0 +1,470 @@ +"""test_community.py - Tests related to the community featureset. +""" + +import json +import pytest +import random +import string + +from ..utils import ( + get_client, + SocialGithubClient, + GithubAdminClient, + cleanup_namespace, +) +from ..utils.legacy import ( + cleanup_social_user, + wait_for_v1_task, +) + + +pytestmark = pytest.mark.qa # noqa: F821 + + +def extract_default_config(ansible_config): + base_cfg = ansible_config('github_user_1') + cfg = {} + cfg['token'] = None + cfg['url'] = base_cfg.get('url') + cfg['auth_url'] = base_cfg.get('auth_url') + cfg['github_url'] = base_cfg.get('github_url') + cfg['github_api_url'] = base_cfg.get('github_api_url') + return cfg + + +@pytest.mark.deployment_community +def test_admin_can_import_legacy_roles(ansible_config): + + github_user = 'jctannerTEST' + github_repo = 'role1' + cleanup_social_user(github_user, ansible_config) + cleanup_social_user(github_user.lower(), ansible_config) + + admin_config = ansible_config("admin") + admin_client = get_client( + config=admin_config, + request_token=False, + require_auth=True + ) + + # do an import with the admin ... + payload = { + 'github_repo': github_repo, + 'github_user': github_user, + } + resp = admin_client('/api/v1/imports/', method='POST', args=payload) + task_id = resp['results'][0]['id'] + res = wait_for_v1_task(task_id=task_id, api_client=admin_client, check=False) + + # it should have failed because of the missing v1+v3 namespaces ... + assert res['results'][0]['summary_fields']['task_messages'][0]['state'] == 'FAILED', res + + # make the legacy namespace + ns_payload = { + 'name': github_user + } + resp = admin_client('/api/v1/namespaces/', method='POST', args=ns_payload) + assert resp['name'] == github_user, resp + assert not resp['summary_fields']['owners'], resp + assert not resp['summary_fields']['provider_namespaces'], resp + v1_id = resp['id'] + + # make the v3 namespace + v3_payload = { + 'name': github_user.lower(), + 'groups': [], + } + resp = admin_client('/api/_ui/v1/namespaces/', method='POST', args=v3_payload) + assert resp['name'] == github_user.lower(), resp + v3_id = resp['id'] + + # bind the v3 namespace to the v1 namespace + v3_bind = { + 'id': v3_id + } + admin_client(f'/api/v1/namespaces/{v1_id}/providers/', method='POST', args=v3_bind) + + # check the providers ... + resp = admin_client(f'/api/v1/namespaces/{v1_id}/providers/') + assert resp[0]['id'] == v3_id, resp + + # try to import again ... + resp = admin_client('/api/v1/imports/', method='POST', args=payload) + task_id = resp['results'][0]['id'] + res = wait_for_v1_task(task_id=task_id, api_client=admin_client, check=False) + assert res['results'][0]['summary_fields']['task_messages'][0]['state'] == 'SUCCESS', res + + +@pytest.mark.skip(reason="not ready yet") +@pytest.mark.deployment_community +def test_social_auth_v3_rbac_workflow(ansible_config): + + # cleanup_social_user('gh01', ansible_config) + base_cfg = ansible_config('github_user_1') + + ga = GithubAdminClient() + suffix = ''.join([random.choice(string.ascii_lowercase) for x in range(0, 5)]) + user_a = ga.create_user(login='0xEEE-32i-' + suffix) + user_a['username'] = user_a['login'] + user_a['token'] = None + user_a['url'] = base_cfg.get('url') + user_a['auth_url'] = base_cfg.get('auth_url') + user_a['github_url'] = base_cfg.get('github_url') + user_a['github_api_url'] = base_cfg.get('github_api_url') + l_ns_name = user_a['username'] + + # login once to confirm v3 namespace creation + with SocialGithubClient(config=user_a) as client: + a_resp = client.get('_ui/v1/me/') + a_ds = a_resp.json() + assert a_ds['username'] == user_a['username'] + + ns_resp = client.get('_ui/v1/my-namespaces/') + ns_ds = ns_resp.json() + assert ns_ds['meta']['count'] == 1, ns_ds + + original_namespace = ns_ds['data'][0]['name'] + + # verify the new legacy namespace has right owner ... + l_ns_name = user_a['username'] + l_resp = client.get(f'v1/namespaces/?name={l_ns_name}') + l_ns_ds = l_resp.json() + assert l_ns_ds['count'] == 1, l_ns_ds + assert l_ns_ds['results'][0]['name'] == l_ns_name, l_ns_ds + assert l_ns_ds['results'][0]['summary_fields']['owners'][0]['username'] == \ + user_a['username'], l_ns_ds + + # verify the legacy provider namespace is the v3 namespace + provider_namespaces = l_ns_ds['results'][0]['summary_fields']['provider_namespaces'] + assert len(provider_namespaces) == 1 + provider_namespace = provider_namespaces[0] + assert provider_namespace['name'].startswith('gh_') + + # make a new login + new_login = user_a['login'] + '-changed' + + # change the user's login and try again + user_b = ga.modify_user(uid=user_a['id'], change=('login', new_login)) + user_b['username'] = user_b['login'] + user_b['token'] = None + user_b['url'] = base_cfg.get('url') + user_b['auth_url'] = base_cfg.get('auth_url') + user_b['github_url'] = base_cfg.get('github_url') + user_b['github_api_url'] = base_cfg.get('github_api_url') + + # current_users = ga.list_users() + + with SocialGithubClient(config=user_b) as client: + b_resp = client.get('_ui/v1/me/') + b_ds = b_resp.json() + + # the UID should have stayed the same + assert b_ds['id'] == a_ds['id'] + + # the backend should have changed the username + assert b_ds['username'] == user_b['username'] + + # now check the namespaces again ... + ns_resp2 = client.get('_ui/v1/my-namespaces/') + ns_ds2 = ns_resp2.json() + + # there should be 2 ... + assert ns_ds2['meta']['count'] == 2, ns_ds2 + + # one of them should be the one created with the previous username + ns_names = [x['name'] for x in ns_ds2['data']] + assert original_namespace in ns_names, ns_names + + # verify the previous legacy namespace has right owner ... + l_resp = client.get(f'v1/namespaces/?name={l_ns_name}') + l_ns_ds = l_resp.json() + assert l_ns_ds['count'] == 1, l_ns_ds + assert l_ns_ds['results'][0]['name'] == l_ns_name, l_ns_ds + assert l_ns_ds['results'][0]['summary_fields']['owners'][0]['username'] == \ + user_b['username'], l_ns_ds + + # verify the new legacy namespace has right owner ... + new_l_ns = user_b['username'] + new_l_resp = client.get(f'v1/namespaces/?name={new_l_ns}') + new_l_ns_ds = new_l_resp.json() + assert new_l_ns_ds['count'] == 1, new_l_ns_ds + assert new_l_ns_ds['results'][0]['name'] == new_l_ns, new_l_ns_ds + assert new_l_ns_ds['results'][0]['summary_fields']['owners'][0]['username'] == \ + user_b['username'], new_l_ns_ds + + # verify the new legacy ns has the new provider ns + provider_namespaces = new_l_ns_ds['results'][0]['summary_fields']['provider_namespaces'] + assert len(provider_namespaces) == 1 + provider_namespace = provider_namespaces[0] + assert provider_namespace['name'].startswith('gh_') + + with SocialGithubClient(config=user_b) as client: + + pass + + # the new login should be able to find all of their legacy namespaces + # import epdb; epdb.st() + + # the new login should be able to import to BOTH legacy namespaces + + # the new login should be able to find all of their v3 namespaces + + # the new login should be able to upload to BOTH v3 namespaces + + # TODO ... what happens with the other owners of a v1 namespace when + # the original owner changes their login? + # TODO ... what happens with the other owners of a v3 namespace when + # the original owner changes their login? + + +@pytest.mark.deployment_community +def test_social_user_with_reclaimed_login(ansible_config): + + ''' + [jtanner@jtw530 galaxy_user_analaysis]$ head -n1 data/user_errors.csv + galaxy_uid,galaxy_username,github_id,github_login,actual_github_id,actual_github_login + [jtanner@jtw530 galaxy_user_analaysis]$ grep sean-m-sullivan data/user_errors.csv + 33901,Wilk42,30054029,sean-m-sullivan,30054029,sean-m-sullivan + ''' + + # a user logged into galaxy once with login "foo" + # the user then changed their name in github to "bar" + # the user then logged back into galaxy ... + # + # 1) social auth updates it's own data based on the github userid + # 2) galaxy's user still has the old login but is linked to the same social account + + # What happens when someone claims the old username ..? + # + # Once they login to galaxy they will get the "foo" login as their username? + # the "foo" namespace will not belong to them + # the "bar" namespace will not belong to them + # do they own -any- namespaces? + + ga = GithubAdminClient() + ga.delete_user(login='Wilk42') + ga.delete_user(login='sean-m-sullivan') + cleanup_social_user('Wilk42', ansible_config) + cleanup_social_user('sean-m-sullivan', ansible_config) + cleanup_social_user('sean-m-sullivan@redhat.com', ansible_config) + + default_cfg = extract_default_config(ansible_config) + + old_login = 'Wilk42' + email = 'sean-m-sullivan@redhat.com' + user_a = ga.create_user(login=old_login, email=email) + user_a.update(default_cfg) + user_a['username'] = old_login + + # login once to make user + with SocialGithubClient(config=user_a) as client: + + a_resp = client.get('_ui/v1/me/') + ns_resp = client.get('_ui/v1/my-namespaces/') + ns_ds = ns_resp.json() + namespace_names_a = [x['name'] for x in ns_ds['data']] + + # make a sean_m_sullivan namespace owned by the old login ... ? + + # make a new login + new_login = 'sean-m-sullivan' + + # change the user's login and try again + user_b = ga.modify_user(uid=user_a['id'], change=('login', new_login)) + user_b.update(default_cfg) + user_b['username'] = new_login + + # sean-m-sullivan never logs in again ... + ''' + with SocialGithubClient(config=user_b) as client: + b_resp = client.get('_ui/v1/me/') + ''' + + # Let some new person claim the old login ... + user_c = ga.create_user(login=old_login, email='shadyperson@haxx.net') + user_c.update(default_cfg) + user_c['username'] = old_login + + with SocialGithubClient(config=user_c) as client: + c_resp = client.get('_ui/v1/me/') + ns_resp_c = client.get('_ui/v1/my-namespaces/') + ns_ds_c = ns_resp_c.json() + namespace_names_c = [x['name'] for x in ns_ds_c['data']] + + # user_a and user_c should have different galaxy uids + assert a_resp.json()['id'] != c_resp.json()['id'] + + # user_c should have the old login + assert c_resp.json()['username'] == old_login + + # user_c should not own the namespaces of user_a + for ns in namespace_names_a: + assert ns not in namespace_names_c + + # now sean logs in with his new login ... + with SocialGithubClient(config=user_b) as client: + b_resp = client.get('_ui/v1/me/') + ns_resp_b = client.get('_ui/v1/my-namespaces/') + ns_ds_b = ns_resp_b.json() + namespace_names_b = [x['name'] for x in ns_ds_b['data']] + + # his ID should be the same + assert b_resp.json()['id'] == a_resp.json()['id'] + + # his username should be the new login + assert b_resp.json()['username'] == new_login + + # he should own his old namespace AND a transformed one from his new login + assert namespace_names_b == ['wilk42', 'sean_m_sullivan'] + + +@pytest.mark.deployment_community +def test_social_user_sync_with_changed_login(ansible_config): + + # https://galaxy.ansible.com/api/v1/users/?username=Wilk42 + # github_id:30054029 + + # https://galaxy.ansible.com/api/v1/roles/21352/ + # Wilk42/kerb_ldap_setup + # Wilk42.kerb_ldap_setup + # https://galaxy.ansible.com/api/v1/namespaces/1838/ + # owner: 33901:Wilk42 + + ga = GithubAdminClient() + ga.delete_user(login='Wilk42') + ga.delete_user(login='sean-m-sullivan') + cleanup_social_user('Wilk42', ansible_config) + cleanup_social_user('wilk42', ansible_config) + cleanup_social_user('wilk420', ansible_config) + cleanup_social_user('Wilk420', ansible_config) + cleanup_social_user('wilk421', ansible_config) + cleanup_social_user('Wilk421', ansible_config) + cleanup_social_user('sean-m-sullivan', ansible_config) + cleanup_social_user('sean-m-sullivan@redhat.com', ansible_config) + cleanup_social_user('30054029@GALAXY.GITHUB.UNVERIFIED.COM', ansible_config) + + default_cfg = extract_default_config(ansible_config) + + # sync the wilk42 v1 namespace ... + admin_config = ansible_config("admin") + admin_client = get_client( + config=admin_config, + request_token=False, + require_auth=True + ) + + # find all the wilk* namespaces and clean them up ... + resp = admin_client('/api/v3/namespaces/') + nsmap = dict((x['name'], x) for x in resp['data']) + for nsname, nsdata in nsmap.items(): + if not nsname.startswith('wilk'): + continue + cleanup_namespace(nsname, api_client=admin_client) + + # v1 sync the user's roles and namespace ... + pargs = json.dumps({"github_user": 'Wilk42', "limit": 1}).encode('utf-8') + resp = admin_client('/api/v1/sync/', method='POST', args=pargs) + wait_for_v1_task(resp=resp, api_client=admin_client) + + # should have same roles + roles_resp = admin_client.request('/api/v1/roles/?namespace=Wilk42') + assert roles_resp['count'] >= 1 + roles = roles_resp['results'] + v1_namespace_summary = roles[0]['summary_fields']['namespace'] + v1_namespace_id = v1_namespace_summary['id'] + v1_namespace_name = v1_namespace_summary['name'] + + assert v1_namespace_name == 'Wilk42' + + # should have a v1 namespace with Wilk42 as the owner? + ns1_owners_resp = admin_client.request(f'/api/v1/namespaces/{v1_namespace_id}/owners/') + assert ns1_owners_resp + ns1_owners_ids = [x['id'] for x in ns1_owners_resp] + ns1_owners_usernames = [x['username'] for x in ns1_owners_resp] + assert 'Wilk42' in ns1_owners_usernames + + # should have a v3 namespace + ns1_resp = admin_client.request(f'/api/v1/namespaces/{v1_namespace_id}/') + provider_namespaces = ns1_resp['summary_fields']['provider_namespaces'] + assert provider_namespaces + provider_namespace = provider_namespaces[0] + # v3_namespace_id = provider_namespace['id'] + v3_namespace_name = provider_namespace['name'] + assert v3_namespace_name == 'wilk42' + + # now some new person who took Wilk42 logs in ... + hacker = ga.create_user(login='Wilk42', email='1337h@xx.net') + hacker['username'] = hacker['login'] + hacker.update(default_cfg) + with SocialGithubClient(config=hacker) as client: + hacker_me_resp = client.get('_ui/v1/me/') + hacker_me_ds = hacker_me_resp.json() + + # check the hacker's namespaces ... + hacker_ns_resp = client.get('_ui/v1/my-namespaces/') + hacker_ns_ds = hacker_ns_resp.json() + hacker_ns_names = [x['name'] for x in hacker_ns_ds['data']] + + # enure the hacker has a new uid + assert hacker_me_ds['id'] != ns1_owners_ids[0] + + # ensure the hacker owns only some newly created v3 namespace + assert hacker_ns_names == ['wilk420'] + + # now sean-m-sullivan who used to be Wilk42 logs in ... + sean = ga.create_user(login='sean-m-sullivan', uid=30054029, email='sean-m-sullivan@redhat.com') + sean['username'] = sean['login'] + sean.update(default_cfg) + + with SocialGithubClient(config=sean) as client: + me_resp = client.get('_ui/v1/me/') + me_ds = me_resp.json() + + # now check the namespaces again ... + ns_resp = client.get('_ui/v1/my-namespaces/') + ns_ds = ns_resp.json() + sean_ns_names = [x['name'] for x in ns_ds['data']] + + # he should have the appropriate username + assert me_ds['username'] == 'sean-m-sullivan' + # he should own the old and a new v3 namespaces + assert sean_ns_names == ['wilk42', 'sean_m_sullivan'] + # his uid should have remained the same + assert me_ds['id'] == ns1_owners_ids[0] + + # he should own the original v1 namespaces + ns1_owners_resp_after = admin_client.request(f'/api/v1/namespaces/{v1_namespace_id}/owners/') + ns1_owner_names_after = [x['username'] for x in ns1_owners_resp_after] + assert ns1_owner_names_after == ['sean-m-sullivan'] + + # he should own a new v1 namespace too ... + ns_resp = admin_client('/api/v1/namespaces/?owner=sean-m-sullivan') + assert ns_resp['count'] == 2 + v1_ns_names = [x['name'] for x in ns_resp['results']] + assert v1_ns_names == ['Wilk42', 'sean-m-sullivan'] + + # what happens when someone installs Wilk42's roles ... + # galaxy.ansible.com shows the role as wilk42/kerb_ldap_setup + # installing with wilk42.kerb_ldap_setup and Wilk42.kerb_ldap_setup should both work + # the backend finds the role with owner__username=wilk42 AND owner__username=Wilk42 + # the role data has github_repo=Wilk42 which is what the client uses + # github then has a redirect from Wilk42/kerb_ldap_setup to sean-m-sullivan/kerb_ldap_setup + # i believe that redirect no longer works after someone claims Wilk42 + + # core code ... + # url = _urljoin(self.api_server, self.available_api_versions['v1'], "roles", + # "?owner__username=%s&name=%s" % (user_name, role_name)) + # archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % + # (role_data["github_user"], role_data["github_repo"], self.version) + + for role in roles: + for owner_username in ['Wilk42', 'wilk42']: + role_name = role['name'] + role_data = admin_client.request( + f'/api/v1/roles/?owner__username={owner_username}&name={role_name}' + ) + assert role_data['count'] == 1 + assert role_data['results'][0]['id'] == role['id'] + assert role_data['results'][0]['github_user'] == 'Wilk42' + assert role_data['results'][0]['github_repo'] == role['github_repo'] + assert role_data['results'][0]['name'] == role['name'] diff --git a/galaxy_ng/tests/integration/community/test_v1_api.py b/galaxy_ng/tests/integration/community/test_v1_api.py index 6b70e2f6ff..ee48d84afd 100644 --- a/galaxy_ng/tests/integration/community/test_v1_api.py +++ b/galaxy_ng/tests/integration/community/test_v1_api.py @@ -6,6 +6,7 @@ from ..utils import ( ansible_galaxy, get_client, + SocialGithubClient, ) from ..utils.legacy import ( cleanup_social_user, @@ -15,6 +16,17 @@ pytestmark = pytest.mark.qa # noqa: F821 +def extract_default_config(ansible_config): + base_cfg = ansible_config('github_user_1') + cfg = {} + cfg['token'] = None + cfg['url'] = base_cfg.get('url') + cfg['auth_url'] = base_cfg.get('auth_url') + cfg['github_url'] = base_cfg.get('github_url') + cfg['github_api_url'] = base_cfg.get('github_api_url') + return cfg + + @pytest.mark.deployment_community def test_v1_owner_username_filter_is_case_insensitive(ansible_config): """" Tests if v1 sync accepts a user&limit arg """ @@ -30,6 +42,15 @@ def test_v1_owner_username_filter_is_case_insensitive(ansible_config): github_repo = 'role1' cleanup_social_user(github_user, ansible_config) + user_cfg = extract_default_config(ansible_config) + user_cfg['username'] = github_user + user_cfg['password'] = 'redhat' + + # Login with the user first to create the v1+v3 namespaces + with SocialGithubClient(config=user_cfg) as client: + me = client.get('_ui/v1/me/') + assert me.json()['username'] == github_user + # Run the import import_pid = ansible_galaxy( f"role import {github_user} {github_repo}", diff --git a/galaxy_ng/tests/integration/utils/__init__.py b/galaxy_ng/tests/integration/utils/__init__.py index a5da4644e9..e1c8234de3 100644 --- a/galaxy_ng/tests/integration/utils/__init__.py +++ b/galaxy_ng/tests/integration/utils/__init__.py @@ -91,5 +91,5 @@ AnsibleDistroAndRepo, delete_all_collections, PulpObjectBase, - GithubAdminClient + GithubAdminClient, ) diff --git a/galaxy_ng/tests/integration/utils/legacy.py b/galaxy_ng/tests/integration/utils/legacy.py index 9f9743a5c8..6695310e9c 100644 --- a/galaxy_ng/tests/integration/utils/legacy.py +++ b/galaxy_ng/tests/integration/utils/legacy.py @@ -8,7 +8,7 @@ ) -def wait_for_v1_task(task_id=None, resp=None, api_client=None): +def wait_for_v1_task(task_id=None, resp=None, api_client=None, check=True): if task_id is None: task_id = resp['task'] @@ -26,7 +26,10 @@ def wait_for_v1_task(task_id=None, resp=None, api_client=None): break time.sleep(.5) - assert state == 'SUCCESS' + if check: + assert state == 'SUCCESS' + + return task_resp def clean_all_roles(ansible_config): diff --git a/profiles/community/github_mock/flaskapp.py b/profiles/community/github_mock/flaskapp.py index 36e5041acb..68527c33d8 100644 --- a/profiles/community/github_mock/flaskapp.py +++ b/profiles/community/github_mock/flaskapp.py @@ -68,10 +68,10 @@ 'email': 'jctannerTEST@gmail.com', }, 'geerlingguy': { - 'id': 1004, + 'id': 481677, 'login': 'geerlingguy', 'password': 'redhat', - 'email': 'geerlingguy@gmail.com', + 'email': 'geerlingguy@nohaxx.me', } }