checkin

dev/scripts.community/check_validated_user_namespace_map.py

@@ -0,0 +1,54 @@
+import gzip
+import json
+import os
+
+from django.contrib.auth import get_user_model
+
+from galaxy_ng.app.models import Namespace
+from galaxy_ng.app.api.v1.models import LegacyNamespace
+
+
+User = get_user_model()
+
+
+def do_check():
+
+    checkmode = True
+    if os.environ.get('CHECK_MODE') == "0":
+        checkmode = False
+
+    fn = 'user_namespace_map_validated.json.gz'
+    with gzip.open(fn, 'rb') as gz_file:
+        raw = gz_file.read()
+    umap = json.loads(raw)
+
+    uids = list(umap.keys())
+    uids = sorted(uids, key=lambda x: int(x))
+    for uid in uids:
+        print(uid)
+
+        old_data = umap[uid]
+        if not old_data.get('github_login_verified'):
+            continue
+
+        galaxy_username = old_data['galaxy_username']
+        github_login = old_data['github_login']
+        if old_data['github_login_new']:
+            github_login = old_data['github_login_new']
+
+        # worry about this later ...
+        if galaxy_username != github_login:
+            continue
+
+        found_user = User.objects.filter(username=galaxy_username).first()
+        if not found_user:
+            print(f'\tFIX - create user {galaxy_username}')
+            if not checkmode:
+                print(f'\t\tcheckmode:{checkmode} do user create ...')
+                found_user, _ = User.objects.get_or_create(username=galaxy_username)
+
+        #import epdb; epdb.st()
+
+
+
+do_check()

dev/scripts.community/cleanup_namespaces.py

@@ -0,0 +1,168 @@
+import argparse
+import re
+
+from django.contrib.auth import get_user_model
+
+from galaxy_ng.app.models import Namespace
+from galaxy_ng.app.api.v1.models import LegacyNamespace
+from galaxy_ng.app.api.v1.models import LegacyRole
+from galaxy_ng.app.utils import rbac
+
+from pulp_ansible.app.models import CollectionVersion
+
+
+User = get_user_model()
+
+
+allowed_to_fix = [
+    'yuvarajcloud',
+    'wpninfra',
+    'vladgh',
+    'tonyskapunk',
+    'shanemcd',
+    'saiello',
+    'rtnp',
+    'rndmh3ro',
+    'rathbunr',
+    'nixocio',
+    'navdeepsachdeva',
+    'mdellweg',
+    'maxamillion',
+    'kubealex',
+    'jorti',
+    'rezizter',
+    'kolesaev',
+    'koalakangaroo',
+    'jorneilander',
+    'jgoos',
+    'jeichler',
+    'himdel',
+    'geoffreyvanwyk',
+    'charlesrocket',
+    'borari',
+]
+
+ignore_list = [
+    'spk',
+    'sigma',
+    'rmasters',
+    'network',
+    'roopamg',
+    'wholeshoot',
+    'tmiller',
+    'kmf',
+    'jdavid',
+]
+
+
+def strip_number_from_string(input_string):
+    #match = re.search(r'(\D+)(\d+)$', input_string)
+    #match = re.search(r'([\w\d_]*)(\d+)$', input_string)
+    match = re.search(r'([\w_]*[A-Za-z_])(\d+)$', input_string)
+    if match:
+        prefix = match.group(1)
+        number = match.group(2)
+        return prefix, int(number)
+    else:
+        return input_string, None
+
+
+def do_cleanup():
+
+
+    # make a list of all namespace names
+    ns_names = []
+    for ns_name in Namespace.objects.values_list('name', flat=True):
+        ns_names.append(ns_name)
+
+    # map out names with a common prefix and a numbered suffix
+    ns_map = {}
+    for ns_name in ns_names:
+        prefix, number = strip_number_from_string(ns_name)
+        #print(f'{prefix} {number}')
+        if number is None:
+            if prefix not in ns_map:
+                ns_map[prefix] = [prefix]
+            continue
+        if prefix not in ns_map:
+            ns_map[prefix] = []
+        ns_map[prefix].append(ns_name)
+
+    # check each ns for content ...
+    ns_keys = sorted(list(ns_map.keys()))
+    counter = 0
+    for ns_key in ns_keys:
+
+        if ns_key in ignore_list:
+            continue
+
+        if len(ns_map[ns_key]) <= 1:
+            continue
+
+        ns = Namespace.objects.filter(name=ns_key).first()
+        if not ns:
+            continue
+
+        if ns:
+            collection_count = CollectionVersion.objects.filter(namespace=ns).count()
+            owners = rbac.get_v3_namespace_owners(ns)
+            legacy_count = LegacyNamespace.objects.filter(namespace=ns).count()
+        else:
+            collection_count = None
+            owners = None
+            legacy_count = None
+
+        # is there a matching user for this namespace ...?
+        found_user = User.objects.filter(username=ns_key).first()
+        if not found_user:
+            continue
+
+        counter += 1
+        print('-' * 100)
+        print(f'{counter}. {ns_key}')
+
+        print('')
+        print(f'\tuser: {found_user}')
+
+        print('')
+        print(f'\tnamespace:{ns} legacy-ns:{legacy_count} collections:{collection_count} owners:{owners}')
+        print('')
+
+        for dupe_name in ns_map[ns_key]:
+
+            if dupe_name == ns_key:
+                continue
+
+            dupe_ns = Namespace.objects.filter(name=dupe_name).first()
+            collection_count = CollectionVersion.objects.filter(namespace=dupe_name).count()
+            dupe_owners = rbac.get_v3_namespace_owners(dupe_ns)
+            dupe_legacy_count = LegacyNamespace.objects.filter(namespace=dupe_ns).count()
+
+            print(f'\t\tdupe:{dupe_name} legacy-ns:{dupe_legacy_count} collections:{collection_count} owners:{dupe_owners}')
+
+            if dupe_legacy_count > 0:
+                for lns in LegacyNamespace.objects.filter(namespace=dupe_ns):
+                    print(f'\t\t\tlegacy:{lns.name} v3:{lns.namespace}')
+                    print(f'\t\t\t\tFIXME - set v1:{lns} provider to {ns}')
+                    if ns_key in allowed_to_fix:
+                        lns.namespace = ns
+                        lns.save()
+
+                    #for role in LegacyRole.objects.filter(namespace=lns):
+                    #    print(f'\t\t\t\t{role.namespace}.{role.name}')
+
+            if (collection_count + dupe_legacy_count) == 0:
+                for dupe_owner in dupe_owners:
+                    if dupe_owner not in owners:
+                        print(f'\t\t\tFIXME - add {dupe_owner} to {ns_key} owners')
+                        if ns_key in allowed_to_fix:
+                            rbac.add_user_to_v3_namespace(dupe_owner, ns)
+
+                print(f'\t\t\tFIXME - delete v3 {dupe_name}')
+                if ns_key in allowed_to_fix:
+                    dupe_ns.delete()
+
+    # import epdb; epdb.st()
+
+
+do_cleanup()

dev/scripts.community/fix_social_users.py

@@ -0,0 +1,148 @@
+from django.contrib.auth import get_user_model
+from social_django.models import UserSocialAuth
+
+from pulp_ansible.app.models import Collection
+from pulp_ansible.app.models import CollectionVersion
+
+from galaxy_ng.app.utils import rbac
+from galaxy_ng.app.utils.galaxy import generate_unverified_email
+from galaxy_ng.app.api.v1.models import LegacyNamespace
+from galaxy_ng.app.api.v1.models import LegacyRole
+from galaxy_ng.app.models import Namespace
+
+
+User = get_user_model()
+
+
+def transform_name(github_login):
+    ns = github_login.lower().replace('-', '_')
+    if ns[0].isdigit():
+        ns = 'gh_' + ns
+    return ns
+
+
+def find_related_v3_namespaces(github_login):
+    transformed = github_login.lower().replace('-', '_')
+    #if transformed[0].isdigit():
+    #    transformed = 'gh_' + transformed
+
+    namespaces = Namespace.objects.filter(name__icontains=transformed)
+    return namespaces
+
+
+for social_user in UserSocialAuth.objects.all().order_by('id'):
+    sid = social_user.id
+    current_user = social_user.user
+    current_username = current_user.username
+    current_email = current_user.email
+    github_login = social_user.extra_data.get('login')
+    github_id = social_user.extra_data.get('id')
+    transformed_login = transform_name(github_login)
+
+    unverified_email = generate_unverified_email(github_id)
+
+    #if current_username == github_login:
+    #    continue
+
+    galaxy_user_by_login = User.objects.filter(username=github_login).first()
+    galaxy_user_by_unverified_email = User.objects.filter(username=unverified_email).first()
+    galaxy_user_by_unverified_name = User.objects.filter(email=unverified_email).first()
+
+    if not galaxy_user_by_unverified_email and not galaxy_user_by_unverified_name:
+        continue
+
+    synced_users = []
+    if galaxy_user_by_unverified_email and galaxy_user_by_unverified_name:
+        if galaxy_user_by_unverified_email.id == galaxy_user_by_unverified_name.id:
+            galaxy_user_by_unverified_name = None
+        if galaxy_user_by_unverified_email:
+            synced_users.append(galaxy_user_by_unverified_email)
+        if galaxy_user_by_unverified_name:
+            synced_users.append(galaxy_user_by_unverified_name)
+
+    # this person got the right user association?
+    synced_users = [x for x in synced_users if x.id != current_user.id]
+    if current_user.email != unverified_email and not synced_users:
+        continue
+
+    print(f'{sid}. {github_id}|{github_login} current:{current_username} current_email:{current_email}')
+
+    for synced_user in synced_users:
+        print(f'\tsynced_user: {synced_user.id}')
+        print(f'\t\t{synced_user}')
+        print(f'\t\t\tusername:{synced_user.username}')
+        print(f'\t\t\temail:{synced_user.email}')
+
+    if len(synced_users) > 1:
+        raise Exception('too many synced users found')
+
+    if not synced_users and current_user.email and current_user.email == unverified_email:
+        print(f'\t FIX - set email to empty string')
+        current_user.email = ''
+        current_user.save()
+        continue
+
+    # proably shouldn't be here if no sync users found?
+    if not synced_users:
+        continue
+    synced_user = synced_users[0]
+
+    # copy ownership ...
+    owned_namespaces = rbac.get_owned_v3_namespaces(synced_user)
+    print(f'\t{synced_user.id} owns {owned_namespaces}')
+    if owned_namespaces:
+        for ns in owned_namespaces:
+            ns_owners = rbac.get_v3_namespace_owners(ns)
+            print(f'\t{ns} owners ...')
+            for xowner in ns_owners:
+                print(f'\t\t{xowner}')
+            if current_user not in ns_owners:
+                print(f'\tFIX - make {current_user} owner of {ns}')
+                rbac.add_user_to_v3_namespace(current_user, ns)
+
+    if synced_user != current_user and \
+            (synced_user.username == unverified_email or synced_user.email == unverified_email):
+        print(f'\tFIX - delete the sync user {synced_user}')
+        synced_user.delete()
+
+    related_namespaces = find_related_v3_namespaces(github_login)
+    for rnamespace in related_namespaces:
+        print(f'\trelated: {rnamespace}')
+
+    # legacy namespaces ... ?
+    provider_namespace = None
+    legacy_namespace = LegacyNamespace.objects.filter(name=github_login).first()
+    print(f'\tlegacy_namespace:{legacy_namespace}')
+    if legacy_namespace:
+        provider_namespace = legacy_namespace.namespace
+        if provider_namespace:
+            print(f'\t\tprovider:{provider_namespace}')
+
+    if legacy_namespace and provider_namespace and \
+        legacy_namespace.name == provider_namespace.name and \
+        provider_namespace in owned_namespaces:
+            continue
+
+    if legacy_namespace and provider_namespace and \
+        transformed_login == provider_namespace.name and \
+        provider_namespace in owned_namespaces:
+            continue
+
+    correct_namespace = Namespace.objects.filter(name=transformed_login).first()
+
+    if provider_namespace.name == transformed_login + '0':
+        print(f'\t\tFIX - set v1:{legacy_namespace} provider namespace to {correct_namespace}')
+        legacy_namespace.namespace = correct_namespace
+        legacy_namespace.save()
+
+        # has content?
+        col_count = Collection.objects.filter(namespace=provider_namespace.name).count()
+        cv_count = CollectionVersion.objects.filter(namespace=provider_namespace.name).count()
+        role_count = LegacyRole.objects.filter(namespace__name=provider_namespace.name).count()
+
+        if (col_count + cv_count + role_count) == 0:
+            print(f'\t\tFIX - delete v3:{provider_namespace}')
+            provider_namespace.delete()
+
+    #print(f'\tFIX - delete the sync user {synced_user}')
+    #synced_user.delete()

dev/scripts.community/testme.py

@@ -0,0 +1,12 @@
+import re
+
+def strip_number_from_string(input_string):
+    #match = re.search(r'(\D+)(\d+)$', input_string)
+    #match = re.search(r'([\w\d_]*)(\d+)$', input_string)
+    match = re.search(r'([\w_]*[A-Za-z_])(\d+)$', input_string)
+    if match:
+        prefix = match.group(1)
+        number = match.group(2)
+        return prefix, int(number)
+    else:
+        return input_string, None