Cleanup.

No-Issue

Signed-off-by: James Tanner <[email protected]>

galaxy_ng/app/management/commands/sync-galaxy-namespaces.py

@@ -26,7 +26,7 @@ def add_arguments(self, parser):
         parser.add_argument("--name", help="find and sync only this namespace name")
         parser.add_argument("--id", help="find and sync only this namespace id")
         parser.add_argument("--limit", type=int)
-        parser.add_argument("--startpage", type=int)
+        parser.add_argument("--start_page", type=int)
 
     def echo(self, message, style=None):
         style = style or self.style.SUCCESS
@@ -49,7 +49,7 @@ def handle(self, *args, **options):
             count = 0
             for total, namespace_info in upstream_namespace_iterator(
                 baseurl=options['baseurl'],
-                start_page=options['startpage'],
+                start_page=options['start_page'],
                 limit=options['limit'],
             ):
 

galaxy_ng/app/utils/galaxy.py

@@ -39,10 +39,7 @@ def safe_fetch(url):
         if rr.status_code < 500:
             return rr
 
-        if counter >= 10:
-            return rr
-
-        time.sleep(5)
+        time.sleep(60)
 
     return rr
 
@@ -55,11 +52,7 @@ def paginated_results(next_url):
     while next_url:
         logger.info(f'fetch {next_url}')
         rr = safe_fetch(next_url)
-
-        try:
-            ds = rr.json()
-        except Exception:
-            raise Exception(next_url)
+        ds = rr.json()
 
         results.extend(ds['results'])
         if 'next' in ds:
@@ -193,17 +186,6 @@ def upstream_namespace_iterator(
             ns_id = ndata['id']
 
             # get the owners too
-            '''
-            owners = []
-            next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
-            while next_owners_url:
-                o_data = requests.get(next_owners_url).json()
-                for owner in o_data['results']:
-                    owners.append(owner)
-                if not o_data.get('next'):
-                    break
-                next_owners_url = _baseurl + o_data['next_link']
-            '''
             ndata['summary_fields']['owners'] = get_namespace_owners_details(_baseurl, ns_id)
 
             # send the collection
@@ -288,17 +270,6 @@ def upstream_collection_iterator(
                 namespace_cache[ns_id] = namespace_data
 
                 # get the owners too
-                '''
-                owners = []
-                next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
-                while next_owners_url:
-                    o_data = requests.get(next_owners_url).json()
-                    for owner in o_data['results']:
-                        owners.append(owner)
-                    if not o_data.get('next'):
-                        break
-                    next_owners_url = _baseurl + o_data['next_link']
-                '''
                 namespace_cache[ns_id]['summary_fields']['owners'] = \
                     get_namespace_owners_details(_baseurl, ns_id)
 
@@ -413,22 +384,12 @@ def upstream_role_iterator(
             if ns_id not in namespace_cache:
                 logger.info(_baseurl + f'/api/v1/namespaces/{ns_id}/')
                 ns_url = _baseurl + f'/api/v1/namespaces/{ns_id}/'
-                namespace_data = safe_fetch(ns_url).json()
-                # logger.info(namespace_data)
+
+                nsd_rr = safe_fetch(ns_url)
+                namespace_data = nsd_rr.json()
                 namespace_cache[ns_id] = namespace_data
 
                 # get the owners too
-                '''
-                owners = []
-                next_owners_url = _baseurl + f'/api/v1/namespaces/{ns_id}/owners/'
-                while next_owners_url:
-                    o_data = requests.get(next_owners_url).json()
-                    for owner in o_data['results']:
-                        owners.append(owner)
-                    if not o_data.get('next'):
-                        break
-                    next_owners_url = _baseurl + o_data['next_link']
-                '''
                 namespace_cache[ns_id]['summary_fields']['owners'] = \
                     get_namespace_owners_details(_baseurl, ns_id)