Read all available releases from code hoster

Clients download first 100 tags to fetch releases. If a repository has
many releases with version-prefixes, it is likely for them to be
ignored due to this restriction.

This commit introduces a generator to fetch all tags until desired
amount of releases is found.

addresses: wbond/packagecontrol.io#143

package_control/clients/bitbucket_client.py

@@ -1,8 +1,8 @@
 import re
-from urllib.parse import quote
+from urllib.parse import urlencode, quote
 
 from ..downloaders.downloader_exception import DownloaderException
-from ..versions import version_sort, version_process
+from ..versions import version_match_prefix
 from .json_api_client import JSONApiClient
 
 
@@ -173,36 +173,35 @@ def download_info_from_tags(self, url, tag_prefix=None):
         if not tags_match:
             return None
 
-        user_repo = tags_match.group(1)
-
-        tags_list = {}
-        tags_url = self._api_url(user_repo, '/refs/tags?pagelen=100')
-        while tags_url:
-            tags_json = self.fetch_json(tags_url)
-            for tag in tags_json['values']:
-                tags_list[tag['name']] = tag['target']['date'][0:19].replace('T', ' ')
-            tags_url = tags_json['next'] if 'next' in tags_json else None
+        def _get_releases(user_repo, tag_prefix, page_size=100):
+            query_string = urlencode({'pagelen': page_size})
+            tags_url = self._api_url(user_repo, '/refs/tags?%s' % query_string)
+            while tags_url:
+                tags_json = self.fetch_json(tags_url)
+                for tag in tags_json['values']:
+                    version = version_match_prefix(tag['name'], tag_prefix)
+                    if version:
+                        yield (
+                            version,
+                            tag['name'],
+                            tag['target']['date'][0:19].replace('T', ' ')
+                        )
+
+                tags_url = tags_json.get('next')
 
-        tag_info = version_process(tags_list.keys(), tag_prefix)
-        tag_info = version_sort(tag_info, reverse=True)
-        if not tag_info:
-            return False
+        user_repo = tags_match.group(1)
 
         max_releases = self.settings.get('max_releases', 0)
+        num_releases = 0
 
         output = []
-        used_versions = set()
-        for info in tag_info:
-            version = info['version']
-            if version in used_versions:
-                continue
-
-            tag = info['prefix'] + version
+        for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True):
+            version, tag, timestamp = release
 
-            output.append(self._make_download_info(user_repo, tag, version, tags_list[tag]))
+            output.append(self._make_download_info(user_repo, tag, str(version), timestamp))
 
-            used_versions.add(version)
-            if max_releases > 0 and len(used_versions) >= max_releases:
+            num_releases += not version.prerelease
+            if max_releases > 0 and num_releases >= max_releases:
                 break
 
         return output

package_control/clients/github_client.py

@@ -2,7 +2,7 @@
 from urllib.parse import urlencode, quote
 
 from ..downloaders.downloader_exception import DownloaderException
-from ..versions import version_sort, version_process
+from ..versions import version_match_prefix
 from .json_api_client import JSONApiClient
 
 
@@ -158,32 +158,35 @@ def download_info_from_tags(self, url, tag_prefix=None):
         if not tags_match:
             return None
 
-        user_repo = tags_match.group(1)
-        tags_url = self._api_url(user_repo, '/tags?per_page=100')
-        tags_json = self.fetch_json(tags_url)
-        tag_urls = {tag['name']: tag['commit']['url'] for tag in tags_json}
-        tag_info = version_process(tag_urls.keys(), tag_prefix)
-        tag_info = version_sort(tag_info, reverse=True)
-        if not tag_info:
-            return False
+        def _get_releases(user_repo, tag_prefix=None, page_size=100):
+            for page in range(100):
+                query_string = urlencode({'page': page * page_size, 'per_page': page_size})
+                tags_url = self._api_url(user_repo, '/tags?%s' % query_string)
+                tags_json = self.fetch_json(tags_url)
+
+                for tag in tags_json:
+                    version = version_match_prefix(tag['name'], tag_prefix)
+                    if version:
+                        yield (version, tag['name'], tag['commit']['url'])
+
+                if len(tags_json) < page_size:
+                    return
 
+        user_repo = tags_match.group(1)
         max_releases = self.settings.get('max_releases', 0)
+        num_releases = 0
 
         output = []
-        used_versions = set()
-        for info in tag_info:
-            version = info['version']
-            if version in used_versions:
-                continue
-
-            tag = info['prefix'] + version
-            tag_info = self.fetch_json(tag_urls[tag])
+        for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True):
+            version, tag, tag_url = release
+
+            tag_info = self.fetch_json(tag_url)
             timestamp = tag_info['commit']['committer']['date'][0:19].replace('T', ' ')
 
-            output.append(self._make_download_info(user_repo, tag, version, timestamp))
+            output.append(self._make_download_info(user_repo, tag, str(version), timestamp))
 
-            used_versions.add(version)
-            if max_releases > 0 and len(used_versions) >= max_releases:
+            num_releases += not version.prerelease
+            if max_releases > 0 and num_releases >= max_releases:
                 break
 
         return output

package_control/clients/gitlab_client.py

@@ -1,8 +1,8 @@
 import re
-from urllib.parse import quote
+from urllib.parse import urlencode, quote
 
 from ..downloaders.downloader_exception import DownloaderException
-from ..versions import version_process, version_sort
+from ..versions import version_match_prefix
 from .json_api_client import JSONApiClient
 
 
@@ -161,34 +161,38 @@ def download_info_from_tags(self, url, tag_prefix=None):
         if not tags_match:
             return None
 
-        user_name, repo_name = tags_match.groups()
-        repo_id = '%s%%2F%s' % (user_name, repo_name)
-        tags_url = self._api_url(repo_id, '/repository/tags?per_page=100')
-        tags_json = self.fetch_json(tags_url)
-        tags_list = {
-            tag['name']: tag['commit']['committed_date'][0:19].replace('T', ' ')
-            for tag in tags_json
-        }
+        def _get_releases(user_repo, tag_prefix=None, page_size=100):
+            for page in range(100):
+                query_string = urlencode({'page': page * page_size, 'per_page': page_size})
+                tags_url = self._api_url(user_repo, '/repository/tags?%s' % query_string)
+                tags_json = self.fetch_json(tags_url)
 
-        tag_info = version_process(tags_list.keys(), tag_prefix)
-        tag_info = version_sort(tag_info, reverse=True)
-        if not tag_info:
-            return False
+                for tag in tags_json:
+                    version = version_match_prefix(tag['name'], tag_prefix)
+                    if version:
+                        yield (
+                            version,
+                            tag['name'],
+                            tag['commit']['committed_date'][0:19].replace('T', ' ')
+                        )
+
+                if len(tags_json) < page_size:
+                    return
+
+        user_name, repo_name = tags_match.groups()
+        user_repo = '%s%%2F%s' % (user_name, repo_name)
 
         max_releases = self.settings.get('max_releases', 0)
+        num_releases = 0
 
         output = []
-        used_versions = set()
-        for info in tag_info:
-            version = info['version']
-            if version in used_versions:
-                continue
+        for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True):
+            version, tag, timestamp = release
 
-            tag = info['prefix'] + version
-            output.append(self._make_download_info(user_name, repo_name, tag, version, tags_list[tag]))
+            output.append(self._make_download_info(user_name, repo_name, tag, str(version), timestamp))
 
-            used_versions.add(version)
-            if max_releases > 0 and len(used_versions) >= max_releases:
+            num_releases += not version.prerelease
+            if max_releases > 0 and num_releases >= max_releases:
                 break
 
         return output

package_control/versions.py

@@ -79,6 +79,26 @@ def version_exclude_prerelease(versions):
     return output
 
 
+def version_match_prefix(version, filter_prefix):
+    """
+    Create a SemVer for a given version, if it matches filter_prefix.
+
+    :param      version:        The version
+    :type       version:        { type_description }
+    :param      filter_prefix:  The filter prefix
+    :type       filter_prefix:  { type_description }
+    """
+    try:
+        if filter_prefix:
+            if version.startswith(filter_prefix):
+                return version_comparable(version[len(filter_prefix):])
+        else:
+            return version_comparable(version)
+    except ValueError:
+        pass
+    return None
+
+
 def version_process(versions, filter_prefix):
     """
     Filter a list of versions to ones that are valid SemVers, if a prefix