From 586322a6f097c730cf533d7737d97053c89b276b Mon Sep 17 00:00:00 2001 From: DeathAxe Date: Fri, 26 Aug 2022 14:56:39 +0200 Subject: [PATCH] Read all available releases from code hoster Clients download first 100 tags to fetch releases. If a repository has many releases with version-prefixes, it is likely for them to be ignored due to this restriction. This commit introduces a generator to fetch all tags until desired amount of releases is found. addresses: https://github.com/wbond/packagecontrol.io/issues/143 --- package_control/clients/bitbucket_client.py | 49 ++++++++++---------- package_control/clients/github_client.py | 43 +++++++++--------- package_control/clients/gitlab_client.py | 50 +++++++++++---------- package_control/versions.py | 20 +++++++++ 4 files changed, 94 insertions(+), 68 deletions(-) diff --git a/package_control/clients/bitbucket_client.py b/package_control/clients/bitbucket_client.py index 5c1e705c..ab848bc8 100644 --- a/package_control/clients/bitbucket_client.py +++ b/package_control/clients/bitbucket_client.py @@ -1,8 +1,8 @@ import re -from urllib.parse import quote +from urllib.parse import urlencode, quote from ..downloaders.downloader_exception import DownloaderException -from ..versions import version_sort, version_process +from ..versions import version_match_prefix from .json_api_client import JSONApiClient @@ -173,36 +173,35 @@ def download_info_from_tags(self, url, tag_prefix=None): if not tags_match: return None - user_repo = tags_match.group(1) - - tags_list = {} - tags_url = self._api_url(user_repo, '/refs/tags?pagelen=100') - while tags_url: - tags_json = self.fetch_json(tags_url) - for tag in tags_json['values']: - tags_list[tag['name']] = tag['target']['date'][0:19].replace('T', ' ') - tags_url = tags_json['next'] if 'next' in tags_json else None + def _get_releases(user_repo, tag_prefix, page_size=100): + query_string = urlencode({'pagelen': page_size}) + tags_url = self._api_url(user_repo, '/refs/tags?%s' % query_string) + while tags_url: + tags_json = self.fetch_json(tags_url) + for tag in tags_json['values']: + version = version_match_prefix(tag['name'], tag_prefix) + if version: + yield ( + version, + tag['name'], + tag['target']['date'][0:19].replace('T', ' ') + ) + + tags_url = tags_json.get('next') - tag_info = version_process(tags_list.keys(), tag_prefix) - tag_info = version_sort(tag_info, reverse=True) - if not tag_info: - return False + user_repo = tags_match.group(1) max_releases = self.settings.get('max_releases', 0) + num_releases = 0 output = [] - used_versions = set() - for info in tag_info: - version = info['version'] - if version in used_versions: - continue - - tag = info['prefix'] + version + for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True): + version, tag, timestamp = release - output.append(self._make_download_info(user_repo, tag, version, tags_list[tag])) + output.append(self._make_download_info(user_repo, tag, str(version), timestamp)) - used_versions.add(version) - if max_releases > 0 and len(used_versions) >= max_releases: + num_releases += not version.prerelease + if max_releases > 0 and num_releases >= max_releases: break return output diff --git a/package_control/clients/github_client.py b/package_control/clients/github_client.py index 906156d6..5667e91b 100644 --- a/package_control/clients/github_client.py +++ b/package_control/clients/github_client.py @@ -2,7 +2,7 @@ from urllib.parse import urlencode, quote from ..downloaders.downloader_exception import DownloaderException -from ..versions import version_sort, version_process +from ..versions import version_match_prefix from .json_api_client import JSONApiClient @@ -158,32 +158,35 @@ def download_info_from_tags(self, url, tag_prefix=None): if not tags_match: return None - user_repo = tags_match.group(1) - tags_url = self._api_url(user_repo, '/tags?per_page=100') - tags_json = self.fetch_json(tags_url) - tag_urls = {tag['name']: tag['commit']['url'] for tag in tags_json} - tag_info = version_process(tag_urls.keys(), tag_prefix) - tag_info = version_sort(tag_info, reverse=True) - if not tag_info: - return False + def _get_releases(user_repo, tag_prefix=None, page_size=100): + for page in range(100): + query_string = urlencode({'page': page * page_size, 'per_page': page_size}) + tags_url = self._api_url(user_repo, '/tags?%s' % query_string) + tags_json = self.fetch_json(tags_url) + + for tag in tags_json: + version = version_match_prefix(tag['name'], tag_prefix) + if version: + yield (version, tag['name'], tag['commit']['url']) + + if len(tags_json) < page_size: + return + user_repo = tags_match.group(1) max_releases = self.settings.get('max_releases', 0) + num_releases = 0 output = [] - used_versions = set() - for info in tag_info: - version = info['version'] - if version in used_versions: - continue - - tag = info['prefix'] + version - tag_info = self.fetch_json(tag_urls[tag]) + for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True): + version, tag, tag_url = release + + tag_info = self.fetch_json(tag_url) timestamp = tag_info['commit']['committer']['date'][0:19].replace('T', ' ') - output.append(self._make_download_info(user_repo, tag, version, timestamp)) + output.append(self._make_download_info(user_repo, tag, str(version), timestamp)) - used_versions.add(version) - if max_releases > 0 and len(used_versions) >= max_releases: + num_releases += not version.prerelease + if max_releases > 0 and num_releases >= max_releases: break return output diff --git a/package_control/clients/gitlab_client.py b/package_control/clients/gitlab_client.py index fec908c3..6fba2381 100644 --- a/package_control/clients/gitlab_client.py +++ b/package_control/clients/gitlab_client.py @@ -1,8 +1,8 @@ import re -from urllib.parse import quote +from urllib.parse import urlencode, quote from ..downloaders.downloader_exception import DownloaderException -from ..versions import version_process, version_sort +from ..versions import version_match_prefix from .json_api_client import JSONApiClient @@ -161,34 +161,38 @@ def download_info_from_tags(self, url, tag_prefix=None): if not tags_match: return None - user_name, repo_name = tags_match.groups() - repo_id = '%s%%2F%s' % (user_name, repo_name) - tags_url = self._api_url(repo_id, '/repository/tags?per_page=100') - tags_json = self.fetch_json(tags_url) - tags_list = { - tag['name']: tag['commit']['committed_date'][0:19].replace('T', ' ') - for tag in tags_json - } + def _get_releases(user_repo, tag_prefix=None, page_size=100): + for page in range(100): + query_string = urlencode({'page': page * page_size, 'per_page': page_size}) + tags_url = self._api_url(user_repo, '/repository/tags?%s' % query_string) + tags_json = self.fetch_json(tags_url) - tag_info = version_process(tags_list.keys(), tag_prefix) - tag_info = version_sort(tag_info, reverse=True) - if not tag_info: - return False + for tag in tags_json: + version = version_match_prefix(tag['name'], tag_prefix) + if version: + yield ( + version, + tag['name'], + tag['commit']['committed_date'][0:19].replace('T', ' ') + ) + + if len(tags_json) < page_size: + return + + user_name, repo_name = tags_match.groups() + user_repo = '%s%%2F%s' % (user_name, repo_name) max_releases = self.settings.get('max_releases', 0) + num_releases = 0 output = [] - used_versions = set() - for info in tag_info: - version = info['version'] - if version in used_versions: - continue + for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True): + version, tag, timestamp = release - tag = info['prefix'] + version - output.append(self._make_download_info(user_name, repo_name, tag, version, tags_list[tag])) + output.append(self._make_download_info(user_name, repo_name, tag, str(version), timestamp)) - used_versions.add(version) - if max_releases > 0 and len(used_versions) >= max_releases: + num_releases += not version.prerelease + if max_releases > 0 and num_releases >= max_releases: break return output diff --git a/package_control/versions.py b/package_control/versions.py index 30556012..602dba03 100644 --- a/package_control/versions.py +++ b/package_control/versions.py @@ -79,6 +79,26 @@ def version_exclude_prerelease(versions): return output +def version_match_prefix(version, filter_prefix): + """ + Create a SemVer for a given version, if it matches filter_prefix. + + :param version: The version + :type version: { type_description } + :param filter_prefix: The filter prefix + :type filter_prefix: { type_description } + """ + try: + if filter_prefix: + if version.startswith(filter_prefix): + return version_comparable(version[len(filter_prefix):]) + else: + return version_comparable(version) + except ValueError: + pass + return None + + def version_process(versions, filter_prefix): """ Filter a list of versions to ones that are valid SemVers, if a prefix