From d82e997e4e53e354b46beaab53ef980d05e18706 Mon Sep 17 00:00:00 2001 From: Bono de Visser Date: Tue, 9 Mar 2021 10:28:36 +0100 Subject: [PATCH 1/2] Also convert non-editable git urls with pip env markers If a non-editable git url is provided (private repo) make sure we also convert this to a URL that is able to clone from private repo's. Refactored some of the stuff to decrease duplication. --- pip_install_privates/install.py | 81 +++++++++++++++++---------------- tests/unit/test_install.py | 68 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 38 deletions(-) diff --git a/pip_install_privates/install.py b/pip_install_privates/install.py index 1ead6c9..c5b10ae 100755 --- a/pip_install_privates/install.py +++ b/pip_install_privates/install.py @@ -41,30 +41,11 @@ def convert_to_github_url_with_token(url, token): return url -def convert_to_editable_github_url_with_token(url, token): - """ - Convert a Github URL to an editable git+https url that identifies via an Oauth token. The Oauth identification - allows for installation of private packages. The editable flag means the package - will be installed as source (i.e. all files will be present instead of it just being installed as a package). - :param url: The url to convert into a Github access token oauth url. - :param token: The Github access token to use for the oauth url. - :return: list: The editable flag for pip and a git+https url with Oauth identification. - """ - url = convert_to_github_url_with_token(url, token) - return ['-e', url] - - -def convert_to_editable_github_url(url): - """ - Convert a Github URL to a git+https url (does not work for private packages). The editable flag means the package - will be installed as source (i.e. all files will be present instead of it just being installed as a package). - :param url: The url to convert into a Github access token oauth url. - :return: list: The editable flag for pip and a git+https url. - """ +def convert_to_github_url(url): for prefix in [GIT_SSH_PREFIX, GIT_GIT_PREFIX]: if url.startswith(prefix): url = 'git+https://github.com/{}'.format(url[len(prefix):]) - return ['-e', url] + return url def can_convert_url(url): @@ -112,25 +93,10 @@ def collect_requirements(fname, transform_with_token=None): # -e 'git+git@github.com:ByteInternet/my-repo.git@20201127.1#egg=my-repo ; python_version=="3.7"' # Do not remove double quotes, since these could be used in the environment marker string i.e. "3.7". stripped_tokens = [token.replace("'", "") for token in tokens] - - if can_convert_url(stripped_tokens[1]): - if transform_with_token: - flag, github_url = convert_to_editable_github_url_with_token( - stripped_tokens[1], transform_with_token) - - github_url = add_potential_pip_environment_markers_to_requirement(stripped_tokens, github_url) - collected += [flag, github_url] - else: - flag, github_url = convert_to_editable_github_url(stripped_tokens[1]) - - github_url = add_potential_pip_environment_markers_to_requirement(stripped_tokens, github_url) - collected += [flag, github_url] - else: - url = add_potential_pip_environment_markers_to_requirement(stripped_tokens, stripped_tokens[1]) - collected += ['-e', url] + collected += convert_potential_editable_github_url(stripped_tokens[1], stripped_tokens, transform_with_token) elif ';' in tokens: - collected += [add_potential_pip_environment_markers_to_requirement(tokens, tokens[0])] + collected += convert_potential_git_url(tokens[0], tokens, transform_with_token) # No special casing for the rest. Just pass everything to pip else: @@ -158,6 +124,45 @@ def add_potential_pip_environment_markers_to_requirement(stripped_tokens, requir return '{} ; {}'.format(requirement, stripped_tokens[environment_index]) +def convert_potential_git_url(requirement, tokens, transform_with_token=False): + """ + Convert a potential Github URL to an git+https url that possibly identifies via an Oauth token. The Oauth + identification allows for installation of private packages. + :param requirement: The potential url to convert into a Github access token oauth url. + :param tokens: All specifications provided for the requirement (i.e. editable flag, the requirement, pip + environment markers, comments) + :param transform_with_token: Whether or not to transform to a github url with OAuth authentication + :return: list: The git+https url with possible Oauth identification. + """ + if can_convert_url(requirement): + if transform_with_token: + github_url = convert_to_github_url_with_token(requirement, transform_with_token) + github_url = add_potential_pip_environment_markers_to_requirement(tokens, github_url) + return [github_url] + else: + github_url = convert_to_github_url(requirement) + github_url = add_potential_pip_environment_markers_to_requirement(tokens, github_url) + return [github_url] + else: + return [add_potential_pip_environment_markers_to_requirement(tokens, requirement)] + + +def convert_potential_editable_github_url(requirement, tokens, transform_with_token=False): + """ + Convert a potential Github URL to an editable git+https url that possibly identifies via an Oauth token. The Oauth + identification allows for installation of private packages. The editable flag means the package + will be installed as source (i.e. all files will be present instead of it just being installed as a package). + :param requirement: The potential url to convert into a Github access token oauth url. + :param tokens: All specifications provided for the requirement (i.e. editable flag, the requirement, pip + environment markers, comments) + :param transform_with_token: Whether or not to transform to a github url with OAuth authentication + :return: list: The editable flag for pip and a git+https url with possible Oauth identification. + """ + requirement_tokens = convert_potential_git_url(requirement, tokens, transform_with_token) + requirement_tokens.insert(0, '-e') + return requirement_tokens + + def install(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/tests/unit/test_install.py b/tests/unit/test_install.py index cd37fe6..5cce3f9 100644 --- a/tests/unit/test_install.py +++ b/tests/unit/test_install.py @@ -285,4 +285,72 @@ def test_transforms_requirement_if_pip_environment_marker_in_tokens_with_inline_ 'fso==0.3.1' ]) + def test_transforms_non_editable_github_url_with_pip_environment_markers_to_correct_requirement(self): + file = self._create_reqs_file([ + 'mock==2.0.0', + 'git+ssh://git@github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7' + ]) + fname = self._create_reqs_file(['-r {}'.format(file), 'fso==0.3.1']) + + ret = collect_requirements(fname) + + self.assertEqual(ret, [ + 'mock==2.0.0', + 'git+https://github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7', + 'fso==0.3.1' + ]) + + def test_transforms_non_editable_github_url_with_pip_environment_markers_to_correct_requirement_with_token(self): + file = self._create_reqs_file([ + 'mock==2.0.0', + 'git+ssh://git@github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7' + ]) + fname = self._create_reqs_file(['-r {}'.format(file), 'fso==0.3.1']) + + ret = collect_requirements(fname, transform_with_token=True) + + self.assertEqual(ret, [ + 'mock==2.0.0', + 'git+https://True:x-oauth-basic@github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7', + 'fso==0.3.1' + ]) + + def test_transforms_non_editable_github_url_with_pip_environment_markers_to_correct_requirement_with_comment(self): + file = self._create_reqs_file([ + 'mock==2.0.0', + 'git+ssh://git@github.com/ByteInternet/... ; python_version=="3.7" # We need this', + 'nose==1.3.7' + ]) + fname = self._create_reqs_file(['-r {}'.format(file), 'fso==0.3.1']) + + ret = collect_requirements(fname) + + self.assertEqual(ret, [ + 'mock==2.0.0', + 'git+https://github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7', + 'fso==0.3.1' + ]) + + def test_transforms_non_editable_github_url_with_pip_environment_markers_to_correct_requirement_with_token_and_comment(self): + file = self._create_reqs_file([ + 'mock==2.0.0', + 'git+ssh://git@github.com/ByteInternet/... ; python_version=="3.7" # We need this because reasons', + 'nose==1.3.7' + ]) + fname = self._create_reqs_file(['-r {}'.format(file), 'fso==0.3.1']) + + ret = collect_requirements(fname, transform_with_token=True) + + self.assertEqual(ret, [ + 'mock==2.0.0', + 'git+https://True:x-oauth-basic@github.com/ByteInternet/... ; python_version=="3.7"', + 'nose==1.3.7', + 'fso==0.3.1' + ]) + From f1b67922a88c4c04ca58073bfc105bf7c1ea2501 Mon Sep 17 00:00:00 2001 From: Bono de Visser Date: Tue, 9 Mar 2021 10:43:08 +0100 Subject: [PATCH 2/2] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 066b33f..5ea3912 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readfile(filename): setup( name='pip_install_privates', - version='0.6.2', + version='0.6.3', description='Install pip packages from private repositories without an ssh agent', long_description=readfile('README.rst'), long_description_content_type='text/x-rst',