From 50356da8393abdb47cc6c64de4fa387d8499cbc4 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 8 Jan 2019 12:13:56 +0000 Subject: [PATCH 01/12] Fix white space handling in file names Previously, when using data_files with a glob that matched a file with whitespace in the name, pip would error with a message that the file does not exist, e.g: error: can't copy 'ansible/roles/yatesr.timezone/templates/timezone-Arch': doesn't exist or not a regular file The problem was that ansible/roles/yatesr.timezone/templates/timezone-Arch was a truncated form of the actual filename: ansible/roles/yatesr.timezone/templates/timezone-Arch Linux.j2 Note the space in the filename and that it has been split on this space. This change allows you to use a glob that matches files with whitespace in the name. It does this by quoting the path. Change-Id: Id2cc32e4e40c1f834b19756e922118d8526358d3 Fixes-Bug: 1810934 --- pbr/hooks/files.py | 24 ++++++-- pbr/tests/test_files.py | 59 +++++++++++++++++-- pbr/tests/test_util.py | 25 ++++++++ pbr/util.py | 10 ++-- ...s-in-data-files-glob-0fe0c398d70dfea8.yaml | 5 ++ 5 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml diff --git a/pbr/hooks/files.py b/pbr/hooks/files.py index 750ac32c..0fe0df55 100644 --- a/pbr/hooks/files.py +++ b/pbr/hooks/files.py @@ -14,6 +14,7 @@ # under the License. import os +import shlex import sys from pbr import find_package @@ -35,6 +36,14 @@ def get_man_section(section): return os.path.join(get_manpath(), 'man%s' % section) +def unquote_path(path): + # unquote the full path, e.g: "'a/full/path'" becomes "a/full/path", also + # strip the quotes off individual path components because os.walk cannot + # handle paths like: "'i like spaces'/'another dir'", so we will pass it + # "i like spaces/another dir" instead. + return "".join(shlex.split(path)) + + class FilesConfig(base.BaseConfig): section = 'files' @@ -57,25 +66,28 @@ def expand_globs(self): target = target.strip() if not target.endswith(os.path.sep): target += os.path.sep - for (dirpath, dirnames, fnames) in os.walk(source_prefix): + unquoted_prefix = unquote_path(source_prefix) + unquoted_target = unquote_path(target) + for (dirpath, dirnames, fnames) in os.walk(unquoted_prefix): # As source_prefix is always matched, using replace with a # a limit of one is always going to replace the path prefix # and not accidentally replace some text in the middle of # the path - new_prefix = dirpath.replace(source_prefix, target, 1) - finished.append("%s = " % new_prefix) + new_prefix = dirpath.replace(unquoted_prefix, + unquoted_target, 1) + finished.append("'%s' = " % new_prefix) finished.extend( - [" %s" % os.path.join(dirpath, f) for f in fnames]) + [" '%s'" % os.path.join(dirpath, f) for f in fnames]) else: finished.append(line) self.data_files = "\n".join(finished) def add_man_path(self, man_path): - self.data_files = "%s\n%s =" % (self.data_files, man_path) + self.data_files = "%s\n'%s' =" % (self.data_files, man_path) def add_man_page(self, man_page): - self.data_files = "%s\n %s" % (self.data_files, man_page) + self.data_files = "%s\n '%s'" % (self.data_files, man_page) def get_man_sections(self): man_sections = dict() diff --git a/pbr/tests/test_files.py b/pbr/tests/test_files.py index ed67f7bc..94a2d9ad 100644 --- a/pbr/tests/test_files.py +++ b/pbr/tests/test_files.py @@ -37,12 +37,17 @@ def setUp(self): pkg_etc = os.path.join(pkg_fixture.base, 'etc') pkg_ansible = os.path.join(pkg_fixture.base, 'ansible', 'kolla-ansible', 'test') + dir_spcs = os.path.join(pkg_fixture.base, 'dir with space') + dir_subdir_spc = os.path.join(pkg_fixture.base, 'multi space', + 'more spaces') pkg_sub = os.path.join(pkg_etc, 'sub') subpackage = os.path.join( pkg_fixture.base, 'fake_package', 'subpackage') os.makedirs(pkg_sub) os.makedirs(subpackage) os.makedirs(pkg_ansible) + os.makedirs(dir_spcs) + os.makedirs(dir_subdir_spc) with open(os.path.join(pkg_etc, "foo"), 'w') as foo_file: foo_file.write("Foo Data") with open(os.path.join(pkg_sub, "bar"), 'w') as foo_file: @@ -51,6 +56,10 @@ def setUp(self): baz_file.write("Baz Data") with open(os.path.join(subpackage, "__init__.py"), 'w') as foo_file: foo_file.write("# empty") + with open(os.path.join(dir_spcs, "file with spc"), 'w') as spc_file: + spc_file.write("# empty") + with open(os.path.join(dir_subdir_spc, "file with spc"), 'w') as file_: + file_.write("# empty") self.useFixture(base.DiveDir(pkg_fixture.base)) @@ -79,9 +88,49 @@ def test_data_files_globbing(self): ) files.FilesConfig(config, 'fake_package').run() self.assertIn( - '\netc/pbr/ = \n etc/foo\netc/pbr/sub = \n etc/sub/bar', + "\n'etc/pbr/' = \n 'etc/foo'\n'etc/pbr/sub' = \n 'etc/sub/bar'", config['files']['data_files']) + def test_data_files_with_spaces(self): + config = dict( + files=dict( + data_files="\n 'i like spaces' = 'dir with space'/*" + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn( + "\n'i like spaces/' = \n 'dir with space/file with spc'", + config['files']['data_files']) + + def test_data_files_with_spaces_subdirectories(self): + # test that we can handle whitespace in subdirectories + data_files = "\n 'one space/two space' = 'multi space/more spaces'/*" + expected = ( + "\n'one space/two space/' = " + "\n 'multi space/more spaces/file with spc'") + config = dict( + files=dict( + data_files=data_files + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn(expected, config['files']['data_files']) + + def test_data_files_with_spaces_quoted_components(self): + # test that we can quote individual path components + data_files = ( + "\n'one space'/'two space' = 'multi space'/'more spaces'/*" + ) + expected = ("\n'one space/two space/' = " + "\n 'multi space/more spaces/file with spc'") + config = dict( + files=dict( + data_files=data_files + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn(expected, config['files']['data_files']) + def test_data_files_globbing_source_prefix_in_directory_name(self): # We want to test that the string, "docs", is not replaced in a # subdirectory name, "sub-docs" @@ -92,8 +141,8 @@ def test_data_files_globbing_source_prefix_in_directory_name(self): ) files.FilesConfig(config, 'fake_package').run() self.assertIn( - '\nshare/ansible/ = ' - '\nshare/ansible/kolla-ansible = ' - '\nshare/ansible/kolla-ansible/test = ' - '\n ansible/kolla-ansible/test/baz', + "\n'share/ansible/' = " + "\n'share/ansible/kolla-ansible' = " + "\n'share/ansible/kolla-ansible/test' = " + "\n 'ansible/kolla-ansible/test/baz'", config['files']['data_files']) diff --git a/pbr/tests/test_util.py b/pbr/tests/test_util.py index 6814ac7b..393bc5cd 100644 --- a/pbr/tests/test_util.py +++ b/pbr/tests/test_util.py @@ -172,3 +172,28 @@ def test_provides_extras(self): config = config_from_ini(ini) kwargs = util.setup_cfg_to_setup_kwargs(config) self.assertEqual(['foo', 'bar'], kwargs['provides_extras']) + + +class TestDataFilesParsing(base.BaseTestCase): + + scenarios = [ + ('data_files', { + 'config_text': """ + [files] + data_files = + 'i like spaces/' = + 'dir with space/file with spc 2' + 'dir with space/file with spc 1' + """, + 'data_files': [ + ('i like spaces/', ['dir with space/file with spc 2', + 'dir with space/file with spc 1']) + ] + })] + + def test_handling_of_whitespace_in_data_files(self): + config = config_from_ini(self.config_text) + kwargs = util.setup_cfg_to_setup_kwargs(config) + + self.assertEqual(self.data_files, + list(kwargs['data_files'])) diff --git a/pbr/util.py b/pbr/util.py index 55d73f8b..e931b5ff 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -64,6 +64,7 @@ from collections import defaultdict import os import re +import shlex import sys import traceback @@ -372,21 +373,22 @@ def setup_cfg_to_setup_kwargs(config, script_args=()): for line in in_cfg_value: if '=' in line: key, value = line.split('=', 1) - key, value = (key.strip(), value.strip()) + key_unquoted = shlex.split(key.strip())[0] + key, value = (key_unquoted, value.strip()) if key in data_files: # Multiple duplicates of the same package name; # this is for backwards compatibility of the old # format prior to d2to1 0.2.6. prev = data_files[key] - prev.extend(value.split()) + prev.extend(shlex.split(value)) else: - prev = data_files[key.strip()] = value.split() + prev = data_files[key.strip()] = shlex.split(value) elif firstline: raise errors.DistutilsOptionError( 'malformed package_data first line %r (misses ' '"=")' % line) else: - prev.extend(line.strip().split()) + prev.extend(shlex.split(line.strip())) firstline = False if arg == 'data_files': # the data_files value is a pointlessly different structure diff --git a/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml b/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml new file mode 100644 index 00000000..793ba73b --- /dev/null +++ b/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the handling of spaces in data_files globs. Please see `bug 1810934 + `_ for more details. From 30926f6dbe05996009d407750acb5ea52a039e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Thu, 16 May 2019 14:33:45 +0200 Subject: [PATCH 02/12] Update Sphinx requirement Sphinx 2.0 no longer works on python 2.7, so we need to start capping it there. Change-Id: Ia13b8a75f11ee997e00e817464829ce8b86d2e16 --- doc/requirements.txt | 3 ++- test-requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index b9c5e1f1..711fb972 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD openstackdocstheme>=1.18.1 # Apache-2.0 reno>=2.5.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 70e4ca04..c30a2b17 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,5 +14,6 @@ virtualenv>=14.0.6 # MIT coverage!=4.4,>=4.0 # Apache-2.0 # optionally exposed by distutils commands -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD testrepository>=0.0.18 # Apache-2.0/BSD From c691c63d83d8ab5c577d83464883971185250688 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Tue, 21 May 2019 16:18:43 +0000 Subject: [PATCH 03/12] Install more dependencies for integration testing When running this locally I needed to install a few more distro packages to provide build dependencies for pip packages. This change adds them to the list of things installed in tools/integration.sh. Change-Id: I2ae04b893c7dfddcb6d94b724f53ee08436bf577 --- tools/integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration.sh b/tools/integration.sh index 3b431e14..6c4dc16e 100644 --- a/tools/integration.sh +++ b/tools/integration.sh @@ -30,7 +30,7 @@ REPODIR=${REPODIR:-$BASE/new} # TODO: Figure out how to get this on to the box properly sudo apt-get update -sudo apt-get install -y --force-yes libvirt-dev libxml2-dev libxslt-dev libmysqlclient-dev libpq-dev libnspr4-dev pkg-config libsqlite3-dev libffi-dev libldap2-dev libsasl2-dev ccache libkrb5-dev liberasurecode-dev libjpeg-dev +sudo apt-get install -y --force-yes libvirt-dev libxml2-dev libxslt-dev libmysqlclient-dev libpq-dev libnspr4-dev pkg-config libsqlite3-dev libffi-dev libldap2-dev libsasl2-dev ccache libkrb5-dev liberasurecode-dev libjpeg-dev libsystemd-dev libnss3-dev libssl-dev # FOR numpy / pyyaml # The source list has been removed from our apt config so rather than From 4428e34dc05eee2da5d5f39243a2201694f96934 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 12 May 2019 09:41:19 +0200 Subject: [PATCH 04/12] Remove neutron-lbaas neutron-lbaas is getting retired, see [1]. Therefore remove use of it from this repo. [1] http://lists.openstack.org/pipermail/openstack-discuss/2019-May/006142.html Change-Id: Icbff2f69cfea78e961a60ae7723ed7dc58f296a7 --- .zuul.yaml | 1 - playbooks/legacy/pbr-installation-devstack/run.yaml | 1 - playbooks/legacy/pbr-installation-upstream-devstack/run.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 0f13f836..e029ea3f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -35,7 +35,6 @@ - openstack/manila-ui - openstack/neutron - openstack/neutron-fwaas - - openstack/neutron-lbaas - openstack/neutron-vpnaas - openstack/nova - openstack/octavia diff --git a/playbooks/legacy/pbr-installation-devstack/run.yaml b/playbooks/legacy/pbr-installation-devstack/run.yaml index c3591cec..96f863ad 100644 --- a/playbooks/legacy/pbr-installation-devstack/run.yaml +++ b/playbooks/legacy/pbr-installation-devstack/run.yaml @@ -63,7 +63,6 @@ export PROJECTS="openstack/zaqar $PROJECTS" export PROJECTS="openstack/neutron $PROJECTS" export PROJECTS="openstack/neutron-fwaas $PROJECTS" - export PROJECTS="openstack/neutron-lbaas $PROJECTS" export PROJECTS="openstack/octavia $PROJECTS" export PROJECTS="openstack/neutron-vpnaas $PROJECTS" export PROJECTS="openstack/nova $PROJECTS" diff --git a/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml index 554d44b2..544dd43b 100644 --- a/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml +++ b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml @@ -63,7 +63,6 @@ export PROJECTS="openstack/zaqar $PROJECTS" export PROJECTS="openstack/neutron $PROJECTS" export PROJECTS="openstack/neutron-fwaas $PROJECTS" - export PROJECTS="openstack/neutron-lbaas $PROJECTS" export PROJECTS="openstack/octavia $PROJECTS" export PROJECTS="openstack/neutron-vpnaas $PROJECTS" export PROJECTS="openstack/nova $PROJECTS" From 713aff286b124aeaa6b2bea3b0d70488b43de4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Mon, 25 Mar 2019 15:34:56 +0100 Subject: [PATCH 05/12] Set subparser argument required When subparser argument is not provided by user argparse return an error message not really useful for user: 'Namespace' object has no attribute 'func' This is due to the fact that when we launch the pbr in cli mode the subparser argument is not mandatory (required) and directly we try to execute a undefined function. Set the subparser required is more helpful for users due to the fact that argparse display the helping message with the available sub-commands that users can use These changes provides the following output if the argument is not passed: usage: pbr [-h] [-v] {sha,info,freeze} ... main.py: error: too few arguments Change-Id: I7982f9d40cb0979ddb89d7bc53964167f8e4b269 --- pbr/cmd/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pbr/cmd/main.py b/pbr/cmd/main.py index 29cd61d7..91ea384e 100644 --- a/pbr/cmd/main.py +++ b/pbr/cmd/main.py @@ -86,7 +86,9 @@ def main(): version=str(pbr.version.VersionInfo('pbr'))) subparsers = parser.add_subparsers( - title='commands', description='valid commands', help='additional help') + title='commands', description='valid commands', help='additional help', + dest='cmd') + subparsers.required = True cmd_sha = subparsers.add_parser('sha', help='print sha of package') cmd_sha.set_defaults(func=get_sha) From 20d95468bcc5df499b17e28b6ce440ad03a9a94f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 15 Oct 2018 12:38:53 -0400 Subject: [PATCH 06/12] Add openstack-tox-py37 job Enables py37 testing while also being more friendly with developer environments where not all python versions may be available. Runs the docs jobs last as that one is the least likely to be needed. Tox min version is needed for the new options used inside. Change-Id: Icc6cb4979b2523a0edfc6375c5c032bb8be76b9f Story: #2004073 Task: #27440 --- .zuul.yaml | 1 + tox.ini | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 0f13f836..403b8006 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -112,6 +112,7 @@ - openstack-python-jobs - openstack-python35-jobs - openstack-python36-jobs + - openstack-python37-jobs - periodic-stable-jobs - publish-openstack-docs-pti check: diff --git a/tox.ini b/tox.ini index 4d460d8b..d029afcb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] -minversion = 2.0 -envlist = py{27,35,36},pep8,docs +minversion = 3.1 +envlist = pep8,py{37,36,35,27},docs +ignore_basepython_conflict = True +skip_missing_interpreters = True [testenv] usedevelop = True From 3b102a551bb2518682a0da4e6065feeb7f20807a Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Fri, 27 Apr 2018 20:11:53 +0000 Subject: [PATCH 07/12] Read description file as utf-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently pbr fails if the description file contains unicode characters. To fix this we need to open the description file as utf-8 explicitly. Since open() in Python 2 doesn't support an encoding parameter, use io.open() which works on both 2 and 3. Co-Authored-By: Hervé Beraud Change-Id: I1bee502ac84b474cc9db5523d2437a8c0a861c00 Closes-Bug: 1704472 --- pbr/tests/test_util.py | 20 ++++++++++++++++++++ pbr/util.py | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pbr/tests/test_util.py b/pbr/tests/test_util.py index 393bc5cd..1cbb2d2a 100644 --- a/pbr/tests/test_util.py +++ b/pbr/tests/test_util.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP) # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,6 +14,7 @@ # under the License. import io +import tempfile import textwrap import six @@ -197,3 +199,21 @@ def test_handling_of_whitespace_in_data_files(self): self.assertEqual(self.data_files, list(kwargs['data_files'])) + + +class TestUTF8DescriptionFile(base.BaseTestCase): + def test_utf8_description_file(self): + _, path = tempfile.mkstemp() + ini_template = """ + [metadata] + description_file = %s + """ + # Two \n's because pbr strips the file content and adds \n\n + # This way we can use it directly as the assert comparison + unicode_description = u'UTF8 description: é"…-ʃŋ\'\n\n' + ini = ini_template % path + with io.open(path, 'w', encoding='utf8') as f: + f.write(unicode_description) + config = config_from_ini(ini) + kwargs = util.setup_cfg_to_setup_kwargs(config) + self.assertEqual(unicode_description, kwargs['long_description']) diff --git a/pbr/util.py b/pbr/util.py index e931b5ff..323747e4 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -62,6 +62,7 @@ import logging # noqa from collections import defaultdict +import io import os import re import shlex @@ -320,7 +321,7 @@ def setup_cfg_to_setup_kwargs(config, script_args=()): in_cfg_value = split_multiline(in_cfg_value) value = '' for filename in in_cfg_value: - description_file = open(filename) + description_file = io.open(filename, encoding='utf-8') try: value += description_file.read().strip() + '\n\n' finally: From de739bb0802a6abee6c745c4bcc74b5e773c4f37 Mon Sep 17 00:00:00 2001 From: Martin Domke Date: Fri, 16 Nov 2018 18:13:17 +0100 Subject: [PATCH 08/12] Allow git-tags to be SemVer compliant This fix allows it to use git-tags with a pre-release suffix that follows the SemVer specification (e.g. 1.2.3-rc1). Change-Id: Ie2e3c0e4145a105aff0a35c8dc6ec2a0cc1fc04e --- pbr/git.py | 7 ++++++- pbr/tests/test_packaging.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pbr/git.py b/pbr/git.py index 6e18adae..6e0e3467 100644 --- a/pbr/git.py +++ b/pbr/git.py @@ -223,6 +223,11 @@ def _iter_log_inner(git_dir): presentation logic to the output - making it suitable for different uses. + .. caution:: this function risk to return a tag that doesn't exist really + inside the git objects list due to replacement made + to tag name to also list pre-release suffix. + Compliant with the SemVer specification (e.g 1.2.3-rc1) + :return: An iterator of (hash, tags_set, 1st_line) tuples. """ log.info('[pbr] Generating ChangeLog') @@ -248,7 +253,7 @@ def _iter_log_inner(git_dir): for tag_string in refname.split("refs/tags/")[1:]: # git tag does not allow : or " " in tag names, so we split # on ", " which is the separator between elements - candidate = tag_string.split(", ")[0] + candidate = tag_string.split(", ")[0].replace("-", ".") if _is_valid_version(candidate): tags.add(candidate) diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py index 853844f4..308ae89c 100644 --- a/pbr/tests/test_packaging.py +++ b/pbr/tests/test_packaging.py @@ -670,6 +670,12 @@ def test_tagged_version_has_tag_version(self): version = packaging._get_version_from_git('1.2.3') self.assertEqual('1.2.3', version) + def test_tagged_version_with_semver_compliant_prerelease(self): + self.repo.commit() + self.repo.tag('1.2.3-rc2') + version = packaging._get_version_from_git() + self.assertEqual('1.2.3.0rc2', version) + def test_non_canonical_tagged_version_bump(self): self.repo.commit() self.repo.tag('1.4') @@ -726,6 +732,13 @@ def test_untagged_version_after_rc_has_dev_version_preversion(self): version = packaging._get_version_from_git('1.2.3') self.assertThat(version, matchers.StartsWith('1.2.3.0a2.dev1')) + def test_untagged_version_after_semver_compliant_prerelease_tag(self): + self.repo.commit() + self.repo.tag('1.2.3-rc2') + self.repo.commit() + version = packaging._get_version_from_git() + self.assertEqual('1.2.3.0rc3.dev1', version) + def test_preversion_too_low_simple(self): # That is, the target version is either already released or not high # enough for the semver requirements given api breaks etc. From a7e5c0202ea00aab29c009f7adef6cea51ac6e25 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Fri, 31 May 2019 15:06:56 +0300 Subject: [PATCH 09/12] Fix Windows support A recent commit [1] broke Windows support by using shlex to unquote paths. The issue with shlex.split is that it doesn't work with Windows paths, treating backslashes as escape characters. We'll just replace backslashes with slashes before using shlex.split, converting them back afterwards. Closes-Bug: #1831242 [1] Id2cc32e4e40c1f834b19756e922118d8526358d3 Change-Id: Icb3abca004a35ab9760db8116fedfa96d012d0d0 --- pbr/hooks/files.py | 7 +++++++ pbr/util.py | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pbr/hooks/files.py b/pbr/hooks/files.py index 0fe0df55..c44af7c4 100644 --- a/pbr/hooks/files.py +++ b/pbr/hooks/files.py @@ -41,6 +41,13 @@ def unquote_path(path): # strip the quotes off individual path components because os.walk cannot # handle paths like: "'i like spaces'/'another dir'", so we will pass it # "i like spaces/another dir" instead. + + if os.name == 'nt': + # shlex cannot handle paths that contain backslashes, treating those + # as escape characters. + path = path.replace("\\", "/") + return "".join(shlex.split(path)).replace("/", "\\") + return "".join(shlex.split(path)) diff --git a/pbr/util.py b/pbr/util.py index e931b5ff..e52f0097 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -161,6 +161,16 @@ CSV_FIELDS = () +def shlex_split(path): + if os.name == 'nt': + # shlex cannot handle paths that contain backslashes, treating those + # as escape characters. + path = path.replace("\\", "/") + return [x.replace("/", "\\") for x in shlex.split(path)] + + return shlex.split(path) + + def resolve_name(name): """Resolve a name like ``module.object`` to an object and return it. @@ -373,22 +383,22 @@ def setup_cfg_to_setup_kwargs(config, script_args=()): for line in in_cfg_value: if '=' in line: key, value = line.split('=', 1) - key_unquoted = shlex.split(key.strip())[0] + key_unquoted = shlex_split(key.strip())[0] key, value = (key_unquoted, value.strip()) if key in data_files: # Multiple duplicates of the same package name; # this is for backwards compatibility of the old # format prior to d2to1 0.2.6. prev = data_files[key] - prev.extend(shlex.split(value)) + prev.extend(shlex_split(value)) else: - prev = data_files[key.strip()] = shlex.split(value) + prev = data_files[key.strip()] = shlex_split(value) elif firstline: raise errors.DistutilsOptionError( 'malformed package_data first line %r (misses ' '"=")' % line) else: - prev.extend(shlex.split(line.strip())) + prev.extend(shlex_split(line.strip())) firstline = False if arg == 'data_files': # the data_files value is a pointlessly different structure From ab3db59c5aeac13f3998de09a2629e9fa52ec908 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Thu, 6 Jun 2019 20:07:18 +0000 Subject: [PATCH 10/12] Make WSGI tests listen on localhost Currently we rely on the wsgi server to choose its own bind address, which tends to result in it listening on a DNS name. This means we have a dependency on that name resolving as expected on the node, which seems to not always be the case. In the interest of removing that environment dependency, let's have the wsgi tests explicitly listen on 127.0.0.1, which should always work and still allows us to verify our part of the WSGI functionality. This also includes a fix to use constraints for building docs because that is also blocking ci on this repo. We need both of these changes for anything to merge without endless rechecks. Change-Id: I48438eb31ee9dd102f06e30b13c6d892a93aba3b Closes-Bug: 1830962 --- pbr/tests/test_wsgi.py | 4 ++-- tox.ini | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pbr/tests/test_wsgi.py b/pbr/tests/test_wsgi.py index f840610d..18732f74 100644 --- a/pbr/tests/test_wsgi.py +++ b/pbr/tests/test_wsgi.py @@ -77,8 +77,8 @@ def test_wsgi_script_run(self): def _test_wsgi(self, cmd_name, output, extra_args=None): cmd = os.path.join(self.temp_dir, 'bin', cmd_name) - print("Running %s -p 0" % cmd) - popen_cmd = [cmd, '-p', '0'] + print("Running %s -p 0 -b 127.0.0.1" % cmd) + popen_cmd = [cmd, '-p', '0', '-b', '127.0.0.1'] if extra_args: popen_cmd.extend(extra_args) diff --git a/tox.ini b/tox.ini index d029afcb..bf7e5c8f 100644 --- a/tox.ini +++ b/tox.ini @@ -23,12 +23,16 @@ commands = flake8 {posargs} [testenv:docs] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -r{toxinidir}/doc/requirements.txt commands = python setup.py build_sphinx [testenv:releasenotes] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -r{toxinidir}/doc/requirements.txt commands = rm -rf releasenotes/build sphinx-build -W -b html -d releasenotes/build/doctrees releasenotes/source releasenotes/build/html From e8e9da3d90d29aec71802f61c5270c3e5cf7cc90 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Fri, 7 Jun 2019 16:05:04 +0000 Subject: [PATCH 11/12] Switch to release.o.o for constraints This is the preferred location to get constraints now. Change-Id: I5dc34e63f94ebf9c19ffb5caeeea24581eca658f --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index bf7e5c8f..e761968c 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ setenv = OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1} OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60} deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt commands = stestr run --suppress-attachments {posargs} @@ -24,14 +24,14 @@ commands = flake8 {posargs} [testenv:docs] basepython = python3 deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt commands = python setup.py build_sphinx [testenv:releasenotes] basepython = python3 deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt commands = rm -rf releasenotes/build From 96b84a90141add9a60153515daea95cfec2f8b53 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Wed, 24 Apr 2019 20:39:19 +0000 Subject: [PATCH 12/12] Stop using pbr sphinx integration Switch to sphinx-build directly and add sphinxcontrib-apidoc for generated api docs. Change-Id: I089a7695986892dcb47c5b5e79a6a227891a2347 --- doc/requirements.txt | 1 + doc/source/conf.py | 10 +++++++++- doc/source/reference/index.rst | 3 +-- doc/source/user/history.rst | 5 ----- doc/source/user/index.rst | 1 - lower-constraints.txt | 1 + setup.cfg | 13 ------------- tox.ini | 6 +++++- 8 files changed, 17 insertions(+), 23 deletions(-) delete mode 100644 doc/source/user/history.rst diff --git a/doc/requirements.txt b/doc/requirements.txt index 711fb972..d0c1f363 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,5 @@ sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD +sphinxcontrib-apidoc>=0.2.0 # BSD openstackdocstheme>=1.18.1 # Apache-2.0 reno>=2.5.0 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index cc7f4b35..13f63a0b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -8,7 +8,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxcontrib.apidoc'] # make openstackdocstheme optional to not increase the needed dependencies try: import openstackdocstheme @@ -72,3 +72,11 @@ '%s Documentation' % project, 'OpenStack Foundation', 'manual'), ] + +# -- sphinxcontrib.apidoc configuration -------------------------------------- + +apidoc_module_dir = '../../pbr' +apidoc_output_dir = 'reference/api' +apidoc_excluded_paths = [ + 'tests', +] diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 68a9c324..3162d671 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -3,6 +3,5 @@ =================== .. toctree:: - :glob: - api/* + api/modules diff --git a/doc/source/user/history.rst b/doc/source/user/history.rst deleted file mode 100644 index 55439e77..00000000 --- a/doc/source/user/history.rst +++ /dev/null @@ -1,5 +0,0 @@ -================= - Release History -================= - -.. include:: ../../../ChangeLog diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 7854dc54..0629e2e5 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -9,4 +9,3 @@ packagers semver compatibility - history diff --git a/lower-constraints.txt b/lower-constraints.txt index 173a299c..fc576a56 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -25,6 +25,7 @@ requests==2.14.2 six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.6.5 +sphinxcontrib-apidoc==0.2.0 sphinxcontrib-websupport==1.0.1 stestr==2.1.0 testrepository==0.0.18 diff --git a/setup.cfg b/setup.cfg index b26be54c..63b782fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,13 +32,6 @@ classifier = packages = pbr -[pbr] -autodoc_tree_index_modules = True -autodoc_tree_excludes = - setup.py - pbr/tests/ -api_doc_dir = reference/api - [entry_points] distutils.setup_keywords = pbr = pbr.core:pbr @@ -47,11 +40,5 @@ egg_info.writers = console_scripts = pbr = pbr.cmd.main:main -[build_sphinx] -all-files = 1 -build-dir = doc/build -source-dir = doc/source -warning-is-error = 1 - [bdist_wheel] universal = 1 diff --git a/tox.ini b/tox.ini index e761968c..5cc32a9a 100644 --- a/tox.ini +++ b/tox.ini @@ -23,13 +23,17 @@ commands = flake8 {posargs} [testenv:docs] basepython = python3 +whitelist_externals = rm deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt -commands = python setup.py build_sphinx +commands = + rm -rf doc/build doc/source/reference/api + sphinx-build -W -b html doc/source doc/build/html {posargs} [testenv:releasenotes] basepython = python3 +whitelist_externals = rm deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt