Merge branch 'master' into custom

.zuul.yaml

@@ -35,7 +35,6 @@
       - openstack/manila-ui
       - openstack/neutron
       - openstack/neutron-fwaas
-      - openstack/neutron-lbaas
       - openstack/neutron-vpnaas
       - openstack/nova
       - openstack/octavia
@@ -112,6 +111,7 @@
       - openstack-python-jobs
       - openstack-python35-jobs
       - openstack-python36-jobs
+      - openstack-python37-jobs
       - periodic-stable-jobs
       - publish-openstack-docs-pti
     check:

doc/requirements.txt

@@ -1,3 +1,5 @@
-sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
+sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD
+sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD
+sphinxcontrib-apidoc>=0.2.0  # BSD
 openstackdocstheme>=1.18.1 # Apache-2.0
 reno>=2.5.0 # Apache-2.0

doc/source/conf.py

@@ -8,7 +8,7 @@
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxcontrib.apidoc']
 # make openstackdocstheme optional to not increase the needed dependencies
 try:
     import openstackdocstheme
@@ -72,3 +72,11 @@
      '%s Documentation' % project,
      'OpenStack Foundation', 'manual'),
 ]
+
+# -- sphinxcontrib.apidoc configuration --------------------------------------
+
+apidoc_module_dir = '../../pbr'
+apidoc_output_dir = 'reference/api'
+apidoc_excluded_paths = [
+    'tests',
+]

doc/source/reference/index.rst

@@ -3,6 +3,5 @@
 ===================
 
 .. toctree::
-   :glob:
 
-   api/*
+   api/modules

doc/source/user/index.rst

@@ -9,4 +9,3 @@
    packagers
    semver
    compatibility
-   history

lower-constraints.txt

@@ -25,6 +25,7 @@ requests==2.14.2
 six==1.10.0
 snowballstemmer==1.2.1
 Sphinx==1.6.5
+sphinxcontrib-apidoc==0.2.0
 sphinxcontrib-websupport==1.0.1
 stestr==2.1.0
 testrepository==0.0.18

pbr/cmd/main.py

@@ -86,7 +86,9 @@ def main():
         version=str(pbr.version.VersionInfo('pbr')))
 
     subparsers = parser.add_subparsers(
-        title='commands', description='valid commands', help='additional help')
+        title='commands', description='valid commands', help='additional help',
+        dest='cmd')
+    subparsers.required = True
 
     cmd_sha = subparsers.add_parser('sha', help='print sha of package')
     cmd_sha.set_defaults(func=get_sha)

pbr/git.py

@@ -224,6 +224,11 @@ def _iter_log_inner(git_dir):
     presentation logic to the output - making it suitable for different
     uses.
 
+    .. caution:: this function risk to return a tag that doesn't exist really
+                 inside the git objects list due to replacement made
+                 to tag name to also list pre-release suffix.
+                 Compliant with the SemVer specification (e.g 1.2.3-rc1)
+
     :return: An iterator of (hash, tags_set, 1st_line) tuples.
     """
     log.info('[pbr] Generating ChangeLog')
@@ -249,7 +254,7 @@ def _iter_log_inner(git_dir):
             for tag_string in refname.split("refs/tags/")[1:]:
                 # git tag does not allow : or " " in tag names, so we split
                 # on ", " which is the separator between elements
-                candidate = tag_string.split(", ")[0]
+                candidate = tag_string.split(", ")[0].replace("-", ".")
                 if _is_valid_version(candidate):
                     tags.add(candidate)
 

pbr/hooks/files.py

@@ -14,6 +14,7 @@
 # under the License.
 
 import os
+import shlex
 import sys
 
 from pbr import find_package
@@ -35,6 +36,21 @@ def get_man_section(section):
     return os.path.join(get_manpath(), 'man%s' % section)
 
 
+def unquote_path(path):
+    # unquote the full path, e.g: "'a/full/path'" becomes "a/full/path", also
+    # strip the quotes off individual path components because os.walk cannot
+    # handle paths like: "'i like spaces'/'another dir'", so we will pass it
+    # "i like spaces/another dir" instead.
+
+    if os.name == 'nt':
+        # shlex cannot handle paths that contain backslashes, treating those
+        # as escape characters.
+        path = path.replace("\\", "/")
+        return "".join(shlex.split(path)).replace("/", "\\")
+
+    return "".join(shlex.split(path))
+
+
 class FilesConfig(base.BaseConfig):
 
     section = 'files'
@@ -57,25 +73,28 @@ def expand_globs(self):
                 target = target.strip()
                 if not target.endswith(os.path.sep):
                     target += os.path.sep
-                for (dirpath, dirnames, fnames) in os.walk(source_prefix):
+                unquoted_prefix = unquote_path(source_prefix)
+                unquoted_target = unquote_path(target)
+                for (dirpath, dirnames, fnames) in os.walk(unquoted_prefix):
                     # As source_prefix is always matched, using replace with a
                     # a limit of one is always going to replace the path prefix
                     # and not accidentally replace some text in the middle of
                     # the path
-                    new_prefix = dirpath.replace(source_prefix, target, 1)
-                    finished.append("%s = " % new_prefix)
+                    new_prefix = dirpath.replace(unquoted_prefix,
+                                                 unquoted_target, 1)
+                    finished.append("'%s' = " % new_prefix)
                     finished.extend(
-                        [" %s" % os.path.join(dirpath, f) for f in fnames])
+                        [" '%s'" % os.path.join(dirpath, f) for f in fnames])
             else:
                 finished.append(line)
 
         self.data_files = "\n".join(finished)
 
     def add_man_path(self, man_path):
-        self.data_files = "%s\n%s =" % (self.data_files, man_path)
+        self.data_files = "%s\n'%s' =" % (self.data_files, man_path)
 
     def add_man_page(self, man_page):
-        self.data_files = "%s\n  %s" % (self.data_files, man_page)
+        self.data_files = "%s\n  '%s'" % (self.data_files, man_page)
 
     def get_man_sections(self):
         man_sections = dict()

pbr/tests/test_files.py

@@ -37,12 +37,17 @@ def setUp(self):
         pkg_etc = os.path.join(pkg_fixture.base, 'etc')
         pkg_ansible = os.path.join(pkg_fixture.base, 'ansible',
                                    'kolla-ansible', 'test')
+        dir_spcs = os.path.join(pkg_fixture.base, 'dir with space')
+        dir_subdir_spc = os.path.join(pkg_fixture.base, 'multi space',
+                                      'more spaces')
         pkg_sub = os.path.join(pkg_etc, 'sub')
         subpackage = os.path.join(
             pkg_fixture.base, 'fake_package', 'subpackage')
         os.makedirs(pkg_sub)
         os.makedirs(subpackage)
         os.makedirs(pkg_ansible)
+        os.makedirs(dir_spcs)
+        os.makedirs(dir_subdir_spc)
         with open(os.path.join(pkg_etc, "foo"), 'w') as foo_file:
             foo_file.write("Foo Data")
         with open(os.path.join(pkg_sub, "bar"), 'w') as foo_file:
@@ -51,6 +56,10 @@ def setUp(self):
             baz_file.write("Baz Data")
         with open(os.path.join(subpackage, "__init__.py"), 'w') as foo_file:
             foo_file.write("# empty")
+        with open(os.path.join(dir_spcs, "file with spc"), 'w') as spc_file:
+            spc_file.write("# empty")
+        with open(os.path.join(dir_subdir_spc, "file with spc"), 'w') as file_:
+            file_.write("# empty")
 
         self.useFixture(base.DiveDir(pkg_fixture.base))
 
@@ -79,9 +88,49 @@ def test_data_files_globbing(self):
         )
         files.FilesConfig(config, 'fake_package').run()
         self.assertIn(
-            '\netc/pbr/ = \n etc/foo\netc/pbr/sub = \n etc/sub/bar',
+            "\n'etc/pbr/' = \n 'etc/foo'\n'etc/pbr/sub' = \n 'etc/sub/bar'",
             config['files']['data_files'])
 
+    def test_data_files_with_spaces(self):
+        config = dict(
+            files=dict(
+                data_files="\n  'i like spaces' = 'dir with space'/*"
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(
+            "\n'i like spaces/' = \n 'dir with space/file with spc'",
+            config['files']['data_files'])
+
+    def test_data_files_with_spaces_subdirectories(self):
+        # test that we can handle whitespace in subdirectories
+        data_files = "\n 'one space/two space' = 'multi space/more spaces'/*"
+        expected = (
+            "\n'one space/two space/' = "
+            "\n 'multi space/more spaces/file with spc'")
+        config = dict(
+            files=dict(
+                data_files=data_files
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(expected, config['files']['data_files'])
+
+    def test_data_files_with_spaces_quoted_components(self):
+        # test that we can quote individual path components
+        data_files = (
+            "\n'one space'/'two space' = 'multi space'/'more spaces'/*"
+        )
+        expected = ("\n'one space/two space/' = "
+                    "\n 'multi space/more spaces/file with spc'")
+        config = dict(
+            files=dict(
+                data_files=data_files
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(expected, config['files']['data_files'])
+
     def test_data_files_globbing_source_prefix_in_directory_name(self):
         # We want to test that the string, "docs", is not replaced in a
         # subdirectory name, "sub-docs"
@@ -92,8 +141,8 @@ def test_data_files_globbing_source_prefix_in_directory_name(self):
         )
         files.FilesConfig(config, 'fake_package').run()
         self.assertIn(
-            '\nshare/ansible/ = '
-            '\nshare/ansible/kolla-ansible = '
-            '\nshare/ansible/kolla-ansible/test = '
-            '\n ansible/kolla-ansible/test/baz',
+            "\n'share/ansible/' = "
+            "\n'share/ansible/kolla-ansible' = "
+            "\n'share/ansible/kolla-ansible/test' = "
+            "\n 'ansible/kolla-ansible/test/baz'",
             config['files']['data_files'])

pbr/tests/test_packaging.py

@@ -670,6 +670,12 @@ def test_tagged_version_has_tag_version(self):
         version = packaging._get_version_from_git('1.2.3')
         self.assertEqual('1.2.3', version)
 
+    def test_tagged_version_with_semver_compliant_prerelease(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3-rc2')
+        version = packaging._get_version_from_git()
+        self.assertEqual('1.2.3.0rc2', version)
+
     def test_non_canonical_tagged_version_bump(self):
         self.repo.commit()
         self.repo.tag('1.4')
@@ -726,6 +732,13 @@ def test_untagged_version_after_rc_has_dev_version_preversion(self):
         version = packaging._get_version_from_git('1.2.3')
         self.assertThat(version, matchers.StartsWith('1.2.3.0a2.dev1'))
 
+    def test_untagged_version_after_semver_compliant_prerelease_tag(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3-rc2')
+        self.repo.commit()
+        version = packaging._get_version_from_git()
+        self.assertEqual('1.2.3.0rc3.dev1', version)
+
     def test_preversion_too_low_simple(self):
         # That is, the target version is either already released or not high
         # enough for the semver requirements given api breaks etc.

pbr/tests/test_util.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,6 +14,7 @@
 # under the License.
 
 import io
+import tempfile
 import textwrap
 
 import six
@@ -172,3 +174,46 @@ def test_provides_extras(self):
         config = config_from_ini(ini)
         kwargs = util.setup_cfg_to_setup_kwargs(config)
         self.assertEqual(['foo', 'bar'], kwargs['provides_extras'])
+
+
+class TestDataFilesParsing(base.BaseTestCase):
+
+    scenarios = [
+        ('data_files', {
+            'config_text': """
+            [files]
+            data_files =
+                'i like spaces/' =
+                    'dir with space/file with spc 2'
+                    'dir with space/file with spc 1'
+            """,
+            'data_files': [
+                ('i like spaces/', ['dir with space/file with spc 2',
+                                    'dir with space/file with spc 1'])
+            ]
+        })]
+
+    def test_handling_of_whitespace_in_data_files(self):
+        config = config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.data_files,
+                         list(kwargs['data_files']))
+
+
+class TestUTF8DescriptionFile(base.BaseTestCase):
+    def test_utf8_description_file(self):
+        _, path = tempfile.mkstemp()
+        ini_template = """
+        [metadata]
+        description_file = %s
+        """
+        # Two \n's because pbr strips the file content and adds \n\n
+        # This way we can use it directly as the assert comparison
+        unicode_description = u'UTF8 description: é"…-ʃŋ\'\n\n'
+        ini = ini_template % path
+        with io.open(path, 'w', encoding='utf8') as f:
+            f.write(unicode_description)
+        config = config_from_ini(ini)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+        self.assertEqual(unicode_description, kwargs['long_description'])

pbr/tests/test_wsgi.py

@@ -77,8 +77,8 @@ def test_wsgi_script_run(self):
 
     def _test_wsgi(self, cmd_name, output, extra_args=None):
         cmd = os.path.join(self.temp_dir, 'bin', cmd_name)
-        print("Running %s -p 0" % cmd)
-        popen_cmd = [cmd, '-p', '0']
+        print("Running %s -p 0 -b 127.0.0.1" % cmd)
+        popen_cmd = [cmd, '-p', '0', '-b', '127.0.0.1']
         if extra_args:
             popen_cmd.extend(extra_args)