Pull latest service module from Oslo

Get latest service module from Oslo to prepare for multi-process API service implementation.
Below are the commits included in this pull.

Changes being pulled into in service module are:
* e7bc8c9 2013-11-20 | Merge "os._exit in _start_child may cause unexpected exception"
* 96a2d4e 2013-11-07 | os._exit in _start_child may cause unexpected exception
* 1771a77 2013-11-05 | Adjust import order according to PEP8 imports rule
* 3110c0f 2013-10-17 | Use multiprocessing.Event to ensure services have started
* b5fba9e 2013-09-18 | Move comment in service.py to correct location
* 11cc74f 2013-08-26 | Fixes issue with SUGHUP in services on Windows
* 825ace5 2013-06-17 | Add service restart function in oslo-incubator
* c935d1c 2013-07-16 | Merge "Allow launchers to be stopped multiple times"
* dc8aa79 2013-07-08 | Allow launchers to be stopped multiple times
* 1a2df89 2013-06-25 | Enable H302 hacking check
* 52e857a 2013-06-19 | Ignore any exceptions from rpc.cleanup().
* 5518ad3 2013-05-16 | Add graceful service shutdown support to Launcher

And these dependent modules
 - cinder/openstack/common/eventlet_backdoor.py
    * 1dcc747 2013-07-15 | Fix stylistic problems with help text
    * 1a2df89 2013-06-25 | Enable H302 hacking check
    * c7c55b2 2013-06-20 | Improve usability when backdoor_port is nonzero
 - cinder/openstack/common/gettextutils.py
    * 3970d46 2013-11-02 | Fix typos in oslo
    * 88db9c8 2013-10-03 | When translating if no locale is given use default locale
 - cinder/openstack/common/jsonutils.py
    * 3d7504b 2013-09-23 | Ensure that Message objects will be sent via RPC in unicode format
    * 1807d32 2013-08-22 | jsonutils: make types py3 compatible
    * bdef862 2013-08-22 | jsonutils: do not require xmlrpclib
    * ded9bd6 2013-08-04 | Make dependency on netaddr optional
    * 7b7566b 2013-06-25 | Add netaddr.IPAddress support to to_primitive()
 - cinder/openstack/common/local.py
    * cb2a2b6 2013-06-28 | Modify local.py to not be dependent on Eventlet
    * 547ab34 2013-03-11 | Fix Copyright Headers - Rename LLC to Foundation
 - cinder/openstack/common/log.py
    * a82e889 2013-11-14 | Merge "Do not name variables as builtins"
    * 2251cb5 2013-11-13 | Do not name variables as builtins
    * 25c5854 2013-11-13 | Adds admin_password as key to be sanitized when logging
    * cbfded9 2013-11-11 | Default iso8601 logging to WARN
    * 76b0cd1 2013-11-04 | Add mask password impl from other projects
 - cinder/openstack/common/loopingcall.py
    * 1a2df89 2013-06-25 | Enable H302 hacking check
 - cinder/openstack/common/threadgroup.py
    * 9d3c34b 2013-10-25 | Add a link method to Thread
    * 1a2df89 2013-06-25 | Enable H302 hacking check
 - cinder/openstack/common/timeutils.py
    * f3b5f17 2013-11-12 | Add helper method total_seconds in timeutils.py
    * 53ebd30 2013-10-18 | python3: use six.text_types for unicode()
    * 3bc6f79 2013-09-19 | Fix timeutils.set_override_time not defaulting to current wall time
    * af76064 2013-08-29 | Optimize timeutils.utcnow_ts()
    * df3f2ba 2013-07-26 | BaseException.message is deprecated since Python 2.6
    * d28fa69 2013-06-27 | python3: Add python3 compatibility.

Partial bp: multi-process-api-service

Change-Id: Ifd25eae9eb2d6ae53bcf1665c3d5b7db4144433c

cinder/openstack/common/eventlet_backdoor.py

@@ -18,8 +18,11 @@
 
 from __future__ import print_function
 
+import errno
 import gc
+import os
 import pprint
+import socket
 import sys
 import traceback
 
@@ -28,14 +31,34 @@
 import greenlet
 from oslo.config import cfg
 
+from cinder.openstack.common.gettextutils import _  # noqa
+from cinder.openstack.common import log as logging
+
+help_for_backdoor_port = (
+    "Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
+    "in listening on a random tcp port number; <port> results in listening "
+    "on the specified port number (and not enabling backdoor if that port "
+    "is in use); and <start>:<end> results in listening on the smallest "
+    "unused port number within the specified range of port numbers.  The "
+    "chosen port is displayed in the service's log file.")
 eventlet_backdoor_opts = [
-    cfg.IntOpt('backdoor_port',
+    cfg.StrOpt('backdoor_port',
                default=None,
-               help='port for eventlet backdoor to listen')
+               help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(eventlet_backdoor_opts)
+LOG = logging.getLogger(__name__)
+
+
+class EventletBackdoorConfigValueError(Exception):
+    def __init__(self, port_range, help_msg, ex):
+        msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
+               '%(help)s' %
+               {'range': port_range, 'ex': ex, 'help': help_msg})
+        super(EventletBackdoorConfigValueError, self).__init__(msg)
+        self.port_range = port_range
 
 
 def _dont_use_this():
@@ -60,6 +83,33 @@ def _print_nativethreads():
         print()
 
 
+def _parse_port_range(port_range):
+    if ':' not in port_range:
+        start, end = port_range, port_range
+    else:
+        start, end = port_range.split(':', 1)
+    try:
+        start, end = int(start), int(end)
+        if end < start:
+            raise ValueError
+        return start, end
+    except ValueError as ex:
+        raise EventletBackdoorConfigValueError(port_range, ex,
+                                               help_for_backdoor_port)
+
+
+def _listen(host, start_port, end_port, listen_func):
+    try_port = start_port
+    while True:
+        try:
+            return listen_func((host, try_port))
+        except socket.error as exc:
+            if (exc.errno != errno.EADDRINUSE or
+               try_port >= end_port):
+                raise
+            try_port += 1
+
+
 def initialize_if_enabled():
     backdoor_locals = {
         'exit': _dont_use_this,      # So we don't exit the entire process
@@ -72,6 +122,8 @@ def initialize_if_enabled():
     if CONF.backdoor_port is None:
         return None
 
+    start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
+
     # NOTE(johannes): The standard sys.displayhook will print the value of
     # the last expression and set it to __builtin__._, which overwrites
     # the __builtin__._ that gettext sets. Let's switch to using pprint
@@ -82,8 +134,13 @@ def displayhook(val):
             pprint.pprint(val)
     sys.displayhook = displayhook
 
-    sock = eventlet.listen(('localhost', CONF.backdoor_port))
+    sock = _listen('localhost', start_port, end_port, eventlet.listen)
+
+    # In the case of backdoor port being zero, a port number is assigned by
+    # listen().  In any case, pull the port number out here.
     port = sock.getsockname()[1]
+    LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
+             {'port': port, 'pid': os.getpid()})
     eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
                      locals=backdoor_locals)
     return port

cinder/openstack/common/gettextutils.py

@@ -317,7 +317,7 @@ def get_available_languages(domain):
     # NOTE(luisg): Babel <1.0 used a function called list(), which was
     # renamed to locale_identifiers() in >=1.0, the requirements master list
     # requires >=0.9.6, uncapped, so defensively work with both. We can remove
-    # this check when the master list updates to >=1.0, and all projects udpate
+    # this check when the master list updates to >=1.0, and update all projects
     list_identifiers = (getattr(localedata, 'list', None) or
                         getattr(localedata, 'locale_identifiers'))
     locale_identifiers = list_identifiers()
@@ -329,13 +329,21 @@ def get_available_languages(domain):
 
 
 def get_localized_message(message, user_locale):
-    """Gets a localized version of the given message in the given locale."""
+    """Gets a localized version of the given message in the given locale.
+
+    If the message is not a Message object the message is returned as-is.
+    If the locale is None the message is translated to the default locale.
+
+    :returns: the translated message in unicode, or the original message if
+              it could not be translated
+    """
+    translated = message
     if isinstance(message, Message):
-        if user_locale:
-            message.locale = user_locale
-        return six.text_type(message)
-    else:
-        return message
+        original_locale = message.locale
+        message.locale = user_locale
+        translated = six.text_type(message)
+        message.locale = original_locale
+    return translated
 
 
 class LocaleHandler(logging.Handler):

cinder/openstack/common/jsonutils.py

@@ -38,21 +38,28 @@
 import inspect
 import itertools
 import json
-import types
-import xmlrpclib
+try:
+    import xmlrpclib
+except ImportError:
+    # NOTE(jd): xmlrpclib is not shipped with Python 3
+    xmlrpclib = None
 
 import six
 
+from cinder.openstack.common import gettextutils
+from cinder.openstack.common import importutils
 from cinder.openstack.common import timeutils
 
+netaddr = importutils.try_import("netaddr")
 
 _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.isfunction, inspect.isgeneratorfunction,
                      inspect.isgenerator, inspect.istraceback, inspect.isframe,
                      inspect.iscode, inspect.isbuiltin, inspect.isroutine,
                      inspect.isabstract]
 
-_simple_types = (types.NoneType, int, basestring, bool, float, long)
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
 
 
 def to_primitive(value, convert_instances=False, convert_datetime=True,
@@ -124,11 +131,13 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
         # It's not clear why xmlrpclib created their own DateTime type, but
         # for our purposes, make it a datetime type which is explicitly
         # handled
-        if isinstance(value, xmlrpclib.DateTime):
+        if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
             value = datetime.datetime(*tuple(value.timetuple())[:6])
 
         if convert_datetime and isinstance(value, datetime.datetime):
             return timeutils.strtime(value)
+        elif isinstance(value, gettextutils.Message):
+            return value.data
         elif hasattr(value, 'iteritems'):
             return recursive(dict(value.iteritems()), level=level + 1)
         elif hasattr(value, '__iter__'):
@@ -137,6 +146,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
             # Likely an instance of something. Watch for cycles.
             # Ignore class member vars.
             return recursive(value.__dict__, level=level + 1)
+        elif netaddr and isinstance(value, netaddr.IPAddress):
+            return six.text_type(value)
         else:
             if any(test(value) for test in _nasty_type_tests):
                 return six.text_type(value)

cinder/openstack/common/local.py

@@ -15,16 +15,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Greenthread local storage of variables using weak references"""
+"""Local storage of variables using weak references"""
 
+import threading
 import weakref
 
-from eventlet import corolocal
 
-
-class WeakLocal(corolocal.local):
+class WeakLocal(threading.local):
     def __getattribute__(self, attr):
-        rval = corolocal.local.__getattribute__(self, attr)
+        rval = super(WeakLocal, self).__getattribute__(attr)
         if rval:
             # NOTE(mikal): this bit is confusing. What is stored is a weak
             # reference, not the value itself. We therefore need to lookup
@@ -34,7 +33,7 @@ def __getattribute__(self, attr):
 
     def __setattr__(self, attr, value):
         value = weakref.ref(value)
-        return corolocal.local.__setattr__(self, attr, value)
+        return super(WeakLocal, self).__setattr__(attr, value)
 
 
 # NOTE(mikal): the name "store" should be deprecated in the future
@@ -45,4 +44,4 @@ def __setattr__(self, attr, value):
 # "strong" store will hold a reference to the object so that it never falls out
 # of scope.
 weak_store = WeakLocal()
-strong_store = corolocal.local
+strong_store = threading.local()

cinder/openstack/common/log.py

@@ -35,6 +35,7 @@
 import logging.config
 import logging.handlers
 import os
+import re
 import sys
 import traceback
 
@@ -50,6 +51,24 @@
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS = []
+_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                    r'(<%(key)s>).*?(</%(key)s>)',
+                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS.append(reg_ex)
+
+
 common_cli_opts = [
     cfg.BoolOpt('debug',
                 short='d',
@@ -136,6 +155,7 @@
                     'qpid=WARN',
                     'sqlalchemy=WARN',
                     'suds=INFO',
+                    'iso8601=WARN',
                 ],
                 help='list of logger=LEVEL pairs'),
     cfg.BoolOpt('publish_errors',
@@ -214,6 +234,39 @@ def _get_log_file_path(binary=None):
     return None
 
 
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords, defaults to "***".
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    secret = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS:
+        message = re.sub(pattern, secret, message)
+    return message
+
+
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
@@ -336,10 +389,10 @@ def format(self, record):
 
 
 def _create_logging_excepthook(product_name):
-    def logging_excepthook(type, value, tb):
+    def logging_excepthook(exc_type, value, tb):
         extra = {}
         if CONF.verbose:
-            extra['exc_info'] = (type, value, tb)
+            extra['exc_info'] = (exc_type, value, tb)
         getLogger(product_name).critical(str(value), **extra)
     return logging_excepthook
 

cinder/openstack/common/loopingcall.py

@@ -22,7 +22,7 @@
 from eventlet import event
 from eventlet import greenthread
 
-from cinder.openstack.common.gettextutils import _
+from cinder.openstack.common.gettextutils import _  # noqa
 from cinder.openstack.common import log as logging
 from cinder.openstack.common import timeutils