Sync with latest module from oslo

- This add a few bugfixes for py3 support.

Change-Id: Ieb73d5f799423fa8abf71634c86601ba6d32df2f

keystoneclient/openstack/common/gettextutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 # Copyright 2013 IBM Corp.
 # All Rights Reserved.
@@ -26,10 +24,13 @@ Usual usage in an openstack.common module:
 
 import copy
 import gettext
-import logging.handlers
+import logging
 import os
 import re
-import UserString
+try:
+    import UserString as _userString
+except ImportError:
+    import collections as _userString
 
 from babel import localedata
 import six
@@ -37,7 +38,7 @@ import six
 _localedir = os.environ.get('keystoneclient'.upper() + '_LOCALEDIR')
 _t = gettext.translation('keystoneclient', localedir=_localedir, fallback=True)
 
-_AVAILABLE_LANGUAGES = []
+_AVAILABLE_LANGUAGES = {}
 USE_LAZY = False
 
 
@@ -57,6 +58,8 @@ def _(msg):
     if USE_LAZY:
         return Message(msg, 'keystoneclient')
     else:
+        if six.PY3:
+            return _t.gettext(msg)
         return _t.ugettext(msg)
 
 
@@ -102,24 +105,28 @@ def install(domain, lazy=False):
             """
             return Message(msg, domain)
 
-        import __builtin__
+        from six import moves
-        __builtin__.__dict__['_'] = _lazy_gettext
+        moves.builtins.__dict__['_'] = _lazy_gettext
     else:
         localedir = '%s_LOCALEDIR' % domain.upper()
-        gettext.install(domain,
+        if six.PY3:
-                        localedir=os.environ.get(localedir),
+            gettext.install(domain,
-                        unicode=True)
+                            localedir=os.environ.get(localedir))
+        else:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir),
+                            unicode=True)
 
 
-class Message(UserString.UserString, object):
+class Message(_userString.UserString, object):
     """Class used to encapsulate translatable messages."""
     def __init__(self, msg, domain):
         # _msg is the gettext msgid and should never change
         self._msg = msg
         self._left_extra_msg = ''
         self._right_extra_msg = ''
+        self._locale = None
         self.params = None
-        self.locale = None
         self.domain = domain
 
     @property
@@ -139,8 +146,13 @@ class Message(UserString.UserString, object):
                                        localedir=localedir,
                                        fallback=True)
 
+        if six.PY3:
+            ugettext = lang.gettext
+        else:
+            ugettext = lang.ugettext
+
         full_msg = (self._left_extra_msg +
-                    lang.ugettext(self._msg) +
+                    ugettext(self._msg) +
                     self._right_extra_msg)
 
         if self.params is not None:
@@ -148,6 +160,33 @@ class Message(UserString.UserString, object):
 
         return six.text_type(full_msg)
 
+    @property
+    def locale(self):
+        return self._locale
+
+    @locale.setter
+    def locale(self, value):
+        self._locale = value
+        if not self.params:
+            return
+
+        # This Message object may have been constructed with one or more
+        # Message objects as substitution parameters, given as a single
+        # Message, or a tuple or Map containing some, so when setting the
+        # locale for this Message we need to set it for those Messages too.
+        if isinstance(self.params, Message):
+            self.params.locale = value
+            return
+        if isinstance(self.params, tuple):
+            for param in self.params:
+                if isinstance(param, Message):
+                    param.locale = value
+            return
+        if isinstance(self.params, dict):
+            for param in self.params.values():
+                if isinstance(param, Message):
+                    param.locale = value
+
     def _save_dictionary_parameter(self, dict_param):
         full_msg = self.data
         # look for %(blah) fields in string;
@@ -166,7 +205,7 @@ class Message(UserString.UserString, object):
                     params[key] = copy.deepcopy(dict_param[key])
                 except TypeError:
                     # cast uncopyable thing to unicode string
-                    params[key] = unicode(dict_param[key])
+                    params[key] = six.text_type(dict_param[key])
 
         return params
 
@@ -185,7 +224,7 @@ class Message(UserString.UserString, object):
             try:
                 self.params = copy.deepcopy(other)
             except TypeError:
-                self.params = unicode(other)
+                self.params = six.text_type(other)
 
         return self
 
@@ -194,11 +233,13 @@ class Message(UserString.UserString, object):
         return self.data
 
     def __str__(self):
+        if six.PY3:
+            return self.__unicode__()
         return self.data.encode('utf-8')
 
     def __getstate__(self):
         to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
-                   'domain', 'params', 'locale']
+                   'domain', 'params', '_locale']
         new_dict = self.__dict__.fromkeys(to_copy)
         for attr in to_copy:
             new_dict[attr] = copy.deepcopy(self.__dict__[attr])
@@ -252,7 +293,7 @@ class Message(UserString.UserString, object):
         if name in ops:
             return getattr(self.data, name)
         else:
-            return UserString.UserString.__getattribute__(self, name)
+            return _userString.UserString.__getattribute__(self, name)
 
 
 def get_available_languages(domain):
@@ -260,8 +301,8 @@ def get_available_languages(domain):
 
     :param domain: the domain to get languages for
     """
-    if _AVAILABLE_LANGUAGES:
+    if domain in _AVAILABLE_LANGUAGES:
-        return _AVAILABLE_LANGUAGES
+        return copy.copy(_AVAILABLE_LANGUAGES[domain])
 
     localedir = '%s_LOCALEDIR' % domain.upper()
     find = lambda x: gettext.find(domain,
@@ -270,28 +311,37 @@ def get_available_languages(domain):
 
     # NOTE(mrodden): en_US should always be available (and first in case
     # order matters) since our in-line message strings are en_US
-    _AVAILABLE_LANGUAGES.append('en_US')
+    language_list = ['en_US']
     # NOTE(luisg): Babel <1.0 used a function called list(), which was
     # renamed to locale_identifiers() in >=1.0, the requirements master list
     # requires >=0.9.6, uncapped, so defensively work with both. We can remove
-    # this check when the master list updates to >=1.0, and all projects udpate
+    # this check when the master list updates to >=1.0, and update all projects
     list_identifiers = (getattr(localedata, 'list', None) or
                         getattr(localedata, 'locale_identifiers'))
     locale_identifiers = list_identifiers()
     for i in locale_identifiers:
         if find(i) is not None:
-            _AVAILABLE_LANGUAGES.append(i)
+            language_list.append(i)
-    return _AVAILABLE_LANGUAGES
+    _AVAILABLE_LANGUAGES[domain] = language_list
+    return copy.copy(language_list)
 
 
 def get_localized_message(message, user_locale):
-    """Gets a localized version of the given message in the given locale."""
+    """Gets a localized version of the given message in the given locale.
-    if (isinstance(message, Message)):
+
-        if user_locale:
+    If the message is not a Message object the message is returned as-is.
-            message.locale = user_locale
+    If the locale is None the message is translated to the default locale.
-        return unicode(message)
+
-    else:
+    :returns: the translated message in unicode, or the original message if
-        return message
+              it could not be translated
+    """
+    translated = message
+    if isinstance(message, Message):
+        original_locale = message.locale
+        message.locale = user_locale
+        translated = six.text_type(message)
+        message.locale = original_locale
+    return translated
 
 
 class LocaleHandler(logging.Handler):

keystoneclient/openstack/common/importutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

keystoneclient/openstack/common/jsonutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara
@@ -41,8 +39,12 @@ import json
 try:
     import xmlrpclib
 except ImportError:
-    # NOTE(jd): xmlrpclib is not shipped with Python 3
+    # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
-    xmlrpclib = None
+    #                 however the function and object call signatures
+    #                 remained the same. This whole try/except block should
+    #                 be removed and replaced with a call to six.moves once
+    #                 six 1.4.2 is released. See http://bit.ly/1bqrVzu
+    import xmlrpc.client as xmlrpclib
 
 import six
 
@@ -124,14 +126,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
                                       level=level,
                                       max_depth=max_depth)
         if isinstance(value, dict):
-            return dict((k, recursive(v)) for k, v in value.iteritems())
+            return dict((k, recursive(v)) for k, v in six.iteritems(value))
         elif isinstance(value, (list, tuple)):
             return [recursive(lv) for lv in value]
 
         # It's not clear why xmlrpclib created their own DateTime type, but
         # for our purposes, make it a datetime type which is explicitly
         # handled
-        if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
+        if isinstance(value, xmlrpclib.DateTime):
             value = datetime.datetime(*tuple(value.timetuple())[:6])
 
         if convert_datetime and isinstance(value, datetime.datetime):

keystoneclient/openstack/common/py3kcompat/__init__.py

@@ -1,4 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # Copyright 2013 Canonical Ltd.
 # All Rights Reserved.

keystoneclient/openstack/common/py3kcompat/urlutils.py

@@ -1,4 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
 #
 # Copyright 2013 Canonical Ltd.
 # All Rights Reserved.
@@ -24,22 +23,34 @@ import six
 
 if six.PY3:
     # python3
+    import urllib.error
     import urllib.parse
+    import urllib.request
 
     urlencode = urllib.parse.urlencode
     urljoin = urllib.parse.urljoin
     quote = urllib.parse.quote
     parse_qsl = urllib.parse.parse_qsl
+    unquote = urllib.parse.unquote
+    unquote_plus = urllib.parse.unquote_plus
     urlparse = urllib.parse.urlparse
     urlsplit = urllib.parse.urlsplit
     urlunsplit = urllib.parse.urlunsplit
+    SplitResult = urllib.parse.SplitResult
+
+    urlopen = urllib.request.urlopen
+    URLError = urllib.error.URLError
+    pathname2url = urllib.request.pathname2url
 else:
     # python2
     import urllib
+    import urllib2
     import urlparse
 
     urlencode = urllib.urlencode
     quote = urllib.quote
+    unquote = urllib.unquote
+    unquote_plus = urllib.unquote_plus
 
     parse = urlparse
     parse_qsl = parse.parse_qsl
@@ -47,3 +58,8 @@ else:
     urlparse = parse.urlparse
     urlsplit = parse.urlsplit
     urlunsplit = parse.urlunsplit
+    SplitResult = parse.SplitResult
+
+    urlopen = urllib2.urlopen
+    URLError = urllib2.URLError
+    pathname2url = urllib.pathname2url

keystoneclient/openstack/common/strutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -101,7 +99,7 @@ def safe_decode(text, incoming=None, errors='strict'):
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a unicode `incoming` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of str
+    :raises TypeError: If text is not an instance of str
     """
     if not isinstance(text, six.string_types):
         raise TypeError("%s can't be decoded" % type(text))
@@ -144,7 +142,7 @@ def safe_encode(text, incoming=None,
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a bytestring `encoding` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of str
+    :raises TypeError: If text is not an instance of str
     """
     if not isinstance(text, six.string_types):
         raise TypeError("%s can't be encoded" % type(text))

keystoneclient/openstack/common/timeutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -21,6 +19,7 @@ Time related utilities and helper functions.
 
 import calendar
 import datetime
+import time
 
 import iso8601
 import six
@@ -49,9 +48,9 @@ def parse_isotime(timestr):
     try:
         return iso8601.parse_date(timestr)
     except iso8601.ParseError as e:
-        raise ValueError(unicode(e))
+        raise ValueError(six.text_type(e))
     except TypeError as e:
-        raise ValueError(unicode(e))
+        raise ValueError(six.text_type(e))
 
 
 def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -90,6 +89,11 @@ def is_newer_than(after, seconds):
 
 def utcnow_ts():
     """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
     return calendar.timegm(utcnow().timetuple())
 
 
@@ -111,12 +115,15 @@ def iso8601_from_timestamp(timestamp):
 utcnow.override_time = None
 
 
-def set_time_override(override_time=datetime.datetime.utcnow()):
+def set_time_override(override_time=None):
     """Overrides utils.utcnow.
 
     Make it return a constant time or a list thereof, one at a time.
+
+    :param override_time: datetime instance or list thereof. If not
+                          given, defaults to the current UTC time.
     """
-    utcnow.override_time = override_time
+    utcnow.override_time = override_time or datetime.datetime.utcnow()
 
 
 def advance_time_delta(timedelta):
@@ -169,6 +176,15 @@ def delta_seconds(before, after):
     datetime objects (as a float, to microsecond resolution).
     """
     delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
     try:
         return delta.total_seconds()
     except AttributeError:

openstack-common.conf

@@ -1,6 +1,6 @@
 [DEFAULT]
 
-# The list of modules to copy from openstack-common
+# The list of modules to copy from oslo-incubator
 module=importutils
 module=install_venv_common
 module=jsonutils

tools/install_venv_common.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2013 OpenStack Foundation
 # Copyright 2013 IBM Corp.
 #
@@ -119,11 +117,7 @@ class InstallVenv(object):
         self.pip_install('setuptools')
         self.pip_install('pbr')
 
-        self.pip_install('-r', self.requirements)
+        self.pip_install('-r', self.requirements, '-r', self.test_requirements)
-        self.pip_install('-r', self.test_requirements)
-
-    def post_process(self):
-        self.get_distro().post_process()
 
     def parse_args(self, argv):
         """Parses command-line arguments."""
@@ -157,14 +151,6 @@ class Distro(InstallVenv):
                  ' requires virtualenv, please install it using your'
                  ' favorite package management tool' % self.project)
 
-    def post_process(self):
-        """Any distribution-specific post-processing gets done here.
-
-        In particular, this is useful for applying patches to code inside
-        the venv.
-        """
-        pass
-
 
 class Fedora(Distro):
     """This covers all Fedora-based distributions.
@@ -176,10 +162,6 @@ class Fedora(Distro):
         return self.run_command_with_code(['rpm', '-q', pkg],
                                           check_exit_code=False)[1] == 0
 
-    def apply_patch(self, originalfile, patchfile):
-        self.run_command(['patch', '-N', originalfile, patchfile],
-                         check_exit_code=False)
-
     def install_virtualenv(self):
         if self.check_cmd('virtualenv'):
             return
@@ -188,27 +170,3 @@ class Fedora(Distro):
             self.die("Please install 'python-virtualenv'.")
 
         super(Fedora, self).install_virtualenv()
-
-    def post_process(self):
-        """Workaround for a bug in eventlet.
-
-        This currently affects RHEL6.1, but the fix can safely be
-        applied to all RHEL and Fedora distributions.
-
-        This can be removed when the fix is applied upstream.
-
-        Nova: https://bugs.launchpad.net/nova/+bug/884915
-        Upstream: https://bitbucket.org/eventlet/eventlet/issue/89
-        RHEL: https://bugzilla.redhat.com/958868
-        """
-
-        if os.path.exists('contrib/redhat-eventlet.patch'):
-            # Install "patch" program if it's not there
-            if not self.check_pkg('patch'):
-                self.die("Please install 'patch'.")
-
-            # Apply the eventlet patch
-            self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
-                                          'site-packages',
-                                          'eventlet/green/subprocess.py'),
-                             'contrib/redhat-eventlet.patch')