Sync modules from oslo, excluding db related

The following modules are updated:
cliutils,gettextutils,importutils,log,test,fixture,config

The following modules are new:
uuidutils

Other chnages:
 * Added uuidutils to openstack-common.conf
 * excluded rally/openstack* from pep8 check in tox.ini

Change-Id: I46fd3dc2e04bb9b5b74721a47689b860a9c68bdb

openstack-common.conf

@@ -10,6 +10,7 @@ module=importutils
 module=log
 module=test
 module=fixture
+module=uuidutils
 
 # The base module to hold the copy of openstack.common
 base=rally

rally/openstack/common/__init__.py

@@ -0,0 +1,17 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+
+six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

rally/openstack/common/cliutils.py

@@ -16,6 +16,8 @@
 # W0621: Redefining name %s from outer scope
 # pylint: disable=W0603,W0621
 
+from __future__ import print_function
+
 import getpass
 import inspect
 import os
@@ -27,7 +29,9 @@ import six
 from six import moves
 
 from rally.openstack.common.apiclient import exceptions
+from rally.openstack.common.gettextutils import _
 from rally.openstack.common import strutils
+from rally.openstack.common import uuidutils
 
 
 def validate_args(fn, *args, **kwargs):
@@ -84,7 +88,7 @@ def env(*args, **kwargs):
     If all are empty, defaults to '' or keyword arg `default`.
     """
     for arg in args:
-        value = os.environ.get(arg, None)
+        value = os.environ.get(arg)
         if value:
             return value
     return kwargs.get('default', '')
@@ -176,9 +180,9 @@ def print_dict(dct, dict_property="Property", wrap=0):
     for k, v in six.iteritems(dct):
         # convert dict to str to check length
         if isinstance(v, dict):
-            v = str(v)
+            v = six.text_type(v)
         if wrap > 0:
-            v = textwrap.fill(str(v), wrap)
+            v = textwrap.fill(six.text_type(v), wrap)
         # if value has a newline, add in multiple rows
         # e.g. fault with stacktrace
         if v and isinstance(v, six.string_types) and r'\n' in v:
@@ -199,7 +203,7 @@ def get_password(max_password_prompts=3):
     if hasattr(sys.stdin, "isatty") and sys.stdin.isatty():
         # Check for Ctrl-D
         try:
-            for _ in moves.range(max_password_prompts):
+            for __ in moves.range(max_password_prompts):
                 pw1 = getpass.getpass("OS Password: ")
                 if verify:
                     pw2 = getpass.getpass("Please verify: ")
@@ -211,3 +215,95 @@ def get_password(max_password_prompts=3):
         except EOFError:
             pass
     return pw
+
+
+def find_resource(manager, name_or_id, **find_args):
+    """Look for resource in a given manager.
+
+    Used as a helper for the _find_* methods.
+    Example:
+
+        def _find_hypervisor(cs, hypervisor):
+            #Get a hypervisor by name or ID.
+            return cliutils.find_resource(cs.hypervisors, hypervisor)
+    """
+    # first try to get entity as integer id
+    try:
+        return manager.get(int(name_or_id))
+    except (TypeError, ValueError, exceptions.NotFound):
+        pass
+
+    # now try to get entity as uuid
+    try:
+        tmp_id = strutils.safe_encode(name_or_id)
+
+        if uuidutils.is_uuid_like(tmp_id):
+            return manager.get(tmp_id)
+    except (TypeError, ValueError, exceptions.NotFound):
+        pass
+
+    # for str id which is not uuid
+    if getattr(manager, 'is_alphanum_id_allowed', False):
+        try:
+            return manager.get(name_or_id)
+        except exceptions.NotFound:
+            pass
+
+    try:
+        try:
+            return manager.find(human_id=name_or_id, **find_args)
+        except exceptions.NotFound:
+            pass
+
+        # finally try to find entity by name
+        try:
+            resource = getattr(manager, 'resource_class', None)
+            name_attr = resource.NAME_ATTR if resource else 'name'
+            kwargs = {name_attr: name_or_id}
+            kwargs.update(find_args)
+            return manager.find(**kwargs)
+        except exceptions.NotFound:
+            msg = _("No %(name)s with a name or "
+                    "ID of '%(name_or_id)s' exists.") % \
+                {
+                    "name": manager.resource_class.__name__.lower(),
+                    "name_or_id": name_or_id
+                }
+            raise exceptions.CommandError(msg)
+    except exceptions.NoUniqueMatch:
+        msg = _("Multiple %(name)s matches found for "
+                "'%(name_or_id)s', use an ID to be more specific.") % \
+            {
+                "name": manager.resource_class.__name__.lower(),
+                "name_or_id": name_or_id
+            }
+        raise exceptions.CommandError(msg)
+
+
+def service_type(stype):
+    """Adds 'service_type' attribute to decorated function.
+
+    Usage:
+        @service_type('volume')
+        def mymethod(f):
+            ...
+    """
+    def inner(f):
+        f.service_type = stype
+        return f
+    return inner
+
+
+def get_service_type(f):
+    """Retrieves service type from function."""
+    return getattr(f, 'service_type', None)
+
+
+def pretty_choice_list(l):
+    return ', '.join("'%s'" % i for i in l)
+
+
+def exit(msg=''):
+    if msg:
+        print (msg, file=sys.stderr)
+    sys.exit(1)

rally/openstack/common/config/generator.py

@@ -1,4 +1,5 @@
 # Copyright 2012 SINA Corporation
+# Copyright 2014 Cisco Systems, Inc.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -18,6 +19,7 @@
 
 from __future__ import print_function
 
+import argparse
 import imp
 import os
 import re
@@ -27,6 +29,7 @@ import textwrap
 
 from oslo.config import cfg
 import six
+import stevedore.named
 
 from rally.openstack.common import gettextutils
 from rally.openstack.common import importutils
@@ -38,6 +41,7 @@ BOOLOPT = "BoolOpt"
 INTOPT = "IntOpt"
 FLOATOPT = "FloatOpt"
 LISTOPT = "ListOpt"
+DICTOPT = "DictOpt"
 MULTISTROPT = "MultiStrOpt"
 
 OPT_TYPES = {
@@ -46,11 +50,12 @@ OPT_TYPES = {
     INTOPT: 'integer value',
     FLOATOPT: 'floating point value',
     LISTOPT: 'list value',
+    DICTOPT: 'dict value',
     MULTISTROPT: 'multi valued',
 }
 
 OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
-                                              FLOATOPT, LISTOPT,
+                                              FLOATOPT, LISTOPT, DICTOPT,
                                               MULTISTROPT]))
 
 PY_EXT = ".py"
@@ -59,34 +64,60 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
 WORDWRAP_WIDTH = 60
 
 
-def generate(srcfiles):
+def raise_extension_exception(extmanager, ep, err):
+    raise
+
+
+def generate(argv):
+    parser = argparse.ArgumentParser(
+        description='generate sample configuration file',
+    )
+    parser.add_argument('-m', dest='modules', action='append')
+    parser.add_argument('-l', dest='libraries', action='append')
+    parser.add_argument('srcfiles', nargs='*')
+    parsed_args = parser.parse_args(argv)
+
     mods_by_pkg = dict()
-    for filepath in srcfiles:
+    for filepath in parsed_args.srcfiles:
         pkg_name = filepath.split(os.sep)[1]
         mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
                             os.path.basename(filepath).split('.')[0]])
         mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
     # NOTE(lzyeval): place top level modules before packages
-    pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
+    pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
-    pkg_names.sort()
+    ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
-    ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
-    ext_names.sort()
     pkg_names.extend(ext_names)
 
     # opts_by_group is a mapping of group name to an options list
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
-    extra_modules = os.getenv("RALLY_CONFIG_GENERATOR_EXTRA_MODULES", "")
+    if parsed_args.modules:
-    if extra_modules:
+        for module_name in parsed_args.modules:
-        for module_name in extra_modules.split(','):
-            module_name = module_name.strip()
             module = _import_module(module_name)
             if module:
                 for group, opts in _list_opts(module):
                     opts_by_group.setdefault(group, []).append((module_name,
                                                                 opts))
 
+    # Look for entry points defined in libraries (or applications) for
+    # option discovery, and include their return values in the output.
+    #
+    # Each entry point should be a function returning an iterable
+    # of pairs with the group name (or None for the default group)
+    # and the list of Opt instances for that group.
+    if parsed_args.libraries:
+        loader = stevedore.named.NamedExtensionManager(
+            'oslo.config.opts',
+            names=list(set(parsed_args.libraries)),
+            invoke_on_load=False,
+            on_load_failure_callback=raise_extension_exception
+        )
+        for ext in loader:
+            for group, opts in ext.plugin():
+                opt_list = opts_by_group.setdefault(group or 'DEFAULT', [])
+                opt_list.append((ext.name, opts))
+
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
         mods.sort()
@@ -120,7 +151,7 @@ def _import_module(mod_str):
 
 def _is_in_group(opt, group):
     "Check if opt is in group."
-    for key, value in group._opts.items():
+    for value in group._opts.values():
         # NOTE(llu): Temporary workaround for bug #1262148, wait until
         # newly released oslo.config support '==' operator.
         if not(value['opt'] != opt):
@@ -134,7 +165,7 @@ def _guess_groups(opt, mod_obj):
         return 'DEFAULT'
 
     # what other groups is it in?
-    for key, value in cfg.CONF.items():
+    for value in cfg.CONF.values():
         if isinstance(value, cfg.CONF.GroupAttr):
             if _is_in_group(opt, value._group):
                 return value._group.name
@@ -203,7 +234,7 @@ def _sanitize_default(name, value):
         return value.replace(BASEDIR, '')
     elif value == _get_my_ip():
         return '10.0.0.1'
-    elif value == socket.gethostname() and 'host' in name:
+    elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name:
         return 'rally'
     elif value.strip() != value:
         return '"%s"' % value
@@ -221,7 +252,8 @@ def _print_opt(opt):
     except (ValueError, AttributeError) as err:
         sys.stderr.write("%s\n" % str(err))
         sys.exit(1)
-    opt_help += ' (' + OPT_TYPES[opt_type] + ')'
+    opt_help = u'%s (%s)' % (opt_help,
+                             OPT_TYPES[opt_type])
     print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
     if opt.deprecated_opts:
         for deprecated_opt in opt.deprecated_opts:
@@ -251,6 +283,11 @@ def _print_opt(opt):
         elif opt_type == LISTOPT:
             assert(isinstance(opt_default, list))
             print('#%s=%s' % (opt_name, ','.join(opt_default)))
+        elif opt_type == DICTOPT:
+            assert(isinstance(opt_default, dict))
+            opt_default_strlist = [str(key) + ':' + str(value)
+                                   for (key, value) in opt_default.items()]
+            print('#%s=%s' % (opt_name, ','.join(opt_default_strlist)))
         elif opt_type == MULTISTROPT:
             assert(isinstance(opt_default, list))
             if not opt_default:

rally/openstack/common/fixture/config.py

@@ -14,22 +14,17 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 import fixtures
 from oslo.config import cfg
 import six
 
 
 class Config(fixtures.Fixture):
-    """Override some configuration values.
+    """Allows overriding configuration settings for the test.
 
-    The keyword arguments are the names of configuration options to
+    `conf` will be reset on cleanup.
-    override and their values.
 
-    If a group argument is supplied, the overrides are applied to
-    the specified configuration option group.
-
-    All overrides are automatically cleared at the end of the current
-    test by the reset() method, which is registered by addCleanup().
     """
 
     def __init__(self, conf=cfg.CONF):
@@ -37,9 +32,54 @@ class Config(fixtures.Fixture):
 
     def setUp(self):
         super(Config, self).setUp()
+        # NOTE(morganfainberg): unregister must be added to cleanup before
+        # reset is because cleanup works in reverse order of registered items,
+        # and a reset must occur before unregistering options can occur.
+        self.addCleanup(self._unregister_config_opts)
         self.addCleanup(self.conf.reset)
+        self._registered_config_opts = {}
 
     def config(self, **kw):
+        """Override configuration values.
+
+        The keyword arguments are the names of configuration options to
+        override and their values.
+
+        If a `group` argument is supplied, the overrides are applied to
+        the specified configuration option group, otherwise the overrides
+        are applied to the ``default`` group.
+
+        """
+
         group = kw.pop('group', None)
         for k, v in six.iteritems(kw):
             self.conf.set_override(k, v, group)
+
+    def _unregister_config_opts(self):
+        for group in self._registered_config_opts:
+            self.conf.unregister_opts(self._registered_config_opts[group],
+                                      group=group)
+
+    def register_opt(self, opt, group=None):
+        """Register a single option for the test run.
+
+        Options registered in this manner will automatically be unregistered
+        during cleanup.
+
+        If a `group` argument is supplied, it will register the new option
+        to that group, otherwise the option is registered to the ``default``
+        group.
+        """
+        self.conf.register_opt(opt, group=group)
+        self._registered_config_opts.setdefault(group, set()).add(opt)
+
+    def register_opts(self, opts, group=None):
+        """Register multiple options for the test run.
+
+        This works in the same manner as register_opt() but takes a list of
+        options as the first argument. All arguments will be registered to the
+        same group if the ``group`` argument is supplied, otherwise all options
+        will be registered to the ``default`` group.
+        """
+        for opt in opts:
+            self.register_opt(opt, group=group)

rally/openstack/common/fixture/lockutils.py

@@ -48,4 +48,4 @@ class LockFixture(fixtures.Fixture):
     def setUp(self):
         super(LockFixture, self).setUp()
         self.addCleanup(self.mgr.__exit__, None, None, None)
-        self.mgr.__enter__()
+        self.lock = self.mgr.__enter__()

rally/openstack/common/fixture/mockpatch.py

@@ -15,6 +15,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+##############################################################################
+##############################################################################
+##
+## DO NOT MODIFY THIS FILE
+##
+## This file is being graduated to the rallytest library. Please make all
+## changes there, and only backport critical fixes here. - dhellmann
+##
+##############################################################################
+##############################################################################
+
 import fixtures
 import mock
 
@@ -22,14 +33,15 @@ import mock
 class PatchObject(fixtures.Fixture):
     """Deal with code around mock."""
 
-    def __init__(self, obj, attr, **kwargs):
+    def __init__(self, obj, attr, new=mock.DEFAULT, **kwargs):
         self.obj = obj
         self.attr = attr
         self.kwargs = kwargs
+        self.new = new
 
     def setUp(self):
         super(PatchObject, self).setUp()
-        _p = mock.patch.object(self.obj, self.attr, **self.kwargs)
+        _p = mock.patch.object(self.obj, self.attr, self.new, **self.kwargs)
         self.mock = _p.start()
         self.addCleanup(_p.stop)
 
@@ -38,12 +50,13 @@ class Patch(fixtures.Fixture):
 
     """Deal with code around mock.patch."""
 
-    def __init__(self, obj, **kwargs):
+    def __init__(self, obj, new=mock.DEFAULT, **kwargs):
         self.obj = obj
         self.kwargs = kwargs
+        self.new = new
 
     def setUp(self):
         super(Patch, self).setUp()
-        _p = mock.patch(self.obj, **self.kwargs)
+        _p = mock.patch(self.obj, self.new, **self.kwargs)
         self.mock = _p.start()
         self.addCleanup(_p.stop)

rally/openstack/common/fixture/moxstubout.py

@@ -15,8 +15,19 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+##############################################################################
+##############################################################################
+##
+## DO NOT MODIFY THIS FILE
+##
+## This file is being graduated to the rallytest library. Please make all
+## changes there, and only backport critical fixes here. - dhellmann
+##
+##############################################################################
+##############################################################################
+
 import fixtures
-import mox
+from six.moves import mox
 
 
 class MoxStubout(fixtures.Fixture):

rally/openstack/common/gettextutils.py

@@ -23,14 +23,11 @@ Usual usage in an openstack.common module:
 """
 
 import copy
+import functools
 import gettext
-import logging
+import locale
+from logging import handlers
 import os
-import re
-try:
-    import UserString as _userString
-except ImportError:
-    import collections as _userString
 
 from babel import localedata
 import six
@@ -38,6 +35,17 @@ import six
 _localedir = os.environ.get('rally'.upper() + '_LOCALEDIR')
 _t = gettext.translation('rally', localedir=_localedir, fallback=True)
 
+# We use separate translation catalogs for each log level, so set up a
+# mapping between the log level name and the translator. The domain
+# for the log level is project_name + "-log-" + log_level so messages
+# for each level end up in their own catalog.
+_t_log_levels = dict(
+    (level, gettext.translation('rally' + '-log-' + level,
+                                localedir=_localedir,
+                                fallback=True))
+    for level in ['info', 'warning', 'error', 'critical']
+)
+
 _AVAILABLE_LANGUAGES = {}
 USE_LAZY = False
 
@@ -56,13 +64,35 @@ def enable_lazy():
 
 def _(msg):
     if USE_LAZY:
-        return Message(msg, 'rally')
+        return Message(msg, domain='rally')
     else:
         if six.PY3:
             return _t.gettext(msg)
         return _t.ugettext(msg)
 
 
+def _log_translation(msg, level):
+    """Build a single translation of a log message
+    """
+    if USE_LAZY:
+        return Message(msg, domain='rally' + '-log-' + level)
+    else:
+        translator = _t_log_levels[level]
+        if six.PY3:
+            return translator.gettext(msg)
+        return translator.ugettext(msg)
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = functools.partial(_log_translation, level='info')
+_LW = functools.partial(_log_translation, level='warning')
+_LE = functools.partial(_log_translation, level='error')
+_LC = functools.partial(_log_translation, level='critical')
+
+
 def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
@@ -88,11 +118,6 @@ def install(domain, lazy=False):
         # messages in OpenStack. We override the standard _() function
         # and % (format string) operation to build Message objects that can
         # later be translated when we have more information.
-        #
-        # Also included below is an example LocaleHandler that translates
-        # Messages to an associated locale, effectively allowing many logs,
-        # each with their own locale.
-
         def _lazy_gettext(msg):
             """Create and return a Message object.
 
@@ -103,7 +128,7 @@ def install(domain, lazy=False):
             Message encapsulates a string so that we can translate
             it later when needed.
             """
-            return Message(msg, domain)
+            return Message(msg, domain=domain)
 
         from six import moves
         moves.builtins.__dict__['_'] = _lazy_gettext
@@ -118,182 +143,144 @@ def install(domain, lazy=False):
                             unicode=True)
 
 
-class Message(_userString.UserString, object):
+class Message(six.text_type):
-    """Class used to encapsulate translatable messages."""
+    """A Message object is a unicode object that can be translated.
-    def __init__(self, msg, domain):
-        # _msg is the gettext msgid and should never change
-        self._msg = msg
-        self._left_extra_msg = ''
-        self._right_extra_msg = ''
-        self._locale = None
-        self.params = None
-        self.domain = domain
 
-    @property
+    Translation of Message is done explicitly using the translate() method.
-    def data(self):
+    For all non-translation intents and purposes, a Message is simply unicode,
-        # NOTE(mrodden): this should always resolve to a unicode string
+    and can be treated as such.
-        # that best represents the state of the message currently
+    """
 
-        localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR')
+    def __new__(cls, msgid, msgtext=None, params=None,
-        if self.locale:
+                domain='rally', *args):
-            lang = gettext.translation(self.domain,
+        """Create a new Message object.
-                                       localedir=localedir,
-                                       languages=[self.locale],
-                                       fallback=True)
-        else:
-            # use system locale for translations
-            lang = gettext.translation(self.domain,
-                                       localedir=localedir,
-                                       fallback=True)
 
+        In order for translation to work gettext requires a message ID, this
+        msgid will be used as the base unicode text. It is also possible
+        for the msgid and the base unicode text to be different by passing
+        the msgtext parameter.
+        """
+        # If the base msgtext is not given, we use the default translation
+        # of the msgid (which is in English) just in case the system locale is
+        # not English, so that the base text will be in that locale by default.
+        if not msgtext:
+            msgtext = Message._translate_msgid(msgid, domain)
+        # We want to initialize the parent unicode with the actual object that
+        # would have been plain unicode if 'Message' was not enabled.
+        msg = super(Message, cls).__new__(cls, msgtext)
+        msg.msgid = msgid
+        msg.domain = domain
+        msg.params = params
+        return msg
+
+    def translate(self, desired_locale=None):
+        """Translate this message to the desired locale.
+
+        :param desired_locale: The desired locale to translate the message to,
+                               if no locale is provided the message will be
+                               translated to the system's default locale.
+
+        :returns: the translated message in unicode
+        """
+
+        translated_message = Message._translate_msgid(self.msgid,
+                                                      self.domain,
+                                                      desired_locale)
+        if self.params is None:
+            # No need for more translation
+            return translated_message
+
+        # This Message object may have been formatted with one or more
+        # Message objects as substitution arguments, given either as a single
+        # argument, part of a tuple, or as one or more values in a dictionary.
+        # When translating this Message we need to translate those Messages too
+        translated_params = _translate_args(self.params, desired_locale)
+
+        translated_message = translated_message % translated_params
+
+        return translated_message
+
+    @staticmethod
+    def _translate_msgid(msgid, domain, desired_locale=None):
+        if not desired_locale:
+            system_locale = locale.getdefaultlocale()
+            # If the system locale is not available to the runtime use English
+            if not system_locale[0]:
+                desired_locale = 'en_US'
+            else:
+                desired_locale = system_locale[0]
+
+        locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
+        lang = gettext.translation(domain,
+                                   localedir=locale_dir,
+                                   languages=[desired_locale],
+                                   fallback=True)
         if six.PY3:
-            ugettext = lang.gettext
+            translator = lang.gettext
         else:
-            ugettext = lang.ugettext
+            translator = lang.ugettext
 
-        full_msg = (self._left_extra_msg +
+        translated_message = translator(msgid)
-                    ugettext(self._msg) +
+        return translated_message
-                    self._right_extra_msg)
-
-        if self.params is not None:
-            full_msg = full_msg % self.params
-
-        return six.text_type(full_msg)
-
-    @property
-    def locale(self):
-        return self._locale
-
-    @locale.setter
-    def locale(self, value):
-        self._locale = value
-        if not self.params:
-            return
-
-        # This Message object may have been constructed with one or more
-        # Message objects as substitution parameters, given as a single
-        # Message, or a tuple or Map containing some, so when setting the
-        # locale for this Message we need to set it for those Messages too.
-        if isinstance(self.params, Message):
-            self.params.locale = value
-            return
-        if isinstance(self.params, tuple):
-            for param in self.params:
-                if isinstance(param, Message):
-                    param.locale = value
-            return
-        if isinstance(self.params, dict):
-            for param in self.params.values():
-                if isinstance(param, Message):
-                    param.locale = value
-
-    def _save_dictionary_parameter(self, dict_param):
-        full_msg = self.data
-        # look for %(blah) fields in string;
-        # ignore %% and deal with the
-        # case where % is first character on the line
-        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
-
-        # if we don't find any %(blah) blocks but have a %s
-        if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
-            # apparently the full dictionary is the parameter
-            params = copy.deepcopy(dict_param)
-        else:
-            params = {}
-            for key in keys:
-                try:
-                    params[key] = copy.deepcopy(dict_param[key])
-                except TypeError:
-                    # cast uncopyable thing to unicode string
-                    params[key] = six.text_type(dict_param[key])
-
-        return params
-
-    def _save_parameters(self, other):
-        # we check for None later to see if
-        # we actually have parameters to inject,
-        # so encapsulate if our parameter is actually None
-        if other is None:
-            self.params = (other, )
-        elif isinstance(other, dict):
-            self.params = self._save_dictionary_parameter(other)
-        else:
-            # fallback to casting to unicode,
-            # this will handle the problematic python code-like
-            # objects that cannot be deep-copied
-            try:
-                self.params = copy.deepcopy(other)
-            except TypeError:
-                self.params = six.text_type(other)
-
-        return self
-
-    # overrides to be more string-like
-    def __unicode__(self):
-        return self.data
-
-    def __str__(self):
-        if six.PY3:
-            return self.__unicode__()
-        return self.data.encode('utf-8')
-
-    def __getstate__(self):
-        to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
-                   'domain', 'params', '_locale']
-        new_dict = self.__dict__.fromkeys(to_copy)
-        for attr in to_copy:
-            new_dict[attr] = copy.deepcopy(self.__dict__[attr])
-
-        return new_dict
-
-    def __setstate__(self, state):
-        for (k, v) in state.items():
-            setattr(self, k, v)
-
-    # operator overloads
-    def __add__(self, other):
-        copied = copy.deepcopy(self)
-        copied._right_extra_msg += other.__str__()
-        return copied
-
-    def __radd__(self, other):
-        copied = copy.deepcopy(self)
-        copied._left_extra_msg += other.__str__()
-        return copied
 
     def __mod__(self, other):
-        # do a format string to catch and raise
+        # When we mod a Message we want the actual operation to be performed
-        # any possible KeyErrors from missing parameters
+        # by the parent class (i.e. unicode()), the only thing  we do here is
-        self.data % other
+        # save the original msgid and the parameters in case of a translation
-        copied = copy.deepcopy(self)
+        params = self._sanitize_mod_params(other)
-        return copied._save_parameters(other)
+        unicode_mod = super(Message, self).__mod__(params)
+        modded = Message(self.msgid,
+                         msgtext=unicode_mod,
+                         params=params,
+                         domain=self.domain)
+        return modded
 
-    def __mul__(self, other):
+    def _sanitize_mod_params(self, other):
-        return self.data * other
+        """Sanitize the object being modded with this Message.
 
-    def __rmul__(self, other):
+        - Add support for modding 'None' so translation supports it
-        return other * self.data
+        - Trim the modded object, which can be a large dictionary, to only
-
+        those keys that would actually be used in a translation
-    def __getitem__(self, key):
+        - Snapshot the object being modded, in case the message is
-        return self.data[key]
+        translated, it will be used as it was when the Message was created
-
+        """
-    def __getslice__(self, start, end):
+        if other is None:
-        return self.data.__getslice__(start, end)
+            params = (other,)
-
+        elif isinstance(other, dict):
-    def __getattribute__(self, name):
+            # Merge the dictionaries
-        # NOTE(mrodden): handle lossy operations that we can't deal with yet
+            # Copy each item in case one does not support deep copy.
-        # These override the UserString implementation, since UserString
+            params = {}
-        # uses our __class__ attribute to try and build a new message
+            if isinstance(self.params, dict):
-        # after running the inner data string through the operation.
+                for key, val in self.params.items():
-        # At that point, we have lost the gettext message id and can just
+                    params[key] = self._copy_param(val)
-        # safely resolve to a string instead.
+            for key, val in other.items():
-        ops = ['capitalize', 'center', 'decode', 'encode',
+                params[key] = self._copy_param(val)
-               'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip',
-               'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
-        if name in ops:
-            return getattr(self.data, name)
         else:
-            return _userString.UserString.__getattribute__(self, name)
+            params = self._copy_param(other)
+        return params
+
+    def _copy_param(self, param):
+        try:
+            return copy.deepcopy(param)
+        except Exception:
+            # Fallback to casting to unicode this will handle the
+            # python code-like objects that can't be deep-copied
+            return six.text_type(param)
+
+    def __add__(self, other):
+        msg = _('Message objects do not support addition.')
+        raise TypeError(msg)
+
+    def __radd__(self, other):
+        return self.__add__(other)
+
+    def __str__(self):
+        # NOTE(luisg): Logging in python 2.6 tries to str() log records,
+        # and it expects specifically a UnicodeError in order to proceed.
+        msg = _('Message objects do not support str() because they may '
+                'contain non-ascii characters. '
+                'Please use unicode() or translate() instead.')
+        raise UnicodeError(msg)
 
 
 def get_available_languages(domain):
@@ -319,53 +306,143 @@ def get_available_languages(domain):
     list_identifiers = (getattr(localedata, 'list', None) or
                         getattr(localedata, 'locale_identifiers'))
     locale_identifiers = list_identifiers()
+
     for i in locale_identifiers:
         if find(i) is not None:
             language_list.append(i)
+
+    # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
+    # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
+    # are perfectly legitimate locales:
+    #     https://github.com/mitsuhiko/babel/issues/37
+    # In Babel 1.3 they fixed the bug and they support these locales, but
+    # they are still not explicitly "listed" by locale_identifiers().
+    # That is  why we add the locales here explicitly if necessary so that
+    # they are listed as supported.
+    aliases = {'zh': 'zh_CN',
+               'zh_Hant_HK': 'zh_HK',
+               'zh_Hant': 'zh_TW',
+               'fil': 'tl_PH'}
+    for (locale, alias) in six.iteritems(aliases):
+        if locale in language_list and alias not in language_list:
+            language_list.append(alias)
+
     _AVAILABLE_LANGUAGES[domain] = language_list
     return copy.copy(language_list)
 
 
-def get_localized_message(message, user_locale):
+def translate(obj, desired_locale=None):
-    """Gets a localized version of the given message in the given locale.
+    """Gets the translated unicode representation of the given object.
 
-    If the message is not a Message object the message is returned as-is.
+    If the object is not translatable it is returned as-is.
-    If the locale is None the message is translated to the default locale.
+    If the locale is None the object is translated to the system locale.
 
-    :returns: the translated message in unicode, or the original message if
+    :param obj: the object to translate
+    :param desired_locale: the locale to translate the message to, if None the
+                           default system locale will be used
+    :returns: the translated object in unicode, or the original object if
               it could not be translated
     """
-    translated = message
+    message = obj
+    if not isinstance(message, Message):
+        # If the object to translate is not already translatable,
+        # let's first get its unicode representation
+        message = six.text_type(obj)
     if isinstance(message, Message):
-        original_locale = message.locale
+        # Even after unicoding() we still need to check if we are
-        message.locale = user_locale
+        # running with translatable unicode before translating
-        translated = six.text_type(message)
+        return message.translate(desired_locale)
-        message.locale = original_locale
+    return obj
-    return translated
 
 
-class LocaleHandler(logging.Handler):
+def _translate_args(args, desired_locale=None):
-    """Handler that can have a locale associated to translate Messages.
+    """Translates all the translatable elements of the given arguments object.
 
-    A quick example of how to utilize the Message class above.
+    This method is used for translating the translatable values in method
-    LocaleHandler takes a locale and a target logging.Handler object
+    arguments which include values of tuples or dictionaries.
-    to forward LogRecord objects to after translating the internal Message.
+    If the object is not a tuple or a dictionary the object itself is
+    translated if it is translatable.
+
+    If the locale is None the object is translated to the system locale.
+
+    :param args: the args to translate
+    :param desired_locale: the locale to translate the args to, if None the
+                           default system locale will be used
+    :returns: a new args object with the translated contents of the original
+    """
+    if isinstance(args, tuple):
+        return tuple(translate(v, desired_locale) for v in args)
+    if isinstance(args, dict):
+        translated_dict = {}
+        for (k, v) in six.iteritems(args):
+            translated_v = translate(v, desired_locale)
+            translated_dict[k] = translated_v
+        return translated_dict
+    return translate(args, desired_locale)
+
+
+class TranslationHandler(handlers.MemoryHandler):
+    """Handler that translates records before logging them.
+
+    The TranslationHandler takes a locale and a target logging.Handler object
+    to forward LogRecord objects to after translating them. This handler
+    depends on Message objects being logged, instead of regular strings.
+
+    The handler can be configured declaratively in the logging.conf as follows:
+
+        [handlers]
+        keys = translatedlog, translator
+
+        [handler_translatedlog]
+        class = handlers.WatchedFileHandler
+        args = ('/var/log/api-localized.log',)
+        formatter = context
+
+        [handler_translator]
+        class = openstack.common.log.TranslationHandler
+        target = translatedlog
+        args = ('zh_CN',)
+
+    If the specified locale is not available in the system, the handler will
+    log in the default locale.
     """
 
-    def __init__(self, locale, target):
+    def __init__(self, locale=None, target=None):
-        """Initialize a LocaleHandler
+        """Initialize a TranslationHandler
 
         :param locale: locale to use for translating messages
         :param target: logging.Handler object to forward
                        LogRecord objects to after translation
         """
-        logging.Handler.__init__(self)
+        # NOTE(luisg): In order to allow this handler to be a wrapper for
+        # other handlers, such as a FileHandler, and still be able to
+        # configure it using logging.conf, this handler has to extend
+        # MemoryHandler because only the MemoryHandlers' logging.conf
+        # parsing is implemented such that it accepts a target handler.
+        handlers.MemoryHandler.__init__(self, capacity=0, target=target)
         self.locale = locale
-        self.target = target
+
+    def setFormatter(self, fmt):
+        self.target.setFormatter(fmt)
 
     def emit(self, record):
-        if isinstance(record.msg, Message):
+        # We save the message from the original record to restore it
-            # set the locale and resolve to a string
+        # after translation, so other handlers are not affected by this
-            record.msg.locale = self.locale
+        original_msg = record.msg
+        original_args = record.args
+
+        try:
+            self._translate_and_log_record(record)
+        finally:
+            record.msg = original_msg
+            record.args = original_args
+
+    def _translate_and_log_record(self, record):
+        record.msg = translate(record.msg, self.locale)
+
+        # In addition to translating the message, we also need to translate
+        # arguments that were passed to the log method that were not part
+        # of the main message e.g., log.info(_('Some message %s'), this_one))
+        record.args = _translate_args(record.args, self.locale)
 
         self.target.emit(record)

rally/openstack/common/importutils.py

@@ -58,6 +58,13 @@ def import_module(import_str):
     return sys.modules[import_str]
 
 
+def import_versioned_module(version, submodule=None):
+    module = 'rally.v%s' % version
+    if submodule:
+        module = '.'.join((module, submodule))
+    return import_module(module)
+
+
 def try_import(import_str, default=None):
     """Try to import a module and if it fails return default."""
     try:

rally/openstack/common/log.py

@@ -15,7 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Openstack logging handler.
+"""OpenStack logging handler.
 
 This module adds to logging functionality by adding the option to specify
 a context object when calling the various log methods.  If the context object
@@ -115,10 +115,21 @@ logging_cli_opts = [
                     '--log-file paths'),
     cfg.BoolOpt('use-syslog',
                 default=False,
-                help='Use syslog for logging.'),
+                help='Use syslog for logging. '
+                     'Existing syslog format is DEPRECATED during I, '
+                     'and then will be changed in J to honor RFC5424'),
+    cfg.BoolOpt('use-syslog-rfc-format',
+                # TODO(bogdando) remove or use True after existing
+                #    syslog format deprecation in J
+                default=False,
+                help='(Optional) Use syslog rfc5424 format for logging. '
+                     'If enabled, will add APP-NAME (RFC5424) before the '
+                     'MSG part of the syslog message.  The old format '
+                     'without APP-NAME is deprecated in I, '
+                     'and will be removed in J.'),
     cfg.StrOpt('syslog-log-facility',
                default='LOG_USER',
-               help='syslog facility to receive log lines')
+               help='Syslog facility to receive log lines')
 ]
 
 generic_log_opts = [
@@ -132,18 +143,18 @@ log_opts = [
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
                        '%(name)s [%(request_id)s %(user_identity)s] '
                        '%(instance)s%(message)s',
-               help='format string to use for log messages with context'),
+               help='Format string to use for log messages with context'),
     cfg.StrOpt('logging_default_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
                        '%(name)s [-] %(instance)s%(message)s',
-               help='format string to use for log messages without context'),
+               help='Format string to use for log messages without context'),
     cfg.StrOpt('logging_debug_format_suffix',
                default='%(funcName)s %(pathname)s:%(lineno)d',
-               help='data to append to log format when level is DEBUG'),
+               help='Data to append to log format when level is DEBUG'),
     cfg.StrOpt('logging_exception_prefix',
                default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
                '%(instance)s',
-               help='prefix each line of exception output with this format'),
+               help='Prefix each line of exception output with this format'),
     cfg.ListOpt('default_log_levels',
                 default=[
                     'amqp=WARN',
@@ -152,15 +163,17 @@ log_opts = [
                     'qpid=WARN',
                     'sqlalchemy=WARN',
                     'suds=INFO',
+                    'oslo.messaging=INFO',
                     'iso8601=WARN',
+                    'requests.packages.urllib3.connectionpool=WARN'
                 ],
-                help='list of logger=LEVEL pairs'),
+                help='List of logger=LEVEL pairs'),
     cfg.BoolOpt('publish_errors',
                 default=False,
-                help='publish error events'),
+                help='Publish error events'),
     cfg.BoolOpt('fatal_deprecations',
                 default=False,
-                help='make deprecations fatal'),
+                help='Make deprecations fatal'),
 
     # NOTE(mikal): there are two options here because sometimes we are handed
     # a full instance (and could include more information), and other times we
@@ -292,18 +305,39 @@ class ContextAdapter(BaseLoggerAdapter):
         self.logger = logger
         self.project = project_name
         self.version = version_string
+        self._deprecated_messages_sent = dict()
 
     @property
     def handlers(self):
         return self.logger.handlers
 
     def deprecated(self, msg, *args, **kwargs):
+        """Call this method when a deprecated feature is used.
+
+        If the system is configured for fatal deprecations then the message
+        is logged at the 'critical' level and :class:`DeprecatedConfig` will
+        be raised.
+
+        Otherwise, the message will be logged (once) at the 'warn' level.
+
+        :raises: :class:`DeprecatedConfig` if the system is configured for
+                 fatal deprecations.
+
+        """
         stdmsg = _("Deprecated: %s") % msg
         if CONF.fatal_deprecations:
             self.critical(stdmsg, *args, **kwargs)
             raise DeprecatedConfig(msg=stdmsg)
-        else:
+
-            self.warn(stdmsg, *args, **kwargs)
+        # Using a list because a tuple with dict can't be stored in a set.
+        sent_args = self._deprecated_messages_sent.setdefault(msg, list())
+
+        if args in sent_args:
+            # Already logged this message, so don't log it again.
+            return
+
+        sent_args.append(args)
+        self.warn(stdmsg, *args, **kwargs)
 
     def process(self, msg, kwargs):
         # NOTE(mrodden): catch any Message/other object and
@@ -324,7 +358,7 @@ class ContextAdapter(BaseLoggerAdapter):
             extra.update(_dictify_context(context))
 
         instance = kwargs.pop('instance', None)
-        instance_uuid = (extra.get('instance_uuid', None) or
+        instance_uuid = (extra.get('instance_uuid') or
                          kwargs.pop('instance_uuid', None))
         instance_extra = ''
         if instance:
@@ -391,9 +425,11 @@ class JSONFormatter(logging.Formatter):
 def _create_logging_excepthook(product_name):
     def logging_excepthook(exc_type, value, tb):
         extra = {}
-        if CONF.verbose:
+        if CONF.verbose or CONF.debug:
             extra['exc_info'] = (exc_type, value, tb)
-        getLogger(product_name).critical(str(value), **extra)
+        getLogger(product_name).critical(
+            "".join(traceback.format_exception_only(exc_type, value)),
+            **extra)
     return logging_excepthook
 
 
@@ -418,12 +454,12 @@ def _load_log_config(log_config_append):
         raise LogConfigError(log_config_append, str(exc))
 
 
-def setup(product_name):
+def setup(product_name, version='unknown'):
     """Setup logging."""
     if CONF.log_config_append:
         _load_log_config(CONF.log_config_append)
     else:
-        _setup_logging_from_conf()
+        _setup_logging_from_conf(product_name, version)
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
@@ -457,15 +493,32 @@ def _find_facility_from_conf():
     return facility
 
 
-def _setup_logging_from_conf():
+class RFCSysLogHandler(logging.handlers.SysLogHandler):
+    def __init__(self, *args, **kwargs):
+        self.binary_name = _get_binary_name()
+        super(RFCSysLogHandler, self).__init__(*args, **kwargs)
+
+    def format(self, record):
+        msg = super(RFCSysLogHandler, self).format(record)
+        msg = self.binary_name + ' ' + msg
+        return msg
+
+
+def _setup_logging_from_conf(project, version):
     log_root = getLogger(None).logger
     for handler in log_root.handlers:
         log_root.removeHandler(handler)
 
     if CONF.use_syslog:
         facility = _find_facility_from_conf()
-        syslog = logging.handlers.SysLogHandler(address='/dev/log',
+        # TODO(bogdando) use the format provided by RFCSysLogHandler
-                                                facility=facility)
+        #   after existing syslog format deprecation in J
+        if CONF.use_syslog_rfc_format:
+            syslog = RFCSysLogHandler(address='/dev/log',
+                                      facility=facility)
+        else:
+            syslog = logging.handlers.SysLogHandler(address='/dev/log',
+                                                    facility=facility)
         log_root.addHandler(syslog)
 
     logpath = _get_log_file_path()
@@ -499,7 +552,9 @@ def _setup_logging_from_conf():
             log_root.info('Deprecated: log_format is now deprecated and will '
                           'be removed in the next release')
         else:
-            handler.setFormatter(ContextFormatter(datefmt=datefmt))
+            handler.setFormatter(ContextFormatter(project=project,
+                                                  version=version,
+                                                  datefmt=datefmt))
 
     if CONF.debug:
         log_root.setLevel(logging.DEBUG)
@@ -543,7 +598,7 @@ class WritableLogger(object):
         self.level = level
 
     def write(self, msg):
-        self.logger.log(self.level, msg)
+        self.logger.log(self.level, msg.rstrip())
 
 
 class ContextFormatter(logging.Formatter):
@@ -557,18 +612,50 @@ class ContextFormatter(logging.Formatter):
     For information about what variables are available for the formatter see:
     http://docs.python.org/library/logging.html#formatter
 
+    If available, uses the context value stored in TLS - local.store.context
+
     """
 
+    def __init__(self, *args, **kwargs):
+        """Initialize ContextFormatter instance
+
+        Takes additional keyword arguments which can be used in the message
+        format string.
+
+        :keyword project: project name
+        :type project: string
+        :keyword version: project version
+        :type version: string
+
+        """
+
+        self.project = kwargs.pop('project', 'unknown')
+        self.version = kwargs.pop('version', 'unknown')
+
+        logging.Formatter.__init__(self, *args, **kwargs)
+
     def format(self, record):
         """Uses contextstring if request_id is set, otherwise default."""
-        # NOTE(sdague): default the fancier formating params
+
+        # store project info
+        record.project = self.project
+        record.version = self.version
+
+        # store request info
+        context = getattr(local.store, 'context', None)
+        if context:
+            d = _dictify_context(context)
+            for k, v in d.items():
+                setattr(record, k, v)
+
+        # NOTE(sdague): default the fancier formatting params
         # to an empty string so we don't throw an exception if
         # they get used
-        for key in ('instance', 'color'):
+        for key in ('instance', 'color', 'user_identity'):
             if key not in record.__dict__:
                 record.__dict__[key] = ''
 
-        if record.__dict__.get('request_id', None):
+        if record.__dict__.get('request_id'):
             self._fmt = CONF.logging_context_format_string
         else:
             self._fmt = CONF.logging_default_format_string
@@ -577,7 +664,7 @@ class ContextFormatter(logging.Formatter):
                 CONF.logging_debug_format_suffix):
             self._fmt += " " + CONF.logging_debug_format_suffix
 
-        # Cache this on the record, Logger will respect our formated copy
+        # Cache this on the record, Logger will respect our formatted copy
         if record.exc_info:
             record.exc_text = self.formatException(record.exc_info, record)
         return logging.Formatter.format(self, record)

rally/openstack/common/test.py

@@ -13,14 +13,28 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+##############################################################################
+##############################################################################
+##
+## DO NOT MODIFY THIS FILE
+##
+## This file is being graduated to the rallytest library. Please make all
+## changes there, and only backport critical fixes here. - dhellmann
+##
+##############################################################################
+##############################################################################
+
 """Common utilities used in testing"""
 
+import logging
 import os
+import tempfile
 
 import fixtures
 import testtools
 
 _TRUE_VALUES = ('True', 'true', '1', 'yes')
+_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s"
 
 
 class BaseTestCase(testtools.TestCase):
@@ -29,9 +43,10 @@ class BaseTestCase(testtools.TestCase):
         super(BaseTestCase, self).setUp()
         self._set_timeout()
         self._fake_output()
-        self.useFixture(fixtures.FakeLogger())
+        self._fake_logs()
         self.useFixture(fixtures.NestedTempfile())
         self.useFixture(fixtures.TempHomeDir())
+        self.tempdirs = []
 
     def _set_timeout(self):
         test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
@@ -50,3 +65,35 @@ class BaseTestCase(testtools.TestCase):
         if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
             stderr = self.useFixture(fixtures.StringStream('stderr')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+
+    def _fake_logs(self):
+        if os.environ.get('OS_DEBUG') in _TRUE_VALUES:
+            level = logging.DEBUG
+        else:
+            level = logging.INFO
+        capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES
+        if capture_logs:
+            self.useFixture(
+                fixtures.FakeLogger(
+                    format=_LOG_FORMAT,
+                    level=level,
+                    nuke_handlers=capture_logs,
+                )
+            )
+        else:
+            logging.basicConfig(format=_LOG_FORMAT, level=level)
+
+    def create_tempfiles(self, files, ext='.conf'):
+        tempfiles = []
+        for (basename, contents) in files:
+            if not os.path.isabs(basename):
+                (fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext)
+            else:
+                path = basename + ext
+                fd = os.open(path, os.O_CREAT | os.O_WRONLY)
+            tempfiles.append(path)
+            try:
+                os.write(fd, contents)
+            finally:
+                os.close(fd)
+        return tempfiles

rally/openstack/common/uuidutils.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2012 Intel Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+UUID related utilities and helper functions.
+"""
+
+import uuid
+
+
+def generate_uuid():
+    return str(uuid.uuid4())
+
+
+def is_uuid_like(val):
+    """Returns validation of a value as a UUID.
+
+    For our purposes, a UUID is a canonical form string:
+    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
+
+    """
+    try:
+        return str(uuid.UUID(val)) == val
+    except (TypeError, ValueError, AttributeError):
+        return False

tools/config/generate_sample.sh

@@ -4,8 +4,8 @@ print_hint() {
     echo "Try \`${0##*/} --help' for more information." >&2
 }
 
-PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \
+PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
-                 --long help,base-dir:,package-name:,output-dir: -- "$@")
+                 --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
 
 if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
 
@@ -21,6 +21,8 @@ while true; do
             echo "-b, --base-dir=DIR        project base directory"
             echo "-p, --package-name=NAME   project package name"
             echo "-o, --output-dir=DIR      file output directory"
+            echo "-m, --module=MOD          extra python module to interrogate for options"
+            echo "-l, --library=LIB         extra library that registers options for discovery"
             exit 0
             ;;
         -b|--base-dir)
@@ -38,6 +40,16 @@ while true; do
             OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
             shift
             ;;
+        -m|--module)
+            shift
+            MODULES="$MODULES -m $1"
+            shift
+            ;;
+        -l|--library)
+            shift
+            LIBRARIES="$LIBRARIES -l $1"
+            shift
+            ;;
         --)
             break
             ;;
@@ -53,7 +65,7 @@ then
     BASEDIR=$(cd "$BASEDIR" && pwd)
 fi
 
-PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}}
+PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)}
 TARGETDIR=$BASEDIR/$PACKAGENAME
 if ! [ -d $TARGETDIR ]
 then
@@ -77,12 +89,20 @@ find $TARGETDIR -type f -name "*.pyc" -delete
 FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
         -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
 
-EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc"
+RC_FILE="`dirname $0`/oslo.config.generator.rc"
-if test -r "$EXTRA_MODULES_FILE"
+if test -r "$RC_FILE"
 then
-    source "$EXTRA_MODULES_FILE"
+    source "$RC_FILE"
 fi
 
+for mod in ${RALLY_CONFIG_GENERATOR_EXTRA_MODULES}; do
+    MODULES="$MODULES -m $mod"
+done
+
+for lib in ${RALLY_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
+    LIBRARIES="$LIBRARIES -l $lib"
+done
+
 export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
@@ -90,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 DEFAULT_MODULEPATH=rally.openstack.common.config.generator
 MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
-python -m $MODULEPATH $FILES > $OUTPUTFILE
+python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
 
 # Hook to allow projects to append custom config file snippets
 CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)

tox.ini

@@ -32,7 +32,7 @@ downloadcache = ~/cache/pip
 ignore = E126,H703
 show-source = true
 builtins = _
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,*rally/verification/verifiers/tempest/openstack-tempest*,build
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,*rally/verification/verifiers/tempest/openstack-tempest*,build,*rally/openstack*
 
 [hacking]
 import_exceptions = rally.openstack.common.gettextutils._