Sync code from oslo incubator

Includes updates in config generator, db.sqlalchemy, gettext utils and
log. New script added: tools/config/check_uptodate.sh

Change-Id: I0afaeb593e1b3425d7953b2a11711801299fe1f9

rally/openstack/common/config/generator.py

@@ -223,6 +223,8 @@ def _get_my_ip():
 
 def _sanitize_default(name, value):
     """Set up a reasonably sensible default for pybasedir, my_ip and host."""
+    hostname = socket.gethostname()
+    fqdn = socket.getfqdn()
     if value.startswith(sys.prefix):
         # NOTE(jd) Don't use os.path.join, because it is likely to think the
         # second part is an absolute pathname and therefore drop the first
@@ -234,8 +236,13 @@ def _sanitize_default(name, value):
         return value.replace(BASEDIR, '')
     elif value == _get_my_ip():
         return '10.0.0.1'
-    elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name:
+    elif value in (hostname, fqdn):
+        if 'host' in name:
             return 'rally'
+    elif value.endswith(hostname):
+        return value.replace(hostname, 'rally')
+    elif value.endswith(fqdn):
+        return value.replace(fqdn, 'rally')
     elif value.strip() != value:
         return '"%s"' % value
     return value

rally/openstack/common/db/sqlalchemy/migration.py

@@ -213,8 +213,15 @@ def _db_schema_sanity_check(engine):
                         'where TABLE_SCHEMA=%s and '
                         'TABLE_COLLATION NOT LIKE "%%utf8%%"')
 
-        table_names = [res[0] for res in engine.execute(onlyutf8_sql,
+        # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic
-                                                        engine.url.database)]
+        # versioning tables from the tables we need to verify utf8 status on.
+        # Non-standard table names are not supported.
+        EXCLUDED_TABLES = ['migrate_version', 'alembic_version']
+
+        table_names = [res[0] for res in
+                       engine.execute(onlyutf8_sql, engine.url.database) if
+                       res[0].lower() not in EXCLUDED_TABLES]
+
         if len(table_names) > 0:
             raise ValueError(_('Tables "%s" have non utf8 collation, '
                                'please make sure all tables are CHARSET=utf8'

rally/openstack/common/db/sqlalchemy/models.py

@@ -70,7 +70,7 @@ class ModelBase(six.Iterator):
         return []
 
     def __iter__(self):
-        columns = dict(object_mapper(self).columns).keys()
+        columns = list(dict(object_mapper(self).columns).keys())
         # NOTE(russellb): Allow models to specify other keys that can be looked
         # up, beyond the actual db columns.  An example would be the 'name'
         # property for an Instance.

rally/openstack/common/db/sqlalchemy/session.py

@@ -367,7 +367,7 @@ def _raise_if_duplicate_entry_error(integrity_error, engine_name):
             return [columns]
         return columns[len(uniqbase):].split("0")[1:]
 
-    if engine_name not in ["ibm_db_sa", "mysql", "sqlite", "postgresql"]:
+    if engine_name not in ("ibm_db_sa", "mysql", "sqlite", "postgresql"):
         return
 
     # FIXME(johannes): The usage of the .message attribute has been
@@ -489,7 +489,7 @@ def _thread_yield(dbapi_con, con_record):
 
 
 def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
-    """Ensures that MySQL and DB2 connections are alive.
+    """Ensures that MySQL, PostgreSQL or DB2 connections are alive.
 
     Borrowed from:
     http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
@@ -645,7 +645,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None,
 
     sqlalchemy.event.listen(engine, 'checkin', _thread_yield)
 
-    if engine.name in ['mysql', 'ibm_db_sa']:
+    if engine.name in ('ibm_db_sa', 'mysql', 'postgresql'):
         ping_callback = functools.partial(_ping_listener, engine)
         sqlalchemy.event.listen(engine, 'checkout', ping_callback)
         if engine.name == 'mysql':

rally/openstack/common/db/sqlalchemy/test_base.py

@@ -18,12 +18,12 @@ import functools
 import os
 
 import fixtures
+from oslotest import base as test_base
 import six
 
+from rally.openstack.common.db.sqlalchemy import provision
 from rally.openstack.common.db.sqlalchemy import session
 from rally.openstack.common.db.sqlalchemy import utils
-from rally.openstack.common.fixture import lockutils
-from tests import test
 
 
 class DbFixture(fixtures.Fixture):
@@ -43,15 +43,17 @@ class DbFixture(fixtures.Fixture):
 
         self.test = test
 
+    def cleanUp(self):
+        self.test.engine.dispose()
+
     def setUp(self):
         super(DbFixture, self).setUp()
 
         self.test.engine = session.create_engine(self._get_uri())
         self.test.sessionmaker = session.get_maker(self.test.engine)
-        self.addCleanup(self.test.engine.dispose)
 
 
-class DbTestCase(test.TestCase):
+class DbTestCase(test_base.BaseTestCase):
     """Base class for testing of DB code.
 
     Using `DbFixture`. Intended to be the main database test case to use all
@@ -103,11 +105,24 @@ class OpportunisticFixture(DbFixture):
     DRIVER = abc.abstractproperty(lambda: None)
     DBNAME = PASSWORD = USERNAME = 'openstack_citest'
 
-    def _get_uri(self):
+    def setUp(self):
-        return utils.get_connect_string(backend=self.DRIVER,
+        self._provisioning_engine = provision.get_engine(
+            utils.get_connect_string(backend=self.DRIVER,
                                      user=self.USERNAME,
                                      passwd=self.PASSWORD,
                                      database=self.DBNAME)
+        )
+        self._uri = provision.create_database(self._provisioning_engine)
+
+        super(OpportunisticFixture, self).setUp()
+
+    def cleanUp(self):
+        super(OpportunisticFixture, self).cleanUp()
+
+        provision.drop_database(self._provisioning_engine, self._uri)
+
+    def _get_uri(self):
+        return self._uri
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -121,9 +136,6 @@ class OpportunisticTestCase(DbTestCase):
     FIXTURE = abc.abstractproperty(lambda: None)
 
     def setUp(self):
-        # TODO(bnemec): Remove this once infra is ready for
-        # https://review.openstack.org/#/c/74963/ to merge.
-        self.useFixture(lockutils.LockFixture('opportunistic-db'))
         credentials = {
             'backend': self.FIXTURE.DRIVER,
             'user': self.FIXTURE.USERNAME,

rally/openstack/common/db/sqlalchemy/test_migrations.py

@@ -20,6 +20,7 @@ import os
 import subprocess
 
 import lockfile
+from oslotest import base as test_base
 from six import moves
 from six.moves.urllib import parse
 import sqlalchemy
@@ -27,7 +28,6 @@ import sqlalchemy.exc
 
 from rally.openstack.common.db.sqlalchemy import utils
 from rally.openstack.common.gettextutils import _LE
-from tests import test
 
 LOG = logging.getLogger(__name__)
 
@@ -68,7 +68,7 @@ def _set_db_lock(lock_path=None, lock_prefix=None):
     return decorator
 
 
-class BaseMigrationTestCase(test.TestCase):
+class BaseMigrationTestCase(test_base.BaseTestCase):
     """Base class fort testing of migration utils."""
 
     def __init__(self, *args, **kwargs):

rally/openstack/common/db/sqlalchemy/utils.py

@@ -254,6 +254,14 @@ def get_table(engine, name):
 
     Needed because the models don't work for us in migrations
     as models will be far out of sync with the current data.
+
+    .. warning::
+
+       Do not use this method when creating ForeignKeys in database migrations
+       because sqlalchemy needs the same MetaData object to hold information
+       about the parent table and the reference table in the ForeignKey. This
+       method uses a unique MetaData object per table object so it won't work
+       with ForeignKey creation.
     """
     metadata = MetaData()
     metadata.bind = engine

rally/openstack/common/gettextutils.py

@@ -32,24 +32,113 @@ import os
 from babel import localedata
 import six
 
-_localedir = os.environ.get('rally'.upper() + '_LOCALEDIR')
-_t = gettext.translation('rally', localedir=_localedir, fallback=True)
-
-# We use separate translation catalogs for each log level, so set up a
-# mapping between the log level name and the translator. The domain
-# for the log level is project_name + "-log-" + log_level so messages
-# for each level end up in their own catalog.
-_t_log_levels = dict(
-    (level, gettext.translation('rally' + '-log-' + level,
-                                localedir=_localedir,
-                                fallback=True))
-    for level in ['info', 'warning', 'error', 'critical']
-)
-
 _AVAILABLE_LANGUAGES = {}
+
+# FIXME(dhellmann): Remove this when moving to rally.i18n.
 USE_LAZY = False
 
 
+class TranslatorFactory(object):
+    """Create translator functions
+    """
+
+    def __init__(self, domain, lazy=False, localedir=None):
+        """Establish a set of translation functions for the domain.
+
+        :param domain: Name of translation domain,
+                       specifying a message catalog.
+        :type domain: str
+        :param lazy: Delays translation until a message is emitted.
+                     Defaults to False.
+        :type lazy: Boolean
+        :param localedir: Directory with translation catalogs.
+        :type localedir: str
+        """
+        self.domain = domain
+        self.lazy = lazy
+        if localedir is None:
+            localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
+        self.localedir = localedir
+
+    def _make_translation_func(self, domain=None):
+        """Return a new translation function ready for use.
+
+        Takes into account whether or not lazy translation is being
+        done.
+
+        The domain can be specified to override the default from the
+        factory, but the localedir from the factory is always used
+        because we assume the log-level translation catalogs are
+        installed in the same directory as the main application
+        catalog.
+
+        """
+        if domain is None:
+            domain = self.domain
+        if self.lazy:
+            return functools.partial(Message, domain=domain)
+        t = gettext.translation(
+            domain,
+            localedir=self.localedir,
+            fallback=True,
+        )
+        if six.PY3:
+            return t.gettext
+        return t.ugettext
+
+    @property
+    def primary(self):
+        "The default translation function."
+        return self._make_translation_func()
+
+    def _make_log_translation_func(self, level):
+        return self._make_translation_func(self.domain + '-log-' + level)
+
+    @property
+    def log_info(self):
+        "Translate info-level log messages."
+        return self._make_log_translation_func('info')
+
+    @property
+    def log_warning(self):
+        "Translate warning-level log messages."
+        return self._make_log_translation_func('warning')
+
+    @property
+    def log_error(self):
+        "Translate error-level log messages."
+        return self._make_log_translation_func('error')
+
+    @property
+    def log_critical(self):
+        "Translate critical-level log messages."
+        return self._make_log_translation_func('critical')
+
+
+# NOTE(dhellmann): When this module moves out of the incubator into
+# rally.i18n, these global variables can be moved to an integration
+# module within each application.
+
+# Create the global translation functions.
+_translators = TranslatorFactory('rally')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical
+
+# NOTE(dhellmann): End of globals that will move to the application's
+# integration module.
+
+
 def enable_lazy():
     """Convenience function for configuring _() to use lazy gettext
 
@@ -58,41 +147,18 @@ def enable_lazy():
     your project is importing _ directly instead of using the
     gettextutils.install() way of importing the _ function.
     """
-    global USE_LAZY
+    # FIXME(dhellmann): This function will be removed in rally.i18n,
+    # because the TranslatorFactory makes it superfluous.
+    global _, _LI, _LW, _LE, _LC, USE_LAZY
+    tf = TranslatorFactory('rally', lazy=True)
+    _ = tf.primary
+    _LI = tf.log_info
+    _LW = tf.log_warning
+    _LE = tf.log_error
+    _LC = tf.log_critical
     USE_LAZY = True
 
 
-def _(msg):
-    if USE_LAZY:
-        return Message(msg, domain='rally')
-    else:
-        if six.PY3:
-            return _t.gettext(msg)
-        return _t.ugettext(msg)
-
-
-def _log_translation(msg, level):
-    """Build a single translation of a log message
-    """
-    if USE_LAZY:
-        return Message(msg, domain='rally' + '-log-' + level)
-    else:
-        translator = _t_log_levels[level]
-        if six.PY3:
-            return translator.gettext(msg)
-        return translator.ugettext(msg)
-
-# Translators for log levels.
-#
-# The abbreviated names are meant to reflect the usual use of a short
-# name like '_'. The "L" is for "log" and the other letter comes from
-# the level.
-_LI = functools.partial(_log_translation, level='info')
-_LW = functools.partial(_log_translation, level='warning')
-_LE = functools.partial(_log_translation, level='error')
-_LC = functools.partial(_log_translation, level='critical')
-
-
 def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
@@ -112,26 +178,9 @@ def install(domain, lazy=False):
                  any available locale.
     """
     if lazy:
-        # NOTE(mrodden): Lazy gettext functionality.
-        #
-        # The following introduces a deferred way to do translations on
-        # messages in OpenStack. We override the standard _() function
-        # and % (format string) operation to build Message objects that can
-        # later be translated when we have more information.
-        def _lazy_gettext(msg):
-            """Create and return a Message object.
-
-            Lazy gettext function for a given domain, it is a factory method
-            for a project/module to get a lazy gettext function for its own
-            translation domain (i.e. nova, glance, cinder, etc.)
-
-            Message encapsulates a string so that we can translate
-            it later when needed.
-            """
-            return Message(msg, domain=domain)
-
         from six import moves
-        moves.builtins.__dict__['_'] = _lazy_gettext
+        tf = TranslatorFactory(domain, lazy=True)
+        moves.builtins.__dict__['_'] = tf.primary
     else:
         localedir = '%s_LOCALEDIR' % domain.upper()
         if six.PY3:
@@ -274,6 +323,7 @@ class Message(six.text_type):
     def __radd__(self, other):
         return self.__add__(other)
 
+    if six.PY2:
         def __str__(self):
             # NOTE(luisg): Logging in python 2.6 tries to str() log records,
             # and it expects specifically a UnicodeError in order to proceed.

rally/openstack/common/log.py

@@ -496,10 +496,16 @@ def _find_facility_from_conf():
 class RFCSysLogHandler(logging.handlers.SysLogHandler):
     def __init__(self, *args, **kwargs):
         self.binary_name = _get_binary_name()
-        super(RFCSysLogHandler, self).__init__(*args, **kwargs)
+        # Do not use super() unless type(logging.handlers.SysLogHandler)
+        #  is 'type' (Python 2.7).
+        # Use old style calls, if the type is 'classobj' (Python 2.6)
+        logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
 
     def format(self, record):
-        msg = super(RFCSysLogHandler, self).format(record)
+        # Do not use super() unless type(logging.handlers.SysLogHandler)
+        #  is 'type' (Python 2.7).
+        # Use old style calls, if the type is 'classobj' (Python 2.6)
+        msg = logging.handlers.SysLogHandler.format(self, record)
         msg = self.binary_name + ' ' + msg
         return msg
 

tools/config/check_uptodate.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+PROJECT_NAME=${PROJECT_NAME:-rally}
+CFGFILE_NAME=${PROJECT_NAME}.conf.sample
+
+if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
+    CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
+elif [ -e etc/${CFGFILE_NAME} ]; then
+    CFGFILE=etc/${CFGFILE_NAME}
+else
+    echo "${0##*/}: can not find config file"
+    exit 1
+fi
+
+TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
+trap "rm -rf $TEMPDIR" EXIT
+
+tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
+
+if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
+then
+   echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
+   echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
+   exit 1
+fi