From 4d8c820cc7be0723333aa3d2bd5adabdce060941 Mon Sep 17 00:00:00 2001 From: Hugh Saunders Date: Mon, 28 Apr 2014 11:56:17 +0000 Subject: [PATCH] Sync code from oslo incubator Includes updates in config generator, db.sqlalchemy, gettext utils and log. New script added: tools/config/check_uptodate.sh Change-Id: I0afaeb593e1b3425d7953b2a11711801299fe1f9 --- rally/openstack/common/config/generator.py | 11 +- .../common/db/sqlalchemy/migration.py | 11 +- .../openstack/common/db/sqlalchemy/models.py | 2 +- .../openstack/common/db/sqlalchemy/session.py | 6 +- .../common/db/sqlalchemy/test_base.py | 34 ++- .../common/db/sqlalchemy/test_migrations.py | 4 +- rally/openstack/common/db/sqlalchemy/utils.py | 8 + rally/openstack/common/gettextutils.py | 194 +++++++++++------- rally/openstack/common/log.py | 10 +- tools/config/check_uptodate.sh | 25 +++ 10 files changed, 210 insertions(+), 95 deletions(-) create mode 100755 tools/config/check_uptodate.sh diff --git a/rally/openstack/common/config/generator.py b/rally/openstack/common/config/generator.py index 24652aa552..e619ad37a9 100644 --- a/rally/openstack/common/config/generator.py +++ b/rally/openstack/common/config/generator.py @@ -223,6 +223,8 @@ def _get_my_ip(): def _sanitize_default(name, value): """Set up a reasonably sensible default for pybasedir, my_ip and host.""" + hostname = socket.gethostname() + fqdn = socket.getfqdn() if value.startswith(sys.prefix): # NOTE(jd) Don't use os.path.join, because it is likely to think the # second part is an absolute pathname and therefore drop the first @@ -234,8 +236,13 @@ def _sanitize_default(name, value): return value.replace(BASEDIR, '') elif value == _get_my_ip(): return '10.0.0.1' - elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name: - return 'rally' + elif value in (hostname, fqdn): + if 'host' in name: + return 'rally' + elif value.endswith(hostname): + return value.replace(hostname, 'rally') + elif value.endswith(fqdn): + return value.replace(fqdn, 'rally') elif value.strip() != value: return '"%s"' % value return value diff --git a/rally/openstack/common/db/sqlalchemy/migration.py b/rally/openstack/common/db/sqlalchemy/migration.py index 73ef73c45f..bca094f161 100644 --- a/rally/openstack/common/db/sqlalchemy/migration.py +++ b/rally/openstack/common/db/sqlalchemy/migration.py @@ -213,8 +213,15 @@ def _db_schema_sanity_check(engine): 'where TABLE_SCHEMA=%s and ' 'TABLE_COLLATION NOT LIKE "%%utf8%%"') - table_names = [res[0] for res in engine.execute(onlyutf8_sql, - engine.url.database)] + # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic + # versioning tables from the tables we need to verify utf8 status on. + # Non-standard table names are not supported. + EXCLUDED_TABLES = ['migrate_version', 'alembic_version'] + + table_names = [res[0] for res in + engine.execute(onlyutf8_sql, engine.url.database) if + res[0].lower() not in EXCLUDED_TABLES] + if len(table_names) > 0: raise ValueError(_('Tables "%s" have non utf8 collation, ' 'please make sure all tables are CHARSET=utf8' diff --git a/rally/openstack/common/db/sqlalchemy/models.py b/rally/openstack/common/db/sqlalchemy/models.py index 46874742d3..e66f4fcd41 100644 --- a/rally/openstack/common/db/sqlalchemy/models.py +++ b/rally/openstack/common/db/sqlalchemy/models.py @@ -70,7 +70,7 @@ class ModelBase(six.Iterator): return [] def __iter__(self): - columns = dict(object_mapper(self).columns).keys() + columns = list(dict(object_mapper(self).columns).keys()) # NOTE(russellb): Allow models to specify other keys that can be looked # up, beyond the actual db columns. An example would be the 'name' # property for an Instance. diff --git a/rally/openstack/common/db/sqlalchemy/session.py b/rally/openstack/common/db/sqlalchemy/session.py index a969548572..ada41631a8 100644 --- a/rally/openstack/common/db/sqlalchemy/session.py +++ b/rally/openstack/common/db/sqlalchemy/session.py @@ -367,7 +367,7 @@ def _raise_if_duplicate_entry_error(integrity_error, engine_name): return [columns] return columns[len(uniqbase):].split("0")[1:] - if engine_name not in ["ibm_db_sa", "mysql", "sqlite", "postgresql"]: + if engine_name not in ("ibm_db_sa", "mysql", "sqlite", "postgresql"): return # FIXME(johannes): The usage of the .message attribute has been @@ -489,7 +489,7 @@ def _thread_yield(dbapi_con, con_record): def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy): - """Ensures that MySQL and DB2 connections are alive. + """Ensures that MySQL, PostgreSQL or DB2 connections are alive. Borrowed from: http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f @@ -645,7 +645,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, sqlalchemy.event.listen(engine, 'checkin', _thread_yield) - if engine.name in ['mysql', 'ibm_db_sa']: + if engine.name in ('ibm_db_sa', 'mysql', 'postgresql'): ping_callback = functools.partial(_ping_listener, engine) sqlalchemy.event.listen(engine, 'checkout', ping_callback) if engine.name == 'mysql': diff --git a/rally/openstack/common/db/sqlalchemy/test_base.py b/rally/openstack/common/db/sqlalchemy/test_base.py index a572b0dbf6..d38f71ad36 100644 --- a/rally/openstack/common/db/sqlalchemy/test_base.py +++ b/rally/openstack/common/db/sqlalchemy/test_base.py @@ -18,12 +18,12 @@ import functools import os import fixtures +from oslotest import base as test_base import six +from rally.openstack.common.db.sqlalchemy import provision from rally.openstack.common.db.sqlalchemy import session from rally.openstack.common.db.sqlalchemy import utils -from rally.openstack.common.fixture import lockutils -from tests import test class DbFixture(fixtures.Fixture): @@ -43,15 +43,17 @@ class DbFixture(fixtures.Fixture): self.test = test + def cleanUp(self): + self.test.engine.dispose() + def setUp(self): super(DbFixture, self).setUp() self.test.engine = session.create_engine(self._get_uri()) self.test.sessionmaker = session.get_maker(self.test.engine) - self.addCleanup(self.test.engine.dispose) -class DbTestCase(test.TestCase): +class DbTestCase(test_base.BaseTestCase): """Base class for testing of DB code. Using `DbFixture`. Intended to be the main database test case to use all @@ -103,11 +105,24 @@ class OpportunisticFixture(DbFixture): DRIVER = abc.abstractproperty(lambda: None) DBNAME = PASSWORD = USERNAME = 'openstack_citest' + def setUp(self): + self._provisioning_engine = provision.get_engine( + utils.get_connect_string(backend=self.DRIVER, + user=self.USERNAME, + passwd=self.PASSWORD, + database=self.DBNAME) + ) + self._uri = provision.create_database(self._provisioning_engine) + + super(OpportunisticFixture, self).setUp() + + def cleanUp(self): + super(OpportunisticFixture, self).cleanUp() + + provision.drop_database(self._provisioning_engine, self._uri) + def _get_uri(self): - return utils.get_connect_string(backend=self.DRIVER, - user=self.USERNAME, - passwd=self.PASSWORD, - database=self.DBNAME) + return self._uri @six.add_metaclass(abc.ABCMeta) @@ -121,9 +136,6 @@ class OpportunisticTestCase(DbTestCase): FIXTURE = abc.abstractproperty(lambda: None) def setUp(self): - # TODO(bnemec): Remove this once infra is ready for - # https://review.openstack.org/#/c/74963/ to merge. - self.useFixture(lockutils.LockFixture('opportunistic-db')) credentials = { 'backend': self.FIXTURE.DRIVER, 'user': self.FIXTURE.USERNAME, diff --git a/rally/openstack/common/db/sqlalchemy/test_migrations.py b/rally/openstack/common/db/sqlalchemy/test_migrations.py index ce25c6c5f6..39fec1960f 100644 --- a/rally/openstack/common/db/sqlalchemy/test_migrations.py +++ b/rally/openstack/common/db/sqlalchemy/test_migrations.py @@ -20,6 +20,7 @@ import os import subprocess import lockfile +from oslotest import base as test_base from six import moves from six.moves.urllib import parse import sqlalchemy @@ -27,7 +28,6 @@ import sqlalchemy.exc from rally.openstack.common.db.sqlalchemy import utils from rally.openstack.common.gettextutils import _LE -from tests import test LOG = logging.getLogger(__name__) @@ -68,7 +68,7 @@ def _set_db_lock(lock_path=None, lock_prefix=None): return decorator -class BaseMigrationTestCase(test.TestCase): +class BaseMigrationTestCase(test_base.BaseTestCase): """Base class fort testing of migration utils.""" def __init__(self, *args, **kwargs): diff --git a/rally/openstack/common/db/sqlalchemy/utils.py b/rally/openstack/common/db/sqlalchemy/utils.py index 7893b4bbfe..2d6f08ccce 100644 --- a/rally/openstack/common/db/sqlalchemy/utils.py +++ b/rally/openstack/common/db/sqlalchemy/utils.py @@ -254,6 +254,14 @@ def get_table(engine, name): Needed because the models don't work for us in migrations as models will be far out of sync with the current data. + + .. warning:: + + Do not use this method when creating ForeignKeys in database migrations + because sqlalchemy needs the same MetaData object to hold information + about the parent table and the reference table in the ForeignKey. This + method uses a unique MetaData object per table object so it won't work + with ForeignKey creation. """ metadata = MetaData() metadata.bind = engine diff --git a/rally/openstack/common/gettextutils.py b/rally/openstack/common/gettextutils.py index fc2285ccd9..99f6944d49 100644 --- a/rally/openstack/common/gettextutils.py +++ b/rally/openstack/common/gettextutils.py @@ -32,24 +32,113 @@ import os from babel import localedata import six -_localedir = os.environ.get('rally'.upper() + '_LOCALEDIR') -_t = gettext.translation('rally', localedir=_localedir, fallback=True) - -# We use separate translation catalogs for each log level, so set up a -# mapping between the log level name and the translator. The domain -# for the log level is project_name + "-log-" + log_level so messages -# for each level end up in their own catalog. -_t_log_levels = dict( - (level, gettext.translation('rally' + '-log-' + level, - localedir=_localedir, - fallback=True)) - for level in ['info', 'warning', 'error', 'critical'] -) - _AVAILABLE_LANGUAGES = {} + +# FIXME(dhellmann): Remove this when moving to rally.i18n. USE_LAZY = False +class TranslatorFactory(object): + """Create translator functions + """ + + def __init__(self, domain, lazy=False, localedir=None): + """Establish a set of translation functions for the domain. + + :param domain: Name of translation domain, + specifying a message catalog. + :type domain: str + :param lazy: Delays translation until a message is emitted. + Defaults to False. + :type lazy: Boolean + :param localedir: Directory with translation catalogs. + :type localedir: str + """ + self.domain = domain + self.lazy = lazy + if localedir is None: + localedir = os.environ.get(domain.upper() + '_LOCALEDIR') + self.localedir = localedir + + def _make_translation_func(self, domain=None): + """Return a new translation function ready for use. + + Takes into account whether or not lazy translation is being + done. + + The domain can be specified to override the default from the + factory, but the localedir from the factory is always used + because we assume the log-level translation catalogs are + installed in the same directory as the main application + catalog. + + """ + if domain is None: + domain = self.domain + if self.lazy: + return functools.partial(Message, domain=domain) + t = gettext.translation( + domain, + localedir=self.localedir, + fallback=True, + ) + if six.PY3: + return t.gettext + return t.ugettext + + @property + def primary(self): + "The default translation function." + return self._make_translation_func() + + def _make_log_translation_func(self, level): + return self._make_translation_func(self.domain + '-log-' + level) + + @property + def log_info(self): + "Translate info-level log messages." + return self._make_log_translation_func('info') + + @property + def log_warning(self): + "Translate warning-level log messages." + return self._make_log_translation_func('warning') + + @property + def log_error(self): + "Translate error-level log messages." + return self._make_log_translation_func('error') + + @property + def log_critical(self): + "Translate critical-level log messages." + return self._make_log_translation_func('critical') + + +# NOTE(dhellmann): When this module moves out of the incubator into +# rally.i18n, these global variables can be moved to an integration +# module within each application. + +# Create the global translation functions. +_translators = TranslatorFactory('rally') + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = _translators.log_info +_LW = _translators.log_warning +_LE = _translators.log_error +_LC = _translators.log_critical + +# NOTE(dhellmann): End of globals that will move to the application's +# integration module. + + def enable_lazy(): """Convenience function for configuring _() to use lazy gettext @@ -58,41 +147,18 @@ def enable_lazy(): your project is importing _ directly instead of using the gettextutils.install() way of importing the _ function. """ - global USE_LAZY + # FIXME(dhellmann): This function will be removed in rally.i18n, + # because the TranslatorFactory makes it superfluous. + global _, _LI, _LW, _LE, _LC, USE_LAZY + tf = TranslatorFactory('rally', lazy=True) + _ = tf.primary + _LI = tf.log_info + _LW = tf.log_warning + _LE = tf.log_error + _LC = tf.log_critical USE_LAZY = True -def _(msg): - if USE_LAZY: - return Message(msg, domain='rally') - else: - if six.PY3: - return _t.gettext(msg) - return _t.ugettext(msg) - - -def _log_translation(msg, level): - """Build a single translation of a log message - """ - if USE_LAZY: - return Message(msg, domain='rally' + '-log-' + level) - else: - translator = _t_log_levels[level] - if six.PY3: - return translator.gettext(msg) - return translator.ugettext(msg) - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = functools.partial(_log_translation, level='info') -_LW = functools.partial(_log_translation, level='warning') -_LE = functools.partial(_log_translation, level='error') -_LC = functools.partial(_log_translation, level='critical') - - def install(domain, lazy=False): """Install a _() function using the given translation domain. @@ -112,26 +178,9 @@ def install(domain, lazy=False): any available locale. """ if lazy: - # NOTE(mrodden): Lazy gettext functionality. - # - # The following introduces a deferred way to do translations on - # messages in OpenStack. We override the standard _() function - # and % (format string) operation to build Message objects that can - # later be translated when we have more information. - def _lazy_gettext(msg): - """Create and return a Message object. - - Lazy gettext function for a given domain, it is a factory method - for a project/module to get a lazy gettext function for its own - translation domain (i.e. nova, glance, cinder, etc.) - - Message encapsulates a string so that we can translate - it later when needed. - """ - return Message(msg, domain=domain) - from six import moves - moves.builtins.__dict__['_'] = _lazy_gettext + tf = TranslatorFactory(domain, lazy=True) + moves.builtins.__dict__['_'] = tf.primary else: localedir = '%s_LOCALEDIR' % domain.upper() if six.PY3: @@ -274,13 +323,14 @@ class Message(six.text_type): def __radd__(self, other): return self.__add__(other) - def __str__(self): - # NOTE(luisg): Logging in python 2.6 tries to str() log records, - # and it expects specifically a UnicodeError in order to proceed. - msg = _('Message objects do not support str() because they may ' - 'contain non-ascii characters. ' - 'Please use unicode() or translate() instead.') - raise UnicodeError(msg) + if six.PY2: + def __str__(self): + # NOTE(luisg): Logging in python 2.6 tries to str() log records, + # and it expects specifically a UnicodeError in order to proceed. + msg = _('Message objects do not support str() because they may ' + 'contain non-ascii characters. ' + 'Please use unicode() or translate() instead.') + raise UnicodeError(msg) def get_available_languages(domain): diff --git a/rally/openstack/common/log.py b/rally/openstack/common/log.py index 084c69d5d6..eeefc5168e 100644 --- a/rally/openstack/common/log.py +++ b/rally/openstack/common/log.py @@ -496,10 +496,16 @@ def _find_facility_from_conf(): class RFCSysLogHandler(logging.handlers.SysLogHandler): def __init__(self, *args, **kwargs): self.binary_name = _get_binary_name() - super(RFCSysLogHandler, self).__init__(*args, **kwargs) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + logging.handlers.SysLogHandler.__init__(self, *args, **kwargs) def format(self, record): - msg = super(RFCSysLogHandler, self).format(record) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + msg = logging.handlers.SysLogHandler.format(self, record) msg = self.binary_name + ' ' + msg return msg diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh new file mode 100755 index 0000000000..b3c6c6335a --- /dev/null +++ b/tools/config/check_uptodate.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +PROJECT_NAME=${PROJECT_NAME:-rally} +CFGFILE_NAME=${PROJECT_NAME}.conf.sample + +if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then + CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} +elif [ -e etc/${CFGFILE_NAME} ]; then + CFGFILE=etc/${CFGFILE_NAME} +else + echo "${0##*/}: can not find config file" + exit 1 +fi + +TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` +trap "rm -rf $TEMPDIR" EXIT + +tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} + +if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} +then + echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." + echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh." + exit 1 +fi