From fe8cb71bd97c6af1bae895c9c0aa07edfd638828 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Fri, 18 Apr 2014 17:04:15 -0500 Subject: [PATCH] Sync with oslo-incubator 74ae271 This syncs Keystone with oslo-incubator commit hash 74ae2713baf82f0155b71c6e528747f644d73355 . In keystone: $ rm -r keystone/openstack/common In oslo-incubator: $ python update.py ../keystone Commits ------- f61a4e7 Fix python26 compatibility for RFCSysLogHandler 54f7e7f Prevent races in opportunistic db test cases 8a0f581 Use oslotest instead of common test module 7f307c2 generator: sanitize values ending with the fqdn 4a591ea Start ping listener also for postgresql f0e50ed Add a warning to not use get_table for working with ForeignKeys fd18c28 Fix safe_encode(): return bytes on Python 3 f0dd798 Remove rendundant parentheses of cfg help strings 6cc96d0 Fix test_gettextutils on Python 3 85d1ce6 Python 3: enable tests/unit/middleware/test_request_id.py 295fcd9 Do not use the 'extend' method on a dict_items object Change-Id: I90dabcfce78544ceeae7d92f5e5b90d854c5da8d --- keystone/openstack/common/config/generator.py | 11 ++- keystone/openstack/common/context.py | 2 +- .../openstack/common/db/sqlalchemy/models.py | 2 +- .../openstack/common/db/sqlalchemy/session.py | 6 +- .../common/db/sqlalchemy/test_base.py | 34 ++++--- .../common/db/sqlalchemy/test_migrations.py | 4 +- .../openstack/common/db/sqlalchemy/utils.py | 8 ++ .../openstack/common/fixture/mockpatch.py | 2 +- .../openstack/common/fixture/moxstubout.py | 2 +- keystone/openstack/common/gettextutils.py | 15 +-- keystone/openstack/common/lockutils.py | 2 +- keystone/openstack/common/log.py | 10 +- keystone/openstack/common/strutils.py | 14 +-- keystone/openstack/common/test.py | 99 ------------------- 14 files changed, 70 insertions(+), 141 deletions(-) delete mode 100644 keystone/openstack/common/test.py diff --git a/keystone/openstack/common/config/generator.py b/keystone/openstack/common/config/generator.py index 2d3a5b7a7..39707b34b 100644 --- a/keystone/openstack/common/config/generator.py +++ b/keystone/openstack/common/config/generator.py @@ -223,6 +223,8 @@ def _get_my_ip(): def _sanitize_default(name, value): """Set up a reasonably sensible default for pybasedir, my_ip and host.""" + hostname = socket.gethostname() + fqdn = socket.getfqdn() if value.startswith(sys.prefix): # NOTE(jd) Don't use os.path.join, because it is likely to think the # second part is an absolute pathname and therefore drop the first @@ -234,8 +236,13 @@ def _sanitize_default(name, value): return value.replace(BASEDIR, '') elif value == _get_my_ip(): return '10.0.0.1' - elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name: - return 'keystone' + elif value in (hostname, fqdn): + if 'host' in name: + return 'keystone' + elif value.endswith(hostname): + return value.replace(hostname, 'keystone') + elif value.endswith(fqdn): + return value.replace(fqdn, 'keystone') elif value.strip() != value: return '"%s"' % value return value diff --git a/keystone/openstack/common/context.py b/keystone/openstack/common/context.py index 09019ee38..3eeb445e4 100644 --- a/keystone/openstack/common/context.py +++ b/keystone/openstack/common/context.py @@ -25,7 +25,7 @@ import uuid def generate_request_id(): - return 'req-%s' % str(uuid.uuid4()) + return b'req-' + str(uuid.uuid4()).encode('ascii') class RequestContext(object): diff --git a/keystone/openstack/common/db/sqlalchemy/models.py b/keystone/openstack/common/db/sqlalchemy/models.py index c22866744..930bf31c9 100644 --- a/keystone/openstack/common/db/sqlalchemy/models.py +++ b/keystone/openstack/common/db/sqlalchemy/models.py @@ -70,7 +70,7 @@ class ModelBase(six.Iterator): return [] def __iter__(self): - columns = dict(object_mapper(self).columns).keys() + columns = list(dict(object_mapper(self).columns).keys()) # NOTE(russellb): Allow models to specify other keys that can be looked # up, beyond the actual db columns. An example would be the 'name' # property for an Instance. diff --git a/keystone/openstack/common/db/sqlalchemy/session.py b/keystone/openstack/common/db/sqlalchemy/session.py index 87da4eb6a..00b9f533e 100644 --- a/keystone/openstack/common/db/sqlalchemy/session.py +++ b/keystone/openstack/common/db/sqlalchemy/session.py @@ -367,7 +367,7 @@ def _raise_if_duplicate_entry_error(integrity_error, engine_name): return [columns] return columns[len(uniqbase):].split("0")[1:] - if engine_name not in ["ibm_db_sa", "mysql", "sqlite", "postgresql"]: + if engine_name not in ("ibm_db_sa", "mysql", "sqlite", "postgresql"): return # FIXME(johannes): The usage of the .message attribute has been @@ -489,7 +489,7 @@ def _thread_yield(dbapi_con, con_record): def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy): - """Ensures that MySQL and DB2 connections are alive. + """Ensures that MySQL, PostgreSQL or DB2 connections are alive. Borrowed from: http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f @@ -645,7 +645,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, sqlalchemy.event.listen(engine, 'checkin', _thread_yield) - if engine.name in ['mysql', 'ibm_db_sa']: + if engine.name in ('ibm_db_sa', 'mysql', 'postgresql'): ping_callback = functools.partial(_ping_listener, engine) sqlalchemy.event.listen(engine, 'checkout', ping_callback) if engine.name == 'mysql': diff --git a/keystone/openstack/common/db/sqlalchemy/test_base.py b/keystone/openstack/common/db/sqlalchemy/test_base.py index ff8d1facc..58c00ccd0 100644 --- a/keystone/openstack/common/db/sqlalchemy/test_base.py +++ b/keystone/openstack/common/db/sqlalchemy/test_base.py @@ -18,12 +18,12 @@ import functools import os import fixtures +from oslotest import base as test_base import six +from keystone.openstack.common.db.sqlalchemy import provision from keystone.openstack.common.db.sqlalchemy import session from keystone.openstack.common.db.sqlalchemy import utils -from keystone.openstack.common.fixture import lockutils -from keystone.openstack.common import test class DbFixture(fixtures.Fixture): @@ -43,15 +43,17 @@ class DbFixture(fixtures.Fixture): self.test = test + def cleanUp(self): + self.test.engine.dispose() + def setUp(self): super(DbFixture, self).setUp() self.test.engine = session.create_engine(self._get_uri()) self.test.sessionmaker = session.get_maker(self.test.engine) - self.addCleanup(self.test.engine.dispose) -class DbTestCase(test.BaseTestCase): +class DbTestCase(test_base.BaseTestCase): """Base class for testing of DB code. Using `DbFixture`. Intended to be the main database test case to use all @@ -103,11 +105,24 @@ class OpportunisticFixture(DbFixture): DRIVER = abc.abstractproperty(lambda: None) DBNAME = PASSWORD = USERNAME = 'openstack_citest' + def setUp(self): + self._provisioning_engine = provision.get_engine( + utils.get_connect_string(backend=self.DRIVER, + user=self.USERNAME, + passwd=self.PASSWORD, + database=self.DBNAME) + ) + self._uri = provision.create_database(self._provisioning_engine) + + super(OpportunisticFixture, self).setUp() + + def cleanUp(self): + super(OpportunisticFixture, self).cleanUp() + + provision.drop_database(self._provisioning_engine, self._uri) + def _get_uri(self): - return utils.get_connect_string(backend=self.DRIVER, - user=self.USERNAME, - passwd=self.PASSWORD, - database=self.DBNAME) + return self._uri @six.add_metaclass(abc.ABCMeta) @@ -121,9 +136,6 @@ class OpportunisticTestCase(DbTestCase): FIXTURE = abc.abstractproperty(lambda: None) def setUp(self): - # TODO(bnemec): Remove this once infra is ready for - # https://review.openstack.org/#/c/74963/ to merge. - self.useFixture(lockutils.LockFixture('opportunistic-db')) credentials = { 'backend': self.FIXTURE.DRIVER, 'user': self.FIXTURE.USERNAME, diff --git a/keystone/openstack/common/db/sqlalchemy/test_migrations.py b/keystone/openstack/common/db/sqlalchemy/test_migrations.py index 7ea6af9e6..a70c80c77 100644 --- a/keystone/openstack/common/db/sqlalchemy/test_migrations.py +++ b/keystone/openstack/common/db/sqlalchemy/test_migrations.py @@ -20,6 +20,7 @@ import os import subprocess import lockfile +from oslotest import base as test_base from six import moves from six.moves.urllib import parse import sqlalchemy @@ -27,7 +28,6 @@ import sqlalchemy.exc from keystone.openstack.common.db.sqlalchemy import utils from keystone.openstack.common.gettextutils import _LE -from keystone.openstack.common import test LOG = logging.getLogger(__name__) @@ -68,7 +68,7 @@ def _set_db_lock(lock_path=None, lock_prefix=None): return decorator -class BaseMigrationTestCase(test.BaseTestCase): +class BaseMigrationTestCase(test_base.BaseTestCase): """Base class fort testing of migration utils.""" def __init__(self, *args, **kwargs): diff --git a/keystone/openstack/common/db/sqlalchemy/utils.py b/keystone/openstack/common/db/sqlalchemy/utils.py index f21f8a679..85b3727f3 100644 --- a/keystone/openstack/common/db/sqlalchemy/utils.py +++ b/keystone/openstack/common/db/sqlalchemy/utils.py @@ -254,6 +254,14 @@ def get_table(engine, name): Needed because the models don't work for us in migrations as models will be far out of sync with the current data. + + .. warning:: + + Do not use this method when creating ForeignKeys in database migrations + because sqlalchemy needs the same MetaData object to hold information + about the parent table and the reference table in the ForeignKey. This + method uses a unique MetaData object per table object so it won't work + with ForeignKey creation. """ metadata = MetaData() metadata.bind = engine diff --git a/keystone/openstack/common/fixture/mockpatch.py b/keystone/openstack/common/fixture/mockpatch.py index 7e0c84c28..4ca6bea6e 100644 --- a/keystone/openstack/common/fixture/mockpatch.py +++ b/keystone/openstack/common/fixture/mockpatch.py @@ -20,7 +20,7 @@ ## ## DO NOT MODIFY THIS FILE ## -## This file is being graduated to the keystonetest library. Please make all +## This file is being graduated to the oslotest library. Please make all ## changes there, and only backport critical fixes here. - dhellmann ## ############################################################################## diff --git a/keystone/openstack/common/fixture/moxstubout.py b/keystone/openstack/common/fixture/moxstubout.py index 2cf6fb9d0..c76c4bdeb 100644 --- a/keystone/openstack/common/fixture/moxstubout.py +++ b/keystone/openstack/common/fixture/moxstubout.py @@ -20,7 +20,7 @@ ## ## DO NOT MODIFY THIS FILE ## -## This file is being graduated to the keystonetest library. Please make all +## This file is being graduated to the oslotest library. Please make all ## changes there, and only backport critical fixes here. - dhellmann ## ############################################################################## diff --git a/keystone/openstack/common/gettextutils.py b/keystone/openstack/common/gettextutils.py index 312118751..701194fd1 100644 --- a/keystone/openstack/common/gettextutils.py +++ b/keystone/openstack/common/gettextutils.py @@ -274,13 +274,14 @@ class Message(six.text_type): def __radd__(self, other): return self.__add__(other) - def __str__(self): - # NOTE(luisg): Logging in python 2.6 tries to str() log records, - # and it expects specifically a UnicodeError in order to proceed. - msg = _('Message objects do not support str() because they may ' - 'contain non-ascii characters. ' - 'Please use unicode() or translate() instead.') - raise UnicodeError(msg) + if six.PY2: + def __str__(self): + # NOTE(luisg): Logging in python 2.6 tries to str() log records, + # and it expects specifically a UnicodeError in order to proceed. + msg = _('Message objects do not support str() because they may ' + 'contain non-ascii characters. ' + 'Please use unicode() or translate() instead.') + raise UnicodeError(msg) def get_available_languages(domain): diff --git a/keystone/openstack/common/lockutils.py b/keystone/openstack/common/lockutils.py index 7af34c867..03c57ad1c 100644 --- a/keystone/openstack/common/lockutils.py +++ b/keystone/openstack/common/lockutils.py @@ -41,7 +41,7 @@ util_opts = [ help='Whether to disable inter-process locks'), cfg.StrOpt('lock_path', default=os.environ.get("KEYSTONE_LOCK_PATH"), - help=('Directory to use for lock files.')) + help='Directory to use for lock files.') ] diff --git a/keystone/openstack/common/log.py b/keystone/openstack/common/log.py index 03a0d124e..5e4206adc 100644 --- a/keystone/openstack/common/log.py +++ b/keystone/openstack/common/log.py @@ -496,10 +496,16 @@ def _find_facility_from_conf(): class RFCSysLogHandler(logging.handlers.SysLogHandler): def __init__(self, *args, **kwargs): self.binary_name = _get_binary_name() - super(RFCSysLogHandler, self).__init__(*args, **kwargs) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + logging.handlers.SysLogHandler.__init__(self, *args, **kwargs) def format(self, record): - msg = super(RFCSysLogHandler, self).format(record) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + msg = logging.handlers.SysLogHandler.format(self, record) msg = self.binary_name + ' ' + msg return msg diff --git a/keystone/openstack/common/strutils.py b/keystone/openstack/common/strutils.py index 09222f45d..4a963feaa 100644 --- a/keystone/openstack/common/strutils.py +++ b/keystone/openstack/common/strutils.py @@ -159,19 +159,13 @@ def safe_encode(text, incoming=None, sys.getdefaultencoding()) if isinstance(text, six.text_type): - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) + return text.encode(encoding, errors) elif text and encoding != incoming: # Decode text before encoding it with `encoding` text = safe_decode(text, incoming, errors) - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) - - return text + return text.encode(encoding, errors) + else: + return text def string_to_bytes(text, unit_system='IEC', return_int=False): diff --git a/keystone/openstack/common/test.py b/keystone/openstack/common/test.py deleted file mode 100644 index 4ab5e9262..000000000 --- a/keystone/openstack/common/test.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -############################################################################## -############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the keystonetest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## -############################################################################## -############################################################################## - -"""Common utilities used in testing""" - -import logging -import os -import tempfile - -import fixtures -import testtools - -_TRUE_VALUES = ('True', 'true', '1', 'yes') -_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s" - - -class BaseTestCase(testtools.TestCase): - - def setUp(self): - super(BaseTestCase, self).setUp() - self._set_timeout() - self._fake_output() - self._fake_logs() - self.useFixture(fixtures.NestedTempfile()) - self.useFixture(fixtures.TempHomeDir()) - self.tempdirs = [] - - def _set_timeout(self): - test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) - try: - test_timeout = int(test_timeout) - except ValueError: - # If timeout value is invalid do not set a timeout. - test_timeout = 0 - if test_timeout > 0: - self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) - - def _fake_output(self): - if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES: - stdout = self.useFixture(fixtures.StringStream('stdout')).stream - self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) - if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: - stderr = self.useFixture(fixtures.StringStream('stderr')).stream - self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) - - def _fake_logs(self): - if os.environ.get('OS_DEBUG') in _TRUE_VALUES: - level = logging.DEBUG - else: - level = logging.INFO - capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES - if capture_logs: - self.useFixture( - fixtures.FakeLogger( - format=_LOG_FORMAT, - level=level, - nuke_handlers=capture_logs, - ) - ) - else: - logging.basicConfig(format=_LOG_FORMAT, level=level) - - def create_tempfiles(self, files, ext='.conf'): - tempfiles = [] - for (basename, contents) in files: - if not os.path.isabs(basename): - (fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext) - else: - path = basename + ext - fd = os.open(path, os.O_CREAT | os.O_WRONLY) - tempfiles.append(path) - try: - os.write(fd, contents) - finally: - os.close(fd) - return tempfiles