Sync with oslo-incubator 74ae271
This syncs Keystone with oslo-incubator commit hash 74ae2713baf82f0155b71c6e528747f644d73355 . In keystone: $ rm -r keystone/openstack/common In oslo-incubator: $ python update.py ../keystone Commits ------- f61a4e7 Fix python26 compatibility for RFCSysLogHandler 54f7e7f Prevent races in opportunistic db test cases 8a0f581 Use oslotest instead of common test module 7f307c2 generator: sanitize values ending with the fqdn 4a591ea Start ping listener also for postgresql f0e50ed Add a warning to not use get_table for working with ForeignKeys fd18c28 Fix safe_encode(): return bytes on Python 3 f0dd798 Remove rendundant parentheses of cfg help strings 6cc96d0 Fix test_gettextutils on Python 3 85d1ce6 Python 3: enable tests/unit/middleware/test_request_id.py 295fcd9 Do not use the 'extend' method on a dict_items object Change-Id: I90dabcfce78544ceeae7d92f5e5b90d854c5da8d
This commit is contained in:
parent
4907779e1c
commit
fe8cb71bd9
@ -223,6 +223,8 @@ def _get_my_ip():
|
|||||||
|
|
||||||
def _sanitize_default(name, value):
|
def _sanitize_default(name, value):
|
||||||
"""Set up a reasonably sensible default for pybasedir, my_ip and host."""
|
"""Set up a reasonably sensible default for pybasedir, my_ip and host."""
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
fqdn = socket.getfqdn()
|
||||||
if value.startswith(sys.prefix):
|
if value.startswith(sys.prefix):
|
||||||
# NOTE(jd) Don't use os.path.join, because it is likely to think the
|
# NOTE(jd) Don't use os.path.join, because it is likely to think the
|
||||||
# second part is an absolute pathname and therefore drop the first
|
# second part is an absolute pathname and therefore drop the first
|
||||||
@ -234,8 +236,13 @@ def _sanitize_default(name, value):
|
|||||||
return value.replace(BASEDIR, '')
|
return value.replace(BASEDIR, '')
|
||||||
elif value == _get_my_ip():
|
elif value == _get_my_ip():
|
||||||
return '10.0.0.1'
|
return '10.0.0.1'
|
||||||
elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name:
|
elif value in (hostname, fqdn):
|
||||||
return 'keystone'
|
if 'host' in name:
|
||||||
|
return 'keystone'
|
||||||
|
elif value.endswith(hostname):
|
||||||
|
return value.replace(hostname, 'keystone')
|
||||||
|
elif value.endswith(fqdn):
|
||||||
|
return value.replace(fqdn, 'keystone')
|
||||||
elif value.strip() != value:
|
elif value.strip() != value:
|
||||||
return '"%s"' % value
|
return '"%s"' % value
|
||||||
return value
|
return value
|
||||||
|
@ -25,7 +25,7 @@ import uuid
|
|||||||
|
|
||||||
|
|
||||||
def generate_request_id():
|
def generate_request_id():
|
||||||
return 'req-%s' % str(uuid.uuid4())
|
return b'req-' + str(uuid.uuid4()).encode('ascii')
|
||||||
|
|
||||||
|
|
||||||
class RequestContext(object):
|
class RequestContext(object):
|
||||||
|
@ -70,7 +70,7 @@ class ModelBase(six.Iterator):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
columns = dict(object_mapper(self).columns).keys()
|
columns = list(dict(object_mapper(self).columns).keys())
|
||||||
# NOTE(russellb): Allow models to specify other keys that can be looked
|
# NOTE(russellb): Allow models to specify other keys that can be looked
|
||||||
# up, beyond the actual db columns. An example would be the 'name'
|
# up, beyond the actual db columns. An example would be the 'name'
|
||||||
# property for an Instance.
|
# property for an Instance.
|
||||||
|
@ -367,7 +367,7 @@ def _raise_if_duplicate_entry_error(integrity_error, engine_name):
|
|||||||
return [columns]
|
return [columns]
|
||||||
return columns[len(uniqbase):].split("0")[1:]
|
return columns[len(uniqbase):].split("0")[1:]
|
||||||
|
|
||||||
if engine_name not in ["ibm_db_sa", "mysql", "sqlite", "postgresql"]:
|
if engine_name not in ("ibm_db_sa", "mysql", "sqlite", "postgresql"):
|
||||||
return
|
return
|
||||||
|
|
||||||
# FIXME(johannes): The usage of the .message attribute has been
|
# FIXME(johannes): The usage of the .message attribute has been
|
||||||
@ -489,7 +489,7 @@ def _thread_yield(dbapi_con, con_record):
|
|||||||
|
|
||||||
|
|
||||||
def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
|
def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
|
||||||
"""Ensures that MySQL and DB2 connections are alive.
|
"""Ensures that MySQL, PostgreSQL or DB2 connections are alive.
|
||||||
|
|
||||||
Borrowed from:
|
Borrowed from:
|
||||||
http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
|
http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
|
||||||
@ -645,7 +645,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None,
|
|||||||
|
|
||||||
sqlalchemy.event.listen(engine, 'checkin', _thread_yield)
|
sqlalchemy.event.listen(engine, 'checkin', _thread_yield)
|
||||||
|
|
||||||
if engine.name in ['mysql', 'ibm_db_sa']:
|
if engine.name in ('ibm_db_sa', 'mysql', 'postgresql'):
|
||||||
ping_callback = functools.partial(_ping_listener, engine)
|
ping_callback = functools.partial(_ping_listener, engine)
|
||||||
sqlalchemy.event.listen(engine, 'checkout', ping_callback)
|
sqlalchemy.event.listen(engine, 'checkout', ping_callback)
|
||||||
if engine.name == 'mysql':
|
if engine.name == 'mysql':
|
||||||
|
@ -18,12 +18,12 @@ import functools
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
import fixtures
|
import fixtures
|
||||||
|
from oslotest import base as test_base
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
from keystone.openstack.common.db.sqlalchemy import provision
|
||||||
from keystone.openstack.common.db.sqlalchemy import session
|
from keystone.openstack.common.db.sqlalchemy import session
|
||||||
from keystone.openstack.common.db.sqlalchemy import utils
|
from keystone.openstack.common.db.sqlalchemy import utils
|
||||||
from keystone.openstack.common.fixture import lockutils
|
|
||||||
from keystone.openstack.common import test
|
|
||||||
|
|
||||||
|
|
||||||
class DbFixture(fixtures.Fixture):
|
class DbFixture(fixtures.Fixture):
|
||||||
@ -43,15 +43,17 @@ class DbFixture(fixtures.Fixture):
|
|||||||
|
|
||||||
self.test = test
|
self.test = test
|
||||||
|
|
||||||
|
def cleanUp(self):
|
||||||
|
self.test.engine.dispose()
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(DbFixture, self).setUp()
|
super(DbFixture, self).setUp()
|
||||||
|
|
||||||
self.test.engine = session.create_engine(self._get_uri())
|
self.test.engine = session.create_engine(self._get_uri())
|
||||||
self.test.sessionmaker = session.get_maker(self.test.engine)
|
self.test.sessionmaker = session.get_maker(self.test.engine)
|
||||||
self.addCleanup(self.test.engine.dispose)
|
|
||||||
|
|
||||||
|
|
||||||
class DbTestCase(test.BaseTestCase):
|
class DbTestCase(test_base.BaseTestCase):
|
||||||
"""Base class for testing of DB code.
|
"""Base class for testing of DB code.
|
||||||
|
|
||||||
Using `DbFixture`. Intended to be the main database test case to use all
|
Using `DbFixture`. Intended to be the main database test case to use all
|
||||||
@ -103,11 +105,24 @@ class OpportunisticFixture(DbFixture):
|
|||||||
DRIVER = abc.abstractproperty(lambda: None)
|
DRIVER = abc.abstractproperty(lambda: None)
|
||||||
DBNAME = PASSWORD = USERNAME = 'openstack_citest'
|
DBNAME = PASSWORD = USERNAME = 'openstack_citest'
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._provisioning_engine = provision.get_engine(
|
||||||
|
utils.get_connect_string(backend=self.DRIVER,
|
||||||
|
user=self.USERNAME,
|
||||||
|
passwd=self.PASSWORD,
|
||||||
|
database=self.DBNAME)
|
||||||
|
)
|
||||||
|
self._uri = provision.create_database(self._provisioning_engine)
|
||||||
|
|
||||||
|
super(OpportunisticFixture, self).setUp()
|
||||||
|
|
||||||
|
def cleanUp(self):
|
||||||
|
super(OpportunisticFixture, self).cleanUp()
|
||||||
|
|
||||||
|
provision.drop_database(self._provisioning_engine, self._uri)
|
||||||
|
|
||||||
def _get_uri(self):
|
def _get_uri(self):
|
||||||
return utils.get_connect_string(backend=self.DRIVER,
|
return self._uri
|
||||||
user=self.USERNAME,
|
|
||||||
passwd=self.PASSWORD,
|
|
||||||
database=self.DBNAME)
|
|
||||||
|
|
||||||
|
|
||||||
@six.add_metaclass(abc.ABCMeta)
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
@ -121,9 +136,6 @@ class OpportunisticTestCase(DbTestCase):
|
|||||||
FIXTURE = abc.abstractproperty(lambda: None)
|
FIXTURE = abc.abstractproperty(lambda: None)
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
# TODO(bnemec): Remove this once infra is ready for
|
|
||||||
# https://review.openstack.org/#/c/74963/ to merge.
|
|
||||||
self.useFixture(lockutils.LockFixture('opportunistic-db'))
|
|
||||||
credentials = {
|
credentials = {
|
||||||
'backend': self.FIXTURE.DRIVER,
|
'backend': self.FIXTURE.DRIVER,
|
||||||
'user': self.FIXTURE.USERNAME,
|
'user': self.FIXTURE.USERNAME,
|
||||||
|
@ -20,6 +20,7 @@ import os
|
|||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
import lockfile
|
import lockfile
|
||||||
|
from oslotest import base as test_base
|
||||||
from six import moves
|
from six import moves
|
||||||
from six.moves.urllib import parse
|
from six.moves.urllib import parse
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
@ -27,7 +28,6 @@ import sqlalchemy.exc
|
|||||||
|
|
||||||
from keystone.openstack.common.db.sqlalchemy import utils
|
from keystone.openstack.common.db.sqlalchemy import utils
|
||||||
from keystone.openstack.common.gettextutils import _LE
|
from keystone.openstack.common.gettextutils import _LE
|
||||||
from keystone.openstack.common import test
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -68,7 +68,7 @@ def _set_db_lock(lock_path=None, lock_prefix=None):
|
|||||||
return decorator
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
class BaseMigrationTestCase(test.BaseTestCase):
|
class BaseMigrationTestCase(test_base.BaseTestCase):
|
||||||
"""Base class fort testing of migration utils."""
|
"""Base class fort testing of migration utils."""
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
@ -254,6 +254,14 @@ def get_table(engine, name):
|
|||||||
|
|
||||||
Needed because the models don't work for us in migrations
|
Needed because the models don't work for us in migrations
|
||||||
as models will be far out of sync with the current data.
|
as models will be far out of sync with the current data.
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
Do not use this method when creating ForeignKeys in database migrations
|
||||||
|
because sqlalchemy needs the same MetaData object to hold information
|
||||||
|
about the parent table and the reference table in the ForeignKey. This
|
||||||
|
method uses a unique MetaData object per table object so it won't work
|
||||||
|
with ForeignKey creation.
|
||||||
"""
|
"""
|
||||||
metadata = MetaData()
|
metadata = MetaData()
|
||||||
metadata.bind = engine
|
metadata.bind = engine
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
##
|
##
|
||||||
## DO NOT MODIFY THIS FILE
|
## DO NOT MODIFY THIS FILE
|
||||||
##
|
##
|
||||||
## This file is being graduated to the keystonetest library. Please make all
|
## This file is being graduated to the oslotest library. Please make all
|
||||||
## changes there, and only backport critical fixes here. - dhellmann
|
## changes there, and only backport critical fixes here. - dhellmann
|
||||||
##
|
##
|
||||||
##############################################################################
|
##############################################################################
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
##
|
##
|
||||||
## DO NOT MODIFY THIS FILE
|
## DO NOT MODIFY THIS FILE
|
||||||
##
|
##
|
||||||
## This file is being graduated to the keystonetest library. Please make all
|
## This file is being graduated to the oslotest library. Please make all
|
||||||
## changes there, and only backport critical fixes here. - dhellmann
|
## changes there, and only backport critical fixes here. - dhellmann
|
||||||
##
|
##
|
||||||
##############################################################################
|
##############################################################################
|
||||||
|
@ -274,13 +274,14 @@ class Message(six.text_type):
|
|||||||
def __radd__(self, other):
|
def __radd__(self, other):
|
||||||
return self.__add__(other)
|
return self.__add__(other)
|
||||||
|
|
||||||
def __str__(self):
|
if six.PY2:
|
||||||
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
|
def __str__(self):
|
||||||
# and it expects specifically a UnicodeError in order to proceed.
|
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
|
||||||
msg = _('Message objects do not support str() because they may '
|
# and it expects specifically a UnicodeError in order to proceed.
|
||||||
'contain non-ascii characters. '
|
msg = _('Message objects do not support str() because they may '
|
||||||
'Please use unicode() or translate() instead.')
|
'contain non-ascii characters. '
|
||||||
raise UnicodeError(msg)
|
'Please use unicode() or translate() instead.')
|
||||||
|
raise UnicodeError(msg)
|
||||||
|
|
||||||
|
|
||||||
def get_available_languages(domain):
|
def get_available_languages(domain):
|
||||||
|
@ -41,7 +41,7 @@ util_opts = [
|
|||||||
help='Whether to disable inter-process locks'),
|
help='Whether to disable inter-process locks'),
|
||||||
cfg.StrOpt('lock_path',
|
cfg.StrOpt('lock_path',
|
||||||
default=os.environ.get("KEYSTONE_LOCK_PATH"),
|
default=os.environ.get("KEYSTONE_LOCK_PATH"),
|
||||||
help=('Directory to use for lock files.'))
|
help='Directory to use for lock files.')
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@ -496,10 +496,16 @@ def _find_facility_from_conf():
|
|||||||
class RFCSysLogHandler(logging.handlers.SysLogHandler):
|
class RFCSysLogHandler(logging.handlers.SysLogHandler):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.binary_name = _get_binary_name()
|
self.binary_name = _get_binary_name()
|
||||||
super(RFCSysLogHandler, self).__init__(*args, **kwargs)
|
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||||
|
# is 'type' (Python 2.7).
|
||||||
|
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||||
|
logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
|
||||||
|
|
||||||
def format(self, record):
|
def format(self, record):
|
||||||
msg = super(RFCSysLogHandler, self).format(record)
|
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||||
|
# is 'type' (Python 2.7).
|
||||||
|
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||||
|
msg = logging.handlers.SysLogHandler.format(self, record)
|
||||||
msg = self.binary_name + ' ' + msg
|
msg = self.binary_name + ' ' + msg
|
||||||
return msg
|
return msg
|
||||||
|
|
||||||
|
@ -159,19 +159,13 @@ def safe_encode(text, incoming=None,
|
|||||||
sys.getdefaultencoding())
|
sys.getdefaultencoding())
|
||||||
|
|
||||||
if isinstance(text, six.text_type):
|
if isinstance(text, six.text_type):
|
||||||
if six.PY3:
|
return text.encode(encoding, errors)
|
||||||
return text.encode(encoding, errors).decode(incoming)
|
|
||||||
else:
|
|
||||||
return text.encode(encoding, errors)
|
|
||||||
elif text and encoding != incoming:
|
elif text and encoding != incoming:
|
||||||
# Decode text before encoding it with `encoding`
|
# Decode text before encoding it with `encoding`
|
||||||
text = safe_decode(text, incoming, errors)
|
text = safe_decode(text, incoming, errors)
|
||||||
if six.PY3:
|
return text.encode(encoding, errors)
|
||||||
return text.encode(encoding, errors).decode(incoming)
|
else:
|
||||||
else:
|
return text
|
||||||
return text.encode(encoding, errors)
|
|
||||||
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
||||||
|
@ -1,99 +0,0 @@
|
|||||||
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
##############################################################################
|
|
||||||
##############################################################################
|
|
||||||
##
|
|
||||||
## DO NOT MODIFY THIS FILE
|
|
||||||
##
|
|
||||||
## This file is being graduated to the keystonetest library. Please make all
|
|
||||||
## changes there, and only backport critical fixes here. - dhellmann
|
|
||||||
##
|
|
||||||
##############################################################################
|
|
||||||
##############################################################################
|
|
||||||
|
|
||||||
"""Common utilities used in testing"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
import fixtures
|
|
||||||
import testtools
|
|
||||||
|
|
||||||
_TRUE_VALUES = ('True', 'true', '1', 'yes')
|
|
||||||
_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s"
|
|
||||||
|
|
||||||
|
|
||||||
class BaseTestCase(testtools.TestCase):
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
super(BaseTestCase, self).setUp()
|
|
||||||
self._set_timeout()
|
|
||||||
self._fake_output()
|
|
||||||
self._fake_logs()
|
|
||||||
self.useFixture(fixtures.NestedTempfile())
|
|
||||||
self.useFixture(fixtures.TempHomeDir())
|
|
||||||
self.tempdirs = []
|
|
||||||
|
|
||||||
def _set_timeout(self):
|
|
||||||
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
|
|
||||||
try:
|
|
||||||
test_timeout = int(test_timeout)
|
|
||||||
except ValueError:
|
|
||||||
# If timeout value is invalid do not set a timeout.
|
|
||||||
test_timeout = 0
|
|
||||||
if test_timeout > 0:
|
|
||||||
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
|
|
||||||
|
|
||||||
def _fake_output(self):
|
|
||||||
if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
|
|
||||||
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
|
|
||||||
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
|
|
||||||
if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
|
|
||||||
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
|
|
||||||
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
|
|
||||||
|
|
||||||
def _fake_logs(self):
|
|
||||||
if os.environ.get('OS_DEBUG') in _TRUE_VALUES:
|
|
||||||
level = logging.DEBUG
|
|
||||||
else:
|
|
||||||
level = logging.INFO
|
|
||||||
capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES
|
|
||||||
if capture_logs:
|
|
||||||
self.useFixture(
|
|
||||||
fixtures.FakeLogger(
|
|
||||||
format=_LOG_FORMAT,
|
|
||||||
level=level,
|
|
||||||
nuke_handlers=capture_logs,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logging.basicConfig(format=_LOG_FORMAT, level=level)
|
|
||||||
|
|
||||||
def create_tempfiles(self, files, ext='.conf'):
|
|
||||||
tempfiles = []
|
|
||||||
for (basename, contents) in files:
|
|
||||||
if not os.path.isabs(basename):
|
|
||||||
(fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext)
|
|
||||||
else:
|
|
||||||
path = basename + ext
|
|
||||||
fd = os.open(path, os.O_CREAT | os.O_WRONLY)
|
|
||||||
tempfiles.append(path)
|
|
||||||
try:
|
|
||||||
os.write(fd, contents)
|
|
||||||
finally:
|
|
||||||
os.close(fd)
|
|
||||||
return tempfiles
|
|
Loading…
Reference in New Issue
Block a user