From ca053e445800b29f94ecd1ff7e7d381e7ddb2927 Mon Sep 17 00:00:00 2001 From: Endre Karlson Date: Mon, 4 Nov 2013 14:54:28 +0100 Subject: [PATCH] [TESTS] Update test code to align with OS * Add oslo test + fixture code * Move components into their respectful subdirectories * Create a base for future service testing like mgm, api etc * Provide missing license headers for some files Change-Id: I353f41a57ee1f45fb76f666c955dc0b9fb9d323b --- libra/api/tests/config.py | 50 ---- libra/openstack/common/fileutils.py | 139 +++++++++ libra/openstack/common/fixture/__init__.py | 0 libra/openstack/common/fixture/config.py | 46 +++ libra/openstack/common/fixture/lockutils.py | 53 ++++ libra/openstack/common/fixture/mockpatch.py | 51 ++++ libra/openstack/common/fixture/moxstubout.py | 34 +++ libra/openstack/common/lockutils.py | 278 ++++++++++++++++++ libra/openstack/common/log.py | 6 +- libra/openstack/common/rpc/impl_zmq.py | 2 +- libra/openstack/common/service.py | 8 +- libra/openstack/common/test.py | 53 ++++ .../admin_api/__init__.py} | 8 - libra/{api/tests => tests/api}/__init__.py | 23 -- libra/tests/api/v1_1/__init__.py | 38 +++ libra/tests/api_base.py | 37 +++ libra/tests/base.py | 143 +++++++++ libra/tests/mgm/___init__.py | 13 + libra/tests/mock_objects.py | 14 + libra/tests/worker/__init__.py | 13 + .../test_controller.py} | 98 +++--- .../test_driver_haproxy.py} | 21 +- libra/tests/{ => worker}/test_lbstats.py | 18 +- openstack-common.conf | 9 +- test-requirements.txt | 2 + 25 files changed, 1019 insertions(+), 138 deletions(-) delete mode 100644 libra/api/tests/config.py create mode 100644 libra/openstack/common/fileutils.py create mode 100644 libra/openstack/common/fixture/__init__.py create mode 100644 libra/openstack/common/fixture/config.py create mode 100644 libra/openstack/common/fixture/lockutils.py create mode 100644 libra/openstack/common/fixture/mockpatch.py create mode 100644 libra/openstack/common/fixture/moxstubout.py create mode 100644 libra/openstack/common/lockutils.py create mode 100644 libra/openstack/common/test.py rename libra/{api/tests/test_units.py => tests/admin_api/__init__.py} (84%) rename libra/{api/tests => tests/api}/__init__.py (53%) create mode 100644 libra/tests/api/v1_1/__init__.py create mode 100644 libra/tests/api_base.py create mode 100644 libra/tests/base.py create mode 100644 libra/tests/mgm/___init__.py create mode 100644 libra/tests/worker/__init__.py rename libra/tests/{test_worker_controller.py => worker/test_controller.py} (88%) rename libra/tests/{test_haproxy_driver.py => worker/test_driver_haproxy.py} (87%) rename libra/tests/{ => worker}/test_lbstats.py (69%) diff --git a/libra/api/tests/config.py b/libra/api/tests/config.py deleted file mode 100644 index 3b3ca392..00000000 --- a/libra/api/tests/config.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Server Specific Configurations -server = { - 'port': '8080', - 'host': '0.0.0.0' -} - -# Pecan Application Configurations -app = { - 'root': 'libra.api.controllers.root.RootController', - 'modules': ['libra.api'], - 'static_root': '%(confdir)s/../../public', - 'template_path': '%(confdir)s/../templates', - 'debug': True, - 'errors': { - '404': '/error/404', - '__force_dict__': True - } -} - -database = { - 'username':'root', - 'password':'', - 'host':'127.0.0.1', - 'schema':'lbaas' -} - -gearman = { - 'server':['localhost:4730'], -} - -# Custom Configurations must be in Python dictionary format:: -# -# foo = {'bar':'baz'} -# -# All configurations are accessible at:: -# pecan.conf diff --git a/libra/openstack/common/fileutils.py b/libra/openstack/common/fileutils.py new file mode 100644 index 00000000..a73086c3 --- /dev/null +++ b/libra/openstack/common/fileutils.py @@ -0,0 +1,139 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import contextlib +import errno +import os +import tempfile + +from libra.openstack.common import excutils +from libra.openstack.common.gettextutils import _ # noqa +from libra.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + +_FILE_CACHE = {} + + +def ensure_tree(path): + """Create a directory (and any ancestor directories required) + + :param path: Directory to create + """ + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST: + if not os.path.isdir(path): + raise + else: + raise + + +def read_cached_file(filename, force_reload=False): + """Read from a file if it has been modified. + + :param force_reload: Whether to reload the file. + :returns: A tuple with a boolean specifying if the data is fresh + or not. + """ + global _FILE_CACHE + + if force_reload and filename in _FILE_CACHE: + del _FILE_CACHE[filename] + + reloaded = False + mtime = os.path.getmtime(filename) + cache_info = _FILE_CACHE.setdefault(filename, {}) + + if not cache_info or mtime > cache_info.get('mtime', 0): + LOG.debug(_("Reloading cached file %s") % filename) + with open(filename) as fap: + cache_info['data'] = fap.read() + cache_info['mtime'] = mtime + reloaded = True + return (reloaded, cache_info['data']) + + +def delete_if_exists(path, remove=os.unlink): + """Delete a file, but ignore file not found error. + + :param path: File to delete + :param remove: Optional function to remove passed path + """ + + try: + remove(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +@contextlib.contextmanager +def remove_path_on_error(path, remove=delete_if_exists): + """Protect code that wants to operate on PATH atomically. + Any exception will cause PATH to be removed. + + :param path: File to work with + :param remove: Optional function to remove passed path + """ + + try: + yield + except Exception: + with excutils.save_and_reraise_exception(): + remove(path) + + +def file_open(*args, **kwargs): + """Open file + + see built-in file() documentation for more details + + Note: The reason this is kept in a separate module is to easily + be able to provide a stub module that doesn't alter system + state at all (for unit tests) + """ + return file(*args, **kwargs) + + +def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): + """Create temporary file or use existing file. + + This util is needed for creating temporary file with + specified content, suffix and prefix. If path is not None, + it will be used for writing content. If the path doesn't + exist it'll be created. + + :param content: content for temporary file. + :param path: same as parameter 'dir' for mkstemp + :param suffix: same as parameter 'suffix' for mkstemp + :param prefix: same as parameter 'prefix' for mkstemp + + For example: it can be used in database tests for creating + configuration files. + """ + if path: + ensure_tree(path) + + (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) + try: + os.write(fd, content) + finally: + os.close(fd) + return path diff --git a/libra/openstack/common/fixture/__init__.py b/libra/openstack/common/fixture/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libra/openstack/common/fixture/config.py b/libra/openstack/common/fixture/config.py new file mode 100644 index 00000000..7b044ef7 --- /dev/null +++ b/libra/openstack/common/fixture/config.py @@ -0,0 +1,46 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2013 Mirantis, Inc. +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import fixtures +from oslo.config import cfg +import six + + +class Config(fixtures.Fixture): + """Override some configuration values. + + The keyword arguments are the names of configuration options to + override and their values. + + If a group argument is supplied, the overrides are applied to + the specified configuration option group. + + All overrides are automatically cleared at the end of the current + test by the reset() method, which is registred by addCleanup(). + """ + + def __init__(self, conf=cfg.CONF): + self.conf = conf + + def setUp(self): + super(Config, self).setUp() + self.addCleanup(self.conf.reset) + + def config(self, **kw): + group = kw.pop('group', None) + for k, v in six.iteritems(kw): + self.conf.set_override(k, v, group) diff --git a/libra/openstack/common/fixture/lockutils.py b/libra/openstack/common/fixture/lockutils.py new file mode 100644 index 00000000..e284de0a --- /dev/null +++ b/libra/openstack/common/fixture/lockutils.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures + +from libra.openstack.common.lockutils import lock + + +class LockFixture(fixtures.Fixture): + """External locking fixture. + + This fixture is basically an alternative to the synchronized decorator with + the external flag so that tearDowns and addCleanups will be included in + the lock context for locking between tests. The fixture is recommended to + be the first line in a test method, like so:: + + def test_method(self): + self.useFixture(LockFixture) + ... + + or the first line in setUp if all the test methods in the class are + required to be serialized. Something like:: + + class TestCase(testtools.testcase): + def setUp(self): + self.useFixture(LockFixture) + super(TestCase, self).setUp() + ... + + This is because addCleanups are put on a LIFO queue that gets run after the + test method exits. (either by completing or raising an exception) + """ + def __init__(self, name, lock_file_prefix=None): + self.mgr = lock(name, lock_file_prefix, True) + + def setUp(self): + super(LockFixture, self).setUp() + self.addCleanup(self.mgr.__exit__, None, None, None) + self.mgr.__enter__() diff --git a/libra/openstack/common/fixture/mockpatch.py b/libra/openstack/common/fixture/mockpatch.py new file mode 100644 index 00000000..cd0d6ca6 --- /dev/null +++ b/libra/openstack/common/fixture/mockpatch.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import mock + + +class PatchObject(fixtures.Fixture): + """Deal with code around mock.""" + + def __init__(self, obj, attr, **kwargs): + self.obj = obj + self.attr = attr + self.kwargs = kwargs + + def setUp(self): + super(PatchObject, self).setUp() + _p = mock.patch.object(self.obj, self.attr, **self.kwargs) + self.mock = _p.start() + self.addCleanup(_p.stop) + + +class Patch(fixtures.Fixture): + + """Deal with code around mock.patch.""" + + def __init__(self, obj, **kwargs): + self.obj = obj + self.kwargs = kwargs + + def setUp(self): + super(Patch, self).setUp() + _p = mock.patch(self.obj, **self.kwargs) + self.mock = _p.start() + self.addCleanup(_p.stop) diff --git a/libra/openstack/common/fixture/moxstubout.py b/libra/openstack/common/fixture/moxstubout.py new file mode 100644 index 00000000..a0e74fd1 --- /dev/null +++ b/libra/openstack/common/fixture/moxstubout.py @@ -0,0 +1,34 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import mox + + +class MoxStubout(fixtures.Fixture): + """Deal with code around mox and stubout as a fixture.""" + + def setUp(self): + super(MoxStubout, self).setUp() + # emulate some of the mox stuff, we can't use the metaclass + # because it screws with our generators + self.mox = mox.Mox() + self.stubs = self.mox.stubs + self.addCleanup(self.mox.UnsetStubs) + self.addCleanup(self.mox.VerifyAll) diff --git a/libra/openstack/common/lockutils.py b/libra/openstack/common/lockutils.py new file mode 100644 index 00000000..fb2d1243 --- /dev/null +++ b/libra/openstack/common/lockutils.py @@ -0,0 +1,278 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import contextlib +import errno +import functools +import os +import threading +import time +import weakref + +from oslo.config import cfg + +from libra.openstack.common import fileutils +from libra.openstack.common.gettextutils import _ # noqa +from libra.openstack.common import local +from libra.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +util_opts = [ + cfg.BoolOpt('disable_process_locking', default=False, + help='Whether to disable inter-process locks'), + cfg.StrOpt('lock_path', + help=('Directory to use for lock files.')) +] + + +CONF = cfg.CONF +CONF.register_opts(util_opts) + + +def set_defaults(lock_path): + cfg.set_defaults(util_opts, lock_path=lock_path) + + +class _InterProcessLock(object): + """Lock implementation which allows multiple locks, working around + issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does + not require any cleanup. Since the lock is always held on a file + descriptor rather than outside of the process, the lock gets dropped + automatically if the process crashes, even if __exit__ is not executed. + + There are no guarantees regarding usage by multiple green threads in a + single process here. This lock works only between processes. Exclusive + access between local threads should be achieved using the semaphores + in the @synchronized decorator. + + Note these locks are released when the descriptor is closed, so it's not + safe to close the file descriptor while another green thread holds the + lock. Just opening and closing the lock file can break synchronisation, + so lock files must be accessed only using this abstraction. + """ + + def __init__(self, name): + self.lockfile = None + self.fname = name + + def __enter__(self): + self.lockfile = open(self.fname, 'w') + + while True: + try: + # Using non-blocking locks since green threads are not + # patched to deal with blocking locking calls. + # Also upon reading the MSDN docs for locking(), it seems + # to have a laughable 10 attempts "blocking" mechanism. + self.trylock() + return self + except IOError as e: + if e.errno in (errno.EACCES, errno.EAGAIN): + # external locks synchronise things like iptables + # updates - give it some time to prevent busy spinning + time.sleep(0.01) + else: + raise + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + self.unlock() + self.lockfile.close() + except IOError: + LOG.exception(_("Could not release the acquired lock `%s`"), + self.fname) + + def trylock(self): + raise NotImplementedError() + + def unlock(self): + raise NotImplementedError() + + +class _WindowsLock(_InterProcessLock): + def trylock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) + + def unlock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) + + +class _PosixLock(_InterProcessLock): + def trylock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + + def unlock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_UN) + + +if os.name == 'nt': + import msvcrt + InterProcessLock = _WindowsLock +else: + import fcntl + InterProcessLock = _PosixLock + +_semaphores = weakref.WeakValueDictionary() + + +@contextlib.contextmanager +def lock(name, lock_file_prefix=None, external=False, lock_path=None): + """Context based lock + + This function yields a `threading.Semaphore` instance (if we don't use + eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is + True, in which case, it'll yield an InterProcessLock instance. + + :param lock_file_prefix: The lock_file_prefix argument is used to provide + lock files on disk with a meaningful prefix. + + :param external: The external keyword argument denotes whether this lock + should work across multiple processes. This means that if two different + workers both run a a method decorated with @synchronized('mylock', + external=True), only one of them will execute at a time. + + :param lock_path: The lock_path keyword argument is used to specify a + special location for external lock files to live. If nothing is set, then + CONF.lock_path is used as a default. + """ + # NOTE(soren): If we ever go natively threaded, this will be racy. + # See http://stackoverflow.com/questions/5390569/dyn + # amically-allocating-and-destroying-mutexes + sem = _semaphores.get(name, threading.Semaphore()) + if name not in _semaphores: + # this check is not racy - we're already holding ref locally + # so GC won't remove the item and there was no IO switch + # (only valid in greenthreads) + _semaphores[name] = sem + + with sem: + LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) + + # NOTE(mikal): I know this looks odd + if not hasattr(local.strong_store, 'locks_held'): + local.strong_store.locks_held = [] + local.strong_store.locks_held.append(name) + + try: + if external and not CONF.disable_process_locking: + LOG.debug(_('Attempting to grab file lock "%(lock)s"'), + {'lock': name}) + + # We need a copy of lock_path because it is non-local + local_lock_path = lock_path or CONF.lock_path + if not local_lock_path: + raise cfg.RequiredOptError('lock_path') + + if not os.path.exists(local_lock_path): + fileutils.ensure_tree(local_lock_path) + LOG.info(_('Created lock path: %s'), local_lock_path) + + def add_prefix(name, prefix): + if not prefix: + return name + sep = '' if prefix.endswith('-') else '-' + return '%s%s%s' % (prefix, sep, name) + + # NOTE(mikal): the lock name cannot contain directory + # separators + lock_file_name = add_prefix(name.replace(os.sep, '_'), + lock_file_prefix) + + lock_file_path = os.path.join(local_lock_path, lock_file_name) + + try: + lock = InterProcessLock(lock_file_path) + with lock as lock: + LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), + {'lock': name, 'path': lock_file_path}) + yield lock + finally: + LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), + {'lock': name, 'path': lock_file_path}) + else: + yield sem + + finally: + local.strong_store.locks_held.remove(name) + + +def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): + """Synchronization decorator. + + Decorating a method like so:: + + @synchronized('mylock') + def foo(self, *args): + ... + + ensures that only one thread will execute the foo method at a time. + + Different methods can share the same lock:: + + @synchronized('mylock') + def foo(self, *args): + ... + + @synchronized('mylock') + def bar(self, *args): + ... + + This way only one of either foo or bar can be executing at a time. + """ + + def wrap(f): + @functools.wraps(f) + def inner(*args, **kwargs): + try: + with lock(name, lock_file_prefix, external, lock_path): + LOG.debug(_('Got semaphore / lock "%(function)s"'), + {'function': f.__name__}) + return f(*args, **kwargs) + finally: + LOG.debug(_('Semaphore / lock released "%(function)s"'), + {'function': f.__name__}) + return inner + return wrap + + +def synchronized_with_prefix(lock_file_prefix): + """Partial object generator for the synchronization decorator. + + Redefine @synchronized in each project like so:: + + (in nova/utils.py) + from nova.openstack.common import lockutils + + synchronized = lockutils.synchronized_with_prefix('nova-') + + + (in nova/foo.py) + from nova import utils + + @utils.synchronized('mylock') + def bar(self, *args): + ... + + The lock_file_prefix argument is used to provide lock files on disk with a + meaningful prefix. + """ + + return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) diff --git a/libra/openstack/common/log.py b/libra/openstack/common/log.py index ae63cfd8..76e79bbe 100644 --- a/libra/openstack/common/log.py +++ b/libra/openstack/common/log.py @@ -127,11 +127,13 @@ log_opts = [ help='prefix each line of exception output with this format'), cfg.ListOpt('default_log_levels', default=[ + 'amqp=WARN', 'amqplib=WARN', - 'sqlalchemy=WARN', 'boto=WARN', - 'suds=INFO', 'keystone=INFO', + 'qpid=WARN', + 'sqlalchemy=WARN', + 'suds=INFO', ], help='list of logger=LEVEL pairs'), cfg.BoolOpt('publish_errors', diff --git a/libra/openstack/common/rpc/impl_zmq.py b/libra/openstack/common/rpc/impl_zmq.py index 784defa7..aea96a90 100644 --- a/libra/openstack/common/rpc/impl_zmq.py +++ b/libra/openstack/common/rpc/impl_zmq.py @@ -192,7 +192,7 @@ class ZmqSocket(object): # it would be much worse if some of the code calling this # were to fail. For now, lets log, and later evaluate # if we can safely raise here. - LOG.error("ZeroMQ socket could not be closed.") + LOG.error(_("ZeroMQ socket could not be closed.")) self.sock = None def recv(self, **kwargs): diff --git a/libra/openstack/common/service.py b/libra/openstack/common/service.py index fad1ae07..9cd0052e 100644 --- a/libra/openstack/common/service.py +++ b/libra/openstack/common/service.py @@ -129,7 +129,7 @@ class ServiceLauncher(Launcher): def handle_signal(self): _set_signals_handler(self._handle_signal) - def _wait_for_exit_or_signal(self): + def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 @@ -137,6 +137,8 @@ class ServiceLauncher(Launcher): CONF.log_opt_values(LOG, std_logging.DEBUG) try: + if ready_callback: + ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) @@ -156,10 +158,10 @@ class ServiceLauncher(Launcher): return status, signo - def wait(self): + def wait(self, ready_callback=None): while True: self.handle_signal() - status, signo = self._wait_for_exit_or_signal() + status, signo = self._wait_for_exit_or_signal(ready_callback) if not _is_sighup(signo): return status self.restart() diff --git a/libra/openstack/common/test.py b/libra/openstack/common/test.py new file mode 100644 index 00000000..84e73bb4 --- /dev/null +++ b/libra/openstack/common/test.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Common utilities used in testing""" + +import os + +import fixtures +import testtools + + +class BaseTestCase(testtools.TestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self._set_timeout() + self._fake_output() + self.useFixture(fixtures.FakeLogger('libra.openstack.common')) + self.useFixture(fixtures.NestedTempfile()) + + def _set_timeout(self): + test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) + try: + test_timeout = int(test_timeout) + except ValueError: + # If timeout value is invalid do not set a timeout. + test_timeout = 0 + if test_timeout > 0: + self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) + + def _fake_output(self): + if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or + os.environ.get('OS_STDOUT_CAPTURE') == '1'): + stdout = self.useFixture(fixtures.StringStream('stdout')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) + if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or + os.environ.get('OS_STDERR_CAPTURE') == '1'): + stderr = self.useFixture(fixtures.StringStream('stderr')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) diff --git a/libra/api/tests/test_units.py b/libra/tests/admin_api/__init__.py similarity index 84% rename from libra/api/tests/test_units.py rename to libra/tests/admin_api/__init__.py index 29a83f18..92bd912f 100644 --- a/libra/api/tests/test_units.py +++ b/libra/tests/admin_api/__init__.py @@ -11,11 +11,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -from unittest import TestCase - - -class TestUnits(TestCase): - - def test_units(self): - assert 5 * 5 == 25 diff --git a/libra/api/tests/__init__.py b/libra/tests/api/__init__.py similarity index 53% rename from libra/api/tests/__init__.py rename to libra/tests/api/__init__.py index b7cc7196..92bd912f 100644 --- a/libra/api/tests/__init__.py +++ b/libra/tests/api/__init__.py @@ -11,26 +11,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -import os -from unittest import TestCase -from pecan import set_config -from pecan.testing import load_test_app - -__all__ = ['FunctionalTest'] - - -class FunctionalTest(TestCase): - """ - Used for functional tests where you need to test your - literal application and its integration with the framework. - """ - - def setUp(self): - self.app = load_test_app(os.path.join( - os.path.dirname(__file__), - 'config.py' - )) - - def tearDown(self): - set_config({}, overwrite=True) diff --git a/libra/tests/api/v1_1/__init__.py b/libra/tests/api/v1_1/__init__.py new file mode 100644 index 00000000..9cd1ad67 --- /dev/null +++ b/libra/tests/api/v1_1/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from libra.tests import api_base + + +""" +Base TestCase for V1.1 API tests. +""" + + +class TestCase(api_base.TestCase): + def setUp(self): + root_dir = self.path_get() + + config = { + 'app': { + 'root': 'libra.api.controllers.root.RootController', + 'modules': ['libra.api'], + 'static_root': '%s/public' % root_dir, + 'template_path': '%s/libra/api/templates' % root_dir, + 'enable_acl': False, + }, + 'wsme': { + 'debug': True, + } + } + self.app = self._make_app(config) diff --git a/libra/tests/api_base.py b/libra/tests/api_base.py new file mode 100644 index 00000000..b98b2c59 --- /dev/null +++ b/libra/tests/api_base.py @@ -0,0 +1,37 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from libra.tests import base +import pecan +import pecan.testing + +from oslo.config import cfg + + +class TestCase(base.TestCase): + """Used for functional tests of Pecan controllers where you need to + test your literal application and its integration with the + framework. + """ + def _make_app(self, config=None, enable_acl=False): + # Determine where we are so we can set up paths in the config + root_dir = self.path_get() + self.config = config or self.config + return pecan.testing.load_test_app(self.config) + + def tearDown(self): + super(FunctionalTest, self).tearDown() + self.app = None + pecan.set_config({}, overwrite=True) diff --git a/libra/tests/base.py b/libra/tests/base.py new file mode 100644 index 00000000..3098c18f --- /dev/null +++ b/libra/tests/base.py @@ -0,0 +1,143 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copied partially from ceilometer + +"""Base classes for our unit tests. + +Allows overriding of config for use of fakes, and some black magic for +inline callbacks. + +""" + +import eventlet +eventlet.monkey_patch(os=False) + +import copy +import os +import shutil +import tempfile +import sys + +import fixtures +import testtools + +#from libra.db import migration +from libra.common import options +from libra.openstack.common import log +from libra.openstack.common import test +from libra.openstack.common.fixture import config +from libra.openstack.common.fixture import moxstubout + +options.CONF.set_override('use_stderr', False) + +# NOTE: Tests fail due to diverse options being required. +options.CONF.import_group('api', 'libra.api') +options.CONF.import_group('mgm', 'libra.mgm') + +log.setup('libra') + +_DB_CACHE = None + + +class Database(fixtures.Fixture): + """ + Fixture for Databases. Handles syncing, tearing down etc. + """ + def __init__(self, db_session, db_migrate, sql_connection, + sqlite_db, sqlite_clean_db): + self.sql_connection = sql_connection + self.sqlite_db = sqlite_db + self.sqlite_clean_db = sqlite_clean_db + + self.engine = db_session.get_engine() + self.engine.dispose() + conn = self.engine.connect() + if sql_connection == "sqlite://": + if db_migrate.db_version() > db_migrate.INIT_VERSION: + return + else: + testdb = os.path.join(CONF.state_path, sqlite_db) + if os.path.exists(testdb): + return + db_migrate.db_sync() +# self.post_migrations() + if sql_connection == "sqlite://": + conn = self.engine.connect() + self._DB = "".join(line for line in conn.connection.iterdump()) + self.engine.dispose() + else: + cleandb = os.path.join(CONF.state_path, sqlite_clean_db) + shutil.copyfile(testdb, cleandb) + + def setUp(self): + super(Database, self).setUp() + + if self.sql_connection == "sqlite://": + conn = self.engine.connect() + conn.connection.executescript(self._DB) + self.addCleanup(self.engine.dispose) + else: + shutil.copyfile( + os.path.join(CONF.state_path, self.sqlite_clean_db), + os.path.join(CONF.state_path, self.sqlite_db)) + + +class TestCase(test.BaseTestCase): + """ + Base test case that holds any "extras" that we use like assertX functions. + """ + def path_get(self, project_file=None): + root = os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', + '..', + ) + ) + if project_file: + return os.path.join(root, project_file) + else: + return root + + +class ServiceTestCase(test.BaseTestCase): + """Base test case for Libra tests.""" + def setUp(self): + super(TestBase, self).setUp() + options.add_common_opts() + self.CONF = self.useFixture(config.Config(options.CONF)).conf + + # NOTE: Provide some fun defaults for testing + self.CONF.set_override('az', 'default', group='mgm') + self.CONF.set_override('nova_secgroup', 'default', group='mgm') + self.CONF.set_override('nova_image', 'image', group='mgm') + self.CONF.set_override('nova_image_size', 'm1.small', group='mgm') + self.CONF.set_override('nova_keyname', 'key', group='mgm') + self.CONF.set_override('nova_user', 'user', group='mgm') + self.CONF.set_override('nova_pass', 'secret', group='mgm') + self.CONF.set_override('nova_auth_url', 'http://localhost:35357/2.0', + group='mgm') + self.CONF.set_override('nova_region', 'region', group='mgm') + + self.CONF.set_override('db_sections', 'test', group='api') + self.CONF.set_override('swift_endpoint', 'test', group='api') + self.CONF.set_override('swift_basepath', 'test', group='api') + + self.CONF.set_override('driver', 'gearman_fake', group='gearman') + + self.CONF([], project='libra') + diff --git a/libra/tests/mgm/___init__.py b/libra/tests/mgm/___init__.py new file mode 100644 index 00000000..92bd912f --- /dev/null +++ b/libra/tests/mgm/___init__.py @@ -0,0 +1,13 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/libra/tests/mock_objects.py b/libra/tests/mock_objects.py index 17b69058..eeb92566 100644 --- a/libra/tests/mock_objects.py +++ b/libra/tests/mock_objects.py @@ -1,3 +1,17 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import json import logging diff --git a/libra/tests/worker/__init__.py b/libra/tests/worker/__init__.py new file mode 100644 index 00000000..92bd912f --- /dev/null +++ b/libra/tests/worker/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/libra/tests/test_worker_controller.py b/libra/tests/worker/test_controller.py similarity index 88% rename from libra/tests/test_worker_controller.py rename to libra/tests/worker/test_controller.py index f958aaa4..ef9ee2d3 100644 --- a/libra/tests/test_worker_controller.py +++ b/libra/tests/worker/test_controller.py @@ -1,5 +1,19 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import logging -import testtools +from libra.tests.base import TestCase import libra.tests.mock_objects from libra import __version__ as libra_version from libra import __release__ as libra_release @@ -8,7 +22,7 @@ from libra.worker.drivers.base import LoadBalancerDriver from libra.worker.drivers.haproxy.driver import HAProxyDriver -class TestWorkerController(testtools.TestCase): +class TestWorkerController(TestCase): def setUp(self): super(TestWorkerController, self).setUp() self.logger = logging.getLogger('test_worker_controller') @@ -182,14 +196,13 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'type': 'CONNECT', - 'delay': 60, - 'timeout': 30, - 'attempts': 1, - 'path': '/healthcheck' - } + 'monitor': { + 'type': 'CONNECT', + 'delay': 60, + 'timeout': 30, + 'attempts': 1, + 'path': '/healthcheck' + } } ] } @@ -211,13 +224,12 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'delay': 60, - 'timeout': 30, - 'attempts': 1, - 'path': '/healthcheck' - } + 'monitor': { + 'delay': 60, + 'timeout': 30, + 'attempts': 1, + 'path': '/healthcheck' + } } ] } @@ -240,13 +252,12 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'type': 'CONNECT', - 'timeout': 30, - 'attempts': 1, - 'path': '/healthcheck' - } + 'monitor': { + 'type': 'CONNECT', + 'timeout': 30, + 'attempts': 1, + 'path': '/healthcheck' + } } ] } @@ -269,13 +280,12 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'type': 'CONNECT', - 'delay': 60, - 'attempts': 1, - 'path': '/healthcheck' - } + 'monitor': { + 'type': 'CONNECT', + 'delay': 60, + 'attempts': 1, + 'path': '/healthcheck' + } } ] } @@ -298,13 +308,12 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'type': 'CONNECT', - 'delay': 60, - 'timeout': 30, - 'path': '/healthcheck' - } + 'monitor': { + 'type': 'CONNECT', + 'delay': 60, + 'timeout': 30, + 'path': '/healthcheck' + } } ] } @@ -327,13 +336,12 @@ class TestWorkerController(testtools.TestCase): 'port': 80 } ], - 'monitor': - { - 'type': 'CONNECT', - 'delay': 60, - 'timeout': 30, - 'attempts': 1 - } + 'monitor': { + 'type': 'CONNECT', + 'delay': 60, + 'timeout': 30, + 'attempts': 1 + } } ] } diff --git a/libra/tests/test_haproxy_driver.py b/libra/tests/worker/test_driver_haproxy.py similarity index 87% rename from libra/tests/test_haproxy_driver.py rename to libra/tests/worker/test_driver_haproxy.py index c609c447..b3a2eb76 100644 --- a/libra/tests/test_haproxy_driver.py +++ b/libra/tests/worker/test_driver_haproxy.py @@ -1,8 +1,22 @@ -import testtools +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from libra.tests.base import TestCase from libra.worker.drivers.haproxy.driver import HAProxyDriver -class TestHAProxyDriver(testtools.TestCase): +class TestHAProxyDriver(TestCase): def setUp(self): super(TestHAProxyDriver, self).setUp() self.driver = HAProxyDriver('libra.tests.mock_objects.FakeOSServices', @@ -38,7 +52,8 @@ class TestHAProxyDriver(testtools.TestCase): self.assertEqual("Unsupported protocol: %s" % proto, e.message) def testAddGaleraRequiresPort(self): - e = self.assertRaises(Exception, self.driver.add_protocol, 'galera', None) + e = self.assertRaises( + Exception, self.driver.add_protocol, 'galera', None) self.assertEqual("Port is required for this protocol.", e.message) def testAddTCPRequiresPort(self): diff --git a/libra/tests/test_lbstats.py b/libra/tests/worker/test_lbstats.py similarity index 69% rename from libra/tests/test_lbstats.py rename to libra/tests/worker/test_lbstats.py index f8506792..6a50a837 100644 --- a/libra/tests/test_lbstats.py +++ b/libra/tests/worker/test_lbstats.py @@ -1,9 +1,23 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import datetime -import testtools +from libra.tests.base import TestCase from libra.common.lbstats import LBStatistics -class TestLBStatistics(testtools.TestCase): +class TestLBStatistics(TestCase): def setUp(self): super(TestLBStatistics, self).setUp() self.stats = LBStatistics() diff --git a/openstack-common.conf b/openstack-common.conf index e857bebe..0f52cddd 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,14 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=importutils,jsonutils,xmlutils,notifier + +module=fixture +module=importutils +module=jsonutils +module=notifier +module=xmlutils +module=test + # The base module to hold the copy of openstack.common base=libra diff --git a/test-requirements.txt b/test-requirements.txt index fd23e5c3..83f50fb1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,5 @@ python-subunit sphinx>=1.1.2 testrepository>=0.0.8 testtools>=0.9.22 +babel +mox \ No newline at end of file