From b9e8f9f9ae9b7398e5c0ccf5d5a58769ee792eb6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 27 Apr 2015 20:17:02 +0000 Subject: [PATCH] Remove oslo namespace package Blueprint remove-namespace-packages Depends-on: Iae62b48993eef3b31420f8cc245a55f5e303c4fc Depends-on: If927fc7d70c625faa072dc8905a7e044f08aad33 both for designate Change-Id: I60ac791560c49514d3a25e2b58b39a12d3da5d00 --- oslo/__init__.py | 13 - oslo/concurrency/__init__.py | 29 -- oslo/concurrency/fixture/__init__.py | 13 - setup.cfg | 5 - tests/__init__.py | 19 - tests/test_import.py | 31 -- tests/test_lockutils.py | 575 --------------------------- tests/test_processutils.py | 540 ------------------------- tests/test_warning.py | 61 --- tests/test_watchdog.py | 75 ---- 10 files changed, 1361 deletions(-) delete mode 100644 oslo/__init__.py delete mode 100644 oslo/concurrency/__init__.py delete mode 100644 oslo/concurrency/fixture/__init__.py delete mode 100644 tests/__init__.py delete mode 100644 tests/test_import.py delete mode 100644 tests/test_lockutils.py delete mode 100644 tests/test_processutils.py delete mode 100644 tests/test_warning.py delete mode 100644 tests/test_watchdog.py diff --git a/oslo/__init__.py b/oslo/__init__.py deleted file mode 100644 index dc130d6..0000000 --- a/oslo/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -__import__('pkg_resources').declare_namespace(__name__) diff --git a/oslo/concurrency/__init__.py b/oslo/concurrency/__init__.py deleted file mode 100644 index b73c33f..0000000 --- a/oslo/concurrency/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import warnings - -from oslo_concurrency import lockutils # noqa -from oslo_concurrency import processutils # noqa - - -def deprecated(): - new_name = __name__.replace('.', '_') - warnings.warn( - ('The oslo namespace package is deprecated. Please use %s instead.' % - new_name), - DeprecationWarning, - stacklevel=3, - ) - - -deprecated() diff --git a/oslo/concurrency/fixture/__init__.py b/oslo/concurrency/fixture/__init__.py deleted file mode 100644 index 07f8bbc..0000000 --- a/oslo/concurrency/fixture/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_concurrency.fixture import lockutils # noqa diff --git a/setup.cfg b/setup.cfg index 5d66c63..0b14cc3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,13 +21,8 @@ classifier = [files] packages = - oslo - oslo.concurrency - oslo.concurrency.fixture oslo_concurrency oslo_concurrency.fixture -namespace_packages = - oslo [entry_points] oslo.config.opts = diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index bd455c4..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -if os.environ.get('TEST_EVENTLET'): - import eventlet - eventlet.monkey_patch() diff --git a/tests/test_import.py b/tests/test_import.py deleted file mode 100644 index cee16fe..0000000 --- a/tests/test_import.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslotest import base as test_base - -# Do NOT change this to new namespace, it's testing the old namespace -# passing hacking. Do NOT add a #noqa line, the point is this has to -# pass without it. -from oslo.concurrency import lockutils - - -class ImportTestCase(test_base.BaseTestCase): - """Test that lockutils can be imported from old namespace. - - This also ensures that hacking rules on this kind of import will - work for the rest of OpenStack. - - """ - - def test_imported(self): - self.assertEqual(len(lockutils._opts), 2, - "Lockutils.opts: %s" % lockutils._opts) diff --git a/tests/test_lockutils.py b/tests/test_lockutils.py deleted file mode 100644 index 22bb041..0000000 --- a/tests/test_lockutils.py +++ /dev/null @@ -1,575 +0,0 @@ -# Copyright 2011 Justin Santa Barbara -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import errno -import fcntl -import multiprocessing -import os -import shutil -import signal -import subprocess -import sys -import tempfile -import threading -import time - -from oslo.config import cfg -from oslotest import base as test_base -import six - -from oslo.concurrency.fixture import lockutils as fixtures -from oslo.concurrency import lockutils -from oslo.config import fixture as config - - -class LockTestCase(test_base.BaseTestCase): - - def setUp(self): - super(LockTestCase, self).setUp() - self.config = self.useFixture(config.Config(lockutils.CONF)).config - - def test_synchronized_wrapped_function_metadata(self): - @lockutils.synchronized('whatever', 'test-') - def foo(): - """Bar.""" - pass - - self.assertEqual(foo.__doc__, 'Bar.', "Wrapped function's docstring " - "got lost") - self.assertEqual(foo.__name__, 'foo', "Wrapped function's name " - "got mangled") - - def test_lock_acquire_release_file_lock(self): - lock_dir = tempfile.mkdtemp() - lock_file = os.path.join(lock_dir, 'lock') - lock = lockutils._FcntlLock(lock_file) - - def try_lock(): - try: - my_lock = lockutils._FcntlLock(lock_file) - my_lock.lockfile = open(lock_file, 'w') - my_lock.trylock() - my_lock.unlock() - os._exit(1) - except IOError: - os._exit(0) - - def attempt_acquire(count): - children = [] - for i in range(count): - child = multiprocessing.Process(target=try_lock) - child.start() - children.append(child) - exit_codes = [] - for child in children: - child.join() - exit_codes.append(child.exitcode) - return sum(exit_codes) - - self.assertTrue(lock.acquire()) - try: - acquired_children = attempt_acquire(10) - self.assertEqual(0, acquired_children) - finally: - lock.release() - - try: - acquired_children = attempt_acquire(5) - self.assertNotEqual(0, acquired_children) - finally: - try: - shutil.rmtree(lock_dir) - except IOError: - pass - - def test_lock_internally(self): - """We can lock across multiple threads.""" - saved_sem_num = len(lockutils._semaphores) - seen_threads = list() - - def f(_id): - with lockutils.lock('testlock2', 'test-', external=False): - for x in range(10): - seen_threads.append(_id) - - threads = [] - for i in range(10): - thread = threading.Thread(target=f, args=(i,)) - threads.append(thread) - thread.start() - - for thread in threads: - thread.join() - - self.assertEqual(len(seen_threads), 100) - # Looking at the seen threads, split it into chunks of 10, and verify - # that the last 9 match the first in each chunk. - for i in range(10): - for j in range(9): - self.assertEqual(seen_threads[i * 10], - seen_threads[i * 10 + 1 + j]) - - self.assertEqual(saved_sem_num, len(lockutils._semaphores), - "Semaphore leak detected") - - def test_nested_synchronized_external_works(self): - """We can nest external syncs.""" - tempdir = tempfile.mkdtemp() - try: - self.config(lock_path=tempdir, group='oslo_concurrency') - sentinel = object() - - @lockutils.synchronized('testlock1', 'test-', external=True) - def outer_lock(): - - @lockutils.synchronized('testlock2', 'test-', external=True) - def inner_lock(): - return sentinel - return inner_lock() - - self.assertEqual(sentinel, outer_lock()) - - finally: - if os.path.exists(tempdir): - shutil.rmtree(tempdir) - - def _do_test_lock_externally(self): - """We can lock across multiple processes.""" - - def lock_files(handles_dir): - - with lockutils.lock('external', 'test-', external=True): - # Open some files we can use for locking - handles = [] - for n in range(50): - path = os.path.join(handles_dir, ('file-%s' % n)) - handles.append(open(path, 'w')) - - # Loop over all the handles and try locking the file - # without blocking, keep a count of how many files we - # were able to lock and then unlock. If the lock fails - # we get an IOError and bail out with bad exit code - count = 0 - for handle in handles: - try: - fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) - count += 1 - fcntl.flock(handle, fcntl.LOCK_UN) - except IOError: - os._exit(2) - finally: - handle.close() - - # Check if we were able to open all files - self.assertEqual(50, count) - - handles_dir = tempfile.mkdtemp() - try: - children = [] - for n in range(50): - pid = os.fork() - if pid: - children.append(pid) - else: - try: - lock_files(handles_dir) - finally: - os._exit(0) - - for child in children: - (pid, status) = os.waitpid(child, 0) - if pid: - self.assertEqual(0, status) - finally: - if os.path.exists(handles_dir): - shutil.rmtree(handles_dir, ignore_errors=True) - - def test_lock_externally(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - try: - self._do_test_lock_externally() - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_lock_externally_lock_dir_not_exist(self): - lock_dir = tempfile.mkdtemp() - os.rmdir(lock_dir) - self.config(lock_path=lock_dir, group='oslo_concurrency') - - try: - self._do_test_lock_externally() - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_synchronized_with_prefix(self): - lock_name = 'mylock' - lock_pfix = 'mypfix-' - - foo = lockutils.synchronized_with_prefix(lock_pfix) - - @foo(lock_name, external=True) - def bar(dirpath, pfix, name): - return True - - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - self.assertTrue(bar(lock_dir, lock_pfix, lock_name)) - - def test_synchronized_without_prefix(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - @lockutils.synchronized('lock', external=True) - def test_without_prefix(): - # We can't check much - pass - - try: - test_without_prefix() - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_synchronized_prefix_without_hypen(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - @lockutils.synchronized('lock', 'hypen', True) - def test_without_hypen(): - # We can't check much - pass - - try: - test_without_hypen() - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_contextlock(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - try: - # Note(flaper87): Lock is not external, which means - # a semaphore will be yielded - with lockutils.lock("test") as sem: - if six.PY2: - self.assertTrue(isinstance(sem, threading._Semaphore)) - else: - self.assertTrue(isinstance(sem, threading.Semaphore)) - - # NOTE(flaper87): Lock is external so an InterProcessLock - # will be yielded. - with lockutils.lock("test2", external=True) as lock: - self.assertTrue(lock.exists()) - - with lockutils.lock("test1", - external=True) as lock1: - self.assertTrue(isinstance(lock1, - lockutils.InterProcessLock)) - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_contextlock_unlocks(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - - sem = None - - try: - with lockutils.lock("test") as sem: - if six.PY2: - self.assertTrue(isinstance(sem, threading._Semaphore)) - else: - self.assertTrue(isinstance(sem, threading.Semaphore)) - - with lockutils.lock("test2", external=True) as lock: - self.assertTrue(lock.exists()) - - # NOTE(flaper87): Lock should be free - with lockutils.lock("test2", external=True) as lock: - self.assertTrue(lock.exists()) - - # NOTE(flaper87): Lock should be free - # but semaphore should already exist. - with lockutils.lock("test") as sem2: - self.assertEqual(sem, sem2) - finally: - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def _test_remove_lock_external_file(self, lock_dir, use_external=False): - lock_name = 'mylock' - lock_pfix = 'mypfix-remove-lock-test-' - - if use_external: - lock_path = lock_dir - else: - lock_path = None - - lockutils.remove_external_lock_file(lock_name, lock_pfix, lock_path) - - for ent in os.listdir(lock_dir): - self.assertRaises(OSError, ent.startswith, lock_pfix) - - if os.path.exists(lock_dir): - shutil.rmtree(lock_dir, ignore_errors=True) - - def test_remove_lock_external_file(self): - lock_dir = tempfile.mkdtemp() - self.config(lock_path=lock_dir, group='oslo_concurrency') - self._test_remove_lock_external_file(lock_dir) - - def test_remove_lock_external_file_lock_path(self): - lock_dir = tempfile.mkdtemp() - self._test_remove_lock_external_file(lock_dir, - use_external=True) - - def test_no_slash_in_b64(self): - # base64(sha1(foobar)) has a slash in it - with lockutils.lock("foobar"): - pass - - def test_deprecated_names(self): - paths = self.create_tempfiles([['fake.conf', '\n'.join([ - '[DEFAULT]', - 'lock_path=foo', - 'disable_process_locking=True']) - ]]) - conf = cfg.ConfigOpts() - conf(['--config-file', paths[0]]) - conf.register_opts(lockutils._opts, 'oslo_concurrency') - self.assertEqual(conf.oslo_concurrency.lock_path, 'foo') - self.assertTrue(conf.oslo_concurrency.disable_process_locking) - - -class BrokenLock(lockutils._FileLock): - def __init__(self, name, errno_code): - super(BrokenLock, self).__init__(name) - self.errno_code = errno_code - - def unlock(self): - pass - - def trylock(self): - err = IOError() - err.errno = self.errno_code - raise err - - -class FileBasedLockingTestCase(test_base.BaseTestCase): - def setUp(self): - super(FileBasedLockingTestCase, self).setUp() - self.lock_dir = tempfile.mkdtemp() - - def test_lock_file_exists(self): - lock_file = os.path.join(self.lock_dir, 'lock-file') - - @lockutils.synchronized('lock-file', external=True, - lock_path=self.lock_dir) - def foo(): - self.assertTrue(os.path.exists(lock_file)) - - foo() - - def test_bad_acquire(self): - lock_file = os.path.join(self.lock_dir, 'lock') - lock = BrokenLock(lock_file, errno.EBUSY) - - self.assertRaises(threading.ThreadError, lock.acquire) - - def test_interprocess_lock(self): - lock_file = os.path.join(self.lock_dir, 'processlock') - - pid = os.fork() - if pid: - # Make sure the child grabs the lock first - start = time.time() - while not os.path.exists(lock_file): - if time.time() - start > 5: - self.fail('Timed out waiting for child to grab lock') - time.sleep(0) - lock1 = lockutils.InterProcessLock('foo') - lock1.lockfile = open(lock_file, 'w') - # NOTE(bnemec): There is a brief window between when the lock file - # is created and when it actually becomes locked. If we happen to - # context switch in that window we may succeed in locking the - # file. Keep retrying until we either get the expected exception - # or timeout waiting. - while time.time() - start < 5: - try: - lock1.trylock() - lock1.unlock() - time.sleep(0) - except IOError: - # This is what we expect to happen - break - else: - self.fail('Never caught expected lock exception') - # We don't need to wait for the full sleep in the child here - os.kill(pid, signal.SIGKILL) - else: - try: - lock2 = lockutils.InterProcessLock('foo') - lock2.lockfile = open(lock_file, 'w') - have_lock = False - while not have_lock: - try: - lock2.trylock() - have_lock = True - except IOError: - pass - finally: - # NOTE(bnemec): This is racy, but I don't want to add any - # synchronization primitives that might mask a problem - # with the one we're trying to test here. - time.sleep(.5) - os._exit(0) - - def test_interthread_external_lock(self): - call_list = [] - - @lockutils.synchronized('foo', external=True, lock_path=self.lock_dir) - def foo(param): - """Simulate a long-running threaded operation.""" - call_list.append(param) - # NOTE(bnemec): This is racy, but I don't want to add any - # synchronization primitives that might mask a problem - # with the one we're trying to test here. - time.sleep(.5) - call_list.append(param) - - def other(param): - foo(param) - - thread = threading.Thread(target=other, args=('other',)) - thread.start() - # Make sure the other thread grabs the lock - # NOTE(bnemec): File locks do not actually work between threads, so - # this test is verifying that the local semaphore is still enforcing - # external locks in that case. This means this test does not have - # the same race problem as the process test above because when the - # file is created the semaphore has already been grabbed. - start = time.time() - while not os.path.exists(os.path.join(self.lock_dir, 'foo')): - if time.time() - start > 5: - self.fail('Timed out waiting for thread to grab lock') - time.sleep(0) - thread1 = threading.Thread(target=other, args=('main',)) - thread1.start() - thread1.join() - thread.join() - self.assertEqual(call_list, ['other', 'other', 'main', 'main']) - - def test_non_destructive(self): - lock_file = os.path.join(self.lock_dir, 'not-destroyed') - with open(lock_file, 'w') as f: - f.write('test') - with lockutils.lock('not-destroyed', external=True, - lock_path=self.lock_dir): - with open(lock_file) as f: - self.assertEqual(f.read(), 'test') - - -class LockutilsModuleTestCase(test_base.BaseTestCase): - - def setUp(self): - super(LockutilsModuleTestCase, self).setUp() - self.old_env = os.environ.get('OSLO_LOCK_PATH') - if self.old_env is not None: - del os.environ['OSLO_LOCK_PATH'] - - def tearDown(self): - if self.old_env is not None: - os.environ['OSLO_LOCK_PATH'] = self.old_env - super(LockutilsModuleTestCase, self).tearDown() - - def test_main(self): - script = '\n'.join([ - 'import os', - 'lock_path = os.environ.get("OSLO_LOCK_PATH")', - 'assert lock_path is not None', - 'assert os.path.isdir(lock_path)', - ]) - argv = ['', sys.executable, '-c', script] - retval = lockutils._lock_wrapper(argv) - self.assertEqual(retval, 0, "Bad OSLO_LOCK_PATH has been set") - - def test_return_value_maintained(self): - script = '\n'.join([ - 'import sys', - 'sys.exit(1)', - ]) - argv = ['', sys.executable, '-c', script] - retval = lockutils._lock_wrapper(argv) - self.assertEqual(retval, 1) - - def test_direct_call_explodes(self): - cmd = [sys.executable, '-m', 'oslo_concurrency.lockutils'] - with open(os.devnull, 'w') as devnull: - retval = subprocess.call(cmd, stderr=devnull) - # 1 for Python 2.7 and 3.x, 255 for 2.6 - self.assertIn(retval, [1, 255]) - - -class TestLockFixture(test_base.BaseTestCase): - - def setUp(self): - super(TestLockFixture, self).setUp() - self.config = self.useFixture(config.Config(lockutils.CONF)).config - self.tempdir = tempfile.mkdtemp() - - def _check_in_lock(self): - self.assertTrue(self.lock.exists()) - - def tearDown(self): - self._check_in_lock() - super(TestLockFixture, self).tearDown() - - def test_lock_fixture(self): - # Setup lock fixture to test that teardown is inside the lock - self.config(lock_path=self.tempdir, group='oslo_concurrency') - fixture = fixtures.LockFixture('test-lock') - self.useFixture(fixture) - self.lock = fixture.lock - - -class TestExternalLockFixture(test_base.BaseTestCase): - def test_fixture(self): - # NOTE(bnemec): This test case is only valid if lockutils-wrapper is - # _not_ in use. Otherwise lock_path will be set on lockutils import - # and this test will pass regardless of whether the fixture is used. - self.useFixture(fixtures.ExternalLockFixture()) - # This will raise an exception if lock_path is not set - with lockutils.external_lock('foo'): - pass - - def test_with_existing_config_fixture(self): - # Make sure the config fixture in the ExternalLockFixture doesn't - # cause any issues for tests using their own config fixture. - conf = self.useFixture(config.Config()) - self.useFixture(fixtures.ExternalLockFixture()) - with lockutils.external_lock('bar'): - conf.register_opt(cfg.StrOpt('foo')) - conf.config(foo='bar') - self.assertEqual(cfg.CONF.foo, 'bar') - # Due to config filter, lock_path should still not be present in - # the global config opt. - self.assertFalse(hasattr(cfg.CONF, 'lock_path')) diff --git a/tests/test_processutils.py b/tests/test_processutils.py deleted file mode 100644 index 22f8928..0000000 --- a/tests/test_processutils.py +++ /dev/null @@ -1,540 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import print_function - -import errno -import logging -import multiprocessing -import os -import stat -import tempfile - -import fixtures -import mock -from oslotest import base as test_base -from oslotest import mockpatch -import six - -from oslo.concurrency import processutils - -PROCESS_EXECUTION_ERROR_LOGGING_TEST = """#!/bin/bash -exit 41""" - -TEST_EXCEPTION_AND_MASKING_SCRIPT = """#!/bin/bash -# This is to test stdout and stderr -# and the command returned in an exception -# when a non-zero exit code is returned -echo onstdout --password='"secret"' -echo onstderr --password='"secret"' 1>&2 -exit 38""" - - -class UtilsTest(test_base.BaseTestCase): - # NOTE(jkoelker) Moar tests from nova need to be ported. But they - # need to be mock'd out. Currently they require actually - # running code. - def test_execute_unknown_kwargs(self): - self.assertRaises(processutils.UnknownArgumentError, - processutils.execute, - hozer=True) - - @mock.patch.object(multiprocessing, 'cpu_count', return_value=8) - def test_get_worker_count(self, mock_cpu_count): - self.assertEqual(8, processutils.get_worker_count()) - - @mock.patch.object(multiprocessing, 'cpu_count', - side_effect=NotImplementedError()) - def test_get_worker_count_cpu_count_not_implemented(self, - mock_cpu_count): - self.assertEqual(1, processutils.get_worker_count()) - - -class ProcessExecutionErrorTest(test_base.BaseTestCase): - - def test_defaults(self): - err = processutils.ProcessExecutionError() - self.assertTrue('None\n' in six.text_type(err)) - self.assertTrue('code: -\n' in six.text_type(err)) - - def test_with_description(self): - description = 'The Narwhal Bacons at Midnight' - err = processutils.ProcessExecutionError(description=description) - self.assertTrue(description in six.text_type(err)) - - def test_with_exit_code(self): - exit_code = 0 - err = processutils.ProcessExecutionError(exit_code=exit_code) - self.assertTrue(str(exit_code) in six.text_type(err)) - - def test_with_cmd(self): - cmd = 'telinit' - err = processutils.ProcessExecutionError(cmd=cmd) - self.assertTrue(cmd in six.text_type(err)) - - def test_with_stdout(self): - stdout = """ - Lo, praise of the prowess of people-kings - of spear-armed Danes, in days long sped, - we have heard, and what honot the athelings won! - Oft Scyld the Scefing from squadroned foes, - from many a tribe, the mead-bench tore, - awing the earls. Since erse he lay - friendless, a foundling, fate repaid him: - for he waxed under welkin, in wealth he trove, - till before him the folk, both far and near, - who house by the whale-path, heard his mandate, - gabe him gits: a good king he! - To him an heir was afterward born, - a son in his halls, whom heaven sent - to favor the fol, feeling their woe - that erst they had lacked an earl for leader - so long a while; the Lord endowed him, - the Wielder of Wonder, with world's renown. - """.strip() - err = processutils.ProcessExecutionError(stdout=stdout) - print(six.text_type(err)) - self.assertTrue('people-kings' in six.text_type(err)) - - def test_with_stderr(self): - stderr = 'Cottonian library' - err = processutils.ProcessExecutionError(stderr=stderr) - self.assertTrue(stderr in six.text_type(err)) - - def test_retry_on_failure(self): - fd, tmpfilename = tempfile.mkstemp() - _, tmpfilename2 = tempfile.mkstemp() - try: - fp = os.fdopen(fd, 'w+') - fp.write('''#!/bin/sh -# If stdin fails to get passed during one of the runs, make a note. -if ! grep -q foo -then - echo 'failure' > "$1" -fi -# If stdin has failed to get passed during this or a previous run, exit early. -if grep failure "$1" -then - exit 1 -fi -runs="$(cat $1)" -if [ -z "$runs" ] -then - runs=0 -fi -runs=$(($runs + 1)) -echo $runs > "$1" -exit 1 -''') - fp.close() - os.chmod(tmpfilename, 0o755) - self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - tmpfilename, tmpfilename2, attempts=10, - process_input=b'foo', - delay_on_retry=False) - fp = open(tmpfilename2, 'r') - runs = fp.read() - fp.close() - self.assertNotEqual(runs.strip(), 'failure', 'stdin did not ' - 'always get passed ' - 'correctly') - runs = int(runs.strip()) - self.assertEqual(runs, 10, 'Ran %d times instead of 10.' % (runs,)) - finally: - os.unlink(tmpfilename) - os.unlink(tmpfilename2) - - def test_unknown_kwargs_raises_error(self): - self.assertRaises(processutils.UnknownArgumentError, - processutils.execute, - '/usr/bin/env', 'true', - this_is_not_a_valid_kwarg=True) - - def test_check_exit_code_boolean(self): - processutils.execute('/usr/bin/env', 'false', check_exit_code=False) - self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - '/usr/bin/env', 'false', check_exit_code=True) - - def test_check_exit_code_list(self): - processutils.execute('/usr/bin/env', 'sh', '-c', 'exit 101', - check_exit_code=(101, 102)) - processutils.execute('/usr/bin/env', 'sh', '-c', 'exit 102', - check_exit_code=(101, 102)) - self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - '/usr/bin/env', 'sh', '-c', 'exit 103', - check_exit_code=(101, 102)) - self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - '/usr/bin/env', 'sh', '-c', 'exit 0', - check_exit_code=(101, 102)) - - def test_no_retry_on_success(self): - fd, tmpfilename = tempfile.mkstemp() - _, tmpfilename2 = tempfile.mkstemp() - try: - fp = os.fdopen(fd, 'w+') - fp.write("""#!/bin/sh -# If we've already run, bail out. -grep -q foo "$1" && exit 1 -# Mark that we've run before. -echo foo > "$1" -# Check that stdin gets passed correctly. -grep foo -""") - fp.close() - os.chmod(tmpfilename, 0o755) - processutils.execute(tmpfilename, - tmpfilename2, - process_input=b'foo', - attempts=2) - finally: - os.unlink(tmpfilename) - os.unlink(tmpfilename2) - - # This test and the one below ensures that when communicate raises - # an OSError, we do the right thing(s) - def test_exception_on_communicate_error(self): - mock = self.useFixture(mockpatch.Patch( - 'subprocess.Popen.communicate', - side_effect=OSError(errno.EAGAIN, 'fake-test'))) - - self.assertRaises(OSError, - processutils.execute, - '/usr/bin/env', - 'false', - check_exit_code=False) - - self.assertEqual(1, mock.mock.call_count) - - def test_retry_on_communicate_error(self): - mock = self.useFixture(mockpatch.Patch( - 'subprocess.Popen.communicate', - side_effect=OSError(errno.EAGAIN, 'fake-test'))) - - self.assertRaises(OSError, - processutils.execute, - '/usr/bin/env', - 'false', - check_exit_code=False, - attempts=5) - - self.assertEqual(5, mock.mock.call_count) - - def _test_and_check_logging_communicate_errors(self, log_errors=None, - attempts=None): - mock = self.useFixture(mockpatch.Patch( - 'subprocess.Popen.communicate', - side_effect=OSError(errno.EAGAIN, 'fake-test'))) - - fixture = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG)) - kwargs = {} - - if log_errors: - kwargs.update({"log_errors": log_errors}) - - if attempts: - kwargs.update({"attempts": attempts}) - - self.assertRaises(OSError, - processutils.execute, - '/usr/bin/env', - 'false', - **kwargs) - - self.assertEqual(attempts if attempts else 1, mock.mock.call_count) - self.assertIn('Got an OSError', fixture.output) - self.assertIn('errno: 11', fixture.output) - self.assertIn("'/usr/bin/env false'", fixture.output) - - def test_logging_on_communicate_error_1(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_FINAL_ERROR, - attempts=None) - - def test_logging_on_communicate_error_2(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_FINAL_ERROR, - attempts=1) - - def test_logging_on_communicate_error_3(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_FINAL_ERROR, - attempts=5) - - def test_logging_on_communicate_error_4(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_ALL_ERRORS, - attempts=None) - - def test_logging_on_communicate_error_5(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_ALL_ERRORS, - attempts=1) - - def test_logging_on_communicate_error_6(self): - self._test_and_check_logging_communicate_errors( - log_errors=processutils.LOG_ALL_ERRORS, - attempts=5) - - def test_with_env_variables(self): - env_vars = {'SUPER_UNIQUE_VAR': 'The answer is 42'} - - out, err = processutils.execute('/usr/bin/env', env_variables=env_vars) - self.assertEqual(type(out), str) - self.assertEqual(type(err), str) - - self.assertIn('SUPER_UNIQUE_VAR=The answer is 42', out) - - def test_as_root(self): - # For the following two tests: processutils.execute() does not - # prepend the root_helper if we are already running with root privs, - # so add it as the first argument to be certain. - out, err = processutils.execute('echo', 'a', 'b', 'c', - run_as_root=True, root_helper='echo') - - self.assertIn('a b c', six.text_type(out)) - - def test_as_root_via_shell(self): - out, err = processutils.execute('echo a b c', run_as_root=True, - root_helper='echo', shell=True) - - self.assertIn('a b c', six.text_type(out)) - - def test_exception_and_masking(self): - tmpfilename = self.create_tempfiles( - [["test_exceptions_and_masking", - TEST_EXCEPTION_AND_MASKING_SCRIPT]], ext='bash')[0] - - os.chmod(tmpfilename, (stat.S_IRWXU | - stat.S_IRGRP | - stat.S_IXGRP | - stat.S_IROTH | - stat.S_IXOTH)) - - err = self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - tmpfilename, 'password="secret"', - 'something') - - self.assertEqual(38, err.exit_code) - self.assertEqual(type(err.stdout), six.text_type) - self.assertEqual(type(err.stderr), six.text_type) - self.assertIn('onstdout --password="***"', err.stdout) - self.assertIn('onstderr --password="***"', err.stderr) - self.assertEqual(err.cmd, ' '.join([tmpfilename, - 'password="***"', - 'something'])) - self.assertNotIn('secret', str(err)) - - -class ProcessExecutionErrorLoggingTest(test_base.BaseTestCase): - def setUp(self): - super(ProcessExecutionErrorLoggingTest, self).setUp() - self.tmpfilename = self.create_tempfiles( - [["process_execution_error_logging_test", - PROCESS_EXECUTION_ERROR_LOGGING_TEST]], - ext='bash')[0] - - os.chmod(self.tmpfilename, (stat.S_IRWXU | stat.S_IRGRP | - stat.S_IXGRP | stat.S_IROTH | - stat.S_IXOTH)) - - def _test_and_check(self, log_errors=None, attempts=None): - fixture = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG)) - kwargs = {} - - if log_errors: - kwargs.update({"log_errors": log_errors}) - - if attempts: - kwargs.update({"attempts": attempts}) - - err = self.assertRaises(processutils.ProcessExecutionError, - processutils.execute, - self.tmpfilename, - **kwargs) - - self.assertEqual(41, err.exit_code) - self.assertIn(self.tmpfilename, fixture.output) - - def test_with_invalid_log_errors(self): - self.assertRaises(processutils.InvalidArgumentError, - processutils.execute, - self.tmpfilename, - log_errors='invalid') - - def test_with_log_errors_NONE(self): - self._test_and_check(log_errors=None, attempts=None) - - def test_with_log_errors_final(self): - self._test_and_check(log_errors=processutils.LOG_FINAL_ERROR, - attempts=None) - - def test_with_log_errors_all(self): - self._test_and_check(log_errors=processutils.LOG_ALL_ERRORS, - attempts=None) - - def test_multiattempt_with_log_errors_NONE(self): - self._test_and_check(log_errors=None, attempts=3) - - def test_multiattempt_with_log_errors_final(self): - self._test_and_check(log_errors=processutils.LOG_FINAL_ERROR, - attempts=3) - - def test_multiattempt_with_log_errors_all(self): - self._test_and_check(log_errors=processutils.LOG_ALL_ERRORS, - attempts=3) - - -def fake_execute(*cmd, **kwargs): - return 'stdout', 'stderr' - - -def fake_execute_raises(*cmd, **kwargs): - raise processutils.ProcessExecutionError(exit_code=42, - stdout='stdout', - stderr='stderr', - cmd=['this', 'is', 'a', - 'command']) - - -class TryCmdTestCase(test_base.BaseTestCase): - def test_keep_warnings(self): - self.useFixture(fixtures.MonkeyPatch( - 'oslo_concurrency.processutils.execute', fake_execute)) - o, e = processutils.trycmd('this is a command'.split(' ')) - self.assertNotEqual('', o) - self.assertNotEqual('', e) - - def test_keep_warnings_from_raise(self): - self.useFixture(fixtures.MonkeyPatch( - 'oslo_concurrency.processutils.execute', fake_execute_raises)) - o, e = processutils.trycmd('this is a command'.split(' '), - discard_warnings=True) - self.assertIsNotNone(o) - self.assertNotEqual('', e) - - def test_discard_warnings(self): - self.useFixture(fixtures.MonkeyPatch( - 'oslo_concurrency.processutils.execute', fake_execute)) - o, e = processutils.trycmd('this is a command'.split(' '), - discard_warnings=True) - self.assertIsNotNone(o) - self.assertEqual('', e) - - -class FakeSshChannel(object): - def __init__(self, rc): - self.rc = rc - - def recv_exit_status(self): - return self.rc - - -class FakeSshStream(six.BytesIO): - def setup_channel(self, rc): - self.channel = FakeSshChannel(rc) - - -class FakeSshConnection(object): - def __init__(self, rc): - self.rc = rc - - def exec_command(self, cmd): - stdout = FakeSshStream(b'stdout') - stdout.setup_channel(self.rc) - return (six.BytesIO(), - stdout, - six.BytesIO(b'stderr')) - - -class SshExecuteTestCase(test_base.BaseTestCase): - def test_invalid_addl_env(self): - self.assertRaises(processutils.InvalidArgumentError, - processutils.ssh_execute, - None, 'ls', addl_env='important') - - def test_invalid_process_input(self): - self.assertRaises(processutils.InvalidArgumentError, - processutils.ssh_execute, - None, 'ls', process_input='important') - - def test_works(self): - o, e = processutils.ssh_execute(FakeSshConnection(0), 'ls') - self.assertEqual('stdout', o) - self.assertEqual('stderr', e) - self.assertEqual(type(o), six.text_type) - self.assertEqual(type(e), six.text_type) - - def test_fails(self): - self.assertRaises(processutils.ProcessExecutionError, - processutils.ssh_execute, FakeSshConnection(1), 'ls') - - def _test_compromising_ssh(self, rc, check): - fixture = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG)) - fake_stdin = six.BytesIO() - - fake_stdout = mock.Mock() - fake_stdout.channel.recv_exit_status.return_value = rc - fake_stdout.read.return_value = b'password="secret"' - - fake_stderr = six.BytesIO(b'password="foobar"') - - command = 'ls --password="bar"' - - connection = mock.Mock() - connection.exec_command.return_value = (fake_stdin, fake_stdout, - fake_stderr) - - if check and rc != -1 and rc != 0: - err = self.assertRaises(processutils.ProcessExecutionError, - processutils.ssh_execute, - connection, command, - check_exit_code=check) - - self.assertEqual(rc, err.exit_code) - self.assertEqual(err.stdout, 'password="***"') - self.assertEqual(err.stderr, 'password="***"') - self.assertEqual(err.cmd, 'ls --password="***"') - self.assertNotIn('secret', str(err)) - self.assertNotIn('foobar', str(err)) - else: - o, e = processutils.ssh_execute(connection, command, - check_exit_code=check) - self.assertEqual('password="***"', o) - self.assertEqual('password="***"', e) - self.assertIn('password="***"', fixture.output) - self.assertNotIn('bar', fixture.output) - - def test_compromising_ssh1(self): - self._test_compromising_ssh(rc=-1, check=True) - - def test_compromising_ssh2(self): - self._test_compromising_ssh(rc=0, check=True) - - def test_compromising_ssh3(self): - self._test_compromising_ssh(rc=1, check=True) - - def test_compromising_ssh4(self): - self._test_compromising_ssh(rc=1, check=False) - - def test_compromising_ssh5(self): - self._test_compromising_ssh(rc=0, check=False) - - def test_compromising_ssh6(self): - self._test_compromising_ssh(rc=-1, check=False) diff --git a/tests/test_warning.py b/tests/test_warning.py deleted file mode 100644 index ec71e0c..0000000 --- a/tests/test_warning.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import imp -import os -import warnings - -import mock -from oslotest import base as test_base -import six - - -class DeprecationWarningTest(test_base.BaseTestCase): - - @mock.patch('warnings.warn') - def test_warning(self, mock_warn): - import oslo.concurrency - imp.reload(oslo.concurrency) - self.assertTrue(mock_warn.called) - args = mock_warn.call_args - self.assertIn('oslo_concurrency', args[0][0]) - self.assertIn('deprecated', args[0][0]) - self.assertTrue(issubclass(args[0][1], DeprecationWarning)) - - def test_real_warning(self): - with warnings.catch_warnings(record=True) as warning_msgs: - warnings.resetwarnings() - warnings.simplefilter('always', DeprecationWarning) - import oslo.concurrency - - # Use a separate function to get the stack level correct - # so we know the message points back to this file. This - # corresponds to an import or reload, which isn't working - # inside the test under Python 3.3. That may be due to a - # difference in the import implementation not triggering - # warnings properly when the module is reloaded, or - # because the warnings module is mostly implemented in C - # and something isn't cleanly resetting the global state - # used to track whether a warning needs to be - # emitted. Whatever the cause, we definitely see the - # warnings.warn() being invoked on a reload (see the test - # above) and warnings are reported on the console when we - # run the tests. A simpler test script run outside of - # testr does correctly report the warnings. - def foo(): - oslo.concurrency.deprecated() - - foo() - self.assertEqual(1, len(warning_msgs)) - msg = warning_msgs[0] - self.assertIn('oslo_concurrency', six.text_type(msg.message)) - self.assertEqual('test_warning.py', os.path.basename(msg.filename)) diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py deleted file mode 100644 index 8bbacc9..0000000 --- a/tests/test_watchdog.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import subprocess -import time - -import fixtures -from oslotest import base as test_base - -from oslo_concurrency import watchdog - -LOG_FORMAT = '%(levelname)s %(message)s' - - -class WatchdogTest(test_base.BaseTestCase): - def setUp(self): - super(WatchdogTest, self).setUp() - # capture the log bits where we can interrogate them - self.logger = logging.getLogger() - self.logger.setLevel(logging.DEBUG) - self.log = self.useFixture( - fixtures.FakeLogger(format=LOG_FORMAT, level=None) - ) - - def test_in_process_delay(self): - with watchdog.watch(self.logger, "in process", after=1.0): - time.sleep(2) - self.assertIn("DEBUG in process not completed after 1", - self.log.output) - loglines = self.log.output.rstrip().split("\n") - self.assertEqual(1, len(loglines), loglines) - - def test_level_setting(self): - with watchdog.watch(self.logger, "in process", - level=logging.ERROR, after=1.0): - time.sleep(2) - self.assertIn("ERROR in process not completed after 1", - self.log.output) - loglines = self.log.output.rstrip().split("\n") - self.assertEqual(1, len(loglines), loglines) - - def test_in_process_delay_no_message(self): - with watchdog.watch(self.logger, "in process", after=1.0): - pass - # wait long enough to know there won't be a message emitted - time.sleep(2) - self.assertEqual('', self.log.output) - - def test_in_process_exploding(self): - try: - with watchdog.watch(self.logger, "ungraceful exit", after=1.0): - raise Exception() - except Exception: - pass - # wait long enough to know there won't be a message emitted - time.sleep(2) - self.assertEqual('', self.log.output) - - def test_subprocess_delay(self): - with watchdog.watch(self.logger, "subprocess", after=0.1): - subprocess.call("sleep 2", shell=True) - self.assertIn("DEBUG subprocess not completed after 0", - self.log.output)