From 372df8a75b33d23a42a8e636090c91668a56a586 Mon Sep 17 00:00:00 2001 From: "ChangBo Guo(gcb)" Date: Mon, 10 Nov 2014 20:45:12 +0800 Subject: [PATCH] Switch Cinder to use oslo.concurrency Let's switch to the newly released oslo library for the processutils and lockutils. We use the config fixture(s) to specify disable_process_locking and lock_path in the CONF variable of oslo.concurrency library for correctly setting the flags. Change-Id: Ib8f3aac5449eba66ea84bc5cad8aea061adab276 --- cinder/backup/drivers/tsm.py | 2 +- cinder/brick/executor.py | 2 +- cinder/brick/initiator/connector.py | 5 +- cinder/brick/initiator/linuxfc.py | 3 +- cinder/brick/initiator/linuxscsi.py | 3 +- cinder/brick/iscsi/iscsi.py | 2 +- cinder/brick/local_dev/lvm.py | 2 +- cinder/brick/remotefs/remotefs.py | 2 +- cinder/image/image_utils.py | 2 +- cinder/openstack/common/lockutils.py | 279 ----------------- cinder/openstack/common/processutils.py | 289 ------------------ cinder/service.py | 2 +- cinder/test.py | 9 +- .../tests/api/contrib/test_admin_actions.py | 13 +- cinder/tests/brick/test_brick_connector.py | 3 +- cinder/tests/brick/test_brick_lvm.py | 2 +- cinder/tests/fake_utils.py | 2 +- cinder/tests/test_backup_ceph.py | 2 +- cinder/tests/test_backup_tsm.py | 3 +- cinder/tests/test_emc_vnxdirect.py | 6 +- cinder/tests/test_eqlx.py | 2 +- cinder/tests/test_glusterfs.py | 2 +- cinder/tests/test_gpfs.py | 2 +- cinder/tests/test_ibmnas.py | 10 +- cinder/tests/test_image_utils.py | 2 +- cinder/tests/test_netapp_utils.py | 2 +- cinder/tests/test_pure.py | 2 +- cinder/tests/test_service.py | 2 +- cinder/tests/test_sheepdog.py | 2 +- cinder/tests/test_storwize_svc.py | 2 +- cinder/tests/test_utils.py | 2 +- cinder/tests/test_volume_utils.py | 2 +- .../test_brcd_fc_zone_client_cli.py | 2 +- .../test_cisco_fc_zone_client_cli.py | 2 +- .../zonemanager/test_cisco_fc_zone_driver.py | 2 +- cinder/utils.py | 4 +- cinder/volume/driver.py | 2 +- cinder/volume/drivers/coraid.py | 2 +- cinder/volume/drivers/emc/emc_vnx_cli.py | 4 +- cinder/volume/drivers/eqlx.py | 2 +- .../volume/drivers/fujitsu_eternus_dx_fc.py | 2 +- .../drivers/fujitsu_eternus_dx_iscsi.py | 2 +- cinder/volume/drivers/glusterfs.py | 2 +- cinder/volume/drivers/hds/nfs.py | 2 +- .../volume/drivers/hitachi/hbsd_basiclib.py | 4 +- cinder/volume/drivers/hitachi/hbsd_horcm.py | 2 +- cinder/volume/drivers/ibm/gpfs.py | 2 +- cinder/volume/drivers/ibm/ibmnas.py | 2 +- cinder/volume/drivers/ibm/storwize_svc/ssh.py | 3 +- cinder/volume/drivers/lvm.py | 2 +- cinder/volume/drivers/netapp/nfs.py | 2 +- cinder/volume/drivers/netapp/utils.py | 2 +- cinder/volume/drivers/nfs.py | 2 +- cinder/volume/drivers/pure.py | 2 +- cinder/volume/drivers/remotefs.py | 2 +- .../drivers/san/hp/hp_lefthand_cliq_proxy.py | 2 +- cinder/volume/drivers/san/san.py | 2 +- cinder/volume/drivers/sheepdog.py | 2 +- cinder/volume/drivers/smbfs.py | 2 +- cinder/volume/flows/manager/create_volume.py | 2 +- cinder/volume/iscsi.py | 3 +- cinder/volume/utils.py | 2 +- .../brocade/brcd_fc_zone_client_cli.py | 2 +- .../drivers/brocade/brcd_fc_zone_driver.py | 2 +- .../cisco/cisco_fc_san_lookup_service.py | 2 +- .../drivers/cisco/cisco_fc_zone_client_cli.py | 2 +- .../drivers/cisco/cisco_fc_zone_driver.py | 2 +- etc/cinder/cinder.conf.sample | 12 - openstack-common.conf | 2 - requirements.txt | 1 + 70 files changed, 97 insertions(+), 661 deletions(-) delete mode 100644 cinder/openstack/common/lockutils.py delete mode 100644 cinder/openstack/common/processutils.py diff --git a/cinder/backup/drivers/tsm.py b/cinder/backup/drivers/tsm.py index bf60c284100..5f5f0573bef 100644 --- a/cinder/backup/drivers/tsm.py +++ b/cinder/backup/drivers/tsm.py @@ -29,13 +29,13 @@ import json import os import stat +from oslo.concurrency import processutils from oslo.config import cfg from cinder.backup.driver import BackupDriver from cinder import exception from cinder.i18n import _LE, _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/executor.py b/cinder/brick/executor.py index de689521cce..ea950055b2b 100644 --- a/cinder/brick/executor.py +++ b/cinder/brick/executor.py @@ -18,7 +18,7 @@ and root_helper settings, so this provides that hook. """ -from cinder.openstack.common import processutils as putils +from oslo.concurrency import processutils as putils class Executor(object): diff --git a/cinder/brick/initiator/connector.py b/cinder/brick/initiator/connector.py index 57974bbf3a4..6f082e0a0d2 100644 --- a/cinder/brick/initiator/connector.py +++ b/cinder/brick/initiator/connector.py @@ -17,6 +17,9 @@ import os import socket import time +from oslo.concurrency import lockutils +from oslo.concurrency import processutils as putils + from cinder.brick import exception from cinder.brick import executor from cinder.brick.initiator import host_driver @@ -24,10 +27,8 @@ from cinder.brick.initiator import linuxfc from cinder.brick.initiator import linuxscsi from cinder.brick.remotefs import remotefs from cinder.i18n import _, _LE -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall -from cinder.openstack.common import processutils as putils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/initiator/linuxfc.py b/cinder/brick/initiator/linuxfc.py index edd9e7c52ac..710db54a02d 100644 --- a/cinder/brick/initiator/linuxfc.py +++ b/cinder/brick/initiator/linuxfc.py @@ -16,10 +16,11 @@ import errno +from oslo.concurrency import processutils as putils + from cinder.brick.initiator import linuxscsi from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/initiator/linuxscsi.py b/cinder/brick/initiator/linuxscsi.py index 9ee74ae3f01..7a1951c2bad 100644 --- a/cinder/brick/initiator/linuxscsi.py +++ b/cinder/brick/initiator/linuxscsi.py @@ -19,10 +19,11 @@ import os import re +from oslo.concurrency import processutils as putils + from cinder.brick import executor from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/iscsi/iscsi.py b/cinder/brick/iscsi/iscsi.py index fee1685177a..14327baab54 100644 --- a/cinder/brick/iscsi/iscsi.py +++ b/cinder/brick/iscsi/iscsi.py @@ -23,6 +23,7 @@ import re import stat import time +from oslo.concurrency import processutils as putils from oslo.config import cfg import six @@ -31,7 +32,6 @@ from cinder.brick import executor from cinder.i18n import _, _LE, _LI, _LW from cinder.openstack.common import fileutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import utils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/local_dev/lvm.py b/cinder/brick/local_dev/lvm.py index 2612b9bc808..2a94892cc37 100644 --- a/cinder/brick/local_dev/lvm.py +++ b/cinder/brick/local_dev/lvm.py @@ -22,13 +22,13 @@ import math import re import time +from oslo.concurrency import processutils as putils from oslo.utils import excutils from cinder.brick import exception from cinder.brick import executor from cinder.i18n import _, _LE, _LW from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils LOG = logging.getLogger(__name__) diff --git a/cinder/brick/remotefs/remotefs.py b/cinder/brick/remotefs/remotefs.py index e951652225b..7a6266ff88d 100644 --- a/cinder/brick/remotefs/remotefs.py +++ b/cinder/brick/remotefs/remotefs.py @@ -19,12 +19,12 @@ import hashlib import os import re +from oslo.concurrency import processutils as putils import six from cinder.brick import exception from cinder.i18n import _, _LI from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils LOG = logging.getLogger(__name__) diff --git a/cinder/image/image_utils.py b/cinder/image/image_utils.py index ebd89b8985d..608993901bd 100644 --- a/cinder/image/image_utils.py +++ b/cinder/image/image_utils.py @@ -28,6 +28,7 @@ import contextlib import os import tempfile +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import timeutils from oslo.utils import units @@ -37,7 +38,6 @@ from cinder.i18n import _ from cinder.openstack.common import fileutils from cinder.openstack.common import imageutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume import utils as volume_utils diff --git a/cinder/openstack/common/lockutils.py b/cinder/openstack/common/lockutils.py deleted file mode 100644 index 3a3100d764b..00000000000 --- a/cinder/openstack/common/lockutils.py +++ /dev/null @@ -1,279 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import errno -import functools -import os -import shutil -import tempfile -import time -import weakref - -from eventlet import semaphore -from oslo.config import cfg - -from cinder.openstack.common import fileutils -from cinder.openstack.common.gettextutils import _ -from cinder.openstack.common import local -from cinder.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - - -util_opts = [ - cfg.BoolOpt('disable_process_locking', default=False, - help='Whether to disable inter-process locks'), - cfg.StrOpt('lock_path', - help=('Directory to use for lock files. Default to a ' - 'temp directory')) -] - - -CONF = cfg.CONF -CONF.register_opts(util_opts) - - -def set_defaults(lock_path): - cfg.set_defaults(util_opts, lock_path=lock_path) - - -class _InterProcessLock(object): - """Lock implementation which allows multiple locks, working around - issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does - not require any cleanup. Since the lock is always held on a file - descriptor rather than outside of the process, the lock gets dropped - automatically if the process crashes, even if __exit__ is not executed. - - There are no guarantees regarding usage by multiple green threads in a - single process here. This lock works only between processes. Exclusive - access between local threads should be achieved using the semaphores - in the @synchronized decorator. - - Note these locks are released when the descriptor is closed, so it's not - safe to close the file descriptor while another green thread holds the - lock. Just opening and closing the lock file can break synchronisation, - so lock files must be accessed only using this abstraction. - """ - - def __init__(self, name): - self.lockfile = None - self.fname = name - - def __enter__(self): - self.lockfile = open(self.fname, 'w') - - while True: - try: - # Using non-blocking locks since green threads are not - # patched to deal with blocking locking calls. - # Also upon reading the MSDN docs for locking(), it seems - # to have a laughable 10 attempts "blocking" mechanism. - self.trylock() - return self - except IOError as e: - if e.errno in (errno.EACCES, errno.EAGAIN): - # external locks synchronise things like iptables - # updates - give it some time to prevent busy spinning - time.sleep(0.01) - else: - raise - - def __exit__(self, exc_type, exc_val, exc_tb): - try: - self.unlock() - self.lockfile.close() - except IOError: - LOG.exception(_("Could not release the acquired lock `%s`"), - self.fname) - - def trylock(self): - raise NotImplementedError() - - def unlock(self): - raise NotImplementedError() - - -class _WindowsLock(_InterProcessLock): - def trylock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) - - def unlock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) - - -class _PosixLock(_InterProcessLock): - def trylock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - def unlock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_UN) - - -if os.name == 'nt': - import msvcrt - InterProcessLock = _WindowsLock -else: - import fcntl - InterProcessLock = _PosixLock - -_semaphores = weakref.WeakValueDictionary() - - -def synchronized(name, lock_file_prefix, external=False, lock_path=None): - """Synchronization decorator. - - Decorating a method like so:: - - @synchronized('mylock') - def foo(self, *args): - ... - - ensures that only one thread will execute the foo method at a time. - - Different methods can share the same lock:: - - @synchronized('mylock') - def foo(self, *args): - ... - - @synchronized('mylock') - def bar(self, *args): - ... - - This way only one of either foo or bar can be executing at a time. - - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. The prefix should end with a - hyphen ('-') if specified. - - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. - - :param lock_path: The lock_path keyword argument is used to specify a - special location for external lock files to live. If nothing is set, then - CONF.lock_path is used as a default. - """ - - def wrap(f): - @functools.wraps(f) - def inner(*args, **kwargs): - # NOTE(soren): If we ever go natively threaded, this will be racy. - # See http://stackoverflow.com/questions/5390569/dyn - # amically-allocating-and-destroying-mutexes - sem = _semaphores.get(name, semaphore.Semaphore()) - if name not in _semaphores: - # this check is not racy - we're already holding ref locally - # so GC won't remove the item and there was no IO switch - # (only valid in greenthreads) - _semaphores[name] = sem - - with sem: - LOG.debug(_('Got semaphore "%(lock)s" for method ' - '"%(method)s"...'), {'lock': name, - 'method': f.__name__}) - - # NOTE(mikal): I know this looks odd - if not hasattr(local.strong_store, 'locks_held'): - local.strong_store.locks_held = [] - local.strong_store.locks_held.append(name) - - try: - if external and not CONF.disable_process_locking: - LOG.debug(_('Attempting to grab file lock "%(lock)s" ' - 'for method "%(method)s"...'), - {'lock': name, 'method': f.__name__}) - cleanup_dir = False - - # We need a copy of lock_path because it is non-local - local_lock_path = lock_path - if not local_lock_path: - local_lock_path = CONF.lock_path - - if not local_lock_path: - cleanup_dir = True - local_lock_path = tempfile.mkdtemp() - - if not os.path.exists(local_lock_path): - fileutils.ensure_tree(local_lock_path) - - # NOTE(mikal): the lock name cannot contain directory - # separators - safe_name = name.replace(os.sep, '_') - lock_file_name = '%s%s' % (lock_file_prefix, safe_name) - lock_file_path = os.path.join(local_lock_path, - lock_file_name) - - try: - lock = InterProcessLock(lock_file_path) - with lock: - LOG.debug(_('Got file lock "%(lock)s" at ' - '%(path)s for method ' - '"%(method)s"...'), - {'lock': name, - 'path': lock_file_path, - 'method': f.__name__}) - retval = f(*args, **kwargs) - finally: - LOG.debug(_('Released file lock "%(lock)s" at ' - '%(path)s for method "%(method)s"...'), - {'lock': name, - 'path': lock_file_path, - 'method': f.__name__}) - # NOTE(vish): This removes the tempdir if we needed - # to create one. This is used to - # cleanup the locks left behind by unit - # tests. - if cleanup_dir: - shutil.rmtree(local_lock_path) - else: - retval = f(*args, **kwargs) - - finally: - local.strong_store.locks_held.remove(name) - - return retval - return inner - return wrap - - -def synchronized_with_prefix(lock_file_prefix): - """Partial object generator for the synchronization decorator. - - Redefine @synchronized in each project like so:: - - (in nova/utils.py) - from nova.openstack.common import lockutils - - synchronized = lockutils.synchronized_with_prefix('nova-') - - - (in nova/foo.py) - from nova import utils - - @utils.synchronized('mylock') - def bar(self, *args): - ... - - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. The prefix should end with a hyphen ('-') if specified. - """ - - return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) diff --git a/cinder/openstack/common/processutils.py b/cinder/openstack/common/processutils.py deleted file mode 100644 index a4dc6718b99..00000000000 --- a/cinder/openstack/common/processutils.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import errno -import logging -import multiprocessing -import os -import random -import shlex -import signal - -from eventlet.green import subprocess -from eventlet import greenthread -import six - -from cinder.openstack.common.gettextutils import _ -from cinder.openstack.common import strutils - - -LOG = logging.getLogger(__name__) - - -class InvalidArgumentError(Exception): - def __init__(self, message=None): - super(InvalidArgumentError, self).__init__(message) - - -class UnknownArgumentError(Exception): - def __init__(self, message=None): - super(UnknownArgumentError, self).__init__(message) - - -class ProcessExecutionError(Exception): - def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - self.exit_code = exit_code - self.stderr = stderr - self.stdout = stdout - self.cmd = cmd - self.description = description - - if description is None: - description = _("Unexpected error while running command.") - if exit_code is None: - exit_code = '-' - message = _('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Stdout: %(stdout)r\n' - 'Stderr: %(stderr)r') % {'description': description, - 'cmd': cmd, - 'exit_code': exit_code, - 'stdout': stdout, - 'stderr': stderr} - super(ProcessExecutionError, self).__init__(message) - - -class NoRootWrapSpecified(Exception): - def __init__(self, message=None): - super(NoRootWrapSpecified, self).__init__(message) - - -def _subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - -def execute(*cmd, **kwargs): - """Helper method to shell out and execute a command through subprocess. - - Allows optional retry. - - :param cmd: Passed to subprocess.Popen. - :type cmd: string - :param process_input: Send to opened process. - :type process_input: string - :param env_variables: Environment variables and their values that - will be set for the process. - :type env_variables: dict - :param check_exit_code: Single bool, int, or list of allowed exit - codes. Defaults to [0]. Raise - :class:`ProcessExecutionError` unless - program exits with one of these code. - :type check_exit_code: boolean, int, or [int] - :param delay_on_retry: True | False. Defaults to True. If set to True, - wait a short amount of time before retrying. - :type delay_on_retry: boolean - :param attempts: How many times to retry cmd. - :type attempts: int - :param run_as_root: True | False. Defaults to False. If set to True, - the command is prefixed by the command specified - in the root_helper kwarg. - :type run_as_root: boolean - :param root_helper: command to prefix to commands called with - run_as_root=True - :type root_helper: string - :param shell: whether or not there should be a shell used to - execute this command. Defaults to false. - :type shell: boolean - :param loglevel: log level for execute commands. - :type loglevel: int. (Should be logging.DEBUG or logging.INFO) - :returns: (stdout, stderr) from process execution - :raises: :class:`UnknownArgumentError` on - receiving unknown arguments - :raises: :class:`ProcessExecutionError` - """ - - process_input = kwargs.pop('process_input', None) - env_variables = kwargs.pop('env_variables', None) - check_exit_code = kwargs.pop('check_exit_code', [0]) - ignore_exit_code = False - delay_on_retry = kwargs.pop('delay_on_retry', True) - attempts = kwargs.pop('attempts', 1) - run_as_root = kwargs.pop('run_as_root', False) - root_helper = kwargs.pop('root_helper', '') - shell = kwargs.pop('shell', False) - loglevel = kwargs.pop('loglevel', logging.DEBUG) - - if isinstance(check_exit_code, bool): - ignore_exit_code = not check_exit_code - check_exit_code = [0] - elif isinstance(check_exit_code, int): - check_exit_code = [check_exit_code] - - if kwargs: - raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs) - - if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: - if not root_helper: - raise NoRootWrapSpecified( - message=_('Command requested root, but did not ' - 'specify a root helper.')) - cmd = shlex.split(root_helper) + list(cmd) - - cmd = map(str, cmd) - sanitized_cmd = strutils.mask_password(' '.join(cmd)) - - while attempts > 0: - attempts -= 1 - try: - LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd) - _PIPE = subprocess.PIPE # pylint: disable=E1101 - - if os.name == 'nt': - preexec_fn = None - close_fds = False - else: - preexec_fn = _subprocess_setup - close_fds = True - - obj = subprocess.Popen(cmd, - stdin=_PIPE, - stdout=_PIPE, - stderr=_PIPE, - close_fds=close_fds, - preexec_fn=preexec_fn, - shell=shell, - env=env_variables) - result = None - for _i in six.moves.range(20): - # NOTE(russellb) 20 is an arbitrary number of retries to - # prevent any chance of looping forever here. - try: - if process_input is not None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - except OSError as e: - if e.errno in (errno.EAGAIN, errno.EINTR): - continue - raise - break - obj.stdin.close() # pylint: disable=E1101 - _returncode = obj.returncode # pylint: disable=E1101 - LOG.log(loglevel, 'Result was %s' % _returncode) - if not ignore_exit_code and _returncode not in check_exit_code: - (stdout, stderr) = result - sanitized_stdout = strutils.mask_password(stdout) - sanitized_stderr = strutils.mask_password(stderr) - raise ProcessExecutionError(exit_code=_returncode, - stdout=sanitized_stdout, - stderr=sanitized_stderr, - cmd=sanitized_cmd) - return result - except ProcessExecutionError: - if not attempts: - raise - else: - LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd) - if delay_on_retry: - greenthread.sleep(random.randint(20, 200) / 100.0) - finally: - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) - - -def trycmd(*args, **kwargs): - """A wrapper around execute() to more easily handle warnings and errors. - - Returns an (out, err) tuple of strings containing the output of - the command's stdout and stderr. If 'err' is not empty then the - command can be considered to have failed. - - :discard_warnings True | False. Defaults to False. If set to True, - then for succeeding commands, stderr is cleared - - """ - discard_warnings = kwargs.pop('discard_warnings', False) - - try: - out, err = execute(*args, **kwargs) - failed = False - except ProcessExecutionError as exn: - out, err = '', six.text_type(exn) - failed = True - - if not failed and discard_warnings and err: - # Handle commands that output to stderr but otherwise succeed - err = '' - - return out, err - - -def ssh_execute(ssh, cmd, process_input=None, - addl_env=None, check_exit_code=True): - sanitized_cmd = strutils.mask_password(cmd) - LOG.debug('Running cmd (SSH): %s', sanitized_cmd) - if addl_env: - raise InvalidArgumentError(_('Environment not supported over SSH')) - - if process_input: - # This is (probably) fixable if we need it... - raise InvalidArgumentError(_('process_input not supported over SSH')) - - stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) - channel = stdout_stream.channel - - # NOTE(justinsb): This seems suspicious... - # ...other SSH clients have buffering issues with this approach - stdout = stdout_stream.read() - sanitized_stdout = strutils.mask_password(stdout) - stderr = stderr_stream.read() - sanitized_stderr = strutils.mask_password(stderr) - - stdin_stream.close() - - exit_status = channel.recv_exit_status() - - # exit_status == -1 if no exit code was returned - if exit_status != -1: - LOG.debug('Result was %s' % exit_status) - if check_exit_code and exit_status != 0: - raise ProcessExecutionError(exit_code=exit_status, - stdout=sanitized_stdout, - stderr=sanitized_stderr, - cmd=sanitized_cmd) - - return (sanitized_stdout, sanitized_stderr) - - -def get_worker_count(): - """Utility to get the default worker count. - - @return: The number of CPUs if that can be determined, else a default - worker count of 1 is returned. - """ - try: - return multiprocessing.cpu_count() - except NotImplementedError: - return 1 diff --git a/cinder/service.py b/cinder/service.py index bfd2f897936..0bb4e6d2e49 100644 --- a/cinder/service.py +++ b/cinder/service.py @@ -22,6 +22,7 @@ import inspect import os import random +from oslo.concurrency import processutils from oslo.config import cfg from oslo.db import exception as db_exc from oslo import messaging @@ -36,7 +37,6 @@ from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall -from cinder.openstack.common import processutils from cinder.openstack.common import service from cinder import rpc from cinder import version diff --git a/cinder/test.py b/cinder/test.py index 16d4860a099..1f64ccc28f5 100644 --- a/cinder/test.py +++ b/cinder/test.py @@ -24,13 +24,14 @@ inline callbacks. import logging import os import shutil -import tempfile import uuid import fixtures import mock import mox +from oslo.concurrency import lockutils from oslo.config import cfg +from oslo.config import fixture as config_fixture from oslo.messaging import conffixture as messaging_conffixture from oslo.utils import strutils from oslo.utils import timeutils @@ -187,7 +188,11 @@ class TestCase(testtools.TestCase): self.override_config('fatal_exception_format_errors', True) # This will be cleaned up by the NestedTempfile fixture - self.override_config('lock_path', tempfile.mkdtemp()) + lock_path = self.useFixture(fixtures.TempDir()).path + self.fixture = self.useFixture( + config_fixture.Config(lockutils.CONF)) + self.fixture.config(lock_path=lock_path, + group='oslo_concurrency') self.override_config('policy_file', os.path.join( os.path.abspath( diff --git a/cinder/tests/api/contrib/test_admin_actions.py b/cinder/tests/api/contrib/test_admin_actions.py index 727f407d6b3..a1f0c54ac40 100644 --- a/cinder/tests/api/contrib/test_admin_actions.py +++ b/cinder/tests/api/contrib/test_admin_actions.py @@ -11,9 +11,11 @@ # under the License. import ast -import tempfile +import fixtures +from oslo.concurrency import lockutils from oslo.config import cfg +from oslo.config import fixture as config_fixture from oslo.serialization import jsonutils from oslo.utils import timeutils import webob @@ -45,10 +47,13 @@ class AdminActionsTest(test.TestCase): def setUp(self): super(AdminActionsTest, self).setUp() - self.tempdir = tempfile.mkdtemp() + self.tempdir = self.useFixture(fixtures.TempDir()).path + self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF)) + self.fixture.config(lock_path=self.tempdir, + group='oslo_concurrency') + self.fixture.config(disable_process_locking=True, + group='oslo_concurrency') self.flags(rpc_backend='cinder.openstack.common.rpc.impl_fake') - self.flags(lock_path=self.tempdir, - disable_process_locking=True) self.volume_api = volume_api.API() cast_as_call.mock_cast_as_call(self.volume_api.volume_rpcapi.client) diff --git a/cinder/tests/brick/test_brick_connector.py b/cinder/tests/brick/test_brick_connector.py index 5f5978b993b..5662eb028eb 100644 --- a/cinder/tests/brick/test_brick_connector.py +++ b/cinder/tests/brick/test_brick_connector.py @@ -16,13 +16,14 @@ import os.path import string import time +from oslo.concurrency import processutils as putils + from cinder.brick import exception from cinder.brick.initiator import connector from cinder.brick.initiator import host_driver from cinder.i18n import _ from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall -from cinder.openstack.common import processutils as putils from cinder import test LOG = logging.getLogger(__name__) diff --git a/cinder/tests/brick/test_brick_lvm.py b/cinder/tests/brick/test_brick_lvm.py index 11a317d9adb..156c9a5ac4a 100644 --- a/cinder/tests/brick/test_brick_lvm.py +++ b/cinder/tests/brick/test_brick_lvm.py @@ -14,11 +14,11 @@ # under the License. import mox +from oslo.concurrency import processutils from cinder.brick import exception from cinder.brick.local_dev import lvm as brick from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf diff --git a/cinder/tests/fake_utils.py b/cinder/tests/fake_utils.py index 8e75e790a1a..2cae17ffc52 100644 --- a/cinder/tests/fake_utils.py +++ b/cinder/tests/fake_utils.py @@ -17,9 +17,9 @@ import re from eventlet import greenthread +from oslo.concurrency import processutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils LOG = logging.getLogger(__name__) diff --git a/cinder/tests/test_backup_ceph.py b/cinder/tests/test_backup_ceph.py index 409a3a983dc..7827b1c20b4 100644 --- a/cinder/tests/test_backup_ceph.py +++ b/cinder/tests/test_backup_ceph.py @@ -21,6 +21,7 @@ import tempfile import uuid import mock +from oslo.concurrency import processutils from oslo.serialization import jsonutils import six @@ -31,7 +32,6 @@ from cinder import db from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.volume.drivers import rbd as rbddriver diff --git a/cinder/tests/test_backup_tsm.py b/cinder/tests/test_backup_tsm.py index aeca6764663..f50af733830 100644 --- a/cinder/tests/test_backup_tsm.py +++ b/cinder/tests/test_backup_tsm.py @@ -22,12 +22,13 @@ import json import os import posix +from oslo.concurrency import processutils as putils + from cinder.backup.drivers import tsm from cinder import context from cinder import db from cinder import exception from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import test from cinder import utils diff --git a/cinder/tests/test_emc_vnxdirect.py b/cinder/tests/test_emc_vnxdirect.py index 6d6200b997a..4148d59d869 100644 --- a/cinder/tests/test_emc_vnxdirect.py +++ b/cinder/tests/test_emc_vnxdirect.py @@ -17,9 +17,9 @@ import os import re import mock +from oslo.concurrency import processutils from cinder import exception -from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers.emc.emc_cli_fc import EMCCLIFCDriver @@ -1155,7 +1155,7 @@ Time Remaining: 0 second(s) fake_cli.assert_has_calls(expect_cmd) @mock.patch( - "cinder.openstack.common.processutils.execute", + "oslo.concurrency.processutils.execute", mock.Mock( return_value=( "fakeportal iqn.1992-04.fake.com:fake.apm00123907237.a8", 0))) @@ -2474,7 +2474,7 @@ class EMCVNXCLIDriverFCTestCase(test.TestCase): return None @mock.patch( - "cinder.openstack.common.processutils.execute", + "oslo.concurrency.processutils.execute", mock.Mock( return_value=( "fakeportal iqn.1992-04.fake.com:fake.apm00123907237.a8", 0))) diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py index 4324bed0505..5139cbbc174 100644 --- a/cinder/tests/test_eqlx.py +++ b/cinder/tests/test_eqlx.py @@ -16,12 +16,12 @@ import time import mox +from oslo.concurrency import processutils import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx diff --git a/cinder/tests/test_glusterfs.py b/cinder/tests/test_glusterfs.py index e839255c8cb..c433f22761e 100644 --- a/cinder/tests/test_glusterfs.py +++ b/cinder/tests/test_glusterfs.py @@ -26,6 +26,7 @@ import mox as mox_lib from mox import IgnoreArg from mox import IsA from mox import stubout +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import units @@ -37,7 +38,6 @@ from cinder import exception from cinder.i18n import _ from cinder.image import image_utils from cinder.openstack.common import imageutils -from cinder.openstack.common import processutils as putils from cinder import test from cinder import utils from cinder.volume import configuration as conf diff --git a/cinder/tests/test_gpfs.py b/cinder/tests/test_gpfs.py index 9fd1f6abbda..54e760b9454 100644 --- a/cinder/tests/test_gpfs.py +++ b/cinder/tests/test_gpfs.py @@ -18,6 +18,7 @@ import shutil import tempfile import mock +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -25,7 +26,6 @@ from cinder import context from cinder import exception from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder import utils from cinder.volume import configuration as conf diff --git a/cinder/tests/test_ibmnas.py b/cinder/tests/test_ibmnas.py index 11b294f42df..a190470849d 100644 --- a/cinder/tests/test_ibmnas.py +++ b/cinder/tests/test_ibmnas.py @@ -180,7 +180,7 @@ class IBMNASDriverTestCase(test.TestCase): @mock.patch('cinder.volume.drivers.ibm.ibmnas.IBMNAS_NFSDriver.' '_ssh_operation') - @mock.patch('cinder.openstack.common.processutils.execute') + @mock.patch('oslo.concurrency.processutils.execute') def test_create_ibmnas_snap_mount_point_provided(self, mock_ssh, mock_execute): """Create ibmnas snap if mount point is provided.""" @@ -195,7 +195,7 @@ class IBMNASDriverTestCase(test.TestCase): @mock.patch('cinder.volume.drivers.ibm.ibmnas.IBMNAS_NFSDriver.' '_ssh_operation') - @mock.patch('cinder.openstack.common.processutils.execute') + @mock.patch('oslo.concurrency.processutils.execute') def test_create_ibmnas_snap_nas_gpfs(self, mock_execute, mock_ssh): """Create ibmnas snap if mount point is provided.""" @@ -300,7 +300,7 @@ class IBMNASDriverTestCase(test.TestCase): self.TEST_EXTEND_SIZE_IN_GB) @mock.patch('cinder.volume.drivers.ibm.ibmnas.IBMNAS_NFSDriver._run_ssh') - @mock.patch('cinder.openstack.common.processutils.execute') + @mock.patch('oslo.concurrency.processutils.execute') def test_delete_snapfiles(self, mock_execute, mock_ssh): """Delete_snapfiles test case.""" @@ -316,7 +316,7 @@ class IBMNASDriverTestCase(test.TestCase): self.TEST_MNT_POINT) @mock.patch('cinder.volume.drivers.ibm.ibmnas.IBMNAS_NFSDriver._run_ssh') - @mock.patch('cinder.openstack.common.processutils.execute') + @mock.patch('oslo.concurrency.processutils.execute') def test_delete_snapfiles_nas_gpfs(self, mock_execute, mock_ssh): """Delete_snapfiles for gpfs-nas platform test case.""" @@ -397,7 +397,7 @@ class IBMNASDriverTestCase(test.TestCase): '_get_provider_location') @mock.patch('cinder.volume.drivers.ibm.ibmnas.IBMNAS_NFSDriver.' '_get_mount_point_for_share') - @mock.patch('cinder.openstack.common.processutils.execute') + @mock.patch('oslo.concurrency.processutils.execute') def test_delete_snapshot(self, mock_execute, mock_mount, mock_provider): """Delete snapshot simple test case.""" diff --git a/cinder/tests/test_image_utils.py b/cinder/tests/test_image_utils.py index 025a12a6c61..f26afd9389e 100644 --- a/cinder/tests/test_image_utils.py +++ b/cinder/tests/test_image_utils.py @@ -20,6 +20,7 @@ import tempfile import mock import mox +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -27,7 +28,6 @@ from cinder import context from cinder import exception from cinder.image import image_utils from cinder.openstack.common import fileutils -from cinder.openstack.common import processutils from cinder import test from cinder import utils from cinder.volume import utils as volume_utils diff --git a/cinder/tests/test_netapp_utils.py b/cinder/tests/test_netapp_utils.py index 0c9ac42a4de..6d2d4c9b592 100644 --- a/cinder/tests/test_netapp_utils.py +++ b/cinder/tests/test_netapp_utils.py @@ -15,8 +15,8 @@ import platform import mock +from oslo.concurrency import processutils as putils -from cinder.openstack.common import processutils as putils from cinder import test from cinder import version from cinder.volume.drivers.netapp import utils as na_utils diff --git a/cinder/tests/test_pure.py b/cinder/tests/test_pure.py index 15ef87f219a..1f871824248 100644 --- a/cinder/tests/test_pure.py +++ b/cinder/tests/test_pure.py @@ -17,10 +17,10 @@ import json import urllib2 import mock +from oslo.concurrency import processutils from oslo.utils import units from cinder import exception -from cinder.openstack.common import processutils from cinder import test from cinder.volume.drivers import pure diff --git a/cinder/tests/test_service.py b/cinder/tests/test_service.py index b547f72ae94..f63271cd2e1 100644 --- a/cinder/tests/test_service.py +++ b/cinder/tests/test_service.py @@ -22,6 +22,7 @@ Unit Tests for remote procedure calls using queue import mock import mox +from oslo.concurrency import processutils from oslo.config import cfg from oslo.db import exception as db_exc @@ -29,7 +30,6 @@ from cinder import context from cinder import db from cinder import exception from cinder import manager -from cinder.openstack.common import processutils from cinder import service from cinder import test from cinder import wsgi diff --git a/cinder/tests/test_sheepdog.py b/cinder/tests/test_sheepdog.py index fe298fe6c59..4bf1f379c18 100644 --- a/cinder/tests/test_sheepdog.py +++ b/cinder/tests/test_sheepdog.py @@ -19,10 +19,10 @@ import contextlib import os import tempfile +from oslo.concurrency import processutils from oslo.utils import units from cinder.image import image_utils -from cinder.openstack.common import processutils from cinder import test from cinder.volume.drivers.sheepdog import SheepdogDriver diff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py index 8c6bfb023d7..0baed6a0514 100644 --- a/cinder/tests/test_storwize_svc.py +++ b/cinder/tests/test_storwize_svc.py @@ -23,6 +23,7 @@ import re import time import mock +from oslo.concurrency import processutils from oslo.utils import excutils from oslo.utils import importutils from oslo.utils import units @@ -31,7 +32,6 @@ from cinder import context from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.tests import utils as testutils from cinder import utils diff --git a/cinder/tests/test_utils.py b/cinder/tests/test_utils.py index 7f0f7912ef7..80decf5980f 100644 --- a/cinder/tests/test_utils.py +++ b/cinder/tests/test_utils.py @@ -21,6 +21,7 @@ import tempfile import uuid import mock +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import timeutils import paramiko @@ -30,7 +31,6 @@ import cinder from cinder.brick.initiator import connector from cinder.brick.initiator import linuxfc from cinder import exception -from cinder.openstack.common import processutils as putils from cinder import ssh_utils from cinder import test from cinder import utils diff --git a/cinder/tests/test_volume_utils.py b/cinder/tests/test_volume_utils.py index 3916692fa98..0c0a1f127f0 100644 --- a/cinder/tests/test_volume_utils.py +++ b/cinder/tests/test_volume_utils.py @@ -19,6 +19,7 @@ import os import re import mock +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import importutils @@ -26,7 +27,6 @@ from cinder import context from cinder import db from cinder import exception from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.tests import fake_notifier from cinder import utils diff --git a/cinder/tests/zonemanager/test_brcd_fc_zone_client_cli.py b/cinder/tests/zonemanager/test_brcd_fc_zone_client_cli.py index 09f20f2e84e..fa3157a475d 100644 --- a/cinder/tests/zonemanager/test_brcd_fc_zone_client_cli.py +++ b/cinder/tests/zonemanager/test_brcd_fc_zone_client_cli.py @@ -21,10 +21,10 @@ import mock from mock import patch +from oslo.concurrency import processutils from cinder import exception from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import test from cinder.zonemanager.drivers.brocade.brcd_fc_zone_client_cli \ import BrcdFCZoneClientCLI diff --git a/cinder/tests/zonemanager/test_cisco_fc_zone_client_cli.py b/cinder/tests/zonemanager/test_cisco_fc_zone_client_cli.py index f1e32486af4..642854f49c5 100644 --- a/cinder/tests/zonemanager/test_cisco_fc_zone_client_cli.py +++ b/cinder/tests/zonemanager/test_cisco_fc_zone_client_cli.py @@ -18,9 +18,9 @@ """Unit tests for Cisco fc zone client cli.""" from mock import patch +from oslo.concurrency import processutils from cinder import exception -from cinder.openstack.common import processutils from cinder import test from cinder.zonemanager.drivers.cisco.cisco_fc_zone_client_cli \ import CiscoFCZoneClientCLI diff --git a/cinder/tests/zonemanager/test_cisco_fc_zone_driver.py b/cinder/tests/zonemanager/test_cisco_fc_zone_driver.py index f0cbb9fc16d..d68385d52cb 100644 --- a/cinder/tests/zonemanager/test_cisco_fc_zone_driver.py +++ b/cinder/tests/zonemanager/test_cisco_fc_zone_driver.py @@ -17,11 +17,11 @@ """Unit tests for Cisco FC zone driver.""" +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import importutils from cinder import exception -from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf diff --git a/cinder/utils.py b/cinder/utils.py index ba27646b9df..15ee88b836d 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -35,6 +35,8 @@ from xml import sax from xml.sax import expatreader from xml.sax import saxutils +from oslo.concurrency import lockutils +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import importutils from oslo.utils import timeutils @@ -43,9 +45,7 @@ import six from cinder.brick.initiator import connector from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils CONF = cfg.CONF diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index c02231e706c..82d5b195859 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -19,6 +19,7 @@ Drivers for volumes. import time +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils @@ -27,7 +28,6 @@ from cinder.i18n import _, _LE from cinder.image import image_utils from cinder.openstack.common import fileutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume import iscsi from cinder.volume import rpcapi as volume_rpcapi diff --git a/cinder/volume/drivers/coraid.py b/cinder/volume/drivers/coraid.py index ab48789e17c..ad350bee05f 100644 --- a/cinder/volume/drivers/coraid.py +++ b/cinder/volume/drivers/coraid.py @@ -26,6 +26,7 @@ import math import urllib import urllib2 +from oslo.concurrency import lockutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import units @@ -33,7 +34,6 @@ import six.moves.urllib.parse as urlparse from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.volume import driver from cinder.volume import volume_types diff --git a/cinder/volume/drivers/emc/emc_vnx_cli.py b/cinder/volume/drivers/emc/emc_vnx_cli.py index eeee3877fad..9cd7bd92760 100644 --- a/cinder/volume/drivers/emc/emc_vnx_cli.py +++ b/cinder/volume/drivers/emc/emc_vnx_cli.py @@ -21,6 +21,8 @@ import random import re import time +from oslo.concurrency import lockutils +from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils as json from oslo.utils import excutils @@ -30,10 +32,8 @@ import six from cinder import exception from cinder.exception import EMCVnxCLICmdError from cinder.i18n import _, _LE, _LI, _LW -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.configuration import Configuration from cinder.volume.drivers.san import san diff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py index 82e91c46b3d..7ce2cc015e0 100644 --- a/cinder/volume/drivers/eqlx.py +++ b/cinder/volume/drivers/eqlx.py @@ -21,13 +21,13 @@ import random import eventlet from eventlet import greenthread import greenlet +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from cinder import exception from cinder.i18n import _, _LE, _LW, _LI from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import ssh_utils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver diff --git a/cinder/volume/drivers/fujitsu_eternus_dx_fc.py b/cinder/volume/drivers/fujitsu_eternus_dx_fc.py index 89569dc67b6..8a17fccf06d 100644 --- a/cinder/volume/drivers/fujitsu_eternus_dx_fc.py +++ b/cinder/volume/drivers/fujitsu_eternus_dx_fc.py @@ -17,10 +17,10 @@ FC Drivers for ETERNUS DX arrays based on SMI-S. """ +from oslo.concurrency import lockutils import six from cinder import context -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.volume import driver from cinder.volume.drivers import fujitsu_eternus_dx_common diff --git a/cinder/volume/drivers/fujitsu_eternus_dx_iscsi.py b/cinder/volume/drivers/fujitsu_eternus_dx_iscsi.py index 10f4a21da56..ab41f6eda75 100644 --- a/cinder/volume/drivers/fujitsu_eternus_dx_iscsi.py +++ b/cinder/volume/drivers/fujitsu_eternus_dx_iscsi.py @@ -17,12 +17,12 @@ ISCSI Drivers for ETERNUS DX arrays based on SMI-S. """ +from oslo.concurrency import lockutils import six from cinder import context from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.volume import driver from cinder.volume.drivers import fujitsu_eternus_dx_common diff --git a/cinder/volume/drivers/glusterfs.py b/cinder/volume/drivers/glusterfs.py index 0661c054c0c..0558cff93ef 100644 --- a/cinder/volume/drivers/glusterfs.py +++ b/cinder/volume/drivers/glusterfs.py @@ -18,6 +18,7 @@ import os import stat import time +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -29,7 +30,6 @@ from cinder.i18n import _, _LE, _LI, _LW from cinder.image import image_utils from cinder.openstack.common import fileutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers import remotefs as remotefs_drv diff --git a/cinder/volume/drivers/hds/nfs.py b/cinder/volume/drivers/hds/nfs.py index a4694f802a8..e48af0b5dd7 100644 --- a/cinder/volume/drivers/hds/nfs.py +++ b/cinder/volume/drivers/hds/nfs.py @@ -21,6 +21,7 @@ import os import time from xml.etree import ElementTree as ETree +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import units @@ -29,7 +30,6 @@ from cinder import exception from cinder.i18n import _, _LE, _LI from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder.volume.drivers.hds.hnas_backend import HnasBackend from cinder.volume.drivers import nfs diff --git a/cinder/volume/drivers/hitachi/hbsd_basiclib.py b/cinder/volume/drivers/hitachi/hbsd_basiclib.py index 3390295f212..3df38b8d31a 100644 --- a/cinder/volume/drivers/hitachi/hbsd_basiclib.py +++ b/cinder/volume/drivers/hitachi/hbsd_basiclib.py @@ -16,14 +16,14 @@ import inspect import os import shlex +from oslo.concurrency import lockutils +from oslo.concurrency import processutils as putils from oslo.utils import excutils import six from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import utils SMPL = 1 diff --git a/cinder/volume/drivers/hitachi/hbsd_horcm.py b/cinder/volume/drivers/hitachi/hbsd_horcm.py index 34bd88bd948..17ba43ceb49 100644 --- a/cinder/volume/drivers/hitachi/hbsd_horcm.py +++ b/cinder/volume/drivers/hitachi/hbsd_horcm.py @@ -20,6 +20,7 @@ import shlex import threading import time +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import excutils import six @@ -28,7 +29,6 @@ from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall -from cinder.openstack.common import processutils as putils from cinder import utils from cinder.volume.drivers.hitachi import hbsd_basiclib as basic_lib diff --git a/cinder/volume/drivers/ibm/gpfs.py b/cinder/volume/drivers/ibm/gpfs.py index 923458b5670..5025495c1cb 100644 --- a/cinder/volume/drivers/ibm/gpfs.py +++ b/cinder/volume/drivers/ibm/gpfs.py @@ -21,6 +21,7 @@ import os import re import shutil +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -29,7 +30,6 @@ from cinder.i18n import _ from cinder.image import image_utils from cinder.openstack.common import fileutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume import driver diff --git a/cinder/volume/drivers/ibm/ibmnas.py b/cinder/volume/drivers/ibm/ibmnas.py index 294af38cbc3..120c553d562 100644 --- a/cinder/volume/drivers/ibm/ibmnas.py +++ b/cinder/volume/drivers/ibm/ibmnas.py @@ -32,6 +32,7 @@ Notes: import os import re +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -39,7 +40,6 @@ from cinder import exception from cinder.i18n import _, _LI, _LW from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers import nfs from cinder.volume.drivers.remotefs import nas_opts diff --git a/cinder/volume/drivers/ibm/storwize_svc/ssh.py b/cinder/volume/drivers/ibm/storwize_svc/ssh.py index 9f90da43a5e..0e58d93becc 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/ssh.py +++ b/cinder/volume/drivers/ibm/storwize_svc/ssh.py @@ -16,10 +16,11 @@ import re +from oslo.concurrency import processutils + from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils LOG = logging.getLogger(__name__) diff --git a/cinder/volume/drivers/lvm.py b/cinder/volume/drivers/lvm.py index a4035f81bcb..95c885902c1 100644 --- a/cinder/volume/drivers/lvm.py +++ b/cinder/volume/drivers/lvm.py @@ -22,6 +22,7 @@ import math import os import socket +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -32,7 +33,6 @@ from cinder.i18n import _, _LE, _LI, _LW from cinder.image import image_utils from cinder.openstack.common import fileutils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume import driver from cinder.volume import utils as volutils diff --git a/cinder/volume/drivers/netapp/nfs.py b/cinder/volume/drivers/netapp/nfs.py index 288a270059e..4bb035f479a 100644 --- a/cinder/volume/drivers/netapp/nfs.py +++ b/cinder/volume/drivers/netapp/nfs.py @@ -23,6 +23,7 @@ from threading import Timer import time import uuid +from oslo.concurrency import processutils from oslo.utils import excutils from oslo.utils import units import six @@ -32,7 +33,6 @@ from cinder import exception from cinder.i18n import _, _LE, _LI, _LW from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.netapp.api import NaApiError from cinder.volume.drivers.netapp.api import NaElement diff --git a/cinder/volume/drivers/netapp/utils.py b/cinder/volume/drivers/netapp/utils.py index a4fd34afd98..da5f85807f0 100644 --- a/cinder/volume/drivers/netapp/utils.py +++ b/cinder/volume/drivers/netapp/utils.py @@ -28,6 +28,7 @@ import platform import socket import uuid +from oslo.concurrency import processutils as putils from oslo.utils import timeutils import six @@ -35,7 +36,6 @@ from cinder import context from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import utils from cinder import version from cinder.volume.drivers.netapp.api import NaApiError diff --git a/cinder/volume/drivers/nfs.py b/cinder/volume/drivers/nfs.py index e671e979851..9276ab9a05f 100644 --- a/cinder/volume/drivers/nfs.py +++ b/cinder/volume/drivers/nfs.py @@ -16,6 +16,7 @@ import errno import os +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import units @@ -24,7 +25,6 @@ from cinder import exception from cinder.i18n import _ from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import utils from cinder.volume.drivers import remotefs diff --git a/cinder/volume/drivers/pure.py b/cinder/volume/drivers/pure.py index 91334975e13..923b91e92c7 100644 --- a/cinder/volume/drivers/pure.py +++ b/cinder/volume/drivers/pure.py @@ -24,6 +24,7 @@ import re import urllib2 import uuid +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import units @@ -31,7 +32,6 @@ from oslo.utils import units from cinder import exception from cinder.i18n import _LE, _LI, _LW from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import san diff --git a/cinder/volume/drivers/remotefs.py b/cinder/volume/drivers/remotefs.py index 6cc809b9580..e8a7b091dc7 100644 --- a/cinder/volume/drivers/remotefs.py +++ b/cinder/volume/drivers/remotefs.py @@ -20,6 +20,7 @@ import os import re import tempfile +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import units @@ -27,7 +28,6 @@ from cinder import exception from cinder.i18n import _, _LE, _LI, _LW from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder.volume import driver LOG = logging.getLogger(__name__) diff --git a/cinder/volume/drivers/san/hp/hp_lefthand_cliq_proxy.py b/cinder/volume/drivers/san/hp/hp_lefthand_cliq_proxy.py index b334002495d..487d39859c1 100644 --- a/cinder/volume/drivers/san/hp/hp_lefthand_cliq_proxy.py +++ b/cinder/volume/drivers/san/hp/hp_lefthand_cliq_proxy.py @@ -21,12 +21,12 @@ operations on the SAN. """ from lxml import etree +from oslo.concurrency import processutils from oslo.utils import units from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder.volume.drivers.san.san import SanISCSIDriver diff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py index 5637203d953..f187f1fde7c 100644 --- a/cinder/volume/drivers/san/san.py +++ b/cinder/volume/drivers/san/san.py @@ -22,13 +22,13 @@ controller on the SAN hardware. We expect to access it over SSH or some API. import random from eventlet import greenthread +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import ssh_utils from cinder import utils from cinder.volume import driver diff --git a/cinder/volume/drivers/sheepdog.py b/cinder/volume/drivers/sheepdog.py index 0ee092efa92..75951852380 100644 --- a/cinder/volume/drivers/sheepdog.py +++ b/cinder/volume/drivers/sheepdog.py @@ -22,6 +22,7 @@ import os import re import tempfile +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units @@ -29,7 +30,6 @@ from cinder import exception from cinder.i18n import _, _LE from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder.volume import driver diff --git a/cinder/volume/drivers/smbfs.py b/cinder/volume/drivers/smbfs.py index a6863b35420..7d2c893fb81 100644 --- a/cinder/volume/drivers/smbfs.py +++ b/cinder/volume/drivers/smbfs.py @@ -16,6 +16,7 @@ import os import re +from oslo.concurrency import processutils as putils from oslo.config import cfg from oslo.utils import units @@ -24,7 +25,6 @@ from cinder import exception from cinder.i18n import _ from cinder.image import image_utils from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder import utils from cinder.volume.drivers import remotefs as remotefs_drv diff --git a/cinder/volume/flows/manager/create_volume.py b/cinder/volume/flows/manager/create_volume.py index fd206816f85..e64d0b59d45 100644 --- a/cinder/volume/flows/manager/create_volume.py +++ b/cinder/volume/flows/manager/create_volume.py @@ -12,6 +12,7 @@ import traceback +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import timeutils import taskflow.engines @@ -23,7 +24,6 @@ from cinder import flow_utils from cinder.i18n import _, _LE, _LI from cinder.image import glance from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import utils from cinder.volume.flows import common from cinder.volume import utils as volume_utils diff --git a/cinder/volume/iscsi.py b/cinder/volume/iscsi.py index e18ce821bc2..c424499f899 100644 --- a/cinder/volume/iscsi.py +++ b/cinder/volume/iscsi.py @@ -16,11 +16,12 @@ import os import re +from oslo.concurrency import processutils as putils + from cinder.brick.iscsi import iscsi from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils as putils from cinder.volume import utils LOG = logging.getLogger(__name__) diff --git a/cinder/volume/utils.py b/cinder/volume/utils.py index 61ae354ade2..ea39be0c0b9 100644 --- a/cinder/volume/utils.py +++ b/cinder/volume/utils.py @@ -18,6 +18,7 @@ import math from Crypto.Random import random +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import strutils from oslo.utils import timeutils @@ -27,7 +28,6 @@ from cinder.brick.local_dev import lvm as brick_lvm from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import rpc from cinder import utils diff --git a/cinder/zonemanager/drivers/brocade/brcd_fc_zone_client_cli.py b/cinder/zonemanager/drivers/brocade/brcd_fc_zone_client_cli.py index e54a50cf72c..fc9122f2976 100644 --- a/cinder/zonemanager/drivers/brocade/brcd_fc_zone_client_cli.py +++ b/cinder/zonemanager/drivers/brocade/brcd_fc_zone_client_cli.py @@ -25,12 +25,12 @@ import random import re from eventlet import greenthread +from oslo.concurrency import processutils from oslo.utils import excutils from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import ssh_utils from cinder import utils import cinder.zonemanager.drivers.brocade.fc_zone_constants as ZoneConstant diff --git a/cinder/zonemanager/drivers/brocade/brcd_fc_zone_driver.py b/cinder/zonemanager/drivers/brocade/brcd_fc_zone_driver.py index 96b1629952f..6740433fbaa 100644 --- a/cinder/zonemanager/drivers/brocade/brcd_fc_zone_driver.py +++ b/cinder/zonemanager/drivers/brocade/brcd_fc_zone_driver.py @@ -30,13 +30,13 @@ add_connection and delete_connection interfaces. """ +from oslo.concurrency import lockutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import importutils from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.zonemanager.drivers.brocade import brcd_fabric_opts as fabric_opts from cinder.zonemanager.drivers.fc_zone_driver import FCZoneDriver diff --git a/cinder/zonemanager/drivers/cisco/cisco_fc_san_lookup_service.py b/cinder/zonemanager/drivers/cisco/cisco_fc_san_lookup_service.py index 659157a8bba..419489030da 100644 --- a/cinder/zonemanager/drivers/cisco/cisco_fc_san_lookup_service.py +++ b/cinder/zonemanager/drivers/cisco/cisco_fc_san_lookup_service.py @@ -18,13 +18,13 @@ import random from eventlet import greenthread +from oslo.concurrency import processutils from oslo.utils import excutils import six from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import ssh_utils from cinder import utils from cinder.zonemanager.drivers.cisco import cisco_fabric_opts as fabric_opts diff --git a/cinder/zonemanager/drivers/cisco/cisco_fc_zone_client_cli.py b/cinder/zonemanager/drivers/cisco/cisco_fc_zone_client_cli.py index 8f010dad7d2..133485fa6bf 100644 --- a/cinder/zonemanager/drivers/cisco/cisco_fc_zone_client_cli.py +++ b/cinder/zonemanager/drivers/cisco/cisco_fc_zone_client_cli.py @@ -22,13 +22,13 @@ import random import re from eventlet import greenthread +from oslo.concurrency import processutils from oslo.utils import excutils import six from cinder import exception from cinder.i18n import _ from cinder.openstack.common import log as logging -from cinder.openstack.common import processutils from cinder import ssh_utils from cinder import utils import cinder.zonemanager.drivers.cisco.fc_zone_constants as ZoneConstant diff --git a/cinder/zonemanager/drivers/cisco/cisco_fc_zone_driver.py b/cinder/zonemanager/drivers/cisco/cisco_fc_zone_driver.py index 15fe0d837bd..04791c5ea8c 100644 --- a/cinder/zonemanager/drivers/cisco/cisco_fc_zone_driver.py +++ b/cinder/zonemanager/drivers/cisco/cisco_fc_zone_driver.py @@ -27,6 +27,7 @@ add_connection and delete_connection interfaces. :zone_name_prefix: Used by: class: 'FCZoneDriver'. Defaults to 'openstack' """ +from oslo.concurrency import lockutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import importutils @@ -34,7 +35,6 @@ import six from cinder import exception from cinder.i18n import _ -from cinder.openstack.common import lockutils from cinder.openstack.common import log as logging from cinder.zonemanager.drivers.cisco import cisco_fabric_opts as fabric_opts from cinder.zonemanager.drivers.fc_zone_driver import FCZoneDriver diff --git a/etc/cinder/cinder.conf.sample b/etc/cinder/cinder.conf.sample index 2aa94b42c30..0e5235c8bea 100644 --- a/etc/cinder/cinder.conf.sample +++ b/etc/cinder/cinder.conf.sample @@ -760,18 +760,6 @@ #backdoor_port= -# -# Options defined in cinder.openstack.common.lockutils -# - -# Whether to disable inter-process locks (boolean value) -#disable_process_locking=false - -# Directory to use for lock files. Default to a temp directory -# (string value) -#lock_path= - - # # Options defined in cinder.openstack.common.log # diff --git a/openstack-common.conf b/openstack-common.conf index 6152d233ac1..7e58f5c8d98 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -10,14 +10,12 @@ module=imageutils module=install_venv_common module=jsonutils module=local -module=lockutils module=log module=log_handler module=loopingcall module=middleware module=periodic_task module=policy -module=processutils module=request_utils module=scheduler module=scheduler.filters diff --git a/requirements.txt b/requirements.txt index 8d5e8676acf..e2dcbe866c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ kombu>=2.5.0 lxml>=2.3 netaddr>=0.7.12 oslo.config>=1.4.0 # Apache-2.0 +oslo.concurrency>=0.1.0 # Apache-2.0 oslo.db>=1.0.0 # Apache-2.0 oslo.messaging>=1.4.0 oslo.rootwrap>=1.3.0