From 8475ba585e5515ebf6eec84f65ae350fa1de6af1 Mon Sep 17 00:00:00 2001 From: Amrith Kumar Date: Thu, 11 Dec 2014 10:50:34 -0500 Subject: [PATCH] Obsolete oslo-incubator modules - processutils This change is part of a multi-part change set to handle obsolete and graduated oslo modules. This commit handles the processutils module which is now oslo.concurrency (oslo_concurrency). The change here is to remove it from openstack-common.conf and add oslo.concurrency to requirements.txt. Since nothing in oslo-incubator depends on processutils, the module is also deleted from the oslo-incubator imports. This change is based on top of https://review.openstack.org/#/c/133068 Change-Id: I5cba2f487b51c877d036acf552df3b64f7c85764 Partial-Bug: #1380789 --- openstack-common.conf | 1 - requirements.txt | 7 +- trove/cmd/api.py | 2 +- trove/cmd/conductor.py | 2 +- trove/cmd/fakemode.py | 2 +- trove/common/exception.py | 3 +- trove/common/utils.py | 2 +- trove/openstack/common/processutils.py | 289 ------------------------- trove/tests/db/migrations.py | 3 +- 9 files changed, 12 insertions(+), 299 deletions(-) delete mode 100644 trove/openstack/common/processutils.py diff --git a/openstack-common.conf b/openstack-common.conf index df12d98acc..22dea12055 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -12,7 +12,6 @@ module=notifier module=pastedeploy module=periodic_task module=policy -module=processutils module=rpc module=service module=threadgroup diff --git a/requirements.txt b/requirements.txt index 7ef08f8b2c..54520ce132 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,10 +27,11 @@ iso8601>=0.1.9 jsonschema>=2.0.0,<3.0.0 Jinja2>=2.6 # BSD License (3 clause) pexpect>=3.1 # ISC License -oslo.config>=1.4.0 # Apache-2.0 +oslo.config>=1.6.0 # Apache-2.0 oslo.i18n>=1.0.0 -oslo.serialization>=1.0.0 -oslo.utils>=1.1.0 +oslo.serialization>=1.2.0 +oslo.utils>=1.2.0 +oslo.concurrency>=0.3.0,!=0.4.0 MySQL-python Babel>=1.3 six>=1.7.0 diff --git a/trove/cmd/api.py b/trove/cmd/api.py index 42cee20429..d119aca57a 100755 --- a/trove/cmd/api.py +++ b/trove/cmd/api.py @@ -12,8 +12,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from oslo_concurrency import processutils from trove.cmd.common import with_initialize -from trove.openstack.common import processutils @with_initialize diff --git a/trove/cmd/conductor.py b/trove/cmd/conductor.py index 25929812a0..c398e860aa 100755 --- a/trove/cmd/conductor.py +++ b/trove/cmd/conductor.py @@ -12,8 +12,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from oslo_concurrency import processutils from trove.cmd.common import with_initialize -from trove.openstack.common import processutils @with_initialize diff --git a/trove/cmd/fakemode.py b/trove/cmd/fakemode.py index 81a3ca83d8..3b61db149a 100755 --- a/trove/cmd/fakemode.py +++ b/trove/cmd/fakemode.py @@ -13,8 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. from oslo.config import cfg as openstack_cfg +from oslo_concurrency import processutils from trove.cmd.common import with_initialize -from trove.openstack.common import processutils opts = [ diff --git a/trove/common/exception.py b/trove/common/exception.py index a125cfb13f..939f480215 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -17,9 +17,10 @@ import re +from oslo_concurrency import processutils + from trove.openstack.common import log as logging from trove.common import base_exception as openstack_exception -from trove.openstack.common import processutils from trove.common.i18n import _ diff --git a/trove/common/utils.py b/trove/common/utils.py index 87071de004..43c8ccb67e 100644 --- a/trove/common/utils.py +++ b/trove/common/utils.py @@ -28,12 +28,12 @@ from passlib import utils as passlib_utils from oslo.utils import importutils from oslo.utils import timeutils +from oslo_concurrency import processutils from trove.common import cfg from trove.common import exception from trove.openstack.common import log as logging from trove.openstack.common import loopingcall -from trove.openstack.common import processutils from trove.openstack.common import utils as openstack_utils from trove.common.i18n import _ diff --git a/trove/openstack/common/processutils.py b/trove/openstack/common/processutils.py deleted file mode 100644 index 58f6b5e1bb..0000000000 --- a/trove/openstack/common/processutils.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import errno -import logging -import multiprocessing -import os -import random -import shlex -import signal - -from eventlet.green import subprocess -from eventlet import greenthread -import six - -from trove.openstack.common.gettextutils import _ -from trove.openstack.common import strutils - - -LOG = logging.getLogger(__name__) - - -class InvalidArgumentError(Exception): - def __init__(self, message=None): - super(InvalidArgumentError, self).__init__(message) - - -class UnknownArgumentError(Exception): - def __init__(self, message=None): - super(UnknownArgumentError, self).__init__(message) - - -class ProcessExecutionError(Exception): - def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - self.exit_code = exit_code - self.stderr = stderr - self.stdout = stdout - self.cmd = cmd - self.description = description - - if description is None: - description = _("Unexpected error while running command.") - if exit_code is None: - exit_code = '-' - message = _('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Stdout: %(stdout)r\n' - 'Stderr: %(stderr)r') % {'description': description, - 'cmd': cmd, - 'exit_code': exit_code, - 'stdout': stdout, - 'stderr': stderr} - super(ProcessExecutionError, self).__init__(message) - - -class NoRootWrapSpecified(Exception): - def __init__(self, message=None): - super(NoRootWrapSpecified, self).__init__(message) - - -def _subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - -def execute(*cmd, **kwargs): - """Helper method to shell out and execute a command through subprocess. - - Allows optional retry. - - :param cmd: Passed to subprocess.Popen. - :type cmd: string - :param process_input: Send to opened process. - :type process_input: string - :param env_variables: Environment variables and their values that - will be set for the process. - :type env_variables: dict - :param check_exit_code: Single bool, int, or list of allowed exit - codes. Defaults to [0]. Raise - :class:`ProcessExecutionError` unless - program exits with one of these code. - :type check_exit_code: boolean, int, or [int] - :param delay_on_retry: True | False. Defaults to True. If set to True, - wait a short amount of time before retrying. - :type delay_on_retry: boolean - :param attempts: How many times to retry cmd. - :type attempts: int - :param run_as_root: True | False. Defaults to False. If set to True, - the command is prefixed by the command specified - in the root_helper kwarg. - :type run_as_root: boolean - :param root_helper: command to prefix to commands called with - run_as_root=True - :type root_helper: string - :param shell: whether or not there should be a shell used to - execute this command. Defaults to false. - :type shell: boolean - :param loglevel: log level for execute commands. - :type loglevel: int. (Should be logging.DEBUG or logging.INFO) - :returns: (stdout, stderr) from process execution - :raises: :class:`UnknownArgumentError` on - receiving unknown arguments - :raises: :class:`ProcessExecutionError` - """ - - process_input = kwargs.pop('process_input', None) - env_variables = kwargs.pop('env_variables', None) - check_exit_code = kwargs.pop('check_exit_code', [0]) - ignore_exit_code = False - delay_on_retry = kwargs.pop('delay_on_retry', True) - attempts = kwargs.pop('attempts', 1) - run_as_root = kwargs.pop('run_as_root', False) - root_helper = kwargs.pop('root_helper', '') - shell = kwargs.pop('shell', False) - loglevel = kwargs.pop('loglevel', logging.DEBUG) - - if isinstance(check_exit_code, bool): - ignore_exit_code = not check_exit_code - check_exit_code = [0] - elif isinstance(check_exit_code, int): - check_exit_code = [check_exit_code] - - if kwargs: - raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs) - - if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: - if not root_helper: - raise NoRootWrapSpecified( - message=_('Command requested root, but did not ' - 'specify a root helper.')) - cmd = shlex.split(root_helper) + list(cmd) - - cmd = map(str, cmd) - sanitized_cmd = strutils.mask_password(' '.join(cmd)) - - while attempts > 0: - attempts -= 1 - try: - LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd) - _PIPE = subprocess.PIPE # pylint: disable=E1101 - - if os.name == 'nt': - preexec_fn = None - close_fds = False - else: - preexec_fn = _subprocess_setup - close_fds = True - - obj = subprocess.Popen(cmd, - stdin=_PIPE, - stdout=_PIPE, - stderr=_PIPE, - close_fds=close_fds, - preexec_fn=preexec_fn, - shell=shell, - env=env_variables) - result = None - for _i in six.moves.range(20): - # NOTE(russellb) 20 is an arbitrary number of retries to - # prevent any chance of looping forever here. - try: - if process_input is not None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - except OSError as e: - if e.errno in (errno.EAGAIN, errno.EINTR): - continue - raise - break - obj.stdin.close() # pylint: disable=E1101 - _returncode = obj.returncode # pylint: disable=E1101 - LOG.log(loglevel, 'Result was %s' % _returncode) - if not ignore_exit_code and _returncode not in check_exit_code: - (stdout, stderr) = result - sanitized_stdout = strutils.mask_password(stdout) - sanitized_stderr = strutils.mask_password(stderr) - raise ProcessExecutionError(exit_code=_returncode, - stdout=sanitized_stdout, - stderr=sanitized_stderr, - cmd=sanitized_cmd) - return result - except ProcessExecutionError: - if not attempts: - raise - else: - LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd) - if delay_on_retry: - greenthread.sleep(random.randint(20, 200) / 100.0) - finally: - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) - - -def trycmd(*args, **kwargs): - """A wrapper around execute() to more easily handle warnings and errors. - - Returns an (out, err) tuple of strings containing the output of - the command's stdout and stderr. If 'err' is not empty then the - command can be considered to have failed. - - :discard_warnings True | False. Defaults to False. If set to True, - then for succeeding commands, stderr is cleared - - """ - discard_warnings = kwargs.pop('discard_warnings', False) - - try: - out, err = execute(*args, **kwargs) - failed = False - except ProcessExecutionError as exn: - out, err = '', six.text_type(exn) - failed = True - - if not failed and discard_warnings and err: - # Handle commands that output to stderr but otherwise succeed - err = '' - - return out, err - - -def ssh_execute(ssh, cmd, process_input=None, - addl_env=None, check_exit_code=True): - sanitized_cmd = strutils.mask_password(cmd) - LOG.debug('Running cmd (SSH): %s', sanitized_cmd) - if addl_env: - raise InvalidArgumentError(_('Environment not supported over SSH')) - - if process_input: - # This is (probably) fixable if we need it... - raise InvalidArgumentError(_('process_input not supported over SSH')) - - stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) - channel = stdout_stream.channel - - # NOTE(justinsb): This seems suspicious... - # ...other SSH clients have buffering issues with this approach - stdout = stdout_stream.read() - sanitized_stdout = strutils.mask_password(stdout) - stderr = stderr_stream.read() - sanitized_stderr = strutils.mask_password(stderr) - - stdin_stream.close() - - exit_status = channel.recv_exit_status() - - # exit_status == -1 if no exit code was returned - if exit_status != -1: - LOG.debug('Result was %s' % exit_status) - if check_exit_code and exit_status != 0: - raise ProcessExecutionError(exit_code=exit_status, - stdout=sanitized_stdout, - stderr=sanitized_stderr, - cmd=sanitized_cmd) - - return (sanitized_stdout, sanitized_stderr) - - -def get_worker_count(): - """Utility to get the default worker count. - - @return: The number of CPUs if that can be determined, else a default - worker count of 1 is returned. - """ - try: - return multiprocessing.cpu_count() - except NotImplementedError: - return 1 diff --git a/trove/tests/db/migrations.py b/trove/tests/db/migrations.py index 1e302333c6..4024d7d1e5 100644 --- a/trove/tests/db/migrations.py +++ b/trove/tests/db/migrations.py @@ -27,6 +27,8 @@ import os from migrate.versioning import repository import migrate.versioning.api as migration_api +from oslo_concurrency import processutils + from proboscis import after_class from proboscis import before_class from proboscis import test @@ -40,7 +42,6 @@ import sqlalchemy.exc import trove.db.sqlalchemy.migrate_repo from trove.common.i18n import _ from trove.openstack.common import log as logging -from trove.openstack.common import processutils from trove.tests.util import event_simulator GROUP = "dbaas.db.migrations"