Merge "Sync processutils from oslo with deps"
This commit is contained in:
commit
72e38f7249
@ -373,8 +373,8 @@ def get_available_languages(domain):
|
||||
'zh_Hant_HK': 'zh_HK',
|
||||
'zh_Hant': 'zh_TW',
|
||||
'fil': 'tl_PH'}
|
||||
for (locale, alias) in six.iteritems(aliases):
|
||||
if locale in language_list and alias not in language_list:
|
||||
for (locale_, alias) in six.iteritems(aliases):
|
||||
if locale_ in language_list and alias not in language_list:
|
||||
language_list.append(alias)
|
||||
|
||||
_AVAILABLE_LANGUAGES[domain] = language_list
|
||||
|
@ -24,10 +24,10 @@ import traceback
|
||||
def import_class(import_str):
|
||||
"""Returns a class from a string including module and class."""
|
||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||
__import__(mod_str)
|
||||
try:
|
||||
__import__(mod_str)
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except (ValueError, AttributeError):
|
||||
except AttributeError:
|
||||
raise ImportError('Class %s cannot be found (%s)' %
|
||||
(class_str,
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
|
@ -168,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs):
|
||||
return json.dumps(value, default=default, **kwargs)
|
||||
|
||||
|
||||
def loads(s, encoding='utf-8'):
|
||||
return json.loads(strutils.safe_decode(s, encoding))
|
||||
def loads(s, encoding='utf-8', **kwargs):
|
||||
return json.loads(strutils.safe_decode(s, encoding), **kwargs)
|
||||
|
||||
|
||||
def load(fp, encoding='utf-8'):
|
||||
return json.load(codecs.getreader(encoding)(fp))
|
||||
def load(fp, encoding='utf-8', **kwargs):
|
||||
return json.load(codecs.getreader(encoding)(fp), **kwargs)
|
||||
|
||||
|
||||
try:
|
||||
|
@ -59,7 +59,10 @@ _SANITIZE_PATTERNS = []
|
||||
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||
r'(<%(key)s>).*?(</%(key)s>)',
|
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
|
||||
'.*?([\'"])',
|
||||
r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
|
||||
|
||||
for key in _SANITIZE_KEYS:
|
||||
for pattern in _FORMAT_PATTERNS:
|
||||
@ -89,7 +92,6 @@ logging_cli_opts = [
|
||||
'files. For details about logging configuration files, '
|
||||
'see the Python logging module documentation.'),
|
||||
cfg.StrOpt('log-format',
|
||||
default=None,
|
||||
metavar='FORMAT',
|
||||
help='DEPRECATED. '
|
||||
'A logging.Formatter log message format string which may '
|
||||
@ -115,7 +117,7 @@ logging_cli_opts = [
|
||||
default=False,
|
||||
help='Use syslog for logging. '
|
||||
'Existing syslog format is DEPRECATED during I, '
|
||||
'and will chang in J to honor RFC5424.'),
|
||||
'and will change in J to honor RFC5424.'),
|
||||
cfg.BoolOpt('use-syslog-rfc-format',
|
||||
# TODO(bogdando) remove or use True after existing
|
||||
# syslog format deprecation in J
|
||||
@ -422,9 +424,7 @@ class JSONFormatter(logging.Formatter):
|
||||
|
||||
def _create_logging_excepthook(product_name):
|
||||
def logging_excepthook(exc_type, value, tb):
|
||||
extra = {}
|
||||
if CONF.verbose or CONF.debug:
|
||||
extra['exc_info'] = (exc_type, value, tb)
|
||||
extra = {'exc_info': (exc_type, value, tb)}
|
||||
getLogger(product_name).critical(
|
||||
"".join(traceback.format_exception_only(exc_type, value)),
|
||||
**extra)
|
||||
@ -462,9 +462,8 @@ def setup(product_name, version='unknown'):
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
cfg.set_defaults(log_opts,
|
||||
logging_context_format_string=
|
||||
logging_context_format_string)
|
||||
cfg.set_defaults(
|
||||
log_opts, logging_context_format_string=logging_context_format_string)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
@ -541,9 +540,14 @@ def _setup_logging_from_conf(project, version):
|
||||
log_root.addHandler(streamlog)
|
||||
|
||||
if CONF.publish_errors:
|
||||
handler = importutils.import_object(
|
||||
"cinder.openstack.common.log_handler.PublishErrorsHandler",
|
||||
logging.ERROR)
|
||||
try:
|
||||
handler = importutils.import_object(
|
||||
"cinder.openstack.common.log_handler.PublishErrorsHandler",
|
||||
logging.ERROR)
|
||||
except ImportError:
|
||||
handler = importutils.import_object(
|
||||
"oslo.messaging.notify.log_handler.PublishErrorsHandler",
|
||||
logging.ERROR)
|
||||
log_root.addHandler(handler)
|
||||
|
||||
datefmt = CONF.log_date_format
|
||||
@ -569,9 +573,15 @@ def _setup_logging_from_conf(project, version):
|
||||
|
||||
for pair in CONF.default_log_levels:
|
||||
mod, _sep, level_name = pair.partition('=')
|
||||
level = logging.getLevelName(level_name)
|
||||
logger = logging.getLogger(mod)
|
||||
logger.setLevel(level)
|
||||
# NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
|
||||
# to integer code.
|
||||
if sys.version_info < (2, 7):
|
||||
level = logging.getLevelName(level_name)
|
||||
logger.setLevel(level)
|
||||
else:
|
||||
logger.setLevel(level_name)
|
||||
|
||||
|
||||
_loggers = {}
|
||||
|
||||
@ -660,14 +670,19 @@ class ContextFormatter(logging.Formatter):
|
||||
record.__dict__[key] = ''
|
||||
|
||||
if record.__dict__.get('request_id'):
|
||||
self._fmt = CONF.logging_context_format_string
|
||||
fmt = CONF.logging_context_format_string
|
||||
else:
|
||||
self._fmt = CONF.logging_default_format_string
|
||||
fmt = CONF.logging_default_format_string
|
||||
|
||||
if (record.levelno == logging.DEBUG and
|
||||
CONF.logging_debug_format_suffix):
|
||||
self._fmt += " " + CONF.logging_debug_format_suffix
|
||||
fmt += " " + CONF.logging_debug_format_suffix
|
||||
|
||||
if sys.version_info < (3, 2):
|
||||
self._fmt = fmt
|
||||
else:
|
||||
self._style = logging.PercentStyle(fmt)
|
||||
self._fmt = self._style._fmt
|
||||
# Cache this on the record, Logger will respect our formatted copy
|
||||
if record.exc_info:
|
||||
record.exc_text = self.formatException(record.exc_info, record)
|
||||
|
@ -1,5 +1,3 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
@ -19,6 +17,9 @@
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging as stdlib_logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import shlex
|
||||
@ -26,6 +27,7 @@ import signal
|
||||
|
||||
from eventlet.green import subprocess
|
||||
from eventlet import greenthread
|
||||
import six
|
||||
|
||||
from cinder.openstack.common.gettextutils import _
|
||||
from cinder.openstack.common import log as logging
|
||||
@ -54,11 +56,18 @@ class ProcessExecutionError(Exception):
|
||||
self.description = description
|
||||
|
||||
if description is None:
|
||||
description = "Unexpected error while running command."
|
||||
description = _("Unexpected error while running command.")
|
||||
if exit_code is None:
|
||||
exit_code = '-'
|
||||
message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
|
||||
% (description, cmd, exit_code, stdout, stderr))
|
||||
message = _('%(description)s\n'
|
||||
'Command: %(cmd)s\n'
|
||||
'Exit code: %(exit_code)s\n'
|
||||
'Stdout: %(stdout)r\n'
|
||||
'Stderr: %(stderr)r') % {'description': description,
|
||||
'cmd': cmd,
|
||||
'exit_code': exit_code,
|
||||
'stdout': stdout,
|
||||
'stderr': stderr}
|
||||
super(ProcessExecutionError, self).__init__(message)
|
||||
|
||||
|
||||
@ -81,7 +90,10 @@ def execute(*cmd, **kwargs):
|
||||
:param cmd: Passed to subprocess.Popen.
|
||||
:type cmd: string
|
||||
:param process_input: Send to opened process.
|
||||
:type proces_input: string
|
||||
:type process_input: string
|
||||
:param env_variables: Environment variables and their values that
|
||||
will be set for the process.
|
||||
:type env_variables: dict
|
||||
:param check_exit_code: Single bool, int, or list of allowed exit
|
||||
codes. Defaults to [0]. Raise
|
||||
:class:`ProcessExecutionError` unless
|
||||
@ -102,6 +114,9 @@ def execute(*cmd, **kwargs):
|
||||
:param shell: whether or not there should be a shell used to
|
||||
execute this command. Defaults to false.
|
||||
:type shell: boolean
|
||||
:param loglevel: log level for execute commands.
|
||||
:type loglevel: int. (Should be stdlib_logging.DEBUG or
|
||||
stdlib_logging.INFO)
|
||||
:returns: (stdout, stderr) from process execution
|
||||
:raises: :class:`UnknownArgumentError` on
|
||||
receiving unknown arguments
|
||||
@ -109,6 +124,7 @@ def execute(*cmd, **kwargs):
|
||||
"""
|
||||
|
||||
process_input = kwargs.pop('process_input', None)
|
||||
env_variables = kwargs.pop('env_variables', None)
|
||||
check_exit_code = kwargs.pop('check_exit_code', [0])
|
||||
ignore_exit_code = False
|
||||
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
||||
@ -116,6 +132,7 @@ def execute(*cmd, **kwargs):
|
||||
run_as_root = kwargs.pop('run_as_root', False)
|
||||
root_helper = kwargs.pop('root_helper', '')
|
||||
shell = kwargs.pop('shell', False)
|
||||
loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
|
||||
|
||||
if isinstance(check_exit_code, bool):
|
||||
ignore_exit_code = not check_exit_code
|
||||
@ -130,8 +147,8 @@ def execute(*cmd, **kwargs):
|
||||
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
|
||||
if not root_helper:
|
||||
raise NoRootWrapSpecified(
|
||||
message=('Command requested root, but did not specify a root '
|
||||
'helper.'))
|
||||
message=_('Command requested root, but did not '
|
||||
'specify a root helper.'))
|
||||
cmd = shlex.split(root_helper) + list(cmd)
|
||||
|
||||
cmd = map(str, cmd)
|
||||
@ -139,7 +156,8 @@ def execute(*cmd, **kwargs):
|
||||
while attempts > 0:
|
||||
attempts -= 1
|
||||
try:
|
||||
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
|
||||
LOG.log(loglevel, 'Running cmd (subprocess): %s',
|
||||
' '.join(logging.mask_password(cmd)))
|
||||
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
||||
|
||||
if os.name == 'nt':
|
||||
@ -155,28 +173,37 @@ def execute(*cmd, **kwargs):
|
||||
stderr=_PIPE,
|
||||
close_fds=close_fds,
|
||||
preexec_fn=preexec_fn,
|
||||
shell=shell)
|
||||
shell=shell,
|
||||
env=env_variables)
|
||||
result = None
|
||||
if process_input is not None:
|
||||
result = obj.communicate(process_input)
|
||||
else:
|
||||
result = obj.communicate()
|
||||
for _i in six.moves.range(20):
|
||||
# NOTE(russellb) 20 is an arbitrary number of retries to
|
||||
# prevent any chance of looping forever here.
|
||||
try:
|
||||
if process_input is not None:
|
||||
result = obj.communicate(process_input)
|
||||
else:
|
||||
result = obj.communicate()
|
||||
except OSError as e:
|
||||
if e.errno in (errno.EAGAIN, errno.EINTR):
|
||||
continue
|
||||
raise
|
||||
break
|
||||
obj.stdin.close() # pylint: disable=E1101
|
||||
_returncode = obj.returncode # pylint: disable=E1101
|
||||
if _returncode:
|
||||
LOG.debug(_('Result was %s') % _returncode)
|
||||
if not ignore_exit_code and _returncode not in check_exit_code:
|
||||
(stdout, stderr) = result
|
||||
raise ProcessExecutionError(exit_code=_returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cmd=' '.join(cmd))
|
||||
LOG.log(loglevel, 'Result was %s' % _returncode)
|
||||
if not ignore_exit_code and _returncode not in check_exit_code:
|
||||
(stdout, stderr) = result
|
||||
raise ProcessExecutionError(exit_code=_returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cmd=' '.join(cmd))
|
||||
return result
|
||||
except ProcessExecutionError:
|
||||
if not attempts:
|
||||
raise
|
||||
else:
|
||||
LOG.debug(_('%r failed. Retrying.'), cmd)
|
||||
LOG.log(loglevel, '%r failed. Retrying.', cmd)
|
||||
if delay_on_retry:
|
||||
greenthread.sleep(random.randint(20, 200) / 100.0)
|
||||
finally:
|
||||
@ -203,7 +230,7 @@ def trycmd(*args, **kwargs):
|
||||
out, err = execute(*args, **kwargs)
|
||||
failed = False
|
||||
except ProcessExecutionError as exn:
|
||||
out, err = '', str(exn)
|
||||
out, err = '', six.text_type(exn)
|
||||
failed = True
|
||||
|
||||
if not failed and discard_warnings and err:
|
||||
@ -215,7 +242,7 @@ def trycmd(*args, **kwargs):
|
||||
|
||||
def ssh_execute(ssh, cmd, process_input=None,
|
||||
addl_env=None, check_exit_code=True):
|
||||
LOG.debug(_('Running cmd (SSH): %s'), cmd)
|
||||
LOG.debug('Running cmd (SSH): %s', cmd)
|
||||
if addl_env:
|
||||
raise InvalidArgumentError(_('Environment not supported over SSH'))
|
||||
|
||||
@ -236,7 +263,7 @@ def ssh_execute(ssh, cmd, process_input=None,
|
||||
|
||||
# exit_status == -1 if no exit code was returned
|
||||
if exit_status != -1:
|
||||
LOG.debug(_('Result was %s') % exit_status)
|
||||
LOG.debug('Result was %s' % exit_status)
|
||||
if check_exit_code and exit_status != 0:
|
||||
raise ProcessExecutionError(exit_code=exit_status,
|
||||
stdout=stdout,
|
||||
@ -244,3 +271,15 @@ def ssh_execute(ssh, cmd, process_input=None,
|
||||
cmd=cmd)
|
||||
|
||||
return (stdout, stderr)
|
||||
|
||||
|
||||
def get_worker_count():
|
||||
"""Utility to get the default worker count.
|
||||
|
||||
@return: The number of CPUs if that can be determined, else a default
|
||||
worker count of 1 is returned.
|
||||
"""
|
||||
try:
|
||||
return multiprocessing.cpu_count()
|
||||
except NotImplementedError:
|
||||
return 1
|
||||
|
@ -78,7 +78,7 @@ def bool_from_string(subject, strict=False, default=False):
|
||||
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
|
||||
"""
|
||||
if not isinstance(subject, six.string_types):
|
||||
subject = str(subject)
|
||||
subject = six.text_type(subject)
|
||||
|
||||
lowered = subject.strip().lower()
|
||||
|
||||
@ -98,7 +98,8 @@ def bool_from_string(subject, strict=False, default=False):
|
||||
|
||||
|
||||
def safe_decode(text, incoming=None, errors='strict'):
|
||||
"""Decodes incoming str using `incoming` if they're not already unicode.
|
||||
"""Decodes incoming text/bytes string using `incoming` if they're not
|
||||
already unicode.
|
||||
|
||||
:param incoming: Text's current encoding
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
@ -107,7 +108,7 @@ def safe_decode(text, incoming=None, errors='strict'):
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, six.string_types):
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be decoded" % type(text))
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
@ -137,7 +138,7 @@ def safe_decode(text, incoming=None, errors='strict'):
|
||||
|
||||
def safe_encode(text, incoming=None,
|
||||
encoding='utf-8', errors='strict'):
|
||||
"""Encodes incoming str/unicode using `encoding`.
|
||||
"""Encodes incoming text/bytes string using `encoding`.
|
||||
|
||||
If incoming is not specified, text is expected to be encoded with
|
||||
current python's default encoding. (`sys.getdefaultencoding`)
|
||||
@ -150,7 +151,7 @@ def safe_encode(text, incoming=None,
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, six.string_types):
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be encoded" % type(text))
|
||||
|
||||
if not incoming:
|
||||
@ -158,19 +159,13 @@ def safe_encode(text, incoming=None,
|
||||
sys.getdefaultencoding())
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
if six.PY3:
|
||||
return text.encode(encoding, errors).decode(incoming)
|
||||
else:
|
||||
return text.encode(encoding, errors)
|
||||
return text.encode(encoding, errors)
|
||||
elif text and encoding != incoming:
|
||||
# Decode text before encoding it with `encoding`
|
||||
text = safe_decode(text, incoming, errors)
|
||||
if six.PY3:
|
||||
return text.encode(encoding, errors).decode(incoming)
|
||||
else:
|
||||
return text.encode(encoding, errors)
|
||||
|
||||
return text
|
||||
return text.encode(encoding, errors)
|
||||
else:
|
||||
return text
|
||||
|
||||
|
||||
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
||||
|
@ -114,7 +114,7 @@ def utcnow():
|
||||
|
||||
|
||||
def iso8601_from_timestamp(timestamp):
|
||||
"""Returns a iso8601 formatted date from timestamp."""
|
||||
"""Returns an iso8601 formatted date from timestamp."""
|
||||
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
||||
|
||||
|
||||
@ -134,7 +134,7 @@ def set_time_override(override_time=None):
|
||||
|
||||
def advance_time_delta(timedelta):
|
||||
"""Advance overridden time using a datetime.timedelta."""
|
||||
assert(not utcnow.override_time is None)
|
||||
assert utcnow.override_time is not None
|
||||
try:
|
||||
for dt in utcnow.override_time:
|
||||
dt += timedelta
|
||||
|
@ -800,7 +800,7 @@
|
||||
#log_dir=<None>
|
||||
|
||||
# Use syslog for logging. Existing syslog format is DEPRECATED
|
||||
# during I, and will chang in J to honor RFC5424. (boolean
|
||||
# during I, and will change in J to honor RFC5424. (boolean
|
||||
# value)
|
||||
#use_syslog=false
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user