From 3a1cd7c8935f0bf05fd505805de7f414a9e4f775 Mon Sep 17 00:00:00 2001
From: Steven Hardy <shardy@redhat.com>
Date: Wed, 22 Oct 2014 11:37:12 +0100
Subject: [PATCH] sync oslo-incubator for request_id shim
Sync 838a2a3 oslo-incubator to get the new request_id shim, so
can introduce a deprecation warning that tells operators upgrading
from Juno that their paste.ini files need to point to oslo.middleware
This also syncs all other modules, except policy, which looks like it
needs some changes to heat thus will be handled via another patch.
Note we shouldn't remove this request_id shim until after Kilo is
branched.
Change-Id: I35125ffa263b0522ff6dd0b80b0beb3cbc79999b
Related-Bug: #1380629
---
etc/heat/heat.conf.sample | 6 +-
heat/openstack/common/__init__.py | 17 --
heat/openstack/common/_i18n.py | 40 +++++
heat/openstack/common/crypto/utils.py | 4 +-
heat/openstack/common/eventlet_backdoor.py | 2 +-
heat/openstack/common/fileutils.py | 18 +-
heat/openstack/common/log.py | 162 ++++++++----------
heat/openstack/common/loopingcall.py | 29 ++--
.../openstack/common/middleware/request_id.py | 28 +--
heat/openstack/common/processutils.py | 55 ++++--
heat/openstack/common/service.py | 10 +-
heat/openstack/common/versionutils.py | 76 ++++++--
requirements.txt | 1 +
13 files changed, 259 insertions(+), 189 deletions(-)
create mode 100644 heat/openstack/common/_i18n.py
diff --git a/etc/heat/heat.conf.sample b/etc/heat/heat.conf.sample
index c14c6051db..4dbdf95c2d 100644
--- a/etc/heat/heat.conf.sample
+++ b/etc/heat/heat.conf.sample
@@ -451,7 +451,7 @@
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# List of logger=LEVEL pairs. (list value)
-#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN
+#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
# Enables or disables publication of error events. (boolean
# value)
@@ -462,11 +462,11 @@
#fatal_deprecations=false
# The format for an instance that is passed with the log
-# message. (string value)
+# message. (string value)
#instance_format="[instance: %(uuid)s] "
# The format for an instance UUID that is passed with the log
-# message. (string value)
+# message. (string value)
#instance_uuid_format="[instance: %(uuid)s] "
# The name of a logging configuration file. This file is
diff --git a/heat/openstack/common/__init__.py b/heat/openstack/common/__init__.py
index d1223eaf76..e69de29bb2 100644
--- a/heat/openstack/common/__init__.py
+++ b/heat/openstack/common/__init__.py
@@ -1,17 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
diff --git a/heat/openstack/common/_i18n.py b/heat/openstack/common/_i18n.py
new file mode 100644
index 0000000000..762a789d78
--- /dev/null
+++ b/heat/openstack/common/_i18n.py
@@ -0,0 +1,40 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='heat')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical
diff --git a/heat/openstack/common/crypto/utils.py b/heat/openstack/common/crypto/utils.py
index 16460b7b64..0dca18e26e 100644
--- a/heat/openstack/common/crypto/utils.py
+++ b/heat/openstack/common/crypto/utils.py
@@ -16,10 +16,10 @@ import base64
from Crypto.Hash import HMAC
from Crypto import Random
+from oslo.utils import importutils
import six
-from heat.openstack.common.gettextutils import _
-from heat.openstack.common import importutils
+from heat.openstack.common._i18n import _
bchr = six.int2byte
diff --git a/heat/openstack/common/eventlet_backdoor.py b/heat/openstack/common/eventlet_backdoor.py
index 4f38ae1e14..a19ca59d67 100644
--- a/heat/openstack/common/eventlet_backdoor.py
+++ b/heat/openstack/common/eventlet_backdoor.py
@@ -29,7 +29,7 @@ import eventlet.backdoor
import greenlet
from oslo.config import cfg
-from heat.openstack.common.gettextutils import _LI
+from heat.openstack.common._i18n import _LI
from heat.openstack.common import log as logging
help_for_backdoor_port = (
diff --git a/heat/openstack/common/fileutils.py b/heat/openstack/common/fileutils.py
index 5983f184cc..59088f6439 100644
--- a/heat/openstack/common/fileutils.py
+++ b/heat/openstack/common/fileutils.py
@@ -18,7 +18,8 @@ import errno
import os
import tempfile
-from heat.openstack.common import excutils
+from oslo.utils import excutils
+
from heat.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@@ -50,8 +51,8 @@ def read_cached_file(filename, force_reload=False):
"""
global _FILE_CACHE
- if force_reload and filename in _FILE_CACHE:
- del _FILE_CACHE[filename]
+ if force_reload:
+ delete_cached_file(filename)
reloaded = False
mtime = os.path.getmtime(filename)
@@ -66,6 +67,17 @@ def read_cached_file(filename, force_reload=False):
return (reloaded, cache_info['data'])
+def delete_cached_file(filename):
+ """Delete cached file if present.
+
+ :param filename: filename to delete
+ """
+ global _FILE_CACHE
+
+ if filename in _FILE_CACHE:
+ del _FILE_CACHE[filename]
+
+
def delete_if_exists(path, remove=os.unlink):
"""Delete a file, but ignore file not found error.
diff --git a/heat/openstack/common/log.py b/heat/openstack/common/log.py
index 972de61485..efdca795bb 100644
--- a/heat/openstack/common/log.py
+++ b/heat/openstack/common/log.py
@@ -33,42 +33,24 @@ import logging
import logging.config
import logging.handlers
import os
-import re
+import socket
import sys
import traceback
from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
import six
from six import moves
-from heat.openstack.common.gettextutils import _
-from heat.openstack.common import importutils
-from heat.openstack.common import jsonutils
+_PY26 = sys.version_info[0:2] == (2, 6)
+
+from heat.openstack.common._i18n import _
from heat.openstack.common import local
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
- r'(<%(key)s>).*?(</%(key)s>)',
- r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
- r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
- r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
- '.*?([\'"])',
- r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
-
-for key in _SANITIZE_KEYS:
- for pattern in _FORMAT_PATTERNS:
- reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
- _SANITIZE_PATTERNS.append(reg_ex)
-
common_cli_opts = [
cfg.BoolOpt('debug',
@@ -138,6 +120,14 @@ generic_log_opts = [
help='Log output to standard error.')
]
+DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
+ 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
+ 'oslo.messaging=INFO', 'iso8601=WARN',
+ 'requests.packages.urllib3.connectionpool=WARN',
+ 'urllib3.connectionpool=WARN', 'websocket=WARN',
+ "keystonemiddleware=WARN", "routes.middleware=WARN",
+ "stevedore=WARN"]
+
log_opts = [
cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@@ -156,17 +146,7 @@ log_opts = [
'%(instance)s',
help='Prefix each line of exception output with this format.'),
cfg.ListOpt('default_log_levels',
- default=[
- 'amqp=WARN',
- 'amqplib=WARN',
- 'boto=WARN',
- 'qpid=WARN',
- 'sqlalchemy=WARN',
- 'suds=INFO',
- 'oslo.messaging=INFO',
- 'iso8601=WARN',
- 'requests.packages.urllib3.connectionpool=WARN'
- ],
+ default=DEFAULT_LOG_LEVELS,
help='List of logger=LEVEL pairs.'),
cfg.BoolOpt('publish_errors',
default=False,
@@ -181,11 +161,11 @@ log_opts = [
cfg.StrOpt('instance_format',
default='[instance: %(uuid)s] ',
help='The format for an instance that is passed with the log '
- 'message. '),
+ 'message.'),
cfg.StrOpt('instance_uuid_format',
default='[instance: %(uuid)s] ',
help='The format for an instance UUID that is passed with the '
- 'log message. '),
+ 'log message.'),
]
CONF = cfg.CONF
@@ -244,45 +224,20 @@ def _get_log_file_path(binary=None):
return None
-def mask_password(message, secret="***"):
- """Replace password with 'secret' in message.
-
- :param message: The string which includes security information.
- :param secret: value with which to replace passwords.
- :returns: The unicode value of message with the password fields masked.
-
- For example:
-
- >>> mask_password("'adminPass' : 'aaaaa'")
- "'adminPass' : '***'"
- >>> mask_password("'admin_pass' : 'aaaaa'")
- "'admin_pass' : '***'"
- >>> mask_password('"password" : "aaaaa"')
- '"password" : "***"'
- >>> mask_password("'original_password' : 'aaaaa'")
- "'original_password' : '***'"
- >>> mask_password("u'original_password' : u'aaaaa'")
- "u'original_password' : u'***'"
- """
- message = six.text_type(message)
-
- # NOTE(ldbragst): Check to see if anything in message contains any key
- # specified in _SANITIZE_KEYS, if not then just return the message since
- # we don't have to mask any passwords.
- if not any(key in message for key in _SANITIZE_KEYS):
- return message
-
- secret = r'\g<1>' + secret + r'\g<2>'
- for pattern in _SANITIZE_PATTERNS:
- message = re.sub(pattern, secret, message)
- return message
-
-
class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs):
self.log(logging.AUDIT, msg, *args, **kwargs)
+ def isEnabledFor(self, level):
+ if _PY26:
+ # This method was added in python 2.7 (and it does the exact
+ # same logic, so we need to do the exact same logic so that
+ # python 2.6 has this capability as well).
+ return self.logger.isEnabledFor(level)
+ else:
+ return super(BaseLoggerAdapter, self).isEnabledFor(level)
+
class LazyAdapter(BaseLoggerAdapter):
def __init__(self, name='unknown', version='unknown'):
@@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter):
def logger(self):
if not self._logger:
self._logger = getLogger(self.name, self.version)
+ if six.PY3:
+ # In Python 3, the code fails because the 'manager' attribute
+ # cannot be found when using a LoggerAdapter as the
+ # underlying logger. Work around this issue.
+ self._logger.manager = self._logger.logger.manager
return self._logger
@@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter):
self.warn(stdmsg, *args, **kwargs)
def process(self, msg, kwargs):
- # NOTE(mrodden): catch any Message/other object and
- # coerce to unicode before they can get
- # to the python logging and possibly
- # cause string encoding trouble
- if not isinstance(msg, six.string_types):
+ # NOTE(jecarey): If msg is not unicode, coerce it into unicode
+ # before it can get to the python logging and
+ # possibly cause string encoding trouble
+ if not isinstance(msg, six.text_type):
msg = six.text_type(msg)
if 'extra' not in kwargs:
@@ -448,7 +407,7 @@ def _load_log_config(log_config_append):
try:
logging.config.fileConfig(log_config_append,
disable_existing_loggers=False)
- except moves.configparser.Error as exc:
+ except (moves.configparser.Error, KeyError) as exc:
raise LogConfigError(log_config_append, six.text_type(exc))
@@ -461,9 +420,20 @@ def setup(product_name, version='unknown'):
sys.excepthook = _create_logging_excepthook(product_name)
-def set_defaults(logging_context_format_string):
- cfg.set_defaults(
- log_opts, logging_context_format_string=logging_context_format_string)
+def set_defaults(logging_context_format_string=None,
+ default_log_levels=None):
+ # Just in case the caller is not setting the
+ # default_log_level. This is insurance because
+ # we introduced the default_log_level parameter
+ # later in a backwards in-compatible change
+ if default_log_levels is not None:
+ cfg.set_defaults(
+ log_opts,
+ default_log_levels=default_log_levels)
+ if logging_context_format_string is not None:
+ cfg.set_defaults(
+ log_opts,
+ logging_context_format_string=logging_context_format_string)
def _find_facility_from_conf():
@@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version):
for handler in log_root.handlers:
log_root.removeHandler(handler)
- if CONF.use_syslog:
- facility = _find_facility_from_conf()
- # TODO(bogdando) use the format provided by RFCSysLogHandler
- # after existing syslog format deprecation in J
- if CONF.use_syslog_rfc_format:
- syslog = RFCSysLogHandler(address='/dev/log',
- facility=facility)
- else:
- syslog = logging.handlers.SysLogHandler(address='/dev/log',
- facility=facility)
- log_root.addHandler(syslog)
-
logpath = _get_log_file_path()
if logpath:
filelog = logging.handlers.WatchedFileHandler(logpath)
@@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version):
else:
logger.setLevel(level_name)
+ if CONF.use_syslog:
+ try:
+ facility = _find_facility_from_conf()
+ # TODO(bogdando) use the format provided by RFCSysLogHandler
+ # after existing syslog format deprecation in J
+ if CONF.use_syslog_rfc_format:
+ syslog = RFCSysLogHandler(facility=facility)
+ else:
+ syslog = logging.handlers.SysLogHandler(facility=facility)
+ log_root.addHandler(syslog)
+ except socket.error:
+ log_root.error('Unable to add syslog handler. Verify that syslog '
+ 'is running.')
+
_loggers = {}
@@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter):
def format(self, record):
"""Uses contextstring if request_id is set, otherwise default."""
+ # NOTE(jecarey): If msg is not unicode, coerce it into unicode
+ # before it can get to the python logging and
+ # possibly cause string encoding trouble
+ if not isinstance(record.msg, six.text_type):
+ record.msg = six.text_type(record.msg)
+
# store project info
record.project = self.project
record.version = self.version
diff --git a/heat/openstack/common/loopingcall.py b/heat/openstack/common/loopingcall.py
index a44ad68b4d..f7a54c5e9b 100644
--- a/heat/openstack/common/loopingcall.py
+++ b/heat/openstack/common/loopingcall.py
@@ -16,16 +16,21 @@
# under the License.
import sys
+import time
from eventlet import event
from eventlet import greenthread
-from heat.openstack.common.gettextutils import _LE, _LW
+from heat.openstack.common._i18n import _LE, _LW
from heat.openstack.common import log as logging
-from heat.openstack.common import timeutils
LOG = logging.getLogger(__name__)
+# NOTE(zyluo): This lambda function was declared to avoid mocking collisions
+# with time.time() called in the standard logging module
+# during unittests.
+_ts = lambda: time.time()
+
class LoopingCallDone(Exception):
"""Exception to break out and stop a LoopingCallBase.
@@ -72,16 +77,17 @@ class FixedIntervalLoopingCall(LoopingCallBase):
try:
while self._running:
- start = timeutils.utcnow()
+ start = _ts()
self.f(*self.args, **self.kw)
- end = timeutils.utcnow()
+ end = _ts()
if not self._running:
break
- delay = interval - timeutils.delta_seconds(start, end)
- if delay <= 0:
- LOG.warn(_LW('task run outlasted interval by %s sec') %
- -delay)
- greenthread.sleep(delay if delay > 0 else 0)
+ delay = end - start - interval
+ if delay > 0:
+ LOG.warn(_LW('task %(func_name)s run outlasted '
+ 'interval by %(delay).2f sec'),
+ {'func_name': repr(self.f), 'delay': delay})
+ greenthread.sleep(-delay if delay < 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
@@ -121,8 +127,9 @@ class DynamicLoopingCall(LoopingCallBase):
if periodic_interval_max is not None:
idle = min(idle, periodic_interval_max)
- LOG.debug('Dynamic looping call sleeping for %.02f '
- 'seconds', idle)
+ LOG.debug('Dynamic looping call %(func_name)s sleeping '
+ 'for %(idle).02f seconds',
+ {'func_name': repr(self.f), 'idle': idle})
greenthread.sleep(idle)
except LoopingCallDone as e:
self.stop()
diff --git a/heat/openstack/common/middleware/request_id.py b/heat/openstack/common/middleware/request_id.py
index 4d7693add7..bb48e669b4 100644
--- a/heat/openstack/common/middleware/request_id.py
+++ b/heat/openstack/common/middleware/request_id.py
@@ -1,6 +1,3 @@
-# Copyright (c) 2013 NEC Corporation
-# All Rights Reserved.
-#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@@ -13,29 +10,18 @@
# License for the specific language governing permissions and limitations
# under the License.
-"""Middleware that ensures request ID.
+"""Compatibility shim for Kilo, while operators migrate to oslo.middleware."""
-It ensures to assign request ID for each API request and set it to
-request environment. The request ID is also added to API response.
-"""
+from oslo.middleware import request_id
-import webob.dec
-
-from heat.openstack.common import context
-from heat.openstack.common.middleware import base
+from heat.openstack.common import versionutils
ENV_REQUEST_ID = 'openstack.request_id'
HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
-class RequestIdMiddleware(base.Middleware):
-
- @webob.dec.wsgify
- def __call__(self, req):
- req_id = context.generate_request_id()
- req.environ[ENV_REQUEST_ID] = req_id
- response = req.get_response(self.application)
- if HTTP_RESP_HEADER_REQUEST_ID not in response.headers:
- response.headers.add(HTTP_RESP_HEADER_REQUEST_ID, req_id)
- return response
+@versionutils.deprecated(as_of=versionutils.deprecated.KILO,
+ in_favor_of='oslo.middleware.RequestId')
+class RequestIdMiddleware(request_id.RequestId):
+ pass
diff --git a/heat/openstack/common/processutils.py b/heat/openstack/common/processutils.py
index a27eaf7376..10e4c27e0e 100644
--- a/heat/openstack/common/processutils.py
+++ b/heat/openstack/common/processutils.py
@@ -18,7 +18,8 @@ System-level utilities and helper functions.
"""
import errno
-import logging as stdlib_logging
+import logging
+import multiprocessing
import os
import random
import shlex
@@ -26,10 +27,10 @@ import signal
from eventlet.green import subprocess
from eventlet import greenthread
+from oslo.utils import strutils
import six
-from heat.openstack.common.gettextutils import _
-from heat.openstack.common import log as logging
+from heat.openstack.common._i18n import _
LOG = logging.getLogger(__name__)
@@ -114,8 +115,7 @@ def execute(*cmd, **kwargs):
execute this command. Defaults to false.
:type shell: boolean
:param loglevel: log level for execute commands.
- :type loglevel: int. (Should be stdlib_logging.DEBUG or
- stdlib_logging.INFO)
+ :type loglevel: int. (Should be logging.DEBUG or logging.INFO)
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
@@ -131,7 +131,7 @@ def execute(*cmd, **kwargs):
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
shell = kwargs.pop('shell', False)
- loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+ loglevel = kwargs.pop('loglevel', logging.DEBUG)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
@@ -140,8 +140,7 @@ def execute(*cmd, **kwargs):
check_exit_code = [check_exit_code]
if kwargs:
- raise UnknownArgumentError(_('Got unknown keyword args '
- 'to utils.execute: %r') % kwargs)
+ raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
if not root_helper:
@@ -151,12 +150,12 @@ def execute(*cmd, **kwargs):
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
+ sanitized_cmd = strutils.mask_password(' '.join(cmd))
while attempts > 0:
attempts -= 1
try:
- LOG.log(loglevel, 'Running cmd (subprocess): %s',
- ' '.join(logging.mask_password(cmd)))
+ LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
_PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
@@ -193,16 +192,18 @@ def execute(*cmd, **kwargs):
LOG.log(loglevel, 'Result was %s' % _returncode)
if not ignore_exit_code and _returncode not in check_exit_code:
(stdout, stderr) = result
+ sanitized_stdout = strutils.mask_password(stdout)
+ sanitized_stderr = strutils.mask_password(stderr)
raise ProcessExecutionError(exit_code=_returncode,
- stdout=stdout,
- stderr=stderr,
- cmd=' '.join(cmd))
+ stdout=sanitized_stdout,
+ stderr=sanitized_stderr,
+ cmd=sanitized_cmd)
return result
except ProcessExecutionError:
if not attempts:
raise
else:
- LOG.log(loglevel, '%r failed. Retrying.', cmd)
+ LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:
@@ -241,7 +242,8 @@ def trycmd(*args, **kwargs):
def ssh_execute(ssh, cmd, process_input=None,
addl_env=None, check_exit_code=True):
- LOG.debug('Running cmd (SSH): %s', cmd)
+ sanitized_cmd = strutils.mask_password(cmd)
+ LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
if addl_env:
raise InvalidArgumentError(_('Environment not supported over SSH'))
@@ -255,7 +257,10 @@ def ssh_execute(ssh, cmd, process_input=None,
# NOTE(justinsb): This seems suspicious...
# ...other SSH clients have buffering issues with this approach
stdout = stdout_stream.read()
+ sanitized_stdout = strutils.mask_password(stdout)
stderr = stderr_stream.read()
+ sanitized_stderr = strutils.mask_password(stderr)
+
stdin_stream.close()
exit_status = channel.recv_exit_status()
@@ -265,8 +270,20 @@ def ssh_execute(ssh, cmd, process_input=None,
LOG.debug('Result was %s' % exit_status)
if check_exit_code and exit_status != 0:
raise ProcessExecutionError(exit_code=exit_status,
- stdout=stdout,
- stderr=stderr,
- cmd=cmd)
+ stdout=sanitized_stdout,
+ stderr=sanitized_stderr,
+ cmd=sanitized_cmd)
- return (stdout, stderr)
+ return (sanitized_stdout, sanitized_stderr)
+
+
+def get_worker_count():
+ """Utility to get the default worker count.
+
+ @return: The number of CPUs if that can be determined, else a default
+ worker count of 1 is returned.
+ """
+ try:
+ return multiprocessing.cpu_count()
+ except NotImplementedError:
+ return 1
diff --git a/heat/openstack/common/service.py b/heat/openstack/common/service.py
index adfc772ad4..89dd67a621 100644
--- a/heat/openstack/common/service.py
+++ b/heat/openstack/common/service.py
@@ -38,14 +38,12 @@ from eventlet import event
from oslo.config import cfg
from heat.openstack.common import eventlet_backdoor
-from heat.openstack.common.gettextutils import _LE, _LI, _LW
-from heat.openstack.common import importutils
+from heat.openstack.common._i18n import _LE, _LI, _LW
from heat.openstack.common import log as logging
from heat.openstack.common import systemd
from heat.openstack.common import threadgroup
-rpc = importutils.try_import('heat.openstack.common.rpc')
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@@ -180,12 +178,6 @@ class ServiceLauncher(Launcher):
status = exc.code
finally:
self.stop()
- if rpc:
- try:
- rpc.cleanup()
- except Exception:
- # We're shutting down, so it doesn't matter at this point.
- LOG.exception(_LE('Exception during rpc cleanup.'))
return status, signo
diff --git a/heat/openstack/common/versionutils.py b/heat/openstack/common/versionutils.py
index 4fa9a965e2..12bf31795c 100644
--- a/heat/openstack/common/versionutils.py
+++ b/heat/openstack/common/versionutils.py
@@ -18,10 +18,12 @@ Helpers for comparing version strings.
"""
import functools
+import inspect
import pkg_resources
+import six
-from heat.openstack.common.gettextutils import _
+from heat.openstack.common._i18n import _
from heat.openstack.common import log as logging
@@ -53,18 +55,36 @@ class deprecated(object):
>>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
... def c(): pass
+ 4. Specifying the deprecated functionality will not be removed:
+ >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0)
+ ... def d(): pass
+
+ 5. Specifying a replacement, deprecated functionality will not be removed:
+ >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0)
+ ... def e(): pass
+
"""
+ # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+ # expected we maintain a gap between Bexar and Folsom in this list.
+ BEXAR = 'B'
FOLSOM = 'F'
GRIZZLY = 'G'
HAVANA = 'H'
ICEHOUSE = 'I'
+ JUNO = 'J'
+ KILO = 'K'
_RELEASES = {
+ # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+ # expected we maintain a gap between Bexar and Folsom in this list.
+ 'B': 'Bexar',
'F': 'Folsom',
'G': 'Grizzly',
'H': 'Havana',
'I': 'Icehouse',
+ 'J': 'Juno',
+ 'K': 'Kilo',
}
_deprecated_msg_with_alternative = _(
@@ -75,6 +95,12 @@ class deprecated(object):
'%(what)s is deprecated as of %(as_of)s and may be '
'removed in %(remove_in)s. It will not be superseded.')
+ _deprecated_msg_with_alternative_no_removal = _(
+ '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.')
+
+ _deprecated_msg_with_no_alternative_no_removal = _(
+ '%(what)s is deprecated as of %(as_of)s. It will not be superseded.')
+
def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
"""Initialize decorator
@@ -92,16 +118,34 @@ class deprecated(object):
self.remove_in = remove_in
self.what = what
- def __call__(self, func):
+ def __call__(self, func_or_cls):
if not self.what:
- self.what = func.__name__ + '()'
+ self.what = func_or_cls.__name__ + '()'
+ msg, details = self._build_message()
- @functools.wraps(func)
- def wrapped(*args, **kwargs):
- msg, details = self._build_message()
- LOG.deprecated(msg, details)
- return func(*args, **kwargs)
- return wrapped
+ if inspect.isfunction(func_or_cls):
+
+ @six.wraps(func_or_cls)
+ def wrapped(*args, **kwargs):
+ LOG.deprecated(msg, details)
+ return func_or_cls(*args, **kwargs)
+ return wrapped
+ elif inspect.isclass(func_or_cls):
+ orig_init = func_or_cls.__init__
+
+ # TODO(tsufiev): change `functools` module to `six` as
+ # soon as six 1.7.4 (with fix for passing `assigned`
+ # argument to underlying `functools.wraps`) is released
+ # and added to the heat-incubator requrements
+ @functools.wraps(orig_init, assigned=('__name__', '__doc__'))
+ def new_init(self, *args, **kwargs):
+ LOG.deprecated(msg, details)
+ orig_init(self, *args, **kwargs)
+ func_or_cls.__init__ = new_init
+ return func_or_cls
+ else:
+ raise TypeError('deprecated can be used only with functions or '
+ 'classes')
def _get_safe_to_remove_release(self, release):
# TODO(dstanek): this method will have to be reimplemented once
@@ -120,9 +164,19 @@ class deprecated(object):
if self.in_favor_of:
details['in_favor_of'] = self.in_favor_of
- msg = self._deprecated_msg_with_alternative
+ if self.remove_in > 0:
+ msg = self._deprecated_msg_with_alternative
+ else:
+ # There are no plans to remove this function, but it is
+ # now deprecated.
+ msg = self._deprecated_msg_with_alternative_no_removal
else:
- msg = self._deprecated_msg_no_alternative
+ if self.remove_in > 0:
+ msg = self._deprecated_msg_no_alternative
+ else:
+ # There are no plans to remove this function, but it is
+ # now deprecated.
+ msg = self._deprecated_msg_with_no_alternative_no_removal
return msg, details
diff --git a/requirements.txt b/requirements.txt
index 2571d8c719..7472f3f7f8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,6 +17,7 @@ oslo.config>=1.4.0 # Apache-2.0
oslo.db>=1.0.0 # Apache-2.0
oslo.i18n>=1.0.0 # Apache-2.0
oslo.messaging>=1.4.0
+oslo.middleware>=0.1.0 # Apache-2.0
oslo.serialization>=1.0.0 # Apache-2.0
oslo.utils>=1.0.0 # Apache-2.0
osprofiler>=0.3.0 # Apache-2.0