sync oslo-incubator for request_id shim

Sync 838a2a3 oslo-incubator to get the new request_id shim, so
can introduce a deprecation warning that tells operators upgrading
from Juno that their paste.ini files need to point to oslo.middleware

This also syncs all other modules, except policy, which looks like it
needs some changes to heat thus will be handled via another patch.

Note we shouldn't remove this request_id shim until after Kilo is
branched.

Change-Id: I35125ffa263b0522ff6dd0b80b0beb3cbc79999b
Related-Bug: #1380629

etc/heat/heat.conf.sample

@@ -451,7 +451,7 @@
 #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
 
 # List of logger=LEVEL pairs. (list value)
-#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN
+#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
 
 # Enables or disables publication of error events. (boolean
 # value)
@@ -462,11 +462,11 @@
 #fatal_deprecations=false
 
 # The format for an instance that is passed with the log
-# message.  (string value)
+# message. (string value)
 #instance_format="[instance: %(uuid)s] "
 
 # The format for an instance UUID that is passed with the log
-# message.  (string value)
+# message. (string value)
 #instance_uuid_format="[instance: %(uuid)s] "
 
 # The name of a logging configuration file. This file is

heat/openstack/common/__init__.py

@@ -1,17 +0,0 @@
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

heat/openstack/common/_i18n.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='heat')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical

heat/openstack/common/crypto/utils.py

@@ -16,10 +16,10 @@ import base64
 
 from Crypto.Hash import HMAC
 from Crypto import Random
+from oslo.utils import importutils
 import six
 
-from heat.openstack.common.gettextutils import _
+from heat.openstack.common._i18n import _
-from heat.openstack.common import importutils
 
 bchr = six.int2byte
 

heat/openstack/common/eventlet_backdoor.py

@@ -29,7 +29,7 @@ import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
-from heat.openstack.common.gettextutils import _LI
+from heat.openstack.common._i18n import _LI
 from heat.openstack.common import log as logging
 
 help_for_backdoor_port = (

heat/openstack/common/fileutils.py

@@ -18,7 +18,8 @@ import errno
 import os
 import tempfile
 
-from heat.openstack.common import excutils
+from oslo.utils import excutils
+
 from heat.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
@@ -50,8 +51,8 @@ def read_cached_file(filename, force_reload=False):
     """
     global _FILE_CACHE
 
-    if force_reload and filename in _FILE_CACHE:
+    if force_reload:
-        del _FILE_CACHE[filename]
+        delete_cached_file(filename)
 
     reloaded = False
     mtime = os.path.getmtime(filename)
@@ -66,6 +67,17 @@ def read_cached_file(filename, force_reload=False):
     return (reloaded, cache_info['data'])
 
 
+def delete_cached_file(filename):
+    """Delete cached file if present.
+
+    :param filename: filename to delete
+    """
+    global _FILE_CACHE
+
+    if filename in _FILE_CACHE:
+        del _FILE_CACHE[filename]
+
+
 def delete_if_exists(path, remove=os.unlink):
     """Delete a file, but ignore file not found error.
 

heat/openstack/common/log.py

@@ -33,42 +33,24 @@ import logging
 import logging.config
 import logging.handlers
 import os
-import re
+import socket
 import sys
 import traceback
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
 import six
 from six import moves
 
-from heat.openstack.common.gettextutils import _
+_PY26 = sys.version_info[0:2] == (2, 6)
-from heat.openstack.common import importutils
+
-from heat.openstack.common import jsonutils
+from heat.openstack.common._i18n import _
 from heat.openstack.common import local
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
-                    r'(<%(key)s>).*?(</%(key)s>)',
-                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
-                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
-                    r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
-                    '.*?([\'"])',
-                    r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
-
-for key in _SANITIZE_KEYS:
-    for pattern in _FORMAT_PATTERNS:
-        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
-        _SANITIZE_PATTERNS.append(reg_ex)
-
 
 common_cli_opts = [
     cfg.BoolOpt('debug',
@@ -138,6 +120,14 @@ generic_log_opts = [
                 help='Log output to standard error.')
 ]
 
+DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
+                      'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
+                      'oslo.messaging=INFO', 'iso8601=WARN',
+                      'requests.packages.urllib3.connectionpool=WARN',
+                      'urllib3.connectionpool=WARN', 'websocket=WARN',
+                      "keystonemiddleware=WARN", "routes.middleware=WARN",
+                      "stevedore=WARN"]
+
 log_opts = [
     cfg.StrOpt('logging_context_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@@ -156,17 +146,7 @@ log_opts = [
                '%(instance)s',
                help='Prefix each line of exception output with this format.'),
     cfg.ListOpt('default_log_levels',
-                default=[
+                default=DEFAULT_LOG_LEVELS,
-                    'amqp=WARN',
-                    'amqplib=WARN',
-                    'boto=WARN',
-                    'qpid=WARN',
-                    'sqlalchemy=WARN',
-                    'suds=INFO',
-                    'oslo.messaging=INFO',
-                    'iso8601=WARN',
-                    'requests.packages.urllib3.connectionpool=WARN'
-                ],
                 help='List of logger=LEVEL pairs.'),
     cfg.BoolOpt('publish_errors',
                 default=False,
@@ -181,11 +161,11 @@ log_opts = [
     cfg.StrOpt('instance_format',
                default='[instance: %(uuid)s] ',
                help='The format for an instance that is passed with the log '
-                    'message. '),
+                    'message.'),
     cfg.StrOpt('instance_uuid_format',
                default='[instance: %(uuid)s] ',
                help='The format for an instance UUID that is passed with the '
-                    'log message. '),
+                    'log message.'),
 ]
 
 CONF = cfg.CONF
@@ -244,45 +224,20 @@ def _get_log_file_path(binary=None):
     return None
 
 
-def mask_password(message, secret="***"):
-    """Replace password with 'secret' in message.
-
-    :param message: The string which includes security information.
-    :param secret: value with which to replace passwords.
-    :returns: The unicode value of message with the password fields masked.
-
-    For example:
-
-    >>> mask_password("'adminPass' : 'aaaaa'")
-    "'adminPass' : '***'"
-    >>> mask_password("'admin_pass' : 'aaaaa'")
-    "'admin_pass' : '***'"
-    >>> mask_password('"password" : "aaaaa"')
-    '"password" : "***"'
-    >>> mask_password("'original_password' : 'aaaaa'")
-    "'original_password' : '***'"
-    >>> mask_password("u'original_password' :   u'aaaaa'")
-    "u'original_password' :   u'***'"
-    """
-    message = six.text_type(message)
-
-    # NOTE(ldbragst): Check to see if anything in message contains any key
-    # specified in _SANITIZE_KEYS, if not then just return the message since
-    # we don't have to mask any passwords.
-    if not any(key in message for key in _SANITIZE_KEYS):
-        return message
-
-    secret = r'\g<1>' + secret + r'\g<2>'
-    for pattern in _SANITIZE_PATTERNS:
-        message = re.sub(pattern, secret, message)
-    return message
-
-
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
         self.log(logging.AUDIT, msg, *args, **kwargs)
 
+    def isEnabledFor(self, level):
+        if _PY26:
+            # This method was added in python 2.7 (and it does the exact
+            # same logic, so we need to do the exact same logic so that
+            # python 2.6 has this capability as well).
+            return self.logger.isEnabledFor(level)
+        else:
+            return super(BaseLoggerAdapter, self).isEnabledFor(level)
+
 
 class LazyAdapter(BaseLoggerAdapter):
     def __init__(self, name='unknown', version='unknown'):
@@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter):
     def logger(self):
         if not self._logger:
             self._logger = getLogger(self.name, self.version)
+            if six.PY3:
+                # In Python 3, the code fails because the 'manager' attribute
+                # cannot be found when using a LoggerAdapter as the
+                # underlying logger. Work around this issue.
+                self._logger.manager = self._logger.logger.manager
         return self._logger
 
 
@@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter):
         self.warn(stdmsg, *args, **kwargs)
 
     def process(self, msg, kwargs):
-        # NOTE(mrodden): catch any Message/other object and
+        # NOTE(jecarey): If msg is not unicode, coerce it into unicode
-        #                coerce to unicode before they can get
+        #                before it can get to the python logging and
-        #                to the python logging and possibly
+        #                possibly cause string encoding trouble
-        #                cause string encoding trouble
+        if not isinstance(msg, six.text_type):
-        if not isinstance(msg, six.string_types):
             msg = six.text_type(msg)
 
         if 'extra' not in kwargs:
@@ -448,7 +407,7 @@ def _load_log_config(log_config_append):
     try:
         logging.config.fileConfig(log_config_append,
                                   disable_existing_loggers=False)
-    except moves.configparser.Error as exc:
+    except (moves.configparser.Error, KeyError) as exc:
         raise LogConfigError(log_config_append, six.text_type(exc))
 
 
@@ -461,9 +420,20 @@ def setup(product_name, version='unknown'):
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
-def set_defaults(logging_context_format_string):
+def set_defaults(logging_context_format_string=None,
-    cfg.set_defaults(
+                 default_log_levels=None):
-        log_opts, logging_context_format_string=logging_context_format_string)
+    # Just in case the caller is not setting the
+    # default_log_level. This is insurance because
+    # we introduced the default_log_level parameter
+    # later in a backwards in-compatible change
+    if default_log_levels is not None:
+        cfg.set_defaults(
+            log_opts,
+            default_log_levels=default_log_levels)
+    if logging_context_format_string is not None:
+        cfg.set_defaults(
+            log_opts,
+            logging_context_format_string=logging_context_format_string)
 
 
 def _find_facility_from_conf():
@@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version):
     for handler in log_root.handlers:
         log_root.removeHandler(handler)
 
-    if CONF.use_syslog:
-        facility = _find_facility_from_conf()
-        # TODO(bogdando) use the format provided by RFCSysLogHandler
-        #   after existing syslog format deprecation in J
-        if CONF.use_syslog_rfc_format:
-            syslog = RFCSysLogHandler(address='/dev/log',
-                                      facility=facility)
-        else:
-            syslog = logging.handlers.SysLogHandler(address='/dev/log',
-                                                    facility=facility)
-        log_root.addHandler(syslog)
-
     logpath = _get_log_file_path()
     if logpath:
         filelog = logging.handlers.WatchedFileHandler(logpath)
@@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version):
         else:
             logger.setLevel(level_name)
 
+    if CONF.use_syslog:
+        try:
+            facility = _find_facility_from_conf()
+            # TODO(bogdando) use the format provided by RFCSysLogHandler
+            #   after existing syslog format deprecation in J
+            if CONF.use_syslog_rfc_format:
+                syslog = RFCSysLogHandler(facility=facility)
+            else:
+                syslog = logging.handlers.SysLogHandler(facility=facility)
+            log_root.addHandler(syslog)
+        except socket.error:
+            log_root.error('Unable to add syslog handler. Verify that syslog '
+                           'is running.')
+
 
 _loggers = {}
 
@@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter):
     def format(self, record):
         """Uses contextstring if request_id is set, otherwise default."""
 
+        # NOTE(jecarey): If msg is not unicode, coerce it into unicode
+        #                before it can get to the python logging and
+        #                possibly cause string encoding trouble
+        if not isinstance(record.msg, six.text_type):
+            record.msg = six.text_type(record.msg)
+
         # store project info
         record.project = self.project
         record.version = self.version

heat/openstack/common/loopingcall.py

@@ -16,16 +16,21 @@
 #    under the License.
 
 import sys
+import time
 
 from eventlet import event
 from eventlet import greenthread
 
-from heat.openstack.common.gettextutils import _LE, _LW
+from heat.openstack.common._i18n import _LE, _LW
 from heat.openstack.common import log as logging
-from heat.openstack.common import timeutils
 
 LOG = logging.getLogger(__name__)
 
+# NOTE(zyluo): This lambda function was declared to avoid mocking collisions
+#              with time.time() called in the standard logging module
+#              during unittests.
+_ts = lambda: time.time()
+
 
 class LoopingCallDone(Exception):
     """Exception to break out and stop a LoopingCallBase.
@@ -72,16 +77,17 @@ class FixedIntervalLoopingCall(LoopingCallBase):
 
             try:
                 while self._running:
-                    start = timeutils.utcnow()
+                    start = _ts()
                     self.f(*self.args, **self.kw)
-                    end = timeutils.utcnow()
+                    end = _ts()
                     if not self._running:
                         break
-                    delay = interval - timeutils.delta_seconds(start, end)
+                    delay = end - start - interval
-                    if delay <= 0:
+                    if delay > 0:
-                        LOG.warn(_LW('task run outlasted interval by %s sec') %
+                        LOG.warn(_LW('task %(func_name)s run outlasted '
-                                 -delay)
+                                     'interval by %(delay).2f sec'),
-                    greenthread.sleep(delay if delay > 0 else 0)
+                                 {'func_name': repr(self.f), 'delay': delay})
+                    greenthread.sleep(-delay if delay < 0 else 0)
             except LoopingCallDone as e:
                 self.stop()
                 done.send(e.retvalue)
@@ -121,8 +127,9 @@ class DynamicLoopingCall(LoopingCallBase):
 
                     if periodic_interval_max is not None:
                         idle = min(idle, periodic_interval_max)
-                    LOG.debug('Dynamic looping call sleeping for %.02f '
+                    LOG.debug('Dynamic looping call %(func_name)s sleeping '
-                              'seconds', idle)
+                              'for %(idle).02f seconds',
+                              {'func_name': repr(self.f), 'idle': idle})
                     greenthread.sleep(idle)
             except LoopingCallDone as e:
                 self.stop()

heat/openstack/common/middleware/request_id.py

@@ -1,6 +1,3 @@
-# Copyright (c) 2013 NEC Corporation
-# All Rights Reserved.
-#
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -13,29 +10,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Middleware that ensures request ID.
+"""Compatibility shim for Kilo, while operators migrate to oslo.middleware."""
 
-It ensures to assign request ID for each API request and set it to
+from oslo.middleware import request_id
-request environment. The request ID is also added to API response.
-"""
 
-import webob.dec
+from heat.openstack.common import versionutils
-
-from heat.openstack.common import context
-from heat.openstack.common.middleware import base
 
 
 ENV_REQUEST_ID = 'openstack.request_id'
 HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
 
 
-class RequestIdMiddleware(base.Middleware):
+@versionutils.deprecated(as_of=versionutils.deprecated.KILO,
-
+                         in_favor_of='oslo.middleware.RequestId')
-    @webob.dec.wsgify
+class RequestIdMiddleware(request_id.RequestId):
-    def __call__(self, req):
+    pass
-        req_id = context.generate_request_id()
-        req.environ[ENV_REQUEST_ID] = req_id
-        response = req.get_response(self.application)
-        if HTTP_RESP_HEADER_REQUEST_ID not in response.headers:
-            response.headers.add(HTTP_RESP_HEADER_REQUEST_ID, req_id)
-        return response

heat/openstack/common/processutils.py

@@ -18,7 +18,8 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
+import multiprocessing
 import os
 import random
 import shlex
@@ -26,10 +27,10 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+from oslo.utils import strutils
 import six
 
-from heat.openstack.common.gettextutils import _
+from heat.openstack.common._i18n import _
-from heat.openstack.common import log as logging
 
 
 LOG = logging.getLogger(__name__)
@@ -114,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
-                            stdlib_logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -131,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -140,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
-                                     'to utils.execute: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:
@@ -151,12 +150,12 @@ def execute(*cmd, **kwargs):
         cmd = shlex.split(root_helper) + list(cmd)
 
     cmd = map(str, cmd)
+    sanitized_cmd = strutils.mask_password(' '.join(cmd))
 
     while attempts > 0:
         attempts -= 1
         try:
-            LOG.log(loglevel, 'Running cmd (subprocess): %s',
+            LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
-                    ' '.join(logging.mask_password(cmd)))
             _PIPE = subprocess.PIPE  # pylint: disable=E1101
 
             if os.name == 'nt':
@@ -193,16 +192,18 @@ def execute(*cmd, **kwargs):
             LOG.log(loglevel, 'Result was %s' % _returncode)
             if not ignore_exit_code and _returncode not in check_exit_code:
                 (stdout, stderr) = result
+                sanitized_stdout = strutils.mask_password(stdout)
+                sanitized_stderr = strutils.mask_password(stderr)
                 raise ProcessExecutionError(exit_code=_returncode,
-                                            stdout=stdout,
+                                            stdout=sanitized_stdout,
-                                            stderr=stderr,
+                                            stderr=sanitized_stderr,
-                                            cmd=' '.join(cmd))
+                                            cmd=sanitized_cmd)
             return result
         except ProcessExecutionError:
             if not attempts:
                 raise
             else:
-                LOG.log(loglevel, '%r failed. Retrying.', cmd)
+                LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
                 if delay_on_retry:
                     greenthread.sleep(random.randint(20, 200) / 100.0)
         finally:
@@ -241,7 +242,8 @@ def trycmd(*args, **kwargs):
 
 def ssh_execute(ssh, cmd, process_input=None,
                 addl_env=None, check_exit_code=True):
-    LOG.debug('Running cmd (SSH): %s', cmd)
+    sanitized_cmd = strutils.mask_password(cmd)
+    LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
     if addl_env:
         raise InvalidArgumentError(_('Environment not supported over SSH'))
 
@@ -255,7 +257,10 @@ def ssh_execute(ssh, cmd, process_input=None,
     # NOTE(justinsb): This seems suspicious...
     # ...other SSH clients have buffering issues with this approach
     stdout = stdout_stream.read()
+    sanitized_stdout = strutils.mask_password(stdout)
     stderr = stderr_stream.read()
+    sanitized_stderr = strutils.mask_password(stderr)
+
     stdin_stream.close()
 
     exit_status = channel.recv_exit_status()
@@ -265,8 +270,20 @@ def ssh_execute(ssh, cmd, process_input=None,
         LOG.debug('Result was %s' % exit_status)
         if check_exit_code and exit_status != 0:
             raise ProcessExecutionError(exit_code=exit_status,
-                                        stdout=stdout,
+                                        stdout=sanitized_stdout,
-                                        stderr=stderr,
+                                        stderr=sanitized_stderr,
-                                        cmd=cmd)
+                                        cmd=sanitized_cmd)
 
-    return (stdout, stderr)
+    return (sanitized_stdout, sanitized_stderr)
+
+
+def get_worker_count():
+    """Utility to get the default worker count.
+
+    @return: The number of CPUs if that can be determined, else a default
+             worker count of 1 is returned.
+    """
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        return 1

heat/openstack/common/service.py

@@ -38,14 +38,12 @@ from eventlet import event
 from oslo.config import cfg
 
 from heat.openstack.common import eventlet_backdoor
-from heat.openstack.common.gettextutils import _LE, _LI, _LW
+from heat.openstack.common._i18n import _LE, _LI, _LW
-from heat.openstack.common import importutils
 from heat.openstack.common import log as logging
 from heat.openstack.common import systemd
 from heat.openstack.common import threadgroup
 
 
-rpc = importutils.try_import('heat.openstack.common.rpc')
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
@@ -180,12 +178,6 @@ class ServiceLauncher(Launcher):
             status = exc.code
         finally:
             self.stop()
-            if rpc:
-                try:
-                    rpc.cleanup()
-                except Exception:
-                    # We're shutting down, so it doesn't matter at this point.
-                    LOG.exception(_LE('Exception during rpc cleanup.'))
 
         return status, signo
 

heat/openstack/common/versionutils.py

@@ -18,10 +18,12 @@ Helpers for comparing version strings.
 """
 
 import functools
+import inspect
 
 import pkg_resources
+import six
 
-from heat.openstack.common.gettextutils import _
+from heat.openstack.common._i18n import _
 from heat.openstack.common import log as logging
 
 
@@ -53,18 +55,36 @@ class deprecated(object):
     >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
     ... def c(): pass
 
+    4. Specifying the deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0)
+    ... def d(): pass
+
+    5. Specifying a replacement, deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0)
+    ... def e(): pass
+
     """
 
+    # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+    # expected we maintain a gap between Bexar and Folsom in this list.
+    BEXAR = 'B'
     FOLSOM = 'F'
     GRIZZLY = 'G'
     HAVANA = 'H'
     ICEHOUSE = 'I'
+    JUNO = 'J'
+    KILO = 'K'
 
     _RELEASES = {
+        # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+        # expected we maintain a gap between Bexar and Folsom in this list.
+        'B': 'Bexar',
         'F': 'Folsom',
         'G': 'Grizzly',
         'H': 'Havana',
         'I': 'Icehouse',
+        'J': 'Juno',
+        'K': 'Kilo',
     }
 
     _deprecated_msg_with_alternative = _(
@@ -75,6 +95,12 @@ class deprecated(object):
         '%(what)s is deprecated as of %(as_of)s and may be '
         'removed in %(remove_in)s. It will not be superseded.')
 
+    _deprecated_msg_with_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.')
+
+    _deprecated_msg_with_no_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s. It will not be superseded.')
+
     def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
         """Initialize decorator
 
@@ -92,16 +118,34 @@ class deprecated(object):
         self.remove_in = remove_in
         self.what = what
 
-    def __call__(self, func):
+    def __call__(self, func_or_cls):
         if not self.what:
-            self.what = func.__name__ + '()'
+            self.what = func_or_cls.__name__ + '()'
+        msg, details = self._build_message()
 
-        @functools.wraps(func)
+        if inspect.isfunction(func_or_cls):
-        def wrapped(*args, **kwargs):
+
-            msg, details = self._build_message()
+            @six.wraps(func_or_cls)
-            LOG.deprecated(msg, details)
+            def wrapped(*args, **kwargs):
-            return func(*args, **kwargs)
+                LOG.deprecated(msg, details)
-        return wrapped
+                return func_or_cls(*args, **kwargs)
+            return wrapped
+        elif inspect.isclass(func_or_cls):
+            orig_init = func_or_cls.__init__
+
+            # TODO(tsufiev): change `functools` module to `six` as
+            # soon as six 1.7.4 (with fix for passing `assigned`
+            # argument to underlying `functools.wraps`) is released
+            # and added to the heat-incubator requrements
+            @functools.wraps(orig_init, assigned=('__name__', '__doc__'))
+            def new_init(self, *args, **kwargs):
+                LOG.deprecated(msg, details)
+                orig_init(self, *args, **kwargs)
+            func_or_cls.__init__ = new_init
+            return func_or_cls
+        else:
+            raise TypeError('deprecated can be used only with functions or '
+                            'classes')
 
     def _get_safe_to_remove_release(self, release):
         # TODO(dstanek): this method will have to be reimplemented once
@@ -120,9 +164,19 @@ class deprecated(object):
 
         if self.in_favor_of:
             details['in_favor_of'] = self.in_favor_of
-            msg = self._deprecated_msg_with_alternative
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_with_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_alternative_no_removal
         else:
-            msg = self._deprecated_msg_no_alternative
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_no_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_no_alternative_no_removal
         return msg, details
 
 

requirements.txt

@@ -17,6 +17,7 @@ oslo.config>=1.4.0  # Apache-2.0
 oslo.db>=1.0.0  # Apache-2.0
 oslo.i18n>=1.0.0  # Apache-2.0
 oslo.messaging>=1.4.0
+oslo.middleware>=0.1.0                  # Apache-2.0
 oslo.serialization>=1.0.0               # Apache-2.0
 oslo.utils>=1.0.0                       # Apache-2.0
 osprofiler>=0.3.0                       # Apache-2.0