From 3a1cd7c8935f0bf05fd505805de7f414a9e4f775 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Wed, 22 Oct 2014 11:37:12 +0100 Subject: [PATCH] sync oslo-incubator for request_id shim Sync 838a2a3 oslo-incubator to get the new request_id shim, so can introduce a deprecation warning that tells operators upgrading from Juno that their paste.ini files need to point to oslo.middleware This also syncs all other modules, except policy, which looks like it needs some changes to heat thus will be handled via another patch. Note we shouldn't remove this request_id shim until after Kilo is branched. Change-Id: I35125ffa263b0522ff6dd0b80b0beb3cbc79999b Related-Bug: #1380629 --- etc/heat/heat.conf.sample | 6 +- heat/openstack/common/__init__.py | 17 -- heat/openstack/common/_i18n.py | 40 +++++ heat/openstack/common/crypto/utils.py | 4 +- heat/openstack/common/eventlet_backdoor.py | 2 +- heat/openstack/common/fileutils.py | 18 +- heat/openstack/common/log.py | 162 ++++++++---------- heat/openstack/common/loopingcall.py | 29 ++-- .../openstack/common/middleware/request_id.py | 28 +-- heat/openstack/common/processutils.py | 55 ++++-- heat/openstack/common/service.py | 10 +- heat/openstack/common/versionutils.py | 76 ++++++-- requirements.txt | 1 + 13 files changed, 259 insertions(+), 189 deletions(-) create mode 100644 heat/openstack/common/_i18n.py diff --git a/etc/heat/heat.conf.sample b/etc/heat/heat.conf.sample index c14c6051d..4dbdf95c2 100644 --- a/etc/heat/heat.conf.sample +++ b/etc/heat/heat.conf.sample @@ -451,7 +451,7 @@ #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s # List of logger=LEVEL pairs. (list value) -#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN +#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN # Enables or disables publication of error events. (boolean # value) @@ -462,11 +462,11 @@ #fatal_deprecations=false # The format for an instance that is passed with the log -# message. (string value) +# message. (string value) #instance_format="[instance: %(uuid)s] " # The format for an instance UUID that is passed with the log -# message. (string value) +# message. (string value) #instance_uuid_format="[instance: %(uuid)s] " # The name of a logging configuration file. This file is diff --git a/heat/openstack/common/__init__.py b/heat/openstack/common/__init__.py index d1223eaf7..e69de29bb 100644 --- a/heat/openstack/common/__init__.py +++ b/heat/openstack/common/__init__.py @@ -1,17 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox')) diff --git a/heat/openstack/common/_i18n.py b/heat/openstack/common/_i18n.py new file mode 100644 index 000000000..762a789d7 --- /dev/null +++ b/heat/openstack/common/_i18n.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""oslo.i18n integration module. + +See http://docs.openstack.org/developer/oslo.i18n/usage.html + +""" + +import oslo.i18n + + +# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the +# application name when this module is synced into the separate +# repository. It is OK to have more than one translation function +# using the same domain, since there will still only be one message +# catalog. +_translators = oslo.i18n.TranslatorFactory(domain='heat') + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = _translators.log_info +_LW = _translators.log_warning +_LE = _translators.log_error +_LC = _translators.log_critical diff --git a/heat/openstack/common/crypto/utils.py b/heat/openstack/common/crypto/utils.py index 16460b7b6..0dca18e26 100644 --- a/heat/openstack/common/crypto/utils.py +++ b/heat/openstack/common/crypto/utils.py @@ -16,10 +16,10 @@ import base64 from Crypto.Hash import HMAC from Crypto import Random +from oslo.utils import importutils import six -from heat.openstack.common.gettextutils import _ -from heat.openstack.common import importutils +from heat.openstack.common._i18n import _ bchr = six.int2byte diff --git a/heat/openstack/common/eventlet_backdoor.py b/heat/openstack/common/eventlet_backdoor.py index 4f38ae1e1..a19ca59d6 100644 --- a/heat/openstack/common/eventlet_backdoor.py +++ b/heat/openstack/common/eventlet_backdoor.py @@ -29,7 +29,7 @@ import eventlet.backdoor import greenlet from oslo.config import cfg -from heat.openstack.common.gettextutils import _LI +from heat.openstack.common._i18n import _LI from heat.openstack.common import log as logging help_for_backdoor_port = ( diff --git a/heat/openstack/common/fileutils.py b/heat/openstack/common/fileutils.py index 5983f184c..59088f643 100644 --- a/heat/openstack/common/fileutils.py +++ b/heat/openstack/common/fileutils.py @@ -18,7 +18,8 @@ import errno import os import tempfile -from heat.openstack.common import excutils +from oslo.utils import excutils + from heat.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -50,8 +51,8 @@ def read_cached_file(filename, force_reload=False): """ global _FILE_CACHE - if force_reload and filename in _FILE_CACHE: - del _FILE_CACHE[filename] + if force_reload: + delete_cached_file(filename) reloaded = False mtime = os.path.getmtime(filename) @@ -66,6 +67,17 @@ def read_cached_file(filename, force_reload=False): return (reloaded, cache_info['data']) +def delete_cached_file(filename): + """Delete cached file if present. + + :param filename: filename to delete + """ + global _FILE_CACHE + + if filename in _FILE_CACHE: + del _FILE_CACHE[filename] + + def delete_if_exists(path, remove=os.unlink): """Delete a file, but ignore file not found error. diff --git a/heat/openstack/common/log.py b/heat/openstack/common/log.py index 972de6148..efdca795b 100644 --- a/heat/openstack/common/log.py +++ b/heat/openstack/common/log.py @@ -33,42 +33,24 @@ import logging import logging.config import logging.handlers import os -import re +import socket import sys import traceback from oslo.config import cfg +from oslo.serialization import jsonutils +from oslo.utils import importutils import six from six import moves -from heat.openstack.common.gettextutils import _ -from heat.openstack.common import importutils -from heat.openstack.common import jsonutils +_PY26 = sys.version_info[0:2] == (2, 6) + +from heat.openstack.common._i18n import _ from heat.openstack.common import local _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS = [] -_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', - r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' - '.*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS.append(reg_ex) - common_cli_opts = [ cfg.BoolOpt('debug', @@ -138,6 +120,14 @@ generic_log_opts = [ help='Log output to standard error.') ] +DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', + 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', + 'oslo.messaging=INFO', 'iso8601=WARN', + 'requests.packages.urllib3.connectionpool=WARN', + 'urllib3.connectionpool=WARN', 'websocket=WARN', + "keystonemiddleware=WARN", "routes.middleware=WARN", + "stevedore=WARN"] + log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' @@ -156,17 +146,7 @@ log_opts = [ '%(instance)s', help='Prefix each line of exception output with this format.'), cfg.ListOpt('default_log_levels', - default=[ - 'amqp=WARN', - 'amqplib=WARN', - 'boto=WARN', - 'qpid=WARN', - 'sqlalchemy=WARN', - 'suds=INFO', - 'oslo.messaging=INFO', - 'iso8601=WARN', - 'requests.packages.urllib3.connectionpool=WARN' - ], + default=DEFAULT_LOG_LEVELS, help='List of logger=LEVEL pairs.'), cfg.BoolOpt('publish_errors', default=False, @@ -181,11 +161,11 @@ log_opts = [ cfg.StrOpt('instance_format', default='[instance: %(uuid)s] ', help='The format for an instance that is passed with the log ' - 'message. '), + 'message.'), cfg.StrOpt('instance_uuid_format', default='[instance: %(uuid)s] ', help='The format for an instance UUID that is passed with the ' - 'log message. '), + 'log message.'), ] CONF = cfg.CONF @@ -244,45 +224,20 @@ def _get_log_file_path(binary=None): return None -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - secret = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS: - message = re.sub(pattern, secret, message) - return message - - class BaseLoggerAdapter(logging.LoggerAdapter): def audit(self, msg, *args, **kwargs): self.log(logging.AUDIT, msg, *args, **kwargs) + def isEnabledFor(self, level): + if _PY26: + # This method was added in python 2.7 (and it does the exact + # same logic, so we need to do the exact same logic so that + # python 2.6 has this capability as well). + return self.logger.isEnabledFor(level) + else: + return super(BaseLoggerAdapter, self).isEnabledFor(level) + class LazyAdapter(BaseLoggerAdapter): def __init__(self, name='unknown', version='unknown'): @@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter): def logger(self): if not self._logger: self._logger = getLogger(self.name, self.version) + if six.PY3: + # In Python 3, the code fails because the 'manager' attribute + # cannot be found when using a LoggerAdapter as the + # underlying logger. Work around this issue. + self._logger.manager = self._logger.logger.manager return self._logger @@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter): self.warn(stdmsg, *args, **kwargs) def process(self, msg, kwargs): - # NOTE(mrodden): catch any Message/other object and - # coerce to unicode before they can get - # to the python logging and possibly - # cause string encoding trouble - if not isinstance(msg, six.string_types): + # NOTE(jecarey): If msg is not unicode, coerce it into unicode + # before it can get to the python logging and + # possibly cause string encoding trouble + if not isinstance(msg, six.text_type): msg = six.text_type(msg) if 'extra' not in kwargs: @@ -448,7 +407,7 @@ def _load_log_config(log_config_append): try: logging.config.fileConfig(log_config_append, disable_existing_loggers=False) - except moves.configparser.Error as exc: + except (moves.configparser.Error, KeyError) as exc: raise LogConfigError(log_config_append, six.text_type(exc)) @@ -461,9 +420,20 @@ def setup(product_name, version='unknown'): sys.excepthook = _create_logging_excepthook(product_name) -def set_defaults(logging_context_format_string): - cfg.set_defaults( - log_opts, logging_context_format_string=logging_context_format_string) +def set_defaults(logging_context_format_string=None, + default_log_levels=None): + # Just in case the caller is not setting the + # default_log_level. This is insurance because + # we introduced the default_log_level parameter + # later in a backwards in-compatible change + if default_log_levels is not None: + cfg.set_defaults( + log_opts, + default_log_levels=default_log_levels) + if logging_context_format_string is not None: + cfg.set_defaults( + log_opts, + logging_context_format_string=logging_context_format_string) def _find_facility_from_conf(): @@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version): for handler in log_root.handlers: log_root.removeHandler(handler) - if CONF.use_syslog: - facility = _find_facility_from_conf() - # TODO(bogdando) use the format provided by RFCSysLogHandler - # after existing syslog format deprecation in J - if CONF.use_syslog_rfc_format: - syslog = RFCSysLogHandler(address='/dev/log', - facility=facility) - else: - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - logpath = _get_log_file_path() if logpath: filelog = logging.handlers.WatchedFileHandler(logpath) @@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version): else: logger.setLevel(level_name) + if CONF.use_syslog: + try: + facility = _find_facility_from_conf() + # TODO(bogdando) use the format provided by RFCSysLogHandler + # after existing syslog format deprecation in J + if CONF.use_syslog_rfc_format: + syslog = RFCSysLogHandler(facility=facility) + else: + syslog = logging.handlers.SysLogHandler(facility=facility) + log_root.addHandler(syslog) + except socket.error: + log_root.error('Unable to add syslog handler. Verify that syslog ' + 'is running.') + _loggers = {} @@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter): def format(self, record): """Uses contextstring if request_id is set, otherwise default.""" + # NOTE(jecarey): If msg is not unicode, coerce it into unicode + # before it can get to the python logging and + # possibly cause string encoding trouble + if not isinstance(record.msg, six.text_type): + record.msg = six.text_type(record.msg) + # store project info record.project = self.project record.version = self.version diff --git a/heat/openstack/common/loopingcall.py b/heat/openstack/common/loopingcall.py index a44ad68b4..f7a54c5e9 100644 --- a/heat/openstack/common/loopingcall.py +++ b/heat/openstack/common/loopingcall.py @@ -16,16 +16,21 @@ # under the License. import sys +import time from eventlet import event from eventlet import greenthread -from heat.openstack.common.gettextutils import _LE, _LW +from heat.openstack.common._i18n import _LE, _LW from heat.openstack.common import log as logging -from heat.openstack.common import timeutils LOG = logging.getLogger(__name__) +# NOTE(zyluo): This lambda function was declared to avoid mocking collisions +# with time.time() called in the standard logging module +# during unittests. +_ts = lambda: time.time() + class LoopingCallDone(Exception): """Exception to break out and stop a LoopingCallBase. @@ -72,16 +77,17 @@ class FixedIntervalLoopingCall(LoopingCallBase): try: while self._running: - start = timeutils.utcnow() + start = _ts() self.f(*self.args, **self.kw) - end = timeutils.utcnow() + end = _ts() if not self._running: break - delay = interval - timeutils.delta_seconds(start, end) - if delay <= 0: - LOG.warn(_LW('task run outlasted interval by %s sec') % - -delay) - greenthread.sleep(delay if delay > 0 else 0) + delay = end - start - interval + if delay > 0: + LOG.warn(_LW('task %(func_name)s run outlasted ' + 'interval by %(delay).2f sec'), + {'func_name': repr(self.f), 'delay': delay}) + greenthread.sleep(-delay if delay < 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) @@ -121,8 +127,9 @@ class DynamicLoopingCall(LoopingCallBase): if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call sleeping for %.02f ' - 'seconds', idle) + LOG.debug('Dynamic looping call %(func_name)s sleeping ' + 'for %(idle).02f seconds', + {'func_name': repr(self.f), 'idle': idle}) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() diff --git a/heat/openstack/common/middleware/request_id.py b/heat/openstack/common/middleware/request_id.py index 4d7693add..bb48e669b 100644 --- a/heat/openstack/common/middleware/request_id.py +++ b/heat/openstack/common/middleware/request_id.py @@ -1,6 +1,3 @@ -# Copyright (c) 2013 NEC Corporation -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,29 +10,18 @@ # License for the specific language governing permissions and limitations # under the License. -"""Middleware that ensures request ID. +"""Compatibility shim for Kilo, while operators migrate to oslo.middleware.""" -It ensures to assign request ID for each API request and set it to -request environment. The request ID is also added to API response. -""" +from oslo.middleware import request_id -import webob.dec - -from heat.openstack.common import context -from heat.openstack.common.middleware import base +from heat.openstack.common import versionutils ENV_REQUEST_ID = 'openstack.request_id' HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id' -class RequestIdMiddleware(base.Middleware): - - @webob.dec.wsgify - def __call__(self, req): - req_id = context.generate_request_id() - req.environ[ENV_REQUEST_ID] = req_id - response = req.get_response(self.application) - if HTTP_RESP_HEADER_REQUEST_ID not in response.headers: - response.headers.add(HTTP_RESP_HEADER_REQUEST_ID, req_id) - return response +@versionutils.deprecated(as_of=versionutils.deprecated.KILO, + in_favor_of='oslo.middleware.RequestId') +class RequestIdMiddleware(request_id.RequestId): + pass diff --git a/heat/openstack/common/processutils.py b/heat/openstack/common/processutils.py index a27eaf737..10e4c27e0 100644 --- a/heat/openstack/common/processutils.py +++ b/heat/openstack/common/processutils.py @@ -18,7 +18,8 @@ System-level utilities and helper functions. """ import errno -import logging as stdlib_logging +import logging +import multiprocessing import os import random import shlex @@ -26,10 +27,10 @@ import signal from eventlet.green import subprocess from eventlet import greenthread +from oslo.utils import strutils import six -from heat.openstack.common.gettextutils import _ -from heat.openstack.common import log as logging +from heat.openstack.common._i18n import _ LOG = logging.getLogger(__name__) @@ -114,8 +115,7 @@ def execute(*cmd, **kwargs): execute this command. Defaults to false. :type shell: boolean :param loglevel: log level for execute commands. - :type loglevel: int. (Should be stdlib_logging.DEBUG or - stdlib_logging.INFO) + :type loglevel: int. (Should be logging.DEBUG or logging.INFO) :returns: (stdout, stderr) from process execution :raises: :class:`UnknownArgumentError` on receiving unknown arguments @@ -131,7 +131,7 @@ def execute(*cmd, **kwargs): run_as_root = kwargs.pop('run_as_root', False) root_helper = kwargs.pop('root_helper', '') shell = kwargs.pop('shell', False) - loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) + loglevel = kwargs.pop('loglevel', logging.DEBUG) if isinstance(check_exit_code, bool): ignore_exit_code = not check_exit_code @@ -140,8 +140,7 @@ def execute(*cmd, **kwargs): check_exit_code = [check_exit_code] if kwargs: - raise UnknownArgumentError(_('Got unknown keyword args ' - 'to utils.execute: %r') % kwargs) + raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs) if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: if not root_helper: @@ -151,12 +150,12 @@ def execute(*cmd, **kwargs): cmd = shlex.split(root_helper) + list(cmd) cmd = map(str, cmd) + sanitized_cmd = strutils.mask_password(' '.join(cmd)) while attempts > 0: attempts -= 1 try: - LOG.log(loglevel, 'Running cmd (subprocess): %s', - ' '.join(logging.mask_password(cmd))) + LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd) _PIPE = subprocess.PIPE # pylint: disable=E1101 if os.name == 'nt': @@ -193,16 +192,18 @@ def execute(*cmd, **kwargs): LOG.log(loglevel, 'Result was %s' % _returncode) if not ignore_exit_code and _returncode not in check_exit_code: (stdout, stderr) = result + sanitized_stdout = strutils.mask_password(stdout) + sanitized_stderr = strutils.mask_password(stderr) raise ProcessExecutionError(exit_code=_returncode, - stdout=stdout, - stderr=stderr, - cmd=' '.join(cmd)) + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=sanitized_cmd) return result except ProcessExecutionError: if not attempts: raise else: - LOG.log(loglevel, '%r failed. Retrying.', cmd) + LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) finally: @@ -241,7 +242,8 @@ def trycmd(*args, **kwargs): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug('Running cmd (SSH): %s', cmd) + sanitized_cmd = strutils.mask_password(cmd) + LOG.debug('Running cmd (SSH): %s', sanitized_cmd) if addl_env: raise InvalidArgumentError(_('Environment not supported over SSH')) @@ -255,7 +257,10 @@ def ssh_execute(ssh, cmd, process_input=None, # NOTE(justinsb): This seems suspicious... # ...other SSH clients have buffering issues with this approach stdout = stdout_stream.read() + sanitized_stdout = strutils.mask_password(stdout) stderr = stderr_stream.read() + sanitized_stderr = strutils.mask_password(stderr) + stdin_stream.close() exit_status = channel.recv_exit_status() @@ -265,8 +270,20 @@ def ssh_execute(ssh, cmd, process_input=None, LOG.debug('Result was %s' % exit_status) if check_exit_code and exit_status != 0: raise ProcessExecutionError(exit_code=exit_status, - stdout=stdout, - stderr=stderr, - cmd=cmd) + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=sanitized_cmd) - return (stdout, stderr) + return (sanitized_stdout, sanitized_stderr) + + +def get_worker_count(): + """Utility to get the default worker count. + + @return: The number of CPUs if that can be determined, else a default + worker count of 1 is returned. + """ + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 diff --git a/heat/openstack/common/service.py b/heat/openstack/common/service.py index adfc772ad..89dd67a62 100644 --- a/heat/openstack/common/service.py +++ b/heat/openstack/common/service.py @@ -38,14 +38,12 @@ from eventlet import event from oslo.config import cfg from heat.openstack.common import eventlet_backdoor -from heat.openstack.common.gettextutils import _LE, _LI, _LW -from heat.openstack.common import importutils +from heat.openstack.common._i18n import _LE, _LI, _LW from heat.openstack.common import log as logging from heat.openstack.common import systemd from heat.openstack.common import threadgroup -rpc = importutils.try_import('heat.openstack.common.rpc') CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -180,12 +178,6 @@ class ServiceLauncher(Launcher): status = exc.code finally: self.stop() - if rpc: - try: - rpc.cleanup() - except Exception: - # We're shutting down, so it doesn't matter at this point. - LOG.exception(_LE('Exception during rpc cleanup.')) return status, signo diff --git a/heat/openstack/common/versionutils.py b/heat/openstack/common/versionutils.py index 4fa9a965e..12bf31795 100644 --- a/heat/openstack/common/versionutils.py +++ b/heat/openstack/common/versionutils.py @@ -18,10 +18,12 @@ Helpers for comparing version strings. """ import functools +import inspect import pkg_resources +import six -from heat.openstack.common.gettextutils import _ +from heat.openstack.common._i18n import _ from heat.openstack.common import log as logging @@ -53,18 +55,36 @@ class deprecated(object): >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1) ... def c(): pass + 4. Specifying the deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0) + ... def d(): pass + + 5. Specifying a replacement, deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0) + ... def e(): pass + """ + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + BEXAR = 'B' FOLSOM = 'F' GRIZZLY = 'G' HAVANA = 'H' ICEHOUSE = 'I' + JUNO = 'J' + KILO = 'K' _RELEASES = { + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + 'B': 'Bexar', 'F': 'Folsom', 'G': 'Grizzly', 'H': 'Havana', 'I': 'Icehouse', + 'J': 'Juno', + 'K': 'Kilo', } _deprecated_msg_with_alternative = _( @@ -75,6 +95,12 @@ class deprecated(object): '%(what)s is deprecated as of %(as_of)s and may be ' 'removed in %(remove_in)s. It will not be superseded.') + _deprecated_msg_with_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.') + + _deprecated_msg_with_no_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s. It will not be superseded.') + def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None): """Initialize decorator @@ -92,16 +118,34 @@ class deprecated(object): self.remove_in = remove_in self.what = what - def __call__(self, func): + def __call__(self, func_or_cls): if not self.what: - self.what = func.__name__ + '()' + self.what = func_or_cls.__name__ + '()' + msg, details = self._build_message() - @functools.wraps(func) - def wrapped(*args, **kwargs): - msg, details = self._build_message() - LOG.deprecated(msg, details) - return func(*args, **kwargs) - return wrapped + if inspect.isfunction(func_or_cls): + + @six.wraps(func_or_cls) + def wrapped(*args, **kwargs): + LOG.deprecated(msg, details) + return func_or_cls(*args, **kwargs) + return wrapped + elif inspect.isclass(func_or_cls): + orig_init = func_or_cls.__init__ + + # TODO(tsufiev): change `functools` module to `six` as + # soon as six 1.7.4 (with fix for passing `assigned` + # argument to underlying `functools.wraps`) is released + # and added to the heat-incubator requrements + @functools.wraps(orig_init, assigned=('__name__', '__doc__')) + def new_init(self, *args, **kwargs): + LOG.deprecated(msg, details) + orig_init(self, *args, **kwargs) + func_or_cls.__init__ = new_init + return func_or_cls + else: + raise TypeError('deprecated can be used only with functions or ' + 'classes') def _get_safe_to_remove_release(self, release): # TODO(dstanek): this method will have to be reimplemented once @@ -120,9 +164,19 @@ class deprecated(object): if self.in_favor_of: details['in_favor_of'] = self.in_favor_of - msg = self._deprecated_msg_with_alternative + if self.remove_in > 0: + msg = self._deprecated_msg_with_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_alternative_no_removal else: - msg = self._deprecated_msg_no_alternative + if self.remove_in > 0: + msg = self._deprecated_msg_no_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_no_alternative_no_removal return msg, details diff --git a/requirements.txt b/requirements.txt index 2571d8c71..7472f3f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ oslo.config>=1.4.0 # Apache-2.0 oslo.db>=1.0.0 # Apache-2.0 oslo.i18n>=1.0.0 # Apache-2.0 oslo.messaging>=1.4.0 +oslo.middleware>=0.1.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 oslo.utils>=1.0.0 # Apache-2.0 osprofiler>=0.3.0 # Apache-2.0