Sync with oslo-incubator
This syncs keystone with oslo-incubator commit hash 5fa2dae429a9e37dfd1a527eb3957cea57a3e8c4 Commits since last sync (e9bb0b59): ----------------------------------- 0506d17 cfgfilter has graduated, remove it e5c78bb log: make set_defaults() tests clean up properly 05ae498 Add JUNO as a target to versionutils module 569979a Fix tests.unit.reports.test_base_report with python3.4 e184dd3 Fix exception message in openstack.common.processutils.execute 3d90045 Backport code for i18n to check lazy at runtime 43b0df3 Updated from global requirements ef37e03 Added missing jsonutils.dump() function 5fd77eb Ability to customize default_log_levels for each project 4d9328c Python 3: enable tests/unit/test_log.py b8db732 Delete 'deprecated' directory a44ad7b Allow test_lockutils to run in isolation 8b5bc62 Allow test_service to run in isolation 80284aa Mark gettextutils obsolete d6b55fb Remove `processutils` dependency on `log` cb5a804 Move `mask_password` to strutils 9502a21 Use the standard python logging in network_utils 3310d8d update new requests logger to default WARN e377393 Changes calcuation of variable delay ab5d5f1 Use timestamp in loopingcall 5c68515 Updated from global requirements 011fc27 Python 3: replace "im_self" by "__self__" 5c11c4d Don't import fcntl on Windows 33afb20 Fix broken formatting of processutils.execute log statement 6751b30 Remove extra whitespace Change-Id: I853469f1fb5f5274ef52caaeb2d12b3164433697 Related-BP: non-persistent-tokens
This commit is contained in:
Morgan Fainberg
parent
7f9d2b65b3
commit
1070afe2cb
|
|
@ -23,7 +23,6 @@ Usual usage in an openstack.common module:
|
|||
"""
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import gettext
|
||||
import locale
|
||||
from logging import handlers
|
||||
|
|
@ -42,7 +41,7 @@ class TranslatorFactory(object):
|
|||
"""Create translator functions
|
||||
"""
|
||||
|
||||
def __init__(self, domain, lazy=False, localedir=None):
|
||||
def __init__(self, domain, localedir=None):
|
||||
"""Establish a set of translation functions for the domain.
|
||||
|
||||
:param domain: Name of translation domain,
|
||||
|
|
@ -55,7 +54,6 @@ class TranslatorFactory(object):
|
|||
:type localedir: str
|
||||
"""
|
||||
self.domain = domain
|
||||
self.lazy = lazy
|
||||
if localedir is None:
|
||||
localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||
self.localedir = localedir
|
||||
|
|
@ -75,16 +73,19 @@ class TranslatorFactory(object):
|
|||
"""
|
||||
if domain is None:
|
||||
domain = self.domain
|
||||
if self.lazy:
|
||||
return functools.partial(Message, domain=domain)
|
||||
t = gettext.translation(
|
||||
domain,
|
||||
localedir=self.localedir,
|
||||
fallback=True,
|
||||
)
|
||||
if six.PY3:
|
||||
return t.gettext
|
||||
return t.ugettext
|
||||
t = gettext.translation(domain,
|
||||
localedir=self.localedir,
|
||||
fallback=True)
|
||||
# Use the appropriate method of the translation object based
|
||||
# on the python version.
|
||||
m = t.gettext if six.PY3 else t.ugettext
|
||||
|
||||
def f(msg):
|
||||
"""oslo.i18n.gettextutils translation function."""
|
||||
if USE_LAZY:
|
||||
return Message(msg, domain=domain)
|
||||
return m(msg)
|
||||
return f
|
||||
|
||||
@property
|
||||
def primary(self):
|
||||
|
|
@ -147,19 +148,11 @@ def enable_lazy():
|
|||
your project is importing _ directly instead of using the
|
||||
gettextutils.install() way of importing the _ function.
|
||||
"""
|
||||
# FIXME(dhellmann): This function will be removed in oslo.i18n,
|
||||
# because the TranslatorFactory makes it superfluous.
|
||||
global _, _LI, _LW, _LE, _LC, USE_LAZY
|
||||
tf = TranslatorFactory('keystone', lazy=True)
|
||||
_ = tf.primary
|
||||
_LI = tf.log_info
|
||||
_LW = tf.log_warning
|
||||
_LE = tf.log_error
|
||||
_LC = tf.log_critical
|
||||
global USE_LAZY
|
||||
USE_LAZY = True
|
||||
|
||||
|
||||
def install(domain, lazy=False):
|
||||
def install(domain):
|
||||
"""Install a _() function using the given translation domain.
|
||||
|
||||
Given a translation domain, install a _() function using gettext's
|
||||
|
|
@ -170,26 +163,14 @@ def install(domain, lazy=False):
|
|||
a translation-domain-specific environment variable (e.g.
|
||||
NOVA_LOCALEDIR).
|
||||
|
||||
Note that to enable lazy translation, enable_lazy must be
|
||||
called.
|
||||
|
||||
:param domain: the translation domain
|
||||
:param lazy: indicates whether or not to install the lazy _() function.
|
||||
The lazy _() introduces a way to do deferred translation
|
||||
of messages by installing a _ that builds Message objects,
|
||||
instead of strings, which can then be lazily translated into
|
||||
any available locale.
|
||||
"""
|
||||
if lazy:
|
||||
from six import moves
|
||||
tf = TranslatorFactory(domain, lazy=True)
|
||||
moves.builtins.__dict__['_'] = tf.primary
|
||||
else:
|
||||
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||
if six.PY3:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir))
|
||||
else:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir),
|
||||
unicode=True)
|
||||
from six import moves
|
||||
tf = TranslatorFactory(domain)
|
||||
moves.builtins.__dict__['_'] = tf.primary
|
||||
|
||||
|
||||
class Message(six.text_type):
|
||||
|
|
|
|||
|
|
@ -168,6 +168,10 @@ def dumps(value, default=to_primitive, **kwargs):
|
|||
return json.dumps(value, default=default, **kwargs)
|
||||
|
||||
|
||||
def dump(obj, fp, *args, **kwargs):
|
||||
return json.dump(obj, fp, *args, **kwargs)
|
||||
|
||||
|
||||
def loads(s, encoding='utf-8', **kwargs):
|
||||
return json.loads(strutils.safe_decode(s, encoding), **kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
|
||||
import contextlib
|
||||
import errno
|
||||
import fcntl
|
||||
import functools
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -195,6 +194,7 @@ if os.name == 'nt':
|
|||
FileLock = _WindowsLock
|
||||
else:
|
||||
import base64
|
||||
import fcntl
|
||||
import hashlib
|
||||
|
||||
import posix_ipc
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import logging
|
|||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
|
|
@ -45,30 +44,13 @@ from keystone.openstack.common.gettextutils import _
|
|||
from keystone.openstack.common import importutils
|
||||
from keystone.openstack.common import jsonutils
|
||||
from keystone.openstack.common import local
|
||||
# NOTE(flaper87): Pls, remove when graduating this module
|
||||
# from the incubator.
|
||||
from keystone.openstack.common.strutils import mask_password # noqa
|
||||
|
||||
|
||||
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
|
||||
|
||||
# NOTE(ldbragst): Let's build a list of regex objects using the list of
|
||||
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
|
||||
# to the list of _SANITIZE_KEYS and we can generate regular expressions
|
||||
# for XML and JSON automatically.
|
||||
_SANITIZE_PATTERNS = []
|
||||
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||
r'(<%(key)s>).*?(</%(key)s>)',
|
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
|
||||
'.*?([\'"])',
|
||||
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
|
||||
|
||||
for key in _SANITIZE_KEYS:
|
||||
for pattern in _FORMAT_PATTERNS:
|
||||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||
_SANITIZE_PATTERNS.append(reg_ex)
|
||||
|
||||
|
||||
common_cli_opts = [
|
||||
cfg.BoolOpt('debug',
|
||||
|
|
@ -138,6 +120,12 @@ generic_log_opts = [
|
|||
help='Log output to standard error.')
|
||||
]
|
||||
|
||||
DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
|
||||
'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
|
||||
'oslo.messaging=INFO', 'iso8601=WARN',
|
||||
'requests.packages.urllib3.connectionpool=WARN',
|
||||
'urllib3.connectionpool=WARN']
|
||||
|
||||
log_opts = [
|
||||
cfg.StrOpt('logging_context_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
|
|
@ -156,17 +144,7 @@ log_opts = [
|
|||
'%(instance)s',
|
||||
help='Prefix each line of exception output with this format.'),
|
||||
cfg.ListOpt('default_log_levels',
|
||||
default=[
|
||||
'amqp=WARN',
|
||||
'amqplib=WARN',
|
||||
'boto=WARN',
|
||||
'qpid=WARN',
|
||||
'sqlalchemy=WARN',
|
||||
'suds=INFO',
|
||||
'oslo.messaging=INFO',
|
||||
'iso8601=WARN',
|
||||
'requests.packages.urllib3.connectionpool=WARN'
|
||||
],
|
||||
default=DEFAULT_LOG_LEVELS,
|
||||
help='List of logger=LEVEL pairs.'),
|
||||
cfg.BoolOpt('publish_errors',
|
||||
default=False,
|
||||
|
|
@ -181,11 +159,11 @@ log_opts = [
|
|||
cfg.StrOpt('instance_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance that is passed with the log '
|
||||
'message. '),
|
||||
'message.'),
|
||||
cfg.StrOpt('instance_uuid_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance UUID that is passed with the '
|
||||
'log message. '),
|
||||
'log message.'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
@ -244,40 +222,6 @@ def _get_log_file_path(binary=None):
|
|||
return None
|
||||
|
||||
|
||||
def mask_password(message, secret="***"):
|
||||
"""Replace password with 'secret' in message.
|
||||
|
||||
:param message: The string which includes security information.
|
||||
:param secret: value with which to replace passwords.
|
||||
:returns: The unicode value of message with the password fields masked.
|
||||
|
||||
For example:
|
||||
|
||||
>>> mask_password("'adminPass' : 'aaaaa'")
|
||||
"'adminPass' : '***'"
|
||||
>>> mask_password("'admin_pass' : 'aaaaa'")
|
||||
"'admin_pass' : '***'"
|
||||
>>> mask_password('"password" : "aaaaa"')
|
||||
'"password" : "***"'
|
||||
>>> mask_password("'original_password' : 'aaaaa'")
|
||||
"'original_password' : '***'"
|
||||
>>> mask_password("u'original_password' : u'aaaaa'")
|
||||
"u'original_password' : u'***'"
|
||||
"""
|
||||
message = six.text_type(message)
|
||||
|
||||
# NOTE(ldbragst): Check to see if anything in message contains any key
|
||||
# specified in _SANITIZE_KEYS, if not then just return the message since
|
||||
# we don't have to mask any passwords.
|
||||
if not any(key in message for key in _SANITIZE_KEYS):
|
||||
return message
|
||||
|
||||
secret = r'\g<1>' + secret + r'\g<2>'
|
||||
for pattern in _SANITIZE_PATTERNS:
|
||||
message = re.sub(pattern, secret, message)
|
||||
return message
|
||||
|
||||
|
||||
class BaseLoggerAdapter(logging.LoggerAdapter):
|
||||
|
||||
def audit(self, msg, *args, **kwargs):
|
||||
|
|
@ -295,6 +239,11 @@ class LazyAdapter(BaseLoggerAdapter):
|
|||
def logger(self):
|
||||
if not self._logger:
|
||||
self._logger = getLogger(self.name, self.version)
|
||||
if six.PY3:
|
||||
# In Python 3, the code fails because the 'manager' attribute
|
||||
# cannot be found when using a LoggerAdapter as the
|
||||
# underlying logger. Work around this issue.
|
||||
self._logger.manager = self._logger.logger.manager
|
||||
return self._logger
|
||||
|
||||
|
||||
|
|
@ -448,7 +397,7 @@ def _load_log_config(log_config_append):
|
|||
try:
|
||||
logging.config.fileConfig(log_config_append,
|
||||
disable_existing_loggers=False)
|
||||
except moves.configparser.Error as exc:
|
||||
except (moves.configparser.Error, KeyError) as exc:
|
||||
raise LogConfigError(log_config_append, six.text_type(exc))
|
||||
|
||||
|
||||
|
|
@ -461,9 +410,18 @@ def setup(product_name, version='unknown'):
|
|||
sys.excepthook = _create_logging_excepthook(product_name)
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
def set_defaults(logging_context_format_string,
|
||||
default_log_levels=None):
|
||||
# Just in case the caller is not setting the
|
||||
# default_log_level. This is insurance because
|
||||
# we introduced the default_log_level parameter
|
||||
# later in a backwards in-compatible change
|
||||
if default_log_levels is None:
|
||||
default_log_levels = DEFAULT_LOG_LEVELS
|
||||
cfg.set_defaults(
|
||||
log_opts, logging_context_format_string=logging_context_format_string)
|
||||
log_opts,
|
||||
logging_context_format_string=logging_context_format_string,
|
||||
default_log_levels=default_log_levels)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
|
|
|
|||
|
|
@ -16,16 +16,21 @@
|
|||
# under the License.
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
from eventlet import event
|
||||
from eventlet import greenthread
|
||||
|
||||
from keystone.openstack.common.gettextutils import _LE, _LW
|
||||
from keystone.openstack.common import log as logging
|
||||
from keystone.openstack.common import timeutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# NOTE(zyluo): This lambda function was declared to avoid mocking collisions
|
||||
# with time.time() called in the standard logging module
|
||||
# during unittests.
|
||||
_ts = lambda: time.time()
|
||||
|
||||
|
||||
class LoopingCallDone(Exception):
|
||||
"""Exception to break out and stop a LoopingCallBase.
|
||||
|
|
@ -72,17 +77,17 @@ class FixedIntervalLoopingCall(LoopingCallBase):
|
|||
|
||||
try:
|
||||
while self._running:
|
||||
start = timeutils.utcnow()
|
||||
start = _ts()
|
||||
self.f(*self.args, **self.kw)
|
||||
end = timeutils.utcnow()
|
||||
end = _ts()
|
||||
if not self._running:
|
||||
break
|
||||
delay = interval - timeutils.delta_seconds(start, end)
|
||||
if delay <= 0:
|
||||
delay = end - start - interval
|
||||
if delay > 0:
|
||||
LOG.warn(_LW('task %(func_name)s run outlasted '
|
||||
'interval by %(delay)s sec'),
|
||||
{'func_name': repr(self.f), 'delay': -delay})
|
||||
greenthread.sleep(delay if delay > 0 else 0)
|
||||
'interval by %(delay).2f sec'),
|
||||
{'func_name': repr(self.f), 'delay': delay})
|
||||
greenthread.sleep(-delay if delay < 0 else 0)
|
||||
except LoopingCallDone as e:
|
||||
self.stop()
|
||||
done.send(e.retvalue)
|
||||
|
|
|
|||
|
|
@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
|
|||
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
|
||||
|
||||
|
||||
# NOTE(flaper87): The following 3 globals are used by `mask_password`
|
||||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
|
||||
|
||||
# NOTE(ldbragst): Let's build a list of regex objects using the list of
|
||||
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
|
||||
# to the list of _SANITIZE_KEYS and we can generate regular expressions
|
||||
# for XML and JSON automatically.
|
||||
_SANITIZE_PATTERNS = []
|
||||
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||
r'(<%(key)s>).*?(</%(key)s>)',
|
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
|
||||
'.*?([\'"])',
|
||||
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
|
||||
|
||||
for key in _SANITIZE_KEYS:
|
||||
for pattern in _FORMAT_PATTERNS:
|
||||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||
_SANITIZE_PATTERNS.append(reg_ex)
|
||||
|
||||
|
||||
def int_from_bool_as_string(subject):
|
||||
"""Interpret a string as a boolean and return either 1 or 0.
|
||||
|
||||
|
|
@ -237,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"):
|
|||
"ascii", "ignore").decode("ascii")
|
||||
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
|
||||
return SLUGIFY_HYPHENATE_RE.sub("-", value)
|
||||
|
||||
|
||||
def mask_password(message, secret="***"):
|
||||
"""Replace password with 'secret' in message.
|
||||
|
||||
:param message: The string which includes security information.
|
||||
:param secret: value with which to replace passwords.
|
||||
:returns: The unicode value of message with the password fields masked.
|
||||
|
||||
For example:
|
||||
|
||||
>>> mask_password("'adminPass' : 'aaaaa'")
|
||||
"'adminPass' : '***'"
|
||||
>>> mask_password("'admin_pass' : 'aaaaa'")
|
||||
"'admin_pass' : '***'"
|
||||
>>> mask_password('"password" : "aaaaa"')
|
||||
'"password" : "***"'
|
||||
>>> mask_password("'original_password' : 'aaaaa'")
|
||||
"'original_password' : '***'"
|
||||
>>> mask_password("u'original_password' : u'aaaaa'")
|
||||
"u'original_password' : u'***'"
|
||||
"""
|
||||
message = six.text_type(message)
|
||||
|
||||
# NOTE(ldbragst): Check to see if anything in message contains any key
|
||||
# specified in _SANITIZE_KEYS, if not then just return the message since
|
||||
# we don't have to mask any passwords.
|
||||
if not any(key in message for key in _SANITIZE_KEYS):
|
||||
return message
|
||||
|
||||
secret = r'\g<1>' + secret + r'\g<2>'
|
||||
for pattern in _SANITIZE_PATTERNS:
|
||||
message = re.sub(pattern, secret, message)
|
||||
return message
|
||||
|
|
|
|||
|
|
@ -55,16 +55,24 @@ class deprecated(object):
|
|||
|
||||
"""
|
||||
|
||||
# NOTE(morganfainberg): Bexar is used for unit test purposes, it is
|
||||
# expected we maintain a gap between Bexar and Folsom in this list.
|
||||
BEXAR = 'B'
|
||||
FOLSOM = 'F'
|
||||
GRIZZLY = 'G'
|
||||
HAVANA = 'H'
|
||||
ICEHOUSE = 'I'
|
||||
JUNO = 'J'
|
||||
|
||||
_RELEASES = {
|
||||
# NOTE(morganfainberg): Bexar is used for unit test purposes, it is
|
||||
# expected we maintain a gap between Bexar and Folsom in this list.
|
||||
'B': 'Bexar',
|
||||
'F': 'Folsom',
|
||||
'G': 'Grizzly',
|
||||
'H': 'Havana',
|
||||
'I': 'Icehouse',
|
||||
'J': 'Juno',
|
||||
}
|
||||
|
||||
_deprecated_msg_with_alternative = _(
|
||||
|
|
|
|||
Loading…
Reference in New Issue