openstack/nova
Code Issues Proposed changes

Partial oslo-incubator sync

Browse Source 
Generated with:
python update.py --base nova --dest-dir ~/Develop/tmp/nova/ --modules
timeutils,sslutils,importutils,memorycache,strutils,context,fileutils,versionutils,systemd

context:
 9b73877 Add a RequestContext.from_dict method
 85d1ce6 Python 3: enable tests/unit/middleware/test_request_id.py
fileutils:
  2b966f9 Fix deletion of cached file for policy enforcer
gettextutils:
 3d90045 Backport code for i18n to check lazy at runtime
jsonutils:
 ef37e03 Added missing jsonutils.dump() function
log:
 5cac11d Merge "Add default log level for websocket"
 433fa0b Make logging_context_format_string optional in log.set_defaults
 ac92c06 Add default log level for websocket
 0aa2bd4 Merge "Ability to customize default_log_levels for each project"
 5fd77eb Ability to customize default_log_levels for each project
 i4d9328c Python 3: enable tests/unit/test_log.py
 722f418 Merge "update new requests logger to default WARN"
 cb5a804 Move `mask_password` to strutils
memorycache:
 90ae24b Remove redundant default=None for config options
 297d772 Raise exception when importing memcache error
strutils:
 cb5a804 Move `mask_password` to strutils
systemd:
 17c4e21 Fix docstring indentation in systemd
 667d1ba Fixed spelling error - occured to occurred
versionutils:
 a2ad3a2 Allow deprecated decorator to specify no plan for removal
 05ae498 Add JUNO as a target to versionutils module

Change-Id: I9e8e8e9b15075d99cf394170c6f5300e7bd0d4cc
This commit is contained in:
Joe Gordon 2014-07-15 14:58:32 +02:00
parent 26dbf73552
commit 74d06db19f
10 changed files with 182 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
nova/openstack/common/context.py
View File

@ -25,7 +25,7 @@ import uuid
def generate_request_id():
return 'req-%s' % str(uuid.uuid4())
return b'req-' + str(uuid.uuid4()).encode('ascii')
class RequestContext(object):
@ -77,6 +77,21 @@ class RequestContext(object):
'instance_uuid': self.instance_uuid,
'user_identity': user_idt}
@classmethod
def from_dict(cls, ctx):
return cls(
auth_token=ctx.get("auth_token"),
user=ctx.get("user"),
tenant=ctx.get("tenant"),
domain=ctx.get("domain"),
user_domain=ctx.get("user_domain"),
project_domain=ctx.get("project_domain"),
is_admin=ctx.get("is_admin", False),
read_only=ctx.get("read_only", False),
show_deleted=ctx.get("show_deleted", False),
request_id=ctx.get("request_id"),
instance_uuid=ctx.get("instance_uuid"))
def get_admin_context(show_deleted=False):
context = RequestContext(None,

15
nova/openstack/common/fileutils.py
View File

@ -50,8 +50,8 @@ def read_cached_file(filename, force_reload=False):
"""
global _FILE_CACHE
if force_reload and filename in _FILE_CACHE:
del _FILE_CACHE[filename]
if force_reload:
delete_cached_file(filename)
reloaded = False
mtime = os.path.getmtime(filename)
@ -66,6 +66,17 @@ def read_cached_file(filename, force_reload=False):
return (reloaded, cache_info['data'])
def delete_cached_file(filename):
"""Delete cached file if present.
:param filename: filename to delete
"""
global _FILE_CACHE
if filename in _FILE_CACHE:
del _FILE_CACHE[filename]
def delete_if_exists(path, remove=os.unlink):
"""Delete a file, but ignore file not found error.

63
nova/openstack/common/gettextutils.py
View File

@ -23,7 +23,6 @@ Usual usage in an openstack.common module:
"""
import copy
import functools
import gettext
import locale
from logging import handlers
@ -42,7 +41,7 @@ class TranslatorFactory(object):
"""Create translator functions
"""
def __init__(self, domain, lazy=False, localedir=None):
def __init__(self, domain, localedir=None):
"""Establish a set of translation functions for the domain.
:param domain: Name of translation domain,
@ -55,7 +54,6 @@ class TranslatorFactory(object):
:type localedir: str
"""
self.domain = domain
self.lazy = lazy
if localedir is None:
localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
self.localedir = localedir
@ -75,16 +73,19 @@ class TranslatorFactory(object):
"""
if domain is None:
domain = self.domain
if self.lazy:
return functools.partial(Message, domain=domain)
t = gettext.translation(
domain,
localedir=self.localedir,
fallback=True,
)
if six.PY3:
return t.gettext
return t.ugettext
t = gettext.translation(domain,
localedir=self.localedir,
fallback=True)
# Use the appropriate method of the translation object based
# on the python version.
m = t.gettext if six.PY3 else t.ugettext
def f(msg):
"""oslo.i18n.gettextutils translation function."""
if USE_LAZY:
return Message(msg, domain=domain)
return m(msg)
return f
@property
def primary(self):
@ -147,19 +148,11 @@ def enable_lazy():
your project is importing _ directly instead of using the
gettextutils.install() way of importing the _ function.
"""
# FIXME(dhellmann): This function will be removed in oslo.i18n,
# because the TranslatorFactory makes it superfluous.
global _, _LI, _LW, _LE, _LC, USE_LAZY
tf = TranslatorFactory('nova', lazy=True)
_ = tf.primary
_LI = tf.log_info
_LW = tf.log_warning
_LE = tf.log_error
_LC = tf.log_critical
global USE_LAZY
USE_LAZY = True
def install(domain, lazy=False):
def install(domain):
"""Install a _() function using the given translation domain.
Given a translation domain, install a _() function using gettext's
@ -170,26 +163,14 @@ def install(domain, lazy=False):
a translation-domain-specific environment variable (e.g.
NOVA_LOCALEDIR).
Note that to enable lazy translation, enable_lazy must be
called.
:param domain: the translation domain
:param lazy: indicates whether or not to install the lazy _() function.
The lazy _() introduces a way to do deferred translation
of messages by installing a _ that builds Message objects,
instead of strings, which can then be lazily translated into
any available locale.
"""
if lazy:
from six import moves
tf = TranslatorFactory(domain, lazy=True)
moves.builtins.__dict__['_'] = tf.primary
else:
localedir = '%s_LOCALEDIR' % domain.upper()
if six.PY3:
gettext.install(domain,
localedir=os.environ.get(localedir))
else:
gettext.install(domain,
localedir=os.environ.get(localedir),
unicode=True)
from six import moves
tf = TranslatorFactory(domain)
moves.builtins.__dict__['_'] = tf.primary
class Message(six.text_type):

4
nova/openstack/common/jsonutils.py
View File

@ -168,6 +168,10 @@ def dumps(value, default=to_primitive, **kwargs):
return json.dumps(value, default=default, **kwargs)
def dump(obj, fp, *args, **kwargs):
return json.dump(obj, fp, *args, **kwargs)
def loads(s, encoding='utf-8', **kwargs):
return json.loads(strutils.safe_decode(s, encoding), **kwargs)

100
nova/openstack/common/log.py
View File

@ -33,7 +33,6 @@ import logging
import logging.config
import logging.handlers
import os
import re
import sys
import traceback
@ -45,30 +44,13 @@ from nova.openstack.common.gettextutils import _
from nova.openstack.common import importutils
from nova.openstack.common import jsonutils
from nova.openstack.common import local
# NOTE(flaper87): Pls, remove when graduating this module
# from the incubator.
from nova.openstack.common.strutils import mask_password # noqa
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
'.*?([\'"])',
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
common_cli_opts = [
cfg.BoolOpt('debug',
@ -138,6 +120,12 @@ generic_log_opts = [
help='Log output to standard error.')
]
DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
'oslo.messaging=INFO', 'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN',
'urllib3.connectionpool=WARN', 'websocket=WARN']
log_opts = [
cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@ -156,17 +144,7 @@ log_opts = [
'%(instance)s',
help='Prefix each line of exception output with this format.'),
cfg.ListOpt('default_log_levels',
default=[
'amqp=WARN',
'amqplib=WARN',
'boto=WARN',
'qpid=WARN',
'sqlalchemy=WARN',
'suds=INFO',
'oslo.messaging=INFO',
'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN'
],
default=DEFAULT_LOG_LEVELS,
help='List of logger=LEVEL pairs.'),
cfg.BoolOpt('publish_errors',
default=False,
@ -244,40 +222,6 @@ def _get_log_file_path(binary=None):
return None
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message
class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs):
@ -295,6 +239,11 @@ class LazyAdapter(BaseLoggerAdapter):
def logger(self):
if not self._logger:
self._logger = getLogger(self.name, self.version)
if six.PY3:
# In Python 3, the code fails because the 'manager' attribute
# cannot be found when using a LoggerAdapter as the
# underlying logger. Work around this issue.
self._logger.manager = self._logger.logger.manager
return self._logger
@ -448,7 +397,7 @@ def _load_log_config(log_config_append):
try:
logging.config.fileConfig(log_config_append,
disable_existing_loggers=False)
except moves.configparser.Error as exc:
except (moves.configparser.Error, KeyError) as exc:
raise LogConfigError(log_config_append, six.text_type(exc))
@ -461,9 +410,20 @@ def setup(product_name, version='unknown'):
sys.excepthook = _create_logging_excepthook(product_name)
def set_defaults(logging_context_format_string):
cfg.set_defaults(
log_opts, logging_context_format_string=logging_context_format_string)
def set_defaults(logging_context_format_string=None,
default_log_levels=None):
# Just in case the caller is not setting the
# default_log_level. This is insurance because
# we introduced the default_log_level parameter
# later in a backwards in-compatible change
if default_log_levels is not None:
cfg.set_defaults(
log_opts,
default_log_levels=default_log_levels)
if logging_context_format_string is not None:
cfg.set_defaults(
log_opts,
logging_context_format_string=logging_context_format_string)
def _find_facility_from_conf():

8
nova/openstack/common/memorycache.py
View File

@ -22,7 +22,6 @@ from nova.openstack.common import timeutils
memcache_opts = [
cfg.ListOpt('memcached_servers',
default=None,
help='Memcached servers or None for in process cache.'),
]
@ -36,11 +35,8 @@ def get_client(memcached_servers=None):
if not memcached_servers:
memcached_servers = CONF.memcached_servers
if memcached_servers:
try:
import memcache
client_cls = memcache.Client
except ImportError:
pass
import memcache
client_cls = memcache.Client
return client_cls(memcached_servers, debug=0)

3
nova/openstack/common/sslutils.py
View File

@ -22,15 +22,12 @@ from nova.openstack.common.gettextutils import _
ssl_opts = [
cfg.StrOpt('ca_file',
default=None,
help="CA certificate file to use to verify "
"connecting clients."),
cfg.StrOpt('cert_file',
default=None,
help="Certificate file to use when starting "
"the server securely."),
cfg.StrOpt('key_file',
default=None,
help="Private key file to use when starting "
"the server securely."),
]

56
nova/openstack/common/strutils.py
View File

@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
# NOTE(flaper87): The following 3 globals are used by `mask_password`
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
'.*?([\'"])',
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
def int_from_bool_as_string(subject):
"""Interpret a string as a boolean and return either 1 or 0.
@ -237,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"):
"ascii", "ignore").decode("ascii")
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
return SLUGIFY_HYPHENATE_RE.sub("-", value)
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message

6
nova/openstack/common/systemd.py
View File

@ -50,14 +50,16 @@ def _sd_notify(unset_env, msg):
def notify():
"""Send notification to Systemd that service is ready.
For details see
http://www.freedesktop.org/software/systemd/man/sd_notify.html
http://www.freedesktop.org/software/systemd/man/sd_notify.html
"""
_sd_notify(False, 'READY=1')
def notify_once():
"""Send notification once to Systemd that service is ready.
Systemd sets NOTIFY_SOCKET environment variable with the name of the
socket listening for notifications from services.
This method removes the NOTIFY_SOCKET environment variable to ensure
@ -75,7 +77,7 @@ def onready(notify_socket, timeout):
:type timeout: float
:returns: 0 service ready
1 service not ready
2 timeout occured
2 timeout occurred
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sock.settimeout(timeout)

37
nova/openstack/common/versionutils.py
View File

@ -18,6 +18,7 @@ Helpers for comparing version strings.
"""
import functools
import pkg_resources
from nova.openstack.common.gettextutils import _
@ -52,18 +53,34 @@ class deprecated(object):
>>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
... def c(): pass
4. Specifying the deprecated functionality will not be removed:
>>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0)
... def d(): pass
5. Specifying a replacement, deprecated functionality will not be removed:
>>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0)
... def e(): pass
"""
# NOTE(morganfainberg): Bexar is used for unit test purposes, it is
# expected we maintain a gap between Bexar and Folsom in this list.
BEXAR = 'B'
FOLSOM = 'F'
GRIZZLY = 'G'
HAVANA = 'H'
ICEHOUSE = 'I'
JUNO = 'J'
_RELEASES = {
# NOTE(morganfainberg): Bexar is used for unit test purposes, it is
# expected we maintain a gap between Bexar and Folsom in this list.
'B': 'Bexar',
'F': 'Folsom',
'G': 'Grizzly',
'H': 'Havana',
'I': 'Icehouse',
'J': 'Juno',
}
_deprecated_msg_with_alternative = _(
@ -74,6 +91,12 @@ class deprecated(object):
'%(what)s is deprecated as of %(as_of)s and may be '
'removed in %(remove_in)s. It will not be superseded.')
_deprecated_msg_with_alternative_no_removal = _(
'%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.')
_deprecated_msg_with_no_alternative_no_removal = _(
'%(what)s is deprecated as of %(as_of)s. It will not be superseded.')
def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
"""Initialize decorator
@ -119,9 +142,19 @@ class deprecated(object):
if self.in_favor_of:
details['in_favor_of'] = self.in_favor_of
msg = self._deprecated_msg_with_alternative
if self.remove_in > 0:
msg = self._deprecated_msg_with_alternative
else:
# There are no plans to remove this function, but it is
# now deprecated.
msg = self._deprecated_msg_with_alternative_no_removal
else:
msg = self._deprecated_msg_no_alternative
if self.remove_in > 0:
msg = self._deprecated_msg_no_alternative
else:
# There are no plans to remove this function, but it is
# now deprecated.
msg = self._deprecated_msg_with_no_alternative_no_removal
return msg, details