openstack/keystone
Code Issues Proposed changes

Sync with oslo-incubator 2fd457b

Browse Source 
This syncs Keystone with oslo-incubator commit hash
2fd457bf2ccbeb2b84ffb204778b6417cd5405ba .

In keystone:

 $ rm -r keystone/openstack/common

In oslo-incubator:

 $ python update.py ../keystone

The newer lockutils requires posix_ipc. It's already in
global-requirements.

Keystone uses gettextutils and versionutils from oslo-incubator
so those should be in openstack-common.conf. They were pulled in
as dependencies of other oslo-incubator modules but Keystone
shouldn't rely on that.

Keystone doesn't use log_handler directly so that module was
removed from openstack-common.conf. log_handler is available in
the oslo.messaging library.

Change-Id: I6082f4e0995ca35372f9b061d8f76890aa93250c
This commit is contained in:
Brant Knudson
2014-03-29 08:28:46 -05:00
parent f630a18304
commit ea7018fe5f
33 changed files with 456 additions and 1674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
etc/keystone.conf.sample
View File

@@ -136,7 +136,7 @@
#allowed_rpc_exception_modules=oslo.messaging.exceptions,nova.exception,cinder.exception,exceptions #allowed_rpc_exception_modules=oslo.messaging.exceptions,nova.exception,cinder.exception,exceptions
# Qpid broker hostname. (string value) # Qpid broker hostname. (string value)
#qpid_hostname=localhost #qpid_hostname=keystone
# Qpid broker port. (integer value) # Qpid broker port. (integer value)
#qpid_port=5672 #qpid_port=5672
@@ -193,7 +193,7 @@
# The RabbitMQ broker address where a single node is used. # The RabbitMQ broker address where a single node is used.
# (string value) # (string value)
#rabbit_host=localhost #rabbit_host=keystone
# The RabbitMQ broker port where a single node is used. # The RabbitMQ broker port where a single node is used.
# (integer value) # (integer value)
@@ -344,21 +344,6 @@
#keystone_ec2_insecure=false #keystone_ec2_insecure=false
#
# Options defined in keystone.openstack.common.eventlet_backdoor
#
# Enable eventlet backdoor. Acceptable values are 0, <port>,
# and <start>:<end>, where 0 results in listening on a random
# tcp port number; <port> results in listening on the
# specified port number (and not enabling backdoor if that
# port is in use); and <start>:<end> results in listening on
# the smallest unused port number within the specified range
# of port numbers. The chosen port is displayed in the
# service's log file. (string value)
#backdoor_port=<None>
# #
# Options defined in keystone.openstack.common.lockutils # Options defined in keystone.openstack.common.lockutils
# #
@@ -402,7 +387,7 @@
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# List of logger=LEVEL pairs (list value) # List of logger=LEVEL pairs (list value)
#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN #default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN
# Publish error events (boolean value) # Publish error events (boolean value)
#publish_errors=false #publish_errors=false

20
keystone/openstack/common/config/generator.py
View File

@@ -1,4 +1,5 @@
# Copyright 2012 SINA Corporation # Copyright 2012 SINA Corporation
# Copyright 2014 Cisco Systems, Inc.
# All Rights Reserved. # All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -40,6 +41,7 @@ BOOLOPT = "BoolOpt"
INTOPT = "IntOpt" INTOPT = "IntOpt"
FLOATOPT = "FloatOpt" FLOATOPT = "FloatOpt"
LISTOPT = "ListOpt" LISTOPT = "ListOpt"
DICTOPT = "DictOpt"
MULTISTROPT = "MultiStrOpt" MULTISTROPT = "MultiStrOpt"
OPT_TYPES = { OPT_TYPES = {
@@ -48,11 +50,12 @@ OPT_TYPES = {
INTOPT: 'integer value', INTOPT: 'integer value',
FLOATOPT: 'floating point value', FLOATOPT: 'floating point value',
LISTOPT: 'list value', LISTOPT: 'list value',
DICTOPT: 'dict value',
MULTISTROPT: 'multi valued', MULTISTROPT: 'multi valued',
} }
OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
FLOATOPT, LISTOPT, FLOATOPT, LISTOPT, DICTOPT,
MULTISTROPT])) MULTISTROPT]))
PY_EXT = ".py" PY_EXT = ".py"
@@ -61,6 +64,10 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
WORDWRAP_WIDTH = 60 WORDWRAP_WIDTH = 60
def raise_extension_exception(extmanager, ep, err):
raise
def generate(argv): def generate(argv):
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='generate sample configuration file', description='generate sample configuration file',
@@ -104,6 +111,7 @@ def generate(argv):
'oslo.config.opts', 'oslo.config.opts',
names=list(set(parsed_args.libraries)), names=list(set(parsed_args.libraries)),
invoke_on_load=False, invoke_on_load=False,
on_load_failure_callback=raise_extension_exception
) )
for ext in loader: for ext in loader:
for group, opts in ext.plugin(): for group, opts in ext.plugin():
@@ -226,7 +234,7 @@ def _sanitize_default(name, value):
return value.replace(BASEDIR, '') return value.replace(BASEDIR, '')
elif value == _get_my_ip(): elif value == _get_my_ip():
return '10.0.0.1' return '10.0.0.1'
elif value == socket.gethostname() and 'host' in name: elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name:
return 'keystone' return 'keystone'
elif value.strip() != value: elif value.strip() != value:
return '"%s"' % value return '"%s"' % value
@@ -244,7 +252,8 @@ def _print_opt(opt):
except (ValueError, AttributeError) as err: except (ValueError, AttributeError) as err:
sys.stderr.write("%s\n" % str(err)) sys.stderr.write("%s\n" % str(err))
sys.exit(1) sys.exit(1)
opt_help += ' (' + OPT_TYPES[opt_type] + ')' opt_help = u'%s (%s)' % (opt_help,
OPT_TYPES[opt_type])
print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
if opt.deprecated_opts: if opt.deprecated_opts:
for deprecated_opt in opt.deprecated_opts: for deprecated_opt in opt.deprecated_opts:
@@ -274,6 +283,11 @@ def _print_opt(opt):
elif opt_type == LISTOPT: elif opt_type == LISTOPT:
assert(isinstance(opt_default, list)) assert(isinstance(opt_default, list))
print('#%s=%s' % (opt_name, ','.join(opt_default))) print('#%s=%s' % (opt_name, ','.join(opt_default)))
elif opt_type == DICTOPT:
assert(isinstance(opt_default, dict))
opt_default_strlist = [str(key) + ':' + str(value)
for (key, value) in opt_default.items()]
print('#%s=%s' % (opt_name, ','.join(opt_default_strlist)))
elif opt_type == MULTISTROPT: elif opt_type == MULTISTROPT:
assert(isinstance(opt_default, list)) assert(isinstance(opt_default, list))
if not opt_default: if not opt_default:

11
keystone/openstack/common/context.py
View File

@@ -98,3 +98,14 @@ def get_context_from_function_and_args(function, args, kwargs):
return arg return arg
return None return None
def is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True

144
keystone/openstack/common/eventlet_backdoor.py
View File

@@ -1,144 +0,0 @@
# Copyright (c) 2012 OpenStack Foundation.
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import errno
import gc
import os
import pprint
import socket
import sys
import traceback
import eventlet
import eventlet.backdoor
import greenlet
from oslo.config import cfg
from keystone.openstack.common.gettextutils import _ # noqa
from keystone.openstack.common import log as logging
help_for_backdoor_port = (
"Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
"in listening on a random tcp port number; <port> results in listening "
"on the specified port number (and not enabling backdoor if that port "
"is in use); and <start>:<end> results in listening on the smallest "
"unused port number within the specified range of port numbers. The "
"chosen port is displayed in the service's log file.")
eventlet_backdoor_opts = [
cfg.StrOpt('backdoor_port',
default=None,
help="Enable eventlet backdoor. %s" % help_for_backdoor_port)
]
CONF = cfg.CONF
CONF.register_opts(eventlet_backdoor_opts)
LOG = logging.getLogger(__name__)
class EventletBackdoorConfigValueError(Exception):
def __init__(self, port_range, help_msg, ex):
msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
'%(help)s' %
{'range': port_range, 'ex': ex, 'help': help_msg})
super(EventletBackdoorConfigValueError, self).__init__(msg)
self.port_range = port_range
def _dont_use_this():
print("Don't use this, just disconnect instead")
def _find_objects(t):
return filter(lambda o: isinstance(o, t), gc.get_objects())
def _print_greenthreads():
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
print(i, gt)
traceback.print_stack(gt.gr_frame)
print()
def _print_nativethreads():
for threadId, stack in sys._current_frames().items():
print(threadId)
traceback.print_stack(stack)
print()
def _parse_port_range(port_range):
if ':' not in port_range:
start, end = port_range, port_range
else:
start, end = port_range.split(':', 1)
try:
start, end = int(start), int(end)
if end < start:
raise ValueError
return start, end
except ValueError as ex:
raise EventletBackdoorConfigValueError(port_range, ex,
help_for_backdoor_port)
def _listen(host, start_port, end_port, listen_func):
try_port = start_port
while True:
try:
return listen_func((host, try_port))
except socket.error as exc:
if (exc.errno != errno.EADDRINUSE or
try_port >= end_port):
raise
try_port += 1
def initialize_if_enabled():
backdoor_locals = {
'exit': _dont_use_this, # So we don't exit the entire process
'quit': _dont_use_this, # So we don't exit the entire process
'fo': _find_objects,
'pgt': _print_greenthreads,
'pnt': _print_nativethreads,
}
if CONF.backdoor_port is None:
return None
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = _listen('localhost', start_port, end_port, eventlet.listen)
# In the case of backdoor port being zero, a port number is assigned by
# listen(). In any case, pull the port number out here.
port = sock.getsockname()[1]
LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
{'port': port, 'pid': os.getpid()})
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
locals=backdoor_locals)
return port

139
keystone/openstack/common/exception.py
View File

@@ -1,139 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Exceptions common to OpenStack projects
"""
import logging
from keystone.openstack.common.gettextutils import _ # noqa
_FATAL_EXCEPTION_FORMAT_ERRORS = False
class Error(Exception):
def __init__(self, message=None):
super(Error, self).__init__(message)
class ApiError(Error):
def __init__(self, message='Unknown', code='Unknown'):
self.api_message = message
self.code = code
super(ApiError, self).__init__('%s: %s' % (code, message))
class NotFound(Error):
pass
class UnknownScheme(Error):
msg_fmt = "Unknown scheme '%s' found in URI"
def __init__(self, scheme):
msg = self.msg_fmt % scheme
super(UnknownScheme, self).__init__(msg)
class BadStoreUri(Error):
msg_fmt = "The Store URI %s was malformed. Reason: %s"
def __init__(self, uri, reason):
msg = self.msg_fmt % (uri, reason)
super(BadStoreUri, self).__init__(msg)
class Duplicate(Error):
pass
class NotAuthorized(Error):
pass
class NotEmpty(Error):
pass
class Invalid(Error):
pass
class BadInputError(Exception):
"""Error resulting from a client sending bad input to a server"""
pass
class MissingArgumentError(Error):
pass
class DatabaseMigrationError(Error):
pass
class ClientConnectionError(Exception):
"""Error resulting from a client connecting to a server"""
pass
def wrap_exception(f):
def _wrap(*args, **kw):
try:
return f(*args, **kw)
except Exception as e:
if not isinstance(e, Error):
logging.exception(_('Uncaught exception'))
raise Error(str(e))
raise
_wrap.func_name = f.func_name
return _wrap
class OpenstackException(Exception):
"""Base Exception class.
To correctly use this class, inherit from it and define
a 'msg_fmt' property. That message will get printf'd
with the keyword arguments provided to the constructor.
"""
msg_fmt = "An unknown exception occurred"
def __init__(self, **kwargs):
try:
self._error_string = self.msg_fmt % kwargs
except Exception:
if _FATAL_EXCEPTION_FORMAT_ERRORS:
raise
else:
# at least get the core message out if something happened
self._error_string = self.msg_fmt
def __str__(self):
return self._error_string
class MalformedRequestBody(OpenstackException):
msg_fmt = "Malformed message body: %(reason)s"
class InvalidContentType(OpenstackException):
msg_fmt = "Invalid content type %(content_type)s"

32
keystone/openstack/common/excutils.py
View File

@@ -24,7 +24,7 @@ import traceback
import six import six
from keystone.openstack.common.gettextutils import _ # noqa from keystone.openstack.common.gettextutils import _LE
class save_and_reraise_exception(object): class save_and_reraise_exception(object):
@@ -49,9 +49,22 @@ class save_and_reraise_exception(object):
decide_if_need_reraise() decide_if_need_reraise()
if not should_be_reraised: if not should_be_reraised:
ctxt.reraise = False ctxt.reraise = False
If another exception occurs and reraise flag is False,
the saved exception will not be logged.
If the caller wants to raise new exception during exception handling
he/she sets reraise to False initially with an ability to set it back to
True if needed::
except Exception:
with save_and_reraise_exception(reraise=False) as ctxt:
[if statements to determine whether to raise a new exception]
# Not raising a new exception, so reraise
ctxt.reraise = True
""" """
def __init__(self): def __init__(self, reraise=True):
self.reraise = True self.reraise = reraise
def __enter__(self): def __enter__(self):
self.type_, self.value, self.tb, = sys.exc_info() self.type_, self.value, self.tb, = sys.exc_info()
@@ -59,10 +72,11 @@ class save_and_reraise_exception(object):
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None: if exc_type is not None:
logging.error(_('Original exception being dropped: %s'), if self.reraise:
traceback.format_exception(self.type_, logging.error(_LE('Original exception being dropped: %s'),
self.value, traceback.format_exception(self.type_,
self.tb)) self.value,
self.tb))
return False return False
if self.reraise: if self.reraise:
six.reraise(self.type_, self.value, self.tb) six.reraise(self.type_, self.value, self.tb)
@@ -88,8 +102,8 @@ def forever_retry_uncaught_exceptions(infunc):
if (cur_time - last_log_time > 60 or if (cur_time - last_log_time > 60 or
this_exc_message != last_exc_message): this_exc_message != last_exc_message):
logging.exception( logging.exception(
_('Unexpected exception occurred %d time(s)... ' _LE('Unexpected exception occurred %d time(s)... '
'retrying.') % exc_count) 'retrying.') % exc_count)
last_log_time = cur_time last_log_time = cur_time
last_exc_message = this_exc_message last_exc_message = this_exc_message
exc_count = 0 exc_count = 0

4
keystone/openstack/common/fileutils.py
View File

@@ -13,14 +13,12 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import contextlib import contextlib
import errno import errno
import os import os
import tempfile import tempfile
from keystone.openstack.common import excutils from keystone.openstack.common import excutils
from keystone.openstack.common.gettextutils import _ # noqa
from keystone.openstack.common import log as logging from keystone.openstack.common import log as logging
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@@ -60,7 +58,7 @@ def read_cached_file(filename, force_reload=False):
cache_info = _FILE_CACHE.setdefault(filename, {}) cache_info = _FILE_CACHE.setdefault(filename, {})
if not cache_info or mtime > cache_info.get('mtime', 0): if not cache_info or mtime > cache_info.get('mtime', 0):
LOG.debug(_("Reloading cached file %s") % filename) LOG.debug("Reloading cached file %s" % filename)
with open(filename) as fap: with open(filename) as fap:
cache_info['data'] = fap.read() cache_info['data'] = fap.read()
cache_info['mtime'] = mtime cache_info['mtime'] = mtime

2
keystone/openstack/common/fixture/lockutils.py
View File

@@ -48,4 +48,4 @@ class LockFixture(fixtures.Fixture):
def setUp(self): def setUp(self):
super(LockFixture, self).setUp() super(LockFixture, self).setUp()
self.addCleanup(self.mgr.__exit__, None, None, None) self.addCleanup(self.mgr.__exit__, None, None, None)
self.mgr.__enter__() self.lock = self.mgr.__enter__()

34
keystone/openstack/common/fixture/logging.py Normal file
View File

@@ -0,0 +1,34 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fixtures
def get_logging_handle_error_fixture():
"""returns a fixture to make logging raise formatting exceptions.
Usage:
self.useFixture(logging.get_logging_handle_error_fixture())
"""
return fixtures.MonkeyPatch('logging.Handler.handleError',
_handleError)
def _handleError(self, record):
"""Monkey patch for logging.Handler.handleError.
The default handleError just logs the error to stderr but we want
the option of actually raising an exception.
"""
raise

11
keystone/openstack/common/fixture/mockpatch.py
View File

@@ -15,6 +15,17 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
##############################################################################
##############################################################################
##
## DO NOT MODIFY THIS FILE
##
## This file is being graduated to the keystonetest library. Please make all
## changes there, and only backport critical fixes here. - dhellmann
##
##############################################################################
##############################################################################
import fixtures import fixtures
import mock import mock

13
keystone/openstack/common/fixture/moxstubout.py
View File

@@ -15,8 +15,19 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
##############################################################################
##############################################################################
##
## DO NOT MODIFY THIS FILE
##
## This file is being graduated to the keystonetest library. Please make all
## changes there, and only backport critical fixes here. - dhellmann
##
##############################################################################
##############################################################################
import fixtures import fixtures
import mox from six.moves import mox
class MoxStubout(fixtures.Fixture): class MoxStubout(fixtures.Fixture):

44
keystone/openstack/common/gettextutils.py
View File

@@ -28,7 +28,6 @@ import gettext
import locale import locale
from logging import handlers from logging import handlers
import os import os
import re
from babel import localedata from babel import localedata
import six import six
@@ -248,47 +247,22 @@ class Message(six.text_type):
if other is None: if other is None:
params = (other,) params = (other,)
elif isinstance(other, dict): elif isinstance(other, dict):
params = self._trim_dictionary_parameters(other) # Merge the dictionaries
# Copy each item in case one does not support deep copy.
params = {}
if isinstance(self.params, dict):
for key, val in self.params.items():
params[key] = self._copy_param(val)
for key, val in other.items():
params[key] = self._copy_param(val)
else: else:
params = self._copy_param(other) params = self._copy_param(other)
return params return params
def _trim_dictionary_parameters(self, dict_param):
"""Return a dict that only has matching entries in the msgid."""
# NOTE(luisg): Here we trim down the dictionary passed as parameters
# to avoid carrying a lot of unnecessary weight around in the message
# object, for example if someone passes in Message() % locals() but
# only some params are used, and additionally we prevent errors for
# non-deepcopyable objects by unicoding() them.
# Look for %(param) keys in msgid;
# Skip %% and deal with the case where % is first character on the line
keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
# If we don't find any %(param) keys but have a %s
if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
# Apparently the full dictionary is the parameter
params = self._copy_param(dict_param)
else:
params = {}
# Save our existing parameters as defaults to protect
# ourselves from losing values if we are called through an
# (erroneous) chain that builds a valid Message with
# arguments, and then does something like "msg % kwds"
# where kwds is an empty dictionary.
src = {}
if isinstance(self.params, dict):
src.update(self.params)
src.update(dict_param)
for key in keys:
params[key] = self._copy_param(src[key])
return params
def _copy_param(self, param): def _copy_param(self, param):
try: try:
return copy.deepcopy(param) return copy.deepcopy(param)
except TypeError: except Exception:
# Fallback to casting to unicode this will handle the # Fallback to casting to unicode this will handle the
# python code-like objects that can't be deep-copied # python code-like objects that can't be deep-copied
return six.text_type(param) return six.text_type(param)

7
keystone/openstack/common/importutils.py
View File

@@ -58,6 +58,13 @@ def import_module(import_str):
return sys.modules[import_str] return sys.modules[import_str]
def import_versioned_module(version, submodule=None):
module = 'keystone.v%s' % version
if submodule:
module = '.'.join((module, submodule))
return import_module(module)
def try_import(import_str, default=None): def try_import(import_str, default=None):
"""Try to import a module and if it fails return default.""" """Try to import a module and if it fails return default."""
try: try:

10
keystone/openstack/common/jsonutils.py
View File

@@ -36,13 +36,9 @@ import functools
import inspect import inspect
import itertools import itertools
import json import json
try:
import xmlrpclib
except ImportError:
# NOTE(jd): xmlrpclib is not shipped with Python 3
xmlrpclib = None
import six import six
import six.moves.xmlrpc_client as xmlrpclib
from keystone.openstack.common import gettextutils from keystone.openstack.common import gettextutils
from keystone.openstack.common import importutils from keystone.openstack.common import importutils
@@ -122,14 +118,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
level=level, level=level,
max_depth=max_depth) max_depth=max_depth)
if isinstance(value, dict): if isinstance(value, dict):
return dict((k, recursive(v)) for k, v in value.iteritems()) return dict((k, recursive(v)) for k, v in six.iteritems(value))
elif isinstance(value, (list, tuple)): elif isinstance(value, (list, tuple)):
return [recursive(lv) for lv in value] return [recursive(lv) for lv in value]
# It's not clear why xmlrpclib created their own DateTime type, but # It's not clear why xmlrpclib created their own DateTime type, but
# for our purposes, make it a datetime type which is explicitly # for our purposes, make it a datetime type which is explicitly
# handled # handled
if xmlrpclib and isinstance(value, xmlrpclib.DateTime): if isinstance(value, xmlrpclib.DateTime):
value = datetime.datetime(*tuple(value.timetuple())[:6]) value = datetime.datetime(*tuple(value.timetuple())[:6])
if convert_datetime and isinstance(value, datetime.datetime): if convert_datetime and isinstance(value, datetime.datetime):

222
keystone/openstack/common/lockutils.py
View File

@@ -13,9 +13,9 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import contextlib import contextlib
import errno import errno
import fcntl
import functools import functools
import os import os
import shutil import shutil
@@ -29,8 +29,7 @@ import weakref
from oslo.config import cfg from oslo.config import cfg
from keystone.openstack.common import fileutils from keystone.openstack.common import fileutils
from keystone.openstack.common.gettextutils import _ # noqa from keystone.openstack.common.gettextutils import _, _LE, _LI
from keystone.openstack.common import local
from keystone.openstack.common import log as logging from keystone.openstack.common import log as logging
@@ -54,7 +53,7 @@ def set_defaults(lock_path):
cfg.set_defaults(util_opts, lock_path=lock_path) cfg.set_defaults(util_opts, lock_path=lock_path)
class _InterProcessLock(object): class _FileLock(object):
"""Lock implementation which allows multiple locks, working around """Lock implementation which allows multiple locks, working around
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
not require any cleanup. Since the lock is always held on a file not require any cleanup. Since the lock is always held on a file
@@ -76,7 +75,13 @@ class _InterProcessLock(object):
self.lockfile = None self.lockfile = None
self.fname = name self.fname = name
def __enter__(self): def acquire(self):
basedir = os.path.dirname(self.fname)
if not os.path.exists(basedir):
fileutils.ensure_tree(basedir)
LOG.info(_LI('Created lock path: %s'), basedir)
self.lockfile = open(self.fname, 'w') self.lockfile = open(self.fname, 'w')
while True: while True:
@@ -86,23 +91,41 @@ class _InterProcessLock(object):
# Also upon reading the MSDN docs for locking(), it seems # Also upon reading the MSDN docs for locking(), it seems
# to have a laughable 10 attempts "blocking" mechanism. # to have a laughable 10 attempts "blocking" mechanism.
self.trylock() self.trylock()
return self LOG.debug('Got file lock "%s"', self.fname)
return True
except IOError as e: except IOError as e:
if e.errno in (errno.EACCES, errno.EAGAIN): if e.errno in (errno.EACCES, errno.EAGAIN):
# external locks synchronise things like iptables # external locks synchronise things like iptables
# updates - give it some time to prevent busy spinning # updates - give it some time to prevent busy spinning
time.sleep(0.01) time.sleep(0.01)
else: else:
raise raise threading.ThreadError(_("Unable to acquire lock on"
" `%(filename)s` due to"
" %(exception)s") %
{
'filename': self.fname,
'exception': e,
})
def __exit__(self, exc_type, exc_val, exc_tb): def __enter__(self):
self.acquire()
return self
def release(self):
try: try:
self.unlock() self.unlock()
self.lockfile.close() self.lockfile.close()
LOG.debug('Released file lock "%s"', self.fname)
except IOError: except IOError:
LOG.exception(_("Could not release the acquired lock `%s`"), LOG.exception(_LE("Could not release the acquired lock `%s`"),
self.fname) self.fname)
def __exit__(self, exc_type, exc_val, exc_tb):
self.release()
def exists(self):
return os.path.exists(self.fname)
def trylock(self): def trylock(self):
raise NotImplementedError() raise NotImplementedError()
@@ -110,7 +133,7 @@ class _InterProcessLock(object):
raise NotImplementedError() raise NotImplementedError()
class _WindowsLock(_InterProcessLock): class _WindowsLock(_FileLock):
def trylock(self): def trylock(self):
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
@@ -118,7 +141,7 @@ class _WindowsLock(_InterProcessLock):
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
class _PosixLock(_InterProcessLock): class _FcntlLock(_FileLock):
def trylock(self): def trylock(self):
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -126,17 +149,120 @@ class _PosixLock(_InterProcessLock):
fcntl.lockf(self.lockfile, fcntl.LOCK_UN) fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
class _PosixLock(object):
def __init__(self, name):
# Hash the name because it's not valid to have POSIX semaphore
# names with things like / in them. Then use base64 to encode
# the digest() instead taking the hexdigest() because the
# result is shorter and most systems can't have shm sempahore
# names longer than 31 characters.
h = hashlib.sha1()
h.update(name.encode('ascii'))
self.name = str((b'/' + base64.urlsafe_b64encode(
h.digest())).decode('ascii'))
def acquire(self, timeout=None):
self.semaphore = posix_ipc.Semaphore(self.name,
flags=posix_ipc.O_CREAT,
initial_value=1)
self.semaphore.acquire(timeout)
return self
def __enter__(self):
self.acquire()
return self
def release(self):
self.semaphore.release()
self.semaphore.close()
def __exit__(self, exc_type, exc_val, exc_tb):
self.release()
def exists(self):
try:
semaphore = posix_ipc.Semaphore(self.name)
except posix_ipc.ExistentialError:
return False
else:
semaphore.close()
return True
if os.name == 'nt': if os.name == 'nt':
import msvcrt import msvcrt
InterProcessLock = _WindowsLock InterProcessLock = _WindowsLock
FileLock = _WindowsLock
else: else:
import fcntl import base64
import hashlib
import posix_ipc
InterProcessLock = _PosixLock InterProcessLock = _PosixLock
FileLock = _FcntlLock
_semaphores = weakref.WeakValueDictionary() _semaphores = weakref.WeakValueDictionary()
_semaphores_lock = threading.Lock() _semaphores_lock = threading.Lock()
def _get_lock_path(name, lock_file_prefix, lock_path=None):
# NOTE(mikal): the lock name cannot contain directory
# separators
name = name.replace(os.sep, '_')
if lock_file_prefix:
sep = '' if lock_file_prefix.endswith('-') else '-'
name = '%s%s%s' % (lock_file_prefix, sep, name)
local_lock_path = lock_path or CONF.lock_path
if not local_lock_path:
# NOTE(bnemec): Create a fake lock path for posix locks so we don't
# unnecessarily raise the RequiredOptError below.
if InterProcessLock is not _PosixLock:
raise cfg.RequiredOptError('lock_path')
local_lock_path = 'posixlock:/'
return os.path.join(local_lock_path, name)
def external_lock(name, lock_file_prefix=None, lock_path=None):
LOG.debug('Attempting to grab external lock "%(lock)s"',
{'lock': name})
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
# NOTE(bnemec): If an explicit lock_path was passed to us then it
# means the caller is relying on file-based locking behavior, so
# we can't use posix locks for those calls.
if lock_path:
return FileLock(lock_file_path)
return InterProcessLock(lock_file_path)
def remove_external_lock_file(name, lock_file_prefix=None):
"""Remove a external lock file when it's not used anymore
This will be helpful when we have a lot of lock files
"""
with internal_lock(name):
lock_file_path = _get_lock_path(name, lock_file_prefix)
try:
os.remove(lock_file_path)
except OSError:
LOG.info(_LI('Failed to remove file %(file)s'),
{'file': lock_file_path})
def internal_lock(name):
with _semaphores_lock:
try:
sem = _semaphores[name]
except KeyError:
sem = threading.Semaphore()
_semaphores[name] = sem
LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
return sem
@contextlib.contextmanager @contextlib.contextmanager
def lock(name, lock_file_prefix=None, external=False, lock_path=None): def lock(name, lock_file_prefix=None, external=False, lock_path=None):
"""Context based lock """Context based lock
@@ -152,67 +278,15 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
should work across multiple processes. This means that if two different should work across multiple processes. This means that if two different
workers both run a a method decorated with @synchronized('mylock', workers both run a a method decorated with @synchronized('mylock',
external=True), only one of them will execute at a time. external=True), only one of them will execute at a time.
:param lock_path: The lock_path keyword argument is used to specify a
special location for external lock files to live. If nothing is set, then
CONF.lock_path is used as a default.
""" """
with _semaphores_lock: int_lock = internal_lock(name)
try: with int_lock:
sem = _semaphores[name] if external and not CONF.disable_process_locking:
except KeyError: ext_lock = external_lock(name, lock_file_prefix, lock_path)
sem = threading.Semaphore() with ext_lock:
_semaphores[name] = sem yield ext_lock
else:
with sem: yield int_lock
LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name})
# NOTE(mikal): I know this looks odd
if not hasattr(local.strong_store, 'locks_held'):
local.strong_store.locks_held = []
local.strong_store.locks_held.append(name)
try:
if external and not CONF.disable_process_locking:
LOG.debug(_('Attempting to grab file lock "%(lock)s"'),
{'lock': name})
# We need a copy of lock_path because it is non-local
local_lock_path = lock_path or CONF.lock_path
if not local_lock_path:
raise cfg.RequiredOptError('lock_path')
if not os.path.exists(local_lock_path):
fileutils.ensure_tree(local_lock_path)
LOG.info(_('Created lock path: %s'), local_lock_path)
def add_prefix(name, prefix):
if not prefix:
return name
sep = '' if prefix.endswith('-') else '-'
return '%s%s%s' % (prefix, sep, name)
# NOTE(mikal): the lock name cannot contain directory
# separators
lock_file_name = add_prefix(name.replace(os.sep, '_'),
lock_file_prefix)
lock_file_path = os.path.join(local_lock_path, lock_file_name)
try:
lock = InterProcessLock(lock_file_path)
with lock as lock:
LOG.debug(_('Got file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
yield lock
finally:
LOG.debug(_('Released file lock "%(lock)s" at %(path)s'),
{'lock': name, 'path': lock_file_path})
else:
yield sem
finally:
local.strong_store.locks_held.remove(name)
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
@@ -244,11 +318,11 @@ def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
def inner(*args, **kwargs): def inner(*args, **kwargs):
try: try:
with lock(name, lock_file_prefix, external, lock_path): with lock(name, lock_file_prefix, external, lock_path):
LOG.debug(_('Got semaphore / lock "%(function)s"'), LOG.debug('Got semaphore / lock "%(function)s"',
{'function': f.__name__}) {'function': f.__name__})
return f(*args, **kwargs) return f(*args, **kwargs)
finally: finally:
LOG.debug(_('Semaphore / lock released "%(function)s"'), LOG.debug('Semaphore / lock released "%(function)s"',
{'function': f.__name__}) {'function': f.__name__})
return inner return inner
return wrap return wrap

9
keystone/openstack/common/log.py
View File

@@ -15,7 +15,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
"""Openstack logging handler. """OpenStack logging handler.
This module adds to logging functionality by adding the option to specify This module adds to logging functionality by adding the option to specify
a context object when calling the various log methods. If the context object a context object when calling the various log methods. If the context object
@@ -163,6 +163,7 @@ log_opts = [
'qpid=WARN', 'qpid=WARN',
'sqlalchemy=WARN', 'sqlalchemy=WARN',
'suds=INFO', 'suds=INFO',
'oslo.messaging=INFO',
'iso8601=WARN', 'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN' 'requests.packages.urllib3.connectionpool=WARN'
], ],
@@ -357,7 +358,7 @@ class ContextAdapter(BaseLoggerAdapter):
extra.update(_dictify_context(context)) extra.update(_dictify_context(context))
instance = kwargs.pop('instance', None) instance = kwargs.pop('instance', None)
instance_uuid = (extra.get('instance_uuid', None) or instance_uuid = (extra.get('instance_uuid') or
kwargs.pop('instance_uuid', None)) kwargs.pop('instance_uuid', None))
instance_extra = '' instance_extra = ''
if instance: if instance:
@@ -650,11 +651,11 @@ class ContextFormatter(logging.Formatter):
# NOTE(sdague): default the fancier formatting params # NOTE(sdague): default the fancier formatting params
# to an empty string so we don't throw an exception if # to an empty string so we don't throw an exception if
# they get used # they get used
for key in ('instance', 'color'): for key in ('instance', 'color', 'user_identity'):
if key not in record.__dict__: if key not in record.__dict__:
record.__dict__[key] = '' record.__dict__[key] = ''
if record.__dict__.get('request_id', None): if record.__dict__.get('request_id'):
self._fmt = CONF.logging_context_format_string self._fmt = CONF.logging_context_format_string
else: else:
self._fmt = CONF.logging_default_format_string self._fmt = CONF.logging_default_format_string

145
keystone/openstack/common/loopingcall.py
View File

@@ -1,145 +0,0 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from eventlet import event
from eventlet import greenthread
from keystone.openstack.common.gettextutils import _ # noqa
from keystone.openstack.common import log as logging
from keystone.openstack.common import timeutils
LOG = logging.getLogger(__name__)
class LoopingCallDone(Exception):
"""Exception to break out and stop a LoopingCall.
The poll-function passed to LoopingCall can raise this exception to
break out of the loop normally. This is somewhat analogous to
StopIteration.
An optional return-value can be included as the argument to the exception;
this return-value will be returned by LoopingCall.wait()
"""
def __init__(self, retvalue=True):
""":param retvalue: Value that LoopingCall.wait() should return."""
self.retvalue = retvalue
class LoopingCallBase(object):
def __init__(self, f=None, *args, **kw):
self.args = args
self.kw = kw
self.f = f
self._running = False
self.done = None
def stop(self):
self._running = False
def wait(self):
return self.done.wait()
class FixedIntervalLoopingCall(LoopingCallBase):
"""A fixed interval looping call."""
def start(self, interval, initial_delay=None):
self._running = True
done = event.Event()
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
self.done = done
greenthread.spawn_n(_inner)
return self.done
# TODO(mikal): this class name is deprecated in Havana and should be removed
# in the I release
LoopingCall = FixedIntervalLoopingCall
class DynamicLoopingCall(LoopingCallBase):
"""A looping call which sleeps until the next known event.
The function called should return how long to sleep for before being
called again.
"""
def start(self, initial_delay=None, periodic_interval_max=None):
self._running = True
done = event.Event()
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
idle = self.f(*self.args, **self.kw)
if not self._running:
break
if periodic_interval_max is not None:
idle = min(idle, periodic_interval_max)
LOG.debug(_('Dynamic looping call sleeping for %.02f '
'seconds'), idle)
greenthread.sleep(idle)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in dynamic looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
self.done = done
greenthread.spawn(_inner)
return self.done

79
keystone/openstack/common/network_utils.py
View File

@@ -1,79 +0,0 @@
# Copyright 2012 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Network-related utilities and helper functions.
"""
from keystone.openstack.common.py3kcompat import urlutils
def parse_host_port(address, default_port=None):
"""Interpret a string as a host:port pair.
An IPv6 address MUST be escaped if accompanied by a port,
because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334
means both [2001:db8:85a3::8a2e:370:7334] and
[2001:db8:85a3::8a2e:370]:7334.
>>> parse_host_port('server01:80')
('server01', 80)
>>> parse_host_port('server01')
('server01', None)
>>> parse_host_port('server01', default_port=1234)
('server01', 1234)
>>> parse_host_port('[::1]:80')
('::1', 80)
>>> parse_host_port('[::1]')
('::1', None)
>>> parse_host_port('[::1]', default_port=1234)
('::1', 1234)
>>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234)
('2001:db8:85a3::8a2e:370:7334', 1234)
"""
if address[0] == '[':
# Escaped ipv6
_host, _port = address[1:].split(']')
host = _host
if ':' in _port:
port = _port.split(':')[1]
else:
port = default_port
else:
if address.count(':') == 1:
host, port = address.split(':')
else:
# 0 means ipv4, >1 means ipv6.
# We prohibit unescaped ipv6 addresses with port.
host = address
port = default_port
return (host, None if port is None else int(port))
def urlsplit(url, scheme='', allow_fragments=True):
"""Parse a URL using urlparse.urlsplit(), splitting query and fragments.
This function papers over Python issue9374 when needed.
The parameters are the same as urlparse.urlsplit.
"""
scheme, netloc, path, query, fragment = urlutils.urlsplit(
url, scheme, allow_fragments)
if allow_fragments and '#' in path:
path, fragment = path.split('#', 1)
if '?' in path:
path, query = path.split('?', 1)
return urlutils.SplitResult(scheme, netloc, path, query, fragment)

63
keystone/openstack/common/policy.py
View File

@@ -46,6 +46,27 @@ policy rule::
project_id:%(project_id)s and not role:dunce project_id:%(project_id)s and not role:dunce
It is possible to perform policy checks on the following user
attributes (obtained through the token): user_id, domain_id or
project_id::
domain_id:<some_value>
Attributes sent along with API calls can be used by the policy engine
(on the right side of the expression), by using the following syntax::
<some_value>:user.id
Contextual attributes of objects identified by their IDs are loaded
from the database. They are also available to the policy engine and
can be checked through the `target` keyword::
<some_value>:target.role.name
All these attributes (related to users, API calls, and context) can be
checked against each other or against constants, be it literals (True,
<a_number>) or strings.
Finally, two special policy checks should be mentioned; the policy Finally, two special policy checks should be mentioned; the policy
check "@" will always accept an access, and the policy check "!" will check "@" will always accept an access, and the policy check "!" will
always reject an access. (Note that if a rule is either the empty always reject an access. (Note that if a rule is either the empty
@@ -64,7 +85,7 @@ import six.moves.urllib.parse as urlparse
import six.moves.urllib.request as urlrequest import six.moves.urllib.request as urlrequest
from keystone.openstack.common import fileutils from keystone.openstack.common import fileutils
from keystone.openstack.common.gettextutils import _ from keystone.openstack.common.gettextutils import _, _LE
from keystone.openstack.common import jsonutils from keystone.openstack.common import jsonutils
from keystone.openstack.common import log as logging from keystone.openstack.common import log as logging
@@ -161,27 +182,31 @@ class Enforcer(object):
is called this will be overwritten. is called this will be overwritten.
:param default_rule: Default rule to use, CONF.default_rule will :param default_rule: Default rule to use, CONF.default_rule will
be used if none is specified. be used if none is specified.
:param use_conf: Whether to load rules from cache or config file.
""" """
def __init__(self, policy_file=None, rules=None, default_rule=None): def __init__(self, policy_file=None, rules=None,
default_rule=None, use_conf=True):
self.rules = Rules(rules, default_rule) self.rules = Rules(rules, default_rule)
self.default_rule = default_rule or CONF.policy_default_rule self.default_rule = default_rule or CONF.policy_default_rule
self.policy_path = None self.policy_path = None
self.policy_file = policy_file or CONF.policy_file self.policy_file = policy_file or CONF.policy_file
self.use_conf = use_conf
def set_rules(self, rules, overwrite=True): def set_rules(self, rules, overwrite=True, use_conf=False):
"""Create a new Rules object based on the provided dict of rules. """Create a new Rules object based on the provided dict of rules.
:param rules: New rules to use. It should be an instance of dict. :param rules: New rules to use. It should be an instance of dict.
:param overwrite: Whether to overwrite current rules or update them :param overwrite: Whether to overwrite current rules or update them
with the new rules. with the new rules.
:param use_conf: Whether to reload rules from cache or config file.
""" """
if not isinstance(rules, dict): if not isinstance(rules, dict):
raise TypeError(_("Rules must be an instance of dict or Rules, " raise TypeError(_("Rules must be an instance of dict or Rules, "
"got %s instead") % type(rules)) "got %s instead") % type(rules))
self.use_conf = use_conf
if overwrite: if overwrite:
self.rules = Rules(rules, self.default_rule) self.rules = Rules(rules, self.default_rule)
else: else:
@@ -201,15 +226,19 @@ class Enforcer(object):
:param force_reload: Whether to overwrite current rules. :param force_reload: Whether to overwrite current rules.
""" """
if not self.policy_path: if force_reload:
self.policy_path = self._get_policy_path() self.use_conf = force_reload
reloaded, data = fileutils.read_cached_file(self.policy_path, if self.use_conf:
force_reload=force_reload) if not self.policy_path:
if reloaded or not self.rules: self.policy_path = self._get_policy_path()
rules = Rules.load_json(data, self.default_rule)
self.set_rules(rules) reloaded, data = fileutils.read_cached_file(
LOG.debug(_("Rules successfully reloaded")) self.policy_path, force_reload=force_reload)
if reloaded or not self.rules:
rules = Rules.load_json(data, self.default_rule)
self.set_rules(rules)
LOG.debug("Rules successfully reloaded")
def _get_policy_path(self): def _get_policy_path(self):
"""Locate the policy json data file. """Locate the policy json data file.
@@ -255,7 +284,7 @@ class Enforcer(object):
# NOTE(flaper87): Not logging target or creds to avoid # NOTE(flaper87): Not logging target or creds to avoid
# potential security issues. # potential security issues.
LOG.debug(_("Rule %s will be now enforced") % rule) LOG.debug("Rule %s will be now enforced" % rule)
self.load_rules() self.load_rules()
@@ -270,7 +299,7 @@ class Enforcer(object):
# Evaluate the rule # Evaluate the rule
result = self.rules[rule](target, creds, self) result = self.rules[rule](target, creds, self)
except KeyError: except KeyError:
LOG.debug(_("Rule [%s] doesn't exist") % rule) LOG.debug("Rule [%s] doesn't exist" % rule)
# If the rule doesn't exist, fail closed # If the rule doesn't exist, fail closed
result = False result = False
@@ -478,7 +507,7 @@ def _parse_check(rule):
try: try:
kind, match = rule.split(':', 1) kind, match = rule.split(':', 1)
except Exception: except Exception:
LOG.exception(_("Failed to understand rule %s") % rule) LOG.exception(_LE("Failed to understand rule %s") % rule)
# If the rule is invalid, we'll fail closed # If the rule is invalid, we'll fail closed
return FalseCheck() return FalseCheck()
@@ -488,7 +517,7 @@ def _parse_check(rule):
elif None in _checks: elif None in _checks:
return _checks[None](kind, match) return _checks[None](kind, match)
else: else:
LOG.error(_("No handler for matches of kind %s") % kind) LOG.error(_LE("No handler for matches of kind %s") % kind)
return FalseCheck() return FalseCheck()
@@ -758,7 +787,7 @@ def _parse_text_rule(rule):
return state.result return state.result
except ValueError: except ValueError:
# Couldn't parse the rule # Couldn't parse the rule
LOG.exception(_("Failed to understand rule %r") % rule) LOG.exception(_LE("Failed to understand rule %r") % rule)
# Fail closed # Fail closed
return FalseCheck() return FalseCheck()

248
keystone/openstack/common/processutils.py
View File

@@ -1,248 +0,0 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import logging as stdlib_logging
import os
import random
import shlex
import signal
from eventlet.green import subprocess
from eventlet import greenthread
from keystone.openstack.common.gettextutils import _ # noqa
from keystone.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class InvalidArgumentError(Exception):
def __init__(self, message=None):
super(InvalidArgumentError, self).__init__(message)
class UnknownArgumentError(Exception):
def __init__(self, message=None):
super(UnknownArgumentError, self).__init__(message)
class ProcessExecutionError(Exception):
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
description=None):
self.exit_code = exit_code
self.stderr = stderr
self.stdout = stdout
self.cmd = cmd
self.description = description
if description is None:
description = "Unexpected error while running command."
if exit_code is None:
exit_code = '-'
message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
% (description, cmd, exit_code, stdout, stderr))
super(ProcessExecutionError, self).__init__(message)
class NoRootWrapSpecified(Exception):
def __init__(self, message=None):
super(NoRootWrapSpecified, self).__init__(message)
def _subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
# non-Python subprocesses expect.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def execute(*cmd, **kwargs):
"""Helper method to shell out and execute a command through subprocess.
Allows optional retry.
:param cmd: Passed to subprocess.Popen.
:type cmd: string
:param process_input: Send to opened process.
:type proces_input: string
:param check_exit_code: Single bool, int, or list of allowed exit
codes. Defaults to [0]. Raise
:class:`ProcessExecutionError` unless
program exits with one of these code.
:type check_exit_code: boolean, int, or [int]
:param delay_on_retry: True | False. Defaults to True. If set to True,
wait a short amount of time before retrying.
:type delay_on_retry: boolean
:param attempts: How many times to retry cmd.
:type attempts: int
:param run_as_root: True | False. Defaults to False. If set to True,
the command is prefixed by the command specified
in the root_helper kwarg.
:type run_as_root: boolean
:param root_helper: command to prefix to commands called with
run_as_root=True
:type root_helper: string
:param shell: whether or not there should be a shell used to
execute this command. Defaults to false.
:type shell: boolean
:param loglevel: log level for execute commands.
:type loglevel: int. (Should be stdlib_logging.DEBUG or
stdlib_logging.INFO)
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
:raises: :class:`ProcessExecutionError`
"""
process_input = kwargs.pop('process_input', None)
check_exit_code = kwargs.pop('check_exit_code', [0])
ignore_exit_code = False
delay_on_retry = kwargs.pop('delay_on_retry', True)
attempts = kwargs.pop('attempts', 1)
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
shell = kwargs.pop('shell', False)
loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
check_exit_code = [0]
elif isinstance(check_exit_code, int):
check_exit_code = [check_exit_code]
if kwargs:
raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs)
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
if not root_helper:
raise NoRootWrapSpecified(
message=('Command requested root, but did not specify a root '
'helper.'))
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
while attempts > 0:
attempts -= 1
try:
LOG.log(loglevel, _('Running cmd (subprocess): %s'), ' '.join(cmd))
_PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
preexec_fn = None
close_fds = False
else:
preexec_fn = _subprocess_setup
close_fds = True
obj = subprocess.Popen(cmd,
stdin=_PIPE,
stdout=_PIPE,
stderr=_PIPE,
close_fds=close_fds,
preexec_fn=preexec_fn,
shell=shell)
result = None
if process_input is not None:
result = obj.communicate(process_input)
else:
result = obj.communicate()
obj.stdin.close() # pylint: disable=E1101
_returncode = obj.returncode # pylint: disable=E1101
LOG.log(loglevel, _('Result was %s') % _returncode)
if not ignore_exit_code and _returncode not in check_exit_code:
(stdout, stderr) = result
raise ProcessExecutionError(exit_code=_returncode,
stdout=stdout,
stderr=stderr,
cmd=' '.join(cmd))
return result
except ProcessExecutionError:
if not attempts:
raise
else:
LOG.log(loglevel, _('%r failed. Retrying.'), cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:
# NOTE(termie): this appears to be necessary to let the subprocess
# call clean something up in between calls, without
# it two execute calls in a row hangs the second one
greenthread.sleep(0)
def trycmd(*args, **kwargs):
"""A wrapper around execute() to more easily handle warnings and errors.
Returns an (out, err) tuple of strings containing the output of
the command's stdout and stderr. If 'err' is not empty then the
command can be considered to have failed.
:discard_warnings True | False. Defaults to False. If set to True,
then for succeeding commands, stderr is cleared
"""
discard_warnings = kwargs.pop('discard_warnings', False)
try:
out, err = execute(*args, **kwargs)
failed = False
except ProcessExecutionError as exn:
out, err = '', str(exn)
failed = True
if not failed and discard_warnings and err:
# Handle commands that output to stderr but otherwise succeed
err = ''
return out, err
def ssh_execute(ssh, cmd, process_input=None,
addl_env=None, check_exit_code=True):
LOG.debug(_('Running cmd (SSH): %s'), cmd)
if addl_env:
raise InvalidArgumentError(_('Environment not supported over SSH'))
if process_input:
# This is (probably) fixable if we need it...
raise InvalidArgumentError(_('process_input not supported over SSH'))
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
channel = stdout_stream.channel
# NOTE(justinsb): This seems suspicious...
# ...other SSH clients have buffering issues with this approach
stdout = stdout_stream.read()
stderr = stderr_stream.read()
stdin_stream.close()
exit_status = channel.recv_exit_status()
# exit_status == -1 if no exit code was returned
if exit_status != -1:
LOG.debug(_('Result was %s') % exit_status)
if check_exit_code and exit_status != 0:
raise ProcessExecutionError(exit_code=exit_status,
stdout=stdout,
stderr=stderr,
cmd=cmd)
return (stdout, stderr)

16
keystone/openstack/common/py3kcompat/__init__.py
View File

@@ -1,16 +0,0 @@
#
# Copyright 2013 Canonical Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

63
keystone/openstack/common/py3kcompat/urlutils.py
View File

@@ -1,63 +0,0 @@
#
# Copyright 2013 Canonical Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""
Python2/Python3 compatibility layer for OpenStack
"""
import six
if six.PY3:
# python3
import urllib.error
import urllib.parse
import urllib.request
urlencode = urllib.parse.urlencode
urljoin = urllib.parse.urljoin
quote = urllib.parse.quote
parse_qsl = urllib.parse.parse_qsl
unquote = urllib.parse.unquote
urlparse = urllib.parse.urlparse
urlsplit = urllib.parse.urlsplit
urlunsplit = urllib.parse.urlunsplit
SplitResult = urllib.parse.SplitResult
urlopen = urllib.request.urlopen
URLError = urllib.error.URLError
pathname2url = urllib.request.pathname2url
else:
# python2
import urllib
import urllib2
import urlparse
urlencode = urllib.urlencode
quote = urllib.quote
unquote = urllib.unquote
parse = urlparse
parse_qsl = parse.parse_qsl
urljoin = parse.urljoin
urlparse = parse.urlparse
urlsplit = parse.urlsplit
urlunsplit = parse.urlunsplit
SplitResult = parse.SplitResult
urlopen = urllib2.urlopen
URLError = urllib2.URLError
pathname2url = urllib.pathname2url

459
keystone/openstack/common/service.py
View File

@@ -1,459 +0,0 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Generic Node base class for all workers that run on hosts."""
import errno
import logging as std_logging
import os
import random
import signal
import sys
import time
import eventlet
from eventlet import event
from oslo.config import cfg
from keystone.openstack.common import eventlet_backdoor
from keystone.openstack.common.gettextutils import _ # noqa
from keystone.openstack.common import importutils
from keystone.openstack.common import log as logging
from keystone.openstack.common import threadgroup
rpc = importutils.try_import('keystone.openstack.common.rpc')
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def _sighup_supported():
return hasattr(signal, 'SIGHUP')
def _is_sighup(signo):
return _sighup_supported() and signo == signal.SIGHUP
def _signo_to_signame(signo):
signals = {signal.SIGTERM: 'SIGTERM',
signal.SIGINT: 'SIGINT'}
if _sighup_supported():
signals[signal.SIGHUP] = 'SIGHUP'
return signals[signo]
def _set_signals_handler(handler):
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
if _sighup_supported():
signal.signal(signal.SIGHUP, handler)
class Launcher(object):
"""Launch one or more services and wait for them to complete."""
def __init__(self):
"""Initialize the service launcher.
:returns: None
"""
self.services = Services()
self.backdoor_port = eventlet_backdoor.initialize_if_enabled()
def launch_service(self, service):
"""Load and start the given service.
:param service: The service you would like to start.
:returns: None
"""
service.backdoor_port = self.backdoor_port
self.services.add(service)
def stop(self):
"""Stop all services which are currently running.
:returns: None
"""
self.services.stop()
def wait(self):
"""Waits until all services have been stopped, and then returns.
:returns: None
"""
self.services.wait()
def restart(self):
"""Reload config files and restart service.
:returns: None
"""
cfg.CONF.reload_config_files()
self.services.restart()
class SignalExit(SystemExit):
def __init__(self, signo, exccode=1):
super(SignalExit, self).__init__(exccode)
self.signo = signo
class ServiceLauncher(Launcher):
def _handle_signal(self, signo, frame):
# Allow the process to be killed again and die from natural causes
_set_signals_handler(signal.SIG_DFL)
raise SignalExit(signo)
def handle_signal(self):
_set_signals_handler(self._handle_signal)
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug(_('Full set of CONF:'))
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
if rpc:
try:
rpc.cleanup()
except Exception:
# We're shutting down, so it doesn't matter at this point.
LOG.exception(_('Exception during rpc cleanup.'))
return status, signo
def wait(self, ready_callback=None):
while True:
self.handle_signal()
status, signo = self._wait_for_exit_or_signal(ready_callback)
if not _is_sighup(signo):
return status
self.restart()
class ServiceWrapper(object):
def __init__(self, service, workers):
self.service = service
self.workers = workers
self.children = set()
self.forktimes = []
class ProcessLauncher(object):
def __init__(self):
self.children = {}
self.sigcaught = None
self.running = True
rfd, self.writepipe = os.pipe()
self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
self.handle_signal()
def handle_signal(self):
_set_signals_handler(self._handle_signal)
def _handle_signal(self, signo, frame):
self.sigcaught = signo
self.running = False
# Allow the process to be killed again and die from natural causes
_set_signals_handler(signal.SIG_DFL)
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read()
LOG.info(_('Parent process has died unexpectedly, exiting'))
sys.exit(1)
def _child_process_handle_signal(self):
# Setup child signal handlers differently
def _sigterm(*args):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
raise SignalExit(signal.SIGTERM)
def _sighup(*args):
signal.signal(signal.SIGHUP, signal.SIG_DFL)
raise SignalExit(signal.SIGHUP)
signal.signal(signal.SIGTERM, _sigterm)
if _sighup_supported():
signal.signal(signal.SIGHUP, _sighup)
# Block SIGINT and let the parent send us a SIGTERM
signal.signal(signal.SIGINT, signal.SIG_IGN)
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def _child_process(self, service):
self._child_process_handle_signal()
# Reopen the eventlet hub to make sure we don't share an epoll
# fd with parent and/or siblings, which would be bad
eventlet.hubs.use_hub()
# Close write to ensure only parent has it open
os.close(self.writepipe)
# Create greenthread to watch for parent to close pipe
eventlet.spawn_n(self._pipe_watcher)
# Reseed random number generator
random.seed()
launcher = Launcher()
launcher.launch_service(service)
return launcher
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup(signo):
break
launcher.restart()
os._exit(status)
LOG.info(_('Started child %d'), pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def launch_service(self, service, workers=1):
wrap = ServiceWrapper(service, workers)
LOG.info(_('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
def _wait_child(self):
try:
# Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def _respawn_children(self):
while self.running:
wrap = self._wait_child()
if not wrap:
# Yield to other threads if no children have exited
# Sleep for a short time to avoid excessive CPU usage
# (see bug #1095346)
eventlet.greenthread.sleep(.01)
continue
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
LOG.debug(_('Full set of CONF:'))
CONF.log_opt_values(LOG, std_logging.DEBUG)
while True:
self.handle_signal()
self._respawn_children()
if self.sigcaught:
signame = _signo_to_signame(self.sigcaught)
LOG.info(_('Caught %s, stopping children'), signame)
if not _is_sighup(self.sigcaught):
break
for pid in self.children:
os.kill(pid, signal.SIGHUP)
self.running = True
self.sigcaught = None
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
class Service(object):
"""Service object for binaries running on hosts."""
def __init__(self, threads=1000):
self.tg = threadgroup.ThreadGroup(threads)
# signal that the service is done shutting itself down:
self._done = event.Event()
def reset(self):
# NOTE(Fengqian): docs for Event.reset() recommend against using it
self._done = event.Event()
def start(self):
pass
def stop(self):
self.tg.stop()
self.tg.wait()
# Signal that service cleanup is done:
if not self._done.ready():
self._done.send()
def wait(self):
self._done.wait()
class Services(object):
def __init__(self):
self.services = []
self.tg = threadgroup.ThreadGroup()
self.done = event.Event()
def add(self, service):
self.services.append(service)
self.tg.add_thread(self.run_service, service, self.done)
def stop(self):
# wait for graceful shutdown of services:
for service in self.services:
service.stop()
service.wait()
# Each service has performed cleanup, now signal that the run_service
# wrapper threads can now die:
if not self.done.ready():
self.done.send()
# reap threads:
self.tg.stop()
def wait(self):
self.tg.wait()
def restart(self):
self.stop()
self.done = event.Event()
for restart_service in self.services:
restart_service.reset()
self.tg.add_thread(self.run_service, restart_service, self.done)
@staticmethod
def run_service(service, done):
"""Service start wrapper.
:param service: service to run
:param done: event to wait on until a shutdown is triggered
:returns: None
"""
service.start()
done.wait()
def launch(service, workers=None):
if workers:
launcher = ProcessLauncher()
launcher.launch_service(service, workers=workers)
else:
launcher = ServiceLauncher()
launcher.launch_service(service)
return launcher

91
keystone/openstack/common/strutils.py
View File

@@ -17,6 +17,7 @@
System-level utilities and helper functions. System-level utilities and helper functions.
""" """
import math
import re import re
import sys import sys
import unicodedata import unicodedata
@@ -26,16 +27,21 @@ import six
from keystone.openstack.common.gettextutils import _ from keystone.openstack.common.gettextutils import _
# Used for looking up extensions of text UNIT_PREFIX_EXPONENT = {
# to their 'multiplied' byte amount 'k': 1,
BYTE_MULTIPLIERS = { 'K': 1,
'': 1, 'Ki': 1,
't': 1024 ** 4, 'M': 2,
'g': 1024 ** 3, 'Mi': 2,
'm': 1024 ** 2, 'G': 3,
'k': 1024, 'Gi': 3,
'T': 4,
'Ti': 4,
}
UNIT_SYSTEM_INFO = {
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
} }
BYTE_REGEX = re.compile(r'(^-?\d+)(\D*)')
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
@@ -92,7 +98,8 @@ def bool_from_string(subject, strict=False, default=False):
def safe_decode(text, incoming=None, errors='strict'): def safe_decode(text, incoming=None, errors='strict'):
"""Decodes incoming str using `incoming` if they're not already unicode. """Decodes incoming text/bytes string using `incoming` if they're not
already unicode.
:param incoming: Text's current encoding :param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid :param errors: Errors handling policy. See here for valid
@@ -101,7 +108,7 @@ def safe_decode(text, incoming=None, errors='strict'):
representation of it. representation of it.
:raises TypeError: If text is not an instance of str :raises TypeError: If text is not an instance of str
""" """
if not isinstance(text, six.string_types): if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be decoded" % type(text)) raise TypeError("%s can't be decoded" % type(text))
if isinstance(text, six.text_type): if isinstance(text, six.text_type):
@@ -131,7 +138,7 @@ def safe_decode(text, incoming=None, errors='strict'):
def safe_encode(text, incoming=None, def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'): encoding='utf-8', errors='strict'):
"""Encodes incoming str/unicode using `encoding`. """Encodes incoming text/bytes string using `encoding`.
If incoming is not specified, text is expected to be encoded with If incoming is not specified, text is expected to be encoded with
current python's default encoding. (`sys.getdefaultencoding`) current python's default encoding. (`sys.getdefaultencoding`)
@@ -144,7 +151,7 @@ def safe_encode(text, incoming=None,
representation of it. representation of it.
:raises TypeError: If text is not an instance of str :raises TypeError: If text is not an instance of str
""" """
if not isinstance(text, six.string_types): if not isinstance(text, (six.string_types, six.binary_type)):
raise TypeError("%s can't be encoded" % type(text)) raise TypeError("%s can't be encoded" % type(text))
if not incoming: if not incoming:
@@ -167,34 +174,50 @@ def safe_encode(text, incoming=None,
return text return text
def to_bytes(text, default=0): def string_to_bytes(text, unit_system='IEC', return_int=False):
"""Converts a string into an integer of bytes. """Converts a string into an float representation of bytes.
Looks at the last characters of the text to determine The units supported for IEC ::
what conversion is needed to turn the input text into a byte number.
Supports "B, K(B), M(B), G(B), and T(B)". (case insensitive) Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
KB, KiB, MB, MiB, GB, GiB, TB, TiB
The units supported for SI ::
kb(it), Mb(it), Gb(it), Tb(it)
kB, MB, GB, TB
Note that the SI unit system does not support capital letter 'K'
:param text: String input for bytes size conversion. :param text: String input for bytes size conversion.
:param default: Default return value when text is blank. :param unit_system: Unit system for byte size conversion.
:param return_int: If True, returns integer representation of text
in bytes. (default: decimal)
:returns: Numerical representation of text in bytes.
:raises ValueError: If text has an invalid value.
""" """
match = BYTE_REGEX.search(text) try:
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
except KeyError:
msg = _('Invalid unit system: "%s"') % unit_system
raise ValueError(msg)
match = reg_ex.match(text)
if match: if match:
magnitude = int(match.group(1)) magnitude = float(match.group(1))
mult_key_org = match.group(2) unit_prefix = match.group(2)
if not mult_key_org: if match.group(3) in ['b', 'bit']:
return magnitude magnitude /= 8
elif text:
msg = _('Invalid string format: %s') % text
raise TypeError(msg)
else: else:
return default msg = _('Invalid string format: %s') % text
mult_key = mult_key_org.lower().replace('b', '', 1) raise ValueError(msg)
multiplier = BYTE_MULTIPLIERS.get(mult_key) if not unit_prefix:
if multiplier is None: res = magnitude
msg = _('Unknown byte multiplier: %s') % mult_key_org else:
raise TypeError(msg) res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
return magnitude * multiplier if return_int:
return int(math.ceil(res))
return res
def to_slug(value, incoming=None, errors="strict"): def to_slug(value, incoming=None, errors="strict"):

49
keystone/openstack/common/test.py
View File

@@ -13,14 +13,28 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
##############################################################################
##############################################################################
##
## DO NOT MODIFY THIS FILE
##
## This file is being graduated to the keystonetest library. Please make all
## changes there, and only backport critical fixes here. - dhellmann
##
##############################################################################
##############################################################################
"""Common utilities used in testing""" """Common utilities used in testing"""
import logging
import os import os
import tempfile
import fixtures import fixtures
import testtools import testtools
_TRUE_VALUES = ('True', 'true', '1', 'yes') _TRUE_VALUES = ('True', 'true', '1', 'yes')
_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s"
class BaseTestCase(testtools.TestCase): class BaseTestCase(testtools.TestCase):
@@ -29,9 +43,10 @@ class BaseTestCase(testtools.TestCase):
super(BaseTestCase, self).setUp() super(BaseTestCase, self).setUp()
self._set_timeout() self._set_timeout()
self._fake_output() self._fake_output()
self.useFixture(fixtures.FakeLogger('keystone.openstack.common')) self._fake_logs()
self.useFixture(fixtures.NestedTempfile()) self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir()) self.useFixture(fixtures.TempHomeDir())
self.tempdirs = []
def _set_timeout(self): def _set_timeout(self):
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
@@ -50,3 +65,35 @@ class BaseTestCase(testtools.TestCase):
if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
stderr = self.useFixture(fixtures.StringStream('stderr')).stream stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
def _fake_logs(self):
if os.environ.get('OS_DEBUG') in _TRUE_VALUES:
level = logging.DEBUG
else:
level = logging.INFO
capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES
if capture_logs:
self.useFixture(
fixtures.FakeLogger(
format=_LOG_FORMAT,
level=level,
nuke_handlers=capture_logs,
)
)
else:
logging.basicConfig(format=_LOG_FORMAT, level=level)
def create_tempfiles(self, files, ext='.conf'):
tempfiles = []
for (basename, contents) in files:
if not os.path.isabs(basename):
(fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext)
else:
path = basename + ext
fd = os.open(path, os.O_CREAT | os.O_WRONLY)
tempfiles.append(path)
try:
os.write(fd, contents)
finally:
os.close(fd)
return tempfiles

123
keystone/openstack/common/threadgroup.py
View File

@@ -1,123 +0,0 @@
# Copyright 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import eventlet
from eventlet import greenpool
from eventlet import greenthread
from keystone.openstack.common import log as logging
from keystone.openstack.common import loopingcall
LOG = logging.getLogger(__name__)
def _thread_done(gt, *args, **kwargs):
"""Callback function to be passed to GreenThread.link() when we spawn()
Calls the :class:`ThreadGroup` to notify if.
"""
kwargs['group'].thread_done(kwargs['thread'])
class Thread(object):
"""Wrapper around a greenthread, that holds a reference to the
:class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
it has done so it can be removed from the threads list.
"""
def __init__(self, thread, group):
self.thread = thread
self.thread.link(_thread_done, group=group, thread=self)
def stop(self):
self.thread.kill()
def wait(self):
return self.thread.wait()
def link(self, func, *args, **kwargs):
self.thread.link(func, *args, **kwargs)
class ThreadGroup(object):
"""The point of the ThreadGroup classis to:
* keep track of timers and greenthreads (making it easier to stop them
when need be).
* provide an easy API to add timers.
"""
def __init__(self, thread_pool_size=10):
self.pool = greenpool.GreenPool(thread_pool_size)
self.threads = []
self.timers = []
def add_dynamic_timer(self, callback, initial_delay=None,
periodic_interval_max=None, *args, **kwargs):
timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs)
timer.start(initial_delay=initial_delay,
periodic_interval_max=periodic_interval_max)
self.timers.append(timer)
def add_timer(self, interval, callback, initial_delay=None,
*args, **kwargs):
pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs)
pulse.start(interval=interval,
initial_delay=initial_delay)
self.timers.append(pulse)
def add_thread(self, callback, *args, **kwargs):
gt = self.pool.spawn(callback, *args, **kwargs)
th = Thread(gt, self)
self.threads.append(th)
return th
def thread_done(self, thread):
self.threads.remove(thread)
def stop(self):
current = greenthread.getcurrent()
for x in self.threads:
if x is current:
# don't kill the current thread.
continue
try:
x.stop()
except Exception as ex:
LOG.exception(ex)
for x in self.timers:
try:
x.stop()
except Exception as ex:
LOG.exception(ex)
self.timers = []
def wait(self):
for x in self.timers:
try:
x.wait()
except eventlet.greenlet.GreenletExit:
pass
except Exception as ex:
LOG.exception(ex)
current = greenthread.getcurrent()
for x in self.threads:
if x is current:
continue
try:
x.wait()
except eventlet.greenlet.GreenletExit:
pass
except Exception as ex:
LOG.exception(ex)

12
keystone/openstack/common/timeutils.py
View File

@@ -77,6 +77,9 @@ def is_older_than(before, seconds):
"""Return True if before is older than seconds.""" """Return True if before is older than seconds."""
if isinstance(before, six.string_types): if isinstance(before, six.string_types):
before = parse_strtime(before).replace(tzinfo=None) before = parse_strtime(before).replace(tzinfo=None)
else:
before = before.replace(tzinfo=None)
return utcnow() - before > datetime.timedelta(seconds=seconds) return utcnow() - before > datetime.timedelta(seconds=seconds)
@@ -84,6 +87,9 @@ def is_newer_than(after, seconds):
"""Return True if after is newer than seconds.""" """Return True if after is newer than seconds."""
if isinstance(after, six.string_types): if isinstance(after, six.string_types):
after = parse_strtime(after).replace(tzinfo=None) after = parse_strtime(after).replace(tzinfo=None)
else:
after = after.replace(tzinfo=None)
return after - utcnow() > datetime.timedelta(seconds=seconds) return after - utcnow() > datetime.timedelta(seconds=seconds)
@@ -108,7 +114,7 @@ def utcnow():
def iso8601_from_timestamp(timestamp): def iso8601_from_timestamp(timestamp):
"""Returns a iso8601 formated date from timestamp.""" """Returns a iso8601 formatted date from timestamp."""
return isotime(datetime.datetime.utcfromtimestamp(timestamp)) return isotime(datetime.datetime.utcfromtimestamp(timestamp))
@@ -195,8 +201,8 @@ def total_seconds(delta):
def is_soon(dt, window): def is_soon(dt, window):
"""Determines if time is going to happen in the next window seconds. """Determines if time is going to happen in the next window seconds.
:params dt: the time :param dt: the time
:params window: minimum seconds to remain to consider the time not soon :param window: minimum seconds to remain to consider the time not soon
:return: True if expiration is within the given duration :return: True if expiration is within the given duration
""" """

39
keystone/openstack/common/uuidutils.py
View File

@@ -1,39 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2012 Intel Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
UUID related utilities and helper functions.
"""
import uuid
def generate_uuid():
return str(uuid.uuid4())
def is_uuid_like(val):
"""Returns validation of a value as a UUID.
For our purposes, a UUID is a canonical form string:
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
"""
try:
return str(uuid.UUID(val)) == val
except (TypeError, ValueError, AttributeError):
return False

12
keystone/openstack/common/versionutils.py
View File

@@ -30,19 +30,19 @@ LOG = logging.getLogger(__name__)
class deprecated(object): class deprecated(object):
"""A decorator to mark callables as deprecated. """A decorator to mark callables as deprecated.
deprecated logs a deprecation message when the callable it decorates This decorator logs a deprecation message when the callable it decorates is
is used. The message will include the release where the callable was used. The message will include the release where the callable was
deprecated, the release where is may be removed and possibly an deprecated, the release where it may be removed and possibly an optional
optional replacement. replacement.
Examples: Examples:
1. Specifing the required deprecated release 1. Specifying the required deprecated release
>>> @deprecated(as_of=deprecated.ICEHOUSE) >>> @deprecated(as_of=deprecated.ICEHOUSE)
... def a(): pass ... def a(): pass
2. Specifing a replacement: 2. Specifying a replacement:
>>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()') >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()')
... def b(): pass ... def b(): pass

3
openstack-common.conf
View File

@@ -6,14 +6,15 @@ module=db.sqlalchemy
module=config module=config
module=colorizer module=colorizer
module=fixture module=fixture
module=gettextutils
module=importutils module=importutils
module=install_venv_common module=install_venv_common
module=jsonutils module=jsonutils
module=log module=log
module=log_handler
module=policy module=policy
module=strutils module=strutils
module=timeutils module=timeutils
module=versionutils
# The base module to hold the copy of openstack.common # The base module to hold the copy of openstack.common
base=keystone base=keystone

1
requirements.txt
View File

@@ -21,3 +21,4 @@ oauthlib>=0.6
dogpile.cache>=0.5.0 dogpile.cache>=0.5.0
jsonschema>=2.0.0,<3.0.0 jsonschema>=2.0.0,<3.0.0
pycadf>=0.4.1 pycadf>=0.4.1
posix_ipc

2
tools/config/check_uptodate.sh
View File

@@ -20,6 +20,6 @@ tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
then then
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
echo "${0##*/}: Please run 'tox -esample_config -r' (see doc/source/developing.rst for more info)" echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
exit 1 exit 1
fi fi

2
tools/config/generate_sample.sh
View File

@@ -65,7 +65,7 @@ then
BASEDIR=$(cd "$BASEDIR" && pwd) BASEDIR=$(cd "$BASEDIR" && pwd)
fi fi
PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}} PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)}
TARGETDIR=$BASEDIR/$PACKAGENAME TARGETDIR=$BASEDIR/$PACKAGENAME
if ! [ -d $TARGETDIR ] if ! [ -d $TARGETDIR ]
then then