From a322a95ab11b3ad45fe870045c0fa7e17f4ed949 Mon Sep 17 00:00:00 2001 From: Thomas Herve Date: Mon, 8 Dec 2014 18:35:33 +0100 Subject: [PATCH] Sync oslo modules Synchronize oslo modules, removing unsed gettextutils and fixing tests for policy module change. Change-Id: I2fbb3668522eabea9a6408ccd2f3c3d76645e2ee --- heat/common/i18n.py | 12 - heat/common/policy.py | 5 +- heat/openstack/common/_i18n.py | 43 +- heat/openstack/common/context.py | 8 +- heat/openstack/common/crypto/utils.py | 13 + heat/openstack/common/eventlet_backdoor.py | 7 + heat/openstack/common/fileutils.py | 3 +- heat/openstack/common/gettextutils.py | 498 --------------------- heat/openstack/common/log.py | 28 +- heat/openstack/common/policy.py | 162 +++++-- heat/openstack/common/threadgroup.py | 2 + heat/tests/test_common_context.py | 6 +- heat/tests/test_common_policy.py | 79 ++-- openstack-common.conf | 1 - 14 files changed, 223 insertions(+), 644 deletions(-) delete mode 100644 heat/openstack/common/gettextutils.py diff --git a/heat/common/i18n.py b/heat/common/i18n.py index b02198fc8..9a49bbeb3 100644 --- a/heat/common/i18n.py +++ b/heat/common/i18n.py @@ -18,8 +18,6 @@ from oslo import i18n -from heat.openstack.common import gettextutils - _translators = i18n.TranslatorFactory(domain='heat') @@ -35,13 +33,3 @@ _LI = _translators.log_info _LW = _translators.log_warning _LE = _translators.log_error _LC = _translators.log_critical - -# FIXME(elynn): Parts in oslo-incubator are still using gettextutils._(), etc., -# from oslo-incubator. Until these parts are changed to use oslo.i18n, we'll -# monkey-patch gettextutils._(), _LI(), etc., to use our oslo.i18n versions. -gettextutils._ = _ -gettextutils._LI = _LI -gettextutils._LW = _LW -gettextutils._LE = _LE -gettextutils._LC = _LC -gettextutils.install = i18n.install diff --git a/heat/common/policy.py b/heat/common/policy.py index 01a4338c7..9e97777ca 100644 --- a/heat/common/policy.py +++ b/heat/common/policy.py @@ -34,11 +34,12 @@ class Enforcer(object): """Responsible for loading and enforcing rules.""" def __init__(self, scope='heat', exc=exception.Forbidden, - default_rule=DEFAULT_RULES['default']): + default_rule=DEFAULT_RULES['default'], policy_file=None): self.scope = scope self.exc = exc self.default_rule = default_rule - self.enforcer = policy.Enforcer(default_rule=default_rule) + self.enforcer = policy.Enforcer( + default_rule=default_rule, policy_file=policy_file) def set_rules(self, rules, overwrite=True): """Create a new Rules object based on the provided dict of rules.""" diff --git a/heat/openstack/common/_i18n.py b/heat/openstack/common/_i18n.py index 762a789d7..15ccf58b8 100644 --- a/heat/openstack/common/_i18n.py +++ b/heat/openstack/common/_i18n.py @@ -16,25 +16,30 @@ See http://docs.openstack.org/developer/oslo.i18n/usage.html """ -import oslo.i18n +try: + import oslo.i18n + # NOTE(dhellmann): This reference to o-s-l-o will be replaced by the + # application name when this module is synced into the separate + # repository. It is OK to have more than one translation function + # using the same domain, since there will still only be one message + # catalog. + _translators = oslo.i18n.TranslatorFactory(domain='heat') -# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the -# application name when this module is synced into the separate -# repository. It is OK to have more than one translation function -# using the same domain, since there will still only be one message -# catalog. -_translators = oslo.i18n.TranslatorFactory(domain='heat') + # The primary translation function using the well-known name "_" + _ = _translators.primary -# The primary translation function using the well-known name "_" -_ = _translators.primary - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = _translators.log_info -_LW = _translators.log_warning -_LE = _translators.log_error -_LC = _translators.log_critical + # Translators for log levels. + # + # The abbreviated names are meant to reflect the usual use of a short + # name like '_'. The "L" is for "log" and the other letter comes from + # the level. + _LI = _translators.log_info + _LW = _translators.log_warning + _LE = _translators.log_error + _LC = _translators.log_critical +except ImportError: + # NOTE(dims): Support for cases where a project wants to use + # code from heat-incubator, but is not ready to be internationalized + # (like tempest) + _ = _LI = _LW = _LE = _LC = lambda x: x diff --git a/heat/openstack/common/context.py b/heat/openstack/common/context.py index b612db714..168989004 100644 --- a/heat/openstack/common/context.py +++ b/heat/openstack/common/context.py @@ -117,10 +117,6 @@ def get_context_from_function_and_args(function, args, kwargs): def is_user_context(context): """Indicates if the request context is a normal user.""" - if not context: + if not context or context.is_admin: return False - if context.is_admin: - return False - if not context.user_id or not context.project_id: - return False - return True + return context.user_id and context.project_id diff --git a/heat/openstack/common/crypto/utils.py b/heat/openstack/common/crypto/utils.py index 0dca18e26..c917d2ebc 100644 --- a/heat/openstack/common/crypto/utils.py +++ b/heat/openstack/common/crypto/utils.py @@ -12,6 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. +######################################################################## +# +# THIS MODULE IS DEPRECATED +# +# Please refer to +# https://etherpad.openstack.org/p/kilo-heat-library-proposals for +# the discussion leading to this deprecation. +# +# We recommend checking out Barbican or the cryptography.py project +# (https://pypi.python.org/pypi/cryptography) instead of this module. +# +######################################################################## + import base64 from Crypto.Hash import HMAC diff --git a/heat/openstack/common/eventlet_backdoor.py b/heat/openstack/common/eventlet_backdoor.py index a19ca59d6..1a67cd78d 100644 --- a/heat/openstack/common/eventlet_backdoor.py +++ b/heat/openstack/common/eventlet_backdoor.py @@ -16,6 +16,7 @@ from __future__ import print_function +import copy import errno import gc import os @@ -49,6 +50,12 @@ CONF.register_opts(eventlet_backdoor_opts) LOG = logging.getLogger(__name__) +def list_opts(): + """Entry point for oslo.config-generator. + """ + return [(None, copy.deepcopy(eventlet_backdoor_opts))] + + class EventletBackdoorConfigValueError(Exception): def __init__(self, port_range, help_msg, ex): msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. ' diff --git a/heat/openstack/common/fileutils.py b/heat/openstack/common/fileutils.py index 59088f643..ec26eaf9c 100644 --- a/heat/openstack/common/fileutils.py +++ b/heat/openstack/common/fileutils.py @@ -15,13 +15,12 @@ import contextlib import errno +import logging import os import tempfile from oslo.utils import excutils -from heat.openstack.common import log as logging - LOG = logging.getLogger(__name__) _FILE_CACHE = {} diff --git a/heat/openstack/common/gettextutils.py b/heat/openstack/common/gettextutils.py deleted file mode 100644 index 1fa18df06..000000000 --- a/heat/openstack/common/gettextutils.py +++ /dev/null @@ -1,498 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# Copyright 2013 IBM Corp. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from heat.openstack.common.gettextutils import _ -""" - -import copy -import functools -import gettext -import locale -from logging import handlers -import os - -from babel import localedata -import six - -_AVAILABLE_LANGUAGES = {} - -# FIXME(dhellmann): Remove this when moving to oslo.i18n. -USE_LAZY = False - - -class TranslatorFactory(object): - """Create translator functions - """ - - def __init__(self, domain, lazy=False, localedir=None): - """Establish a set of translation functions for the domain. - - :param domain: Name of translation domain, - specifying a message catalog. - :type domain: str - :param lazy: Delays translation until a message is emitted. - Defaults to False. - :type lazy: Boolean - :param localedir: Directory with translation catalogs. - :type localedir: str - """ - self.domain = domain - self.lazy = lazy - if localedir is None: - localedir = os.environ.get(domain.upper() + '_LOCALEDIR') - self.localedir = localedir - - def _make_translation_func(self, domain=None): - """Return a new translation function ready for use. - - Takes into account whether or not lazy translation is being - done. - - The domain can be specified to override the default from the - factory, but the localedir from the factory is always used - because we assume the log-level translation catalogs are - installed in the same directory as the main application - catalog. - - """ - if domain is None: - domain = self.domain - if self.lazy: - return functools.partial(Message, domain=domain) - t = gettext.translation( - domain, - localedir=self.localedir, - fallback=True, - ) - if six.PY3: - return t.gettext - return t.ugettext - - @property - def primary(self): - "The default translation function." - return self._make_translation_func() - - def _make_log_translation_func(self, level): - return self._make_translation_func(self.domain + '-log-' + level) - - @property - def log_info(self): - "Translate info-level log messages." - return self._make_log_translation_func('info') - - @property - def log_warning(self): - "Translate warning-level log messages." - return self._make_log_translation_func('warning') - - @property - def log_error(self): - "Translate error-level log messages." - return self._make_log_translation_func('error') - - @property - def log_critical(self): - "Translate critical-level log messages." - return self._make_log_translation_func('critical') - - -# NOTE(dhellmann): When this module moves out of the incubator into -# oslo.i18n, these global variables can be moved to an integration -# module within each application. - -# Create the global translation functions. -_translators = TranslatorFactory('heat') - -# The primary translation function using the well-known name "_" -_ = _translators.primary - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = _translators.log_info -_LW = _translators.log_warning -_LE = _translators.log_error -_LC = _translators.log_critical - -# NOTE(dhellmann): End of globals that will move to the application's -# integration module. - - -def enable_lazy(): - """Convenience function for configuring _() to use lazy gettext - - Call this at the start of execution to enable the gettextutils._ - function to use lazy gettext functionality. This is useful if - your project is importing _ directly instead of using the - gettextutils.install() way of importing the _ function. - """ - # FIXME(dhellmann): This function will be removed in oslo.i18n, - # because the TranslatorFactory makes it superfluous. - global _, _LI, _LW, _LE, _LC, USE_LAZY - tf = TranslatorFactory('heat', lazy=True) - _ = tf.primary - _LI = tf.log_info - _LW = tf.log_warning - _LE = tf.log_error - _LC = tf.log_critical - USE_LAZY = True - - -def install(domain, lazy=False): - """Install a _() function using the given translation domain. - - Given a translation domain, install a _() function using gettext's - install() function. - - The main difference from gettext.install() is that we allow - overriding the default localedir (e.g. /usr/share/locale) using - a translation-domain-specific environment variable (e.g. - NOVA_LOCALEDIR). - - :param domain: the translation domain - :param lazy: indicates whether or not to install the lazy _() function. - The lazy _() introduces a way to do deferred translation - of messages by installing a _ that builds Message objects, - instead of strings, which can then be lazily translated into - any available locale. - """ - if lazy: - from six import moves - tf = TranslatorFactory(domain, lazy=True) - moves.builtins.__dict__['_'] = tf.primary - else: - localedir = '%s_LOCALEDIR' % domain.upper() - if six.PY3: - gettext.install(domain, - localedir=os.environ.get(localedir)) - else: - gettext.install(domain, - localedir=os.environ.get(localedir), - unicode=True) - - -class Message(six.text_type): - """A Message object is a unicode object that can be translated. - - Translation of Message is done explicitly using the translate() method. - For all non-translation intents and purposes, a Message is simply unicode, - and can be treated as such. - """ - - def __new__(cls, msgid, msgtext=None, params=None, - domain='heat', *args): - """Create a new Message object. - - In order for translation to work gettext requires a message ID, this - msgid will be used as the base unicode text. It is also possible - for the msgid and the base unicode text to be different by passing - the msgtext parameter. - """ - # If the base msgtext is not given, we use the default translation - # of the msgid (which is in English) just in case the system locale is - # not English, so that the base text will be in that locale by default. - if not msgtext: - msgtext = Message._translate_msgid(msgid, domain) - # We want to initialize the parent unicode with the actual object that - # would have been plain unicode if 'Message' was not enabled. - msg = super(Message, cls).__new__(cls, msgtext) - msg.msgid = msgid - msg.domain = domain - msg.params = params - return msg - - def translate(self, desired_locale=None): - """Translate this message to the desired locale. - - :param desired_locale: The desired locale to translate the message to, - if no locale is provided the message will be - translated to the system's default locale. - - :returns: the translated message in unicode - """ - - translated_message = Message._translate_msgid(self.msgid, - self.domain, - desired_locale) - if self.params is None: - # No need for more translation - return translated_message - - # This Message object may have been formatted with one or more - # Message objects as substitution arguments, given either as a single - # argument, part of a tuple, or as one or more values in a dictionary. - # When translating this Message we need to translate those Messages too - translated_params = _translate_args(self.params, desired_locale) - - translated_message = translated_message % translated_params - - return translated_message - - @staticmethod - def _translate_msgid(msgid, domain, desired_locale=None): - if not desired_locale: - system_locale = locale.getdefaultlocale() - # If the system locale is not available to the runtime use English - if not system_locale[0]: - desired_locale = 'en_US' - else: - desired_locale = system_locale[0] - - locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR') - lang = gettext.translation(domain, - localedir=locale_dir, - languages=[desired_locale], - fallback=True) - if six.PY3: - translator = lang.gettext - else: - translator = lang.ugettext - - translated_message = translator(msgid) - return translated_message - - def __mod__(self, other): - # When we mod a Message we want the actual operation to be performed - # by the parent class (i.e. unicode()), the only thing we do here is - # save the original msgid and the parameters in case of a translation - params = self._sanitize_mod_params(other) - unicode_mod = super(Message, self).__mod__(params) - modded = Message(self.msgid, - msgtext=unicode_mod, - params=params, - domain=self.domain) - return modded - - def _sanitize_mod_params(self, other): - """Sanitize the object being modded with this Message. - - - Add support for modding 'None' so translation supports it - - Trim the modded object, which can be a large dictionary, to only - those keys that would actually be used in a translation - - Snapshot the object being modded, in case the message is - translated, it will be used as it was when the Message was created - """ - if other is None: - params = (other,) - elif isinstance(other, dict): - # Merge the dictionaries - # Copy each item in case one does not support deep copy. - params = {} - if isinstance(self.params, dict): - for key, val in self.params.items(): - params[key] = self._copy_param(val) - for key, val in other.items(): - params[key] = self._copy_param(val) - else: - params = self._copy_param(other) - return params - - def _copy_param(self, param): - try: - return copy.deepcopy(param) - except Exception: - # Fallback to casting to unicode this will handle the - # python code-like objects that can't be deep-copied - return six.text_type(param) - - def __add__(self, other): - msg = _('Message objects do not support addition.') - raise TypeError(msg) - - def __radd__(self, other): - return self.__add__(other) - - if six.PY2: - def __str__(self): - # NOTE(luisg): Logging in python 2.6 tries to str() log records, - # and it expects specifically a UnicodeError in order to proceed. - msg = _('Message objects do not support str() because they may ' - 'contain non-ascii characters. ' - 'Please use unicode() or translate() instead.') - raise UnicodeError(msg) - - -def get_available_languages(domain): - """Lists the available languages for the given translation domain. - - :param domain: the domain to get languages for - """ - if domain in _AVAILABLE_LANGUAGES: - return copy.copy(_AVAILABLE_LANGUAGES[domain]) - - localedir = '%s_LOCALEDIR' % domain.upper() - find = lambda x: gettext.find(domain, - localedir=os.environ.get(localedir), - languages=[x]) - - # NOTE(mrodden): en_US should always be available (and first in case - # order matters) since our in-line message strings are en_US - language_list = ['en_US'] - # NOTE(luisg): Babel <1.0 used a function called list(), which was - # renamed to locale_identifiers() in >=1.0, the requirements master list - # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and update all projects - list_identifiers = (getattr(localedata, 'list', None) or - getattr(localedata, 'locale_identifiers')) - locale_identifiers = list_identifiers() - - for i in locale_identifiers: - if find(i) is not None: - language_list.append(i) - - # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported - # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they - # are perfectly legitimate locales: - # https://github.com/mitsuhiko/babel/issues/37 - # In Babel 1.3 they fixed the bug and they support these locales, but - # they are still not explicitly "listed" by locale_identifiers(). - # That is why we add the locales here explicitly if necessary so that - # they are listed as supported. - aliases = {'zh': 'zh_CN', - 'zh_Hant_HK': 'zh_HK', - 'zh_Hant': 'zh_TW', - 'fil': 'tl_PH'} - for (locale_, alias) in six.iteritems(aliases): - if locale_ in language_list and alias not in language_list: - language_list.append(alias) - - _AVAILABLE_LANGUAGES[domain] = language_list - return copy.copy(language_list) - - -def translate(obj, desired_locale=None): - """Gets the translated unicode representation of the given object. - - If the object is not translatable it is returned as-is. - If the locale is None the object is translated to the system locale. - - :param obj: the object to translate - :param desired_locale: the locale to translate the message to, if None the - default system locale will be used - :returns: the translated object in unicode, or the original object if - it could not be translated - """ - message = obj - if not isinstance(message, Message): - # If the object to translate is not already translatable, - # let's first get its unicode representation - message = six.text_type(obj) - if isinstance(message, Message): - # Even after unicoding() we still need to check if we are - # running with translatable unicode before translating - return message.translate(desired_locale) - return obj - - -def _translate_args(args, desired_locale=None): - """Translates all the translatable elements of the given arguments object. - - This method is used for translating the translatable values in method - arguments which include values of tuples or dictionaries. - If the object is not a tuple or a dictionary the object itself is - translated if it is translatable. - - If the locale is None the object is translated to the system locale. - - :param args: the args to translate - :param desired_locale: the locale to translate the args to, if None the - default system locale will be used - :returns: a new args object with the translated contents of the original - """ - if isinstance(args, tuple): - return tuple(translate(v, desired_locale) for v in args) - if isinstance(args, dict): - translated_dict = {} - for (k, v) in six.iteritems(args): - translated_v = translate(v, desired_locale) - translated_dict[k] = translated_v - return translated_dict - return translate(args, desired_locale) - - -class TranslationHandler(handlers.MemoryHandler): - """Handler that translates records before logging them. - - The TranslationHandler takes a locale and a target logging.Handler object - to forward LogRecord objects to after translating them. This handler - depends on Message objects being logged, instead of regular strings. - - The handler can be configured declaratively in the logging.conf as follows: - - [handlers] - keys = translatedlog, translator - - [handler_translatedlog] - class = handlers.WatchedFileHandler - args = ('/var/log/api-localized.log',) - formatter = context - - [handler_translator] - class = openstack.common.log.TranslationHandler - target = translatedlog - args = ('zh_CN',) - - If the specified locale is not available in the system, the handler will - log in the default locale. - """ - - def __init__(self, locale=None, target=None): - """Initialize a TranslationHandler - - :param locale: locale to use for translating messages - :param target: logging.Handler object to forward - LogRecord objects to after translation - """ - # NOTE(luisg): In order to allow this handler to be a wrapper for - # other handlers, such as a FileHandler, and still be able to - # configure it using logging.conf, this handler has to extend - # MemoryHandler because only the MemoryHandlers' logging.conf - # parsing is implemented such that it accepts a target handler. - handlers.MemoryHandler.__init__(self, capacity=0, target=target) - self.locale = locale - - def setFormatter(self, fmt): - self.target.setFormatter(fmt) - - def emit(self, record): - # We save the message from the original record to restore it - # after translation, so other handlers are not affected by this - original_msg = record.msg - original_args = record.args - - try: - self._translate_and_log_record(record) - finally: - record.msg = original_msg - record.args = original_args - - def _translate_and_log_record(self, record): - record.msg = translate(record.msg, self.locale) - - # In addition to translating the message, we also need to translate - # arguments that were passed to the log method that were not part - # of the main message e.g., log.info(_('Some message %s'), this_one)) - record.args = _translate_args(record.args, self.locale) - - self.target.emit(record) diff --git a/heat/openstack/common/log.py b/heat/openstack/common/log.py index efdca795b..fd3315190 100644 --- a/heat/openstack/common/log.py +++ b/heat/openstack/common/log.py @@ -27,6 +27,7 @@ It also allows setting of formatting information through conf. """ +import copy import inspect import itertools import logging @@ -174,6 +175,16 @@ CONF.register_cli_opts(logging_cli_opts) CONF.register_opts(generic_log_opts) CONF.register_opts(log_opts) + +def list_opts(): + """Entry point for oslo.config-generator.""" + return [(None, copy.deepcopy(common_cli_opts)), + (None, copy.deepcopy(logging_cli_opts)), + (None, copy.deepcopy(generic_log_opts)), + (None, copy.deepcopy(log_opts)), + ] + + # our new audit level # NOTE(jkoelker) Since we synthesized an audit level, make the logging # module aware of it so it acts like other levels. @@ -498,14 +509,9 @@ def _setup_logging_from_conf(project, version): log_root.addHandler(streamlog) if CONF.publish_errors: - try: - handler = importutils.import_object( - "heat.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) - except ImportError: - handler = importutils.import_object( - "oslo.messaging.notify.log_handler.PublishErrorsHandler", - logging.ERROR) + handler = importutils.import_object( + "oslo.messaging.notify.log_handler.PublishErrorsHandler", + logging.ERROR) log_root.addHandler(handler) datefmt = CONF.log_date_format @@ -546,9 +552,11 @@ def _setup_logging_from_conf(project, version): # TODO(bogdando) use the format provided by RFCSysLogHandler # after existing syslog format deprecation in J if CONF.use_syslog_rfc_format: - syslog = RFCSysLogHandler(facility=facility) + syslog = RFCSysLogHandler(address='/dev/log', + facility=facility) else: - syslog = logging.handlers.SysLogHandler(facility=facility) + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) log_root.addHandler(syslog) except socket.error: log_root.error('Unable to add syslog handler. Verify that syslog ' diff --git a/heat/openstack/common/policy.py b/heat/openstack/common/policy.py index de48dceae..afeac2970 100644 --- a/heat/openstack/common/policy.py +++ b/heat/openstack/common/policy.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# # Copyright (c) 2012 OpenStack Foundation. # All Rights Reserved. # @@ -22,22 +24,43 @@ string written in the new policy language. In the list-of-lists representation, each check inside the innermost list is combined as with an "and" conjunction--for that check to pass, all the specified checks must pass. These innermost lists are then -combined as with an "or" conjunction. This is the original way of -expressing policies, but there now exists a new way: the policy -language. - -In the policy language, each check is specified the same way as in the -list-of-lists representation: a simple "a:b" pair that is matched to -the correct code to perform that check. However, conjunction -operators are available, allowing for more expressiveness in crafting -policies. - -As an example, take the following rule, expressed in the list-of-lists -representation:: +combined as with an "or" conjunction. As an example, take the following +rule, expressed in the list-of-lists representation:: [["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]] -In the policy language, this becomes:: +This is the original way of expressing policies, but there now exists a +new way: the policy language. + +In the policy language, each check is specified the same way as in the +list-of-lists representation: a simple "a:b" pair that is matched to +the correct class to perform that check:: + + +===========================================================================+ + | TYPE | SYNTAX | + +===========================================================================+ + |User's Role | role:admin | + +---------------------------------------------------------------------------+ + |Rules already defined on policy | rule:admin_required | + +---------------------------------------------------------------------------+ + |Against URL's¹ | http://my-url.org/check | + +---------------------------------------------------------------------------+ + |User attributes² | project_id:%(target.project.id)s | + +---------------------------------------------------------------------------+ + |Strings | :'xpto2035abc' | + | | 'myproject': | + +---------------------------------------------------------------------------+ + | | project_id:xpto2035abc | + |Literals | domain_id:20 | + | | True:%(user.enabled)s | + +===========================================================================+ + +¹URL checking must return 'True' to be valid +²User attributes (obtained through the token): user_id, domain_id or project_id + +Conjunction operators are available, allowing for more expressiveness +in crafting policies. So, in the policy language, the previous check in +list-of-lists becomes:: role:admin or (project_id:%(project_id)s and role:projectadmin) @@ -46,26 +69,16 @@ policy rule:: project_id:%(project_id)s and not role:dunce -It is possible to perform policy checks on the following user -attributes (obtained through the token): user_id, domain_id or -project_id:: - - domain_id: - Attributes sent along with API calls can be used by the policy engine (on the right side of the expression), by using the following syntax:: - :user.id + :%(user.id)s Contextual attributes of objects identified by their IDs are loaded from the database. They are also available to the policy engine and can be checked through the `target` keyword:: - :target.role.name - -All these attributes (related to users, API calls, and context) can be -checked against each other or against constants, be it literals (True, -) or strings. + :%(target.role.name)s Finally, two special policy checks should be mentioned; the policy check "@" will always accept an access, and the policy check "!" will @@ -77,16 +90,18 @@ as it allows particular rules to be explicitly disabled. import abc import ast +import copy +import os import re from oslo.config import cfg +from oslo.serialization import jsonutils import six import six.moves.urllib.parse as urlparse import six.moves.urllib.request as urlrequest from heat.openstack.common import fileutils -from heat.openstack.common.gettextutils import _, _LE -from heat.openstack.common import jsonutils +from heat.openstack.common._i18n import _, _LE, _LI from heat.openstack.common import log as logging @@ -98,6 +113,14 @@ policy_opts = [ default='default', help=_('Default rule. Enforced when a requested rule is not ' 'found.')), + cfg.MultiStrOpt('policy_dirs', + default=['policy.d'], + help=_('Directories where policy configuration files are ' + 'stored. They can be relative to any directory ' + 'in the search path defined by the config_dir ' + 'option, or absolute paths. The file defined by ' + 'policy_file must exist for these directories to ' + 'be searched.')), ] CONF = cfg.CONF @@ -108,6 +131,11 @@ LOG = logging.getLogger(__name__) _checks = {} +def list_opts(): + """Entry point for oslo.config-generator.""" + return [(None, copy.deepcopy(policy_opts))] + + class PolicyNotAuthorized(Exception): def __init__(self, rule): @@ -184,16 +212,19 @@ class Enforcer(object): :param default_rule: Default rule to use, CONF.default_rule will be used if none is specified. :param use_conf: Whether to load rules from cache or config file. + :param overwrite: Whether to overwrite existing rules when reload rules + from config file. """ def __init__(self, policy_file=None, rules=None, - default_rule=None, use_conf=True): - self.rules = Rules(rules, default_rule) + default_rule=None, use_conf=True, overwrite=True): self.default_rule = default_rule or CONF.policy_default_rule + self.rules = Rules(rules, self.default_rule) self.policy_path = None self.policy_file = policy_file or CONF.policy_file self.use_conf = use_conf + self.overwrite = overwrite def set_rules(self, rules, overwrite=True, use_conf=False): """Create a new Rules object based on the provided dict of rules. @@ -216,6 +247,7 @@ class Enforcer(object): def clear(self): """Clears Enforcer rules, policy's cache and policy's path.""" self.set_rules({}) + fileutils.delete_cached_file(self.policy_path) self.default_rule = None self.policy_path = None @@ -224,7 +256,7 @@ class Enforcer(object): Policy file is cached and will be reloaded if modified. - :param force_reload: Whether to overwrite current rules. + :param force_reload: Whether to reload rules from config file. """ if force_reload: @@ -232,31 +264,55 @@ class Enforcer(object): if self.use_conf: if not self.policy_path: - self.policy_path = self._get_policy_path() + self.policy_path = self._get_policy_path(self.policy_file) + self._load_policy_file(self.policy_path, force_reload, + overwrite=self.overwrite) + for path in CONF.policy_dirs: + try: + path = self._get_policy_path(path) + except cfg.ConfigFilesNotFoundError: + LOG.info(_LI("Can not find policy directory: %s"), path) + continue + self._walk_through_policy_directory(path, + self._load_policy_file, + force_reload, False) + + @staticmethod + def _walk_through_policy_directory(path, func, *args): + # We do not iterate over sub-directories. + policy_files = next(os.walk(path))[2] + policy_files.sort() + for policy_file in [p for p in policy_files if not p.startswith('.')]: + func(os.path.join(path, policy_file), *args) + + def _load_policy_file(self, path, force_reload, overwrite=True): reloaded, data = fileutils.read_cached_file( - self.policy_path, force_reload=force_reload) - if reloaded or not self.rules: + path, force_reload=force_reload) + if reloaded or not self.rules or not overwrite: rules = Rules.load_json(data, self.default_rule) - self.set_rules(rules) + self.set_rules(rules, overwrite=overwrite, use_conf=True) LOG.debug("Rules successfully reloaded") - def _get_policy_path(self): - """Locate the policy json data file. + def _get_policy_path(self, path): + """Locate the policy json data file/path. - :param policy_file: Custom policy file to locate. + :param path: It's value can be a full path or related path. When + full path specified, this function just returns the full + path. When related path specified, this function will + search configuration directories to find one that exists. :returns: The policy path - :raises: ConfigFilesNotFoundError if the file couldn't + :raises: ConfigFilesNotFoundError if the file/path couldn't be located. """ - policy_file = CONF.find_file(self.policy_file) + policy_path = CONF.find_file(path) - if policy_file: - return policy_file + if policy_path: + return policy_path - raise cfg.ConfigFilesNotFoundError((self.policy_file,)) + raise cfg.ConfigFilesNotFoundError((path,)) def enforce(self, rule, target, creds, do_raise=False, exc=None, *args, **kwargs): @@ -271,7 +327,7 @@ class Enforcer(object): :param do_raise: Whether to raise an exception or not if check fails. :param exc: Class of the exception to raise if the check fails. - Any remaining arguments passed to check() (both + Any remaining arguments passed to enforce() (both positional and keyword arguments) will be passed to the exception class. If not specified, PolicyNotAuthorized will be used. @@ -784,7 +840,7 @@ def _parse_text_rule(rule): return state.result except ValueError: # Couldn't parse the rule - LOG.exception(_LE("Failed to understand rule %r") % rule) + LOG.exception(_LE("Failed to understand rule %s") % rule) # Fail closed return FalseCheck() @@ -855,7 +911,17 @@ class HttpCheck(Check): """ url = ('http:' + self.match) % target - data = {'target': jsonutils.dumps(target), + + # Convert instances of object() in target temporarily to + # empty dict to avoid circular reference detection + # errors in jsonutils.dumps(). + temp_target = copy.deepcopy(target) + for key in target.keys(): + element = target.get(key) + if type(element) is object: + temp_target[key] = {} + + data = {'target': jsonutils.dumps(temp_target), 'credentials': jsonutils.dumps(creds)} post_data = urlparse.urlencode(data) f = urlrequest.urlopen(url, post_data) @@ -875,7 +941,6 @@ class GenericCheck(Check): 'Member':%(role.name)s """ - # TODO(termie): do dict inspection via dot syntax try: match = self.match % target except KeyError: @@ -888,7 +953,10 @@ class GenericCheck(Check): leftval = ast.literal_eval(self.kind) except ValueError: try: - leftval = creds[self.kind] + kind_parts = self.kind.split('.') + leftval = creds + for kind_part in kind_parts: + leftval = leftval[kind_part] except KeyError: return False return match == six.text_type(leftval) diff --git a/heat/openstack/common/threadgroup.py b/heat/openstack/common/threadgroup.py index a77414201..d64998ee5 100644 --- a/heat/openstack/common/threadgroup.py +++ b/heat/openstack/common/threadgroup.py @@ -96,6 +96,8 @@ class ThreadGroup(object): continue try: x.stop() + except eventlet.greenlet.GreenletExit: + pass except Exception as ex: LOG.exception(ex) diff --git a/heat/tests/test_common_context.py b/heat/tests/test_common_context.py index 00101528a..32590492e 100644 --- a/heat/tests/test_common_context.py +++ b/heat/tests/test_common_context.py @@ -19,7 +19,6 @@ import webob from heat.common import context from heat.common import exception -from heat.openstack.common import policy as base_policy from heat.tests import common policy_path = os.path.dirname(os.path.realpath(__file__)) + "/policy/" @@ -244,10 +243,7 @@ class RequestContextMiddlewareTest(common.HeatTestCase): cfg.StrOpt('project', default='heat'), ] cfg.CONF.register_opts(opts) - pf = policy_path + 'check_admin.json' - self.m.StubOutWithMock(base_policy.Enforcer, '_get_policy_path') - base_policy.Enforcer._get_policy_path().MultipleTimes().AndReturn(pf) - self.m.ReplayAll() + cfg.CONF.set_override('policy_file', 'check_admin.json') def test_context_middleware(self): diff --git a/heat/tests/test_common_policy.py b/heat/tests/test_common_policy.py index 9d1a3b677..01b1cb314 100644 --- a/heat/tests/test_common_policy.py +++ b/heat/tests/test_common_policy.py @@ -49,16 +49,13 @@ class TestPolicyEnforcer(common.HeatTestCase): cfg.CONF.register_opts(opts) self.addCleanup(self.m.VerifyAll) - def stub_policyfile(self, filename): - pf = policy_path + filename - self.m.StubOutWithMock(base_policy.Enforcer, '_get_policy_path') - base_policy.Enforcer._get_policy_path().MultipleTimes().AndReturn(pf) - self.m.ReplayAll() + def get_policy_file(self, filename): + return policy_path + filename def test_policy_cfn_default(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer(scope='cloudformation') + enforcer = policy.Enforcer( + scope='cloudformation', + policy_file=self.get_policy_file('deny_stack_user.json')) ctx = utils.dummy_context(roles=[]) for action in self.cfn_actions: @@ -66,9 +63,9 @@ class TestPolicyEnforcer(common.HeatTestCase): enforcer.enforce(ctx, action) def test_policy_cfn_notallowed(self): - self.stub_policyfile('notallowed.json') - - enforcer = policy.Enforcer(scope='cloudformation') + enforcer = policy.Enforcer( + scope='cloudformation', + policy_file=self.get_policy_file('notallowed.json')) ctx = utils.dummy_context(roles=[]) for action in self.cfn_actions: @@ -77,9 +74,9 @@ class TestPolicyEnforcer(common.HeatTestCase): action, {}) def test_policy_cfn_deny_stack_user(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer(scope='cloudformation') + enforcer = policy.Enforcer( + scope='cloudformation', + policy_file=self.get_policy_file('deny_stack_user.json')) ctx = utils.dummy_context(roles=['heat_stack_user']) for action in self.cfn_actions: @@ -91,9 +88,9 @@ class TestPolicyEnforcer(common.HeatTestCase): action, {}) def test_policy_cfn_allow_non_stack_user(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer(scope='cloudformation') + enforcer = policy.Enforcer( + scope='cloudformation', + policy_file=self.get_policy_file('deny_stack_user.json')) ctx = utils.dummy_context(roles=['not_a_stack_user']) for action in self.cfn_actions: @@ -101,9 +98,9 @@ class TestPolicyEnforcer(common.HeatTestCase): enforcer.enforce(ctx, action) def test_policy_cw_deny_stack_user(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer(scope='cloudwatch') + enforcer = policy.Enforcer( + scope='cloudwatch', + policy_file=self.get_policy_file('deny_stack_user.json')) ctx = utils.dummy_context(roles=['heat_stack_user']) for action in self.cw_actions: @@ -115,9 +112,9 @@ class TestPolicyEnforcer(common.HeatTestCase): action, {}) def test_policy_cw_allow_non_stack_user(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer(scope='cloudwatch') + enforcer = policy.Enforcer( + scope='cloudwatch', + policy_file=self.get_policy_file('deny_stack_user.json')) ctx = utils.dummy_context(roles=['not_a_stack_user']) for action in self.cw_actions: @@ -125,52 +122,50 @@ class TestPolicyEnforcer(common.HeatTestCase): enforcer.enforce(ctx, action) def test_set_rules_overwrite_true(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer() + enforcer = policy.Enforcer( + policy_file=self.get_policy_file('deny_stack_user.json')) enforcer.load_rules(True) enforcer.set_rules({'test_heat_rule': 1}, True) self.assertEqual({'test_heat_rule': 1}, enforcer.enforcer.rules) def test_set_rules_overwrite_false(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer() + enforcer = policy.Enforcer( + policy_file=self.get_policy_file('deny_stack_user.json')) + enforcer.load_rules(True) enforcer.load_rules(True) enforcer.set_rules({'test_heat_rule': 1}, False) self.assertIn('test_heat_rule', enforcer.enforcer.rules) def test_load_rules_force_reload_true(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer() + enforcer = policy.Enforcer( + policy_file=self.get_policy_file('deny_stack_user.json')) + enforcer.load_rules(True) enforcer.set_rules({'test_heat_rule': 'test'}) enforcer.load_rules(True) self.assertNotIn({'test_heat_rule': 'test'}, enforcer.enforcer.rules) def test_load_rules_force_reload_false(self): - self.stub_policyfile('deny_stack_user.json') - - enforcer = policy.Enforcer() + enforcer = policy.Enforcer( + policy_file=self.get_policy_file('deny_stack_user.json')) + enforcer.load_rules(True) enforcer.load_rules(True) enforcer.set_rules({'test_heat_rule': 'test'}) enforcer.load_rules(False) self.assertIn('test_heat_rule', enforcer.enforcer.rules) def test_default_rule(self): - self.stub_policyfile('deny_stack_user.json') - ctx = utils.dummy_context(roles=['not_a_stack_user']) default_rule = base_policy.FalseCheck() - enforcer = policy.Enforcer(scope='cloudformation', - exc=None, default_rule=default_rule) + enforcer = policy.Enforcer( + scope='cloudformation', + policy_file=self.get_policy_file('deny_stack_user.json'), + exc=None, default_rule=default_rule) action = 'no_such_action' self.assertFalse(enforcer.enforce(ctx, action)) def test_check_admin(self): - self.stub_policyfile('check_admin.json') - - enforcer = policy.Enforcer() + enforcer = policy.Enforcer( + policy_file=self.get_policy_file('check_admin.json')) ctx = utils.dummy_context(roles=[]) self.assertFalse(enforcer.check_is_admin(ctx)) diff --git a/openstack-common.conf b/openstack-common.conf index 4a29992f6..ae1eb32f6 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -5,7 +5,6 @@ module=config module=context module=crypto module=eventlet_backdoor -module=gettextutils module=local module=log module=loopingcall