From 636a5287e79494ff066529b128b53a86458b4c6f Mon Sep 17 00:00:00 2001 From: lin-hua-cheng Date: Fri, 10 Oct 2014 16:45:49 -0700 Subject: [PATCH] Sync oslo-incubator for fileutils, log and policy Sync from oslo-incubator commit 4504e4f4917d99c4889a8a5b7907d6b23ced2ffc Additional changes: - remove excutils since this already moved to oslo-utils - added _i18n.py workaround from oslo-incubator, required while transitioning to oslo.i18n - fix openstack_dashboard/policy.py to work with latest policy code Change-Id: I904b9af517cc85e0cc041a1467d6fda599d8d3a1 Partially-Implements: blueprint improve-oslo-usage --- openstack-common.conf | 2 +- .../openstack/common/__init__.py | 17 -- openstack_dashboard/openstack/common/_i18n.py | 40 +++ .../openstack/common/excutils.py | 113 -------- .../openstack/common/fileutils.py | 22 +- openstack_dashboard/openstack/common/log.py | 255 +++++++++--------- .../openstack/common/policy.py | 71 +++-- openstack_dashboard/policy.py | 5 + 8 files changed, 238 insertions(+), 287 deletions(-) create mode 100644 openstack_dashboard/openstack/common/_i18n.py delete mode 100644 openstack_dashboard/openstack/common/excutils.py diff --git a/openstack-common.conf b/openstack-common.conf index 91f9ccf18..5d098da59 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,5 +1,5 @@ [DEFAULT] -module=excutils +module=_i18n module=fileutils module=gettextutils module=importutils diff --git a/openstack_dashboard/openstack/common/__init__.py b/openstack_dashboard/openstack/common/__init__.py index d1223eaf7..e69de29bb 100644 --- a/openstack_dashboard/openstack/common/__init__.py +++ b/openstack_dashboard/openstack/common/__init__.py @@ -1,17 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox')) diff --git a/openstack_dashboard/openstack/common/_i18n.py b/openstack_dashboard/openstack/common/_i18n.py new file mode 100644 index 000000000..78f1eb9a3 --- /dev/null +++ b/openstack_dashboard/openstack/common/_i18n.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""oslo.i18n integration module. + +See http://docs.openstack.org/developer/oslo.i18n/usage.html + +""" + +import oslo.i18n + + +# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the +# application name when this module is synced into the separate +# repository. It is OK to have more than one translation function +# using the same domain, since there will still only be one message +# catalog. +_translators = oslo.i18n.TranslatorFactory(domain='openstack_dashboard') + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = _translators.log_info +_LW = _translators.log_warning +_LE = _translators.log_error +_LC = _translators.log_critical diff --git a/openstack_dashboard/openstack/common/excutils.py b/openstack_dashboard/openstack/common/excutils.py deleted file mode 100644 index 197579ed9..000000000 --- a/openstack_dashboard/openstack/common/excutils.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# Copyright 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exception related utilities. -""" - -import logging -import sys -import time -import traceback - -import six - -from openstack_dashboard.openstack.common.gettextutils import _LE - - -class save_and_reraise_exception(object): - """Save current exception, run some code and then re-raise. - - In some cases the exception context can be cleared, resulting in None - being attempted to be re-raised after an exception handler is run. This - can happen when eventlet switches greenthreads or when running an - exception handler, code raises and catches an exception. In both - cases the exception context will be cleared. - - To work around this, we save the exception state, run handler code, and - then re-raise the original exception. If another exception occurs, the - saved exception is logged and the new exception is re-raised. - - In some cases the caller may not want to re-raise the exception, and - for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example:: - - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False - - If another exception occurs and reraise flag is False, - the saved exception will not be logged. - - If the caller wants to raise new exception during exception handling - he/she sets reraise to False initially with an ability to set it back to - True if needed:: - - except Exception: - with save_and_reraise_exception(reraise=False) as ctxt: - [if statements to determine whether to raise a new exception] - # Not raising a new exception, so reraise - ctxt.reraise = True - """ - def __init__(self, reraise=True): - self.reraise = reraise - - def __enter__(self): - self.type_, self.value, self.tb, = sys.exc_info() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None: - if self.reraise: - logging.error(_LE('Original exception being dropped: %s'), - traceback.format_exception(self.type_, - self.value, - self.tb)) - return False - if self.reraise: - six.reraise(self.type_, self.value, self.tb) - - -def forever_retry_uncaught_exceptions(infunc): - def inner_func(*args, **kwargs): - last_log_time = 0 - last_exc_message = None - exc_count = 0 - while True: - try: - return infunc(*args, **kwargs) - except Exception as exc: - this_exc_message = six.u(str(exc)) - if this_exc_message == last_exc_message: - exc_count += 1 - else: - exc_count = 1 - # Do not log any more frequently than once a minute unless - # the exception message changes - cur_time = int(time.time()) - if (cur_time - last_log_time > 60 or - this_exc_message != last_exc_message): - logging.exception( - _LE('Unexpected exception occurred %d time(s)... ' - 'retrying.') % exc_count) - last_log_time = cur_time - last_exc_message = this_exc_message - exc_count = 0 - # This should be a very rare event. In case it isn't, do - # a sleep. - time.sleep(1) - return inner_func diff --git a/openstack_dashboard/openstack/common/fileutils.py b/openstack_dashboard/openstack/common/fileutils.py index e3765f695..0d5c84392 100644 --- a/openstack_dashboard/openstack/common/fileutils.py +++ b/openstack_dashboard/openstack/common/fileutils.py @@ -18,7 +18,8 @@ import errno import os import tempfile -from openstack_dashboard.openstack.common import excutils +from oslo.utils import excutils + from openstack_dashboard.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -50,8 +51,8 @@ def read_cached_file(filename, force_reload=False): """ global _FILE_CACHE - if force_reload and filename in _FILE_CACHE: - del _FILE_CACHE[filename] + if force_reload: + delete_cached_file(filename) reloaded = False mtime = os.path.getmtime(filename) @@ -66,6 +67,17 @@ def read_cached_file(filename, force_reload=False): return (reloaded, cache_info['data']) +def delete_cached_file(filename): + """Delete cached file if present. + + :param filename: filename to delete + """ + global _FILE_CACHE + + if filename in _FILE_CACHE: + del _FILE_CACHE[filename] + + def delete_if_exists(path, remove=os.unlink): """Delete a file, but ignore file not found error. @@ -99,13 +111,13 @@ def remove_path_on_error(path, remove=delete_if_exists): def file_open(*args, **kwargs): """Open file - see built-in file() documentation for more details + see built-in open() documentation for more details Note: The reason this is kept in a separate module is to easily be able to provide a stub module that doesn't alter system state at all (for unit tests) """ - return file(*args, **kwargs) + return open(*args, **kwargs) def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): diff --git a/openstack_dashboard/openstack/common/log.py b/openstack_dashboard/openstack/common/log.py index febd793ab..b7b40f904 100644 --- a/openstack_dashboard/openstack/common/log.py +++ b/openstack_dashboard/openstack/common/log.py @@ -33,39 +33,24 @@ import logging import logging.config import logging.handlers import os -import re +import socket import sys import traceback from oslo.config import cfg +from oslo.serialization import jsonutils +from oslo.utils import importutils import six from six import moves -from openstack_dashboard.openstack.common.gettextutils import _ -from openstack_dashboard.openstack.common import importutils -from openstack_dashboard.openstack.common import jsonutils +_PY26 = sys.version_info[0:2] == (2, 6) + +from openstack_dashboard.openstack.common._i18n import _ from openstack_dashboard.openstack.common import local _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS = [] -_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS.append(reg_ex) - common_cli_opts = [ cfg.BoolOpt('debug', @@ -84,14 +69,11 @@ logging_cli_opts = [ cfg.StrOpt('log-config-append', metavar='PATH', deprecated_name='log-config', - help='The name of logging configuration file. It does not ' - 'disable existing loggers, but just appends specified ' - 'logging configuration to any other existing logging ' - 'options. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), + help='The name of a logging configuration file. This file ' + 'is appended to any existing logging configuration ' + 'files. For details about logging configuration files, ' + 'see the Python logging module documentation.'), cfg.StrOpt('log-format', - default=None, metavar='FORMAT', help='DEPRECATED. ' 'A logging.Formatter log message format string which may ' @@ -103,7 +85,7 @@ logging_cli_opts = [ default=_DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s'), + 'Default: %(default)s .'), cfg.StrOpt('log-file', metavar='PATH', deprecated_name='logfile', @@ -112,80 +94,78 @@ logging_cli_opts = [ cfg.StrOpt('log-dir', deprecated_name='logdir', help='(Optional) The base directory used for relative ' - '--log-file paths'), + '--log-file paths.'), cfg.BoolOpt('use-syslog', default=False, help='Use syslog for logging. ' 'Existing syslog format is DEPRECATED during I, ' - 'and then will be changed in J to honor RFC5424'), + 'and will change in J to honor RFC5424.'), cfg.BoolOpt('use-syslog-rfc-format', # TODO(bogdando) remove or use True after existing # syslog format deprecation in J default=False, - help='(Optional) Use syslog rfc5424 format for logging. ' - 'If enabled, will add APP-NAME (RFC5424) before the ' - 'MSG part of the syslog message. The old format ' - 'without APP-NAME is deprecated in I, ' + help='(Optional) Enables or disables syslog rfc5424 format ' + 'for logging. If enabled, prefixes the MSG part of the ' + 'syslog message with APP-NAME (RFC5424). The ' + 'format without the APP-NAME is deprecated in I, ' 'and will be removed in J.'), cfg.StrOpt('syslog-log-facility', default='LOG_USER', - help='Syslog facility to receive log lines') + help='Syslog facility to receive log lines.') ] generic_log_opts = [ cfg.BoolOpt('use_stderr', default=True, - help='Log output to standard error') + help='Log output to standard error.') ] +DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', + 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', + 'oslo.messaging=INFO', 'iso8601=WARN', + 'requests.packages.urllib3.connectionpool=WARN', + 'urllib3.connectionpool=WARN', 'websocket=WARN', + "keystonemiddleware=WARN", "routes.middleware=WARN", + "stevedore=WARN"] + log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', - help='Format string to use for log messages with context'), + help='Format string to use for log messages with context.'), cfg.StrOpt('logging_default_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [-] %(instance)s%(message)s', - help='Format string to use for log messages without context'), + help='Format string to use for log messages without context.'), cfg.StrOpt('logging_debug_format_suffix', default='%(funcName)s %(pathname)s:%(lineno)d', - help='Data to append to log format when level is DEBUG'), + help='Data to append to log format when level is DEBUG.'), cfg.StrOpt('logging_exception_prefix', default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' '%(instance)s', - help='Prefix each line of exception output with this format'), + help='Prefix each line of exception output with this format.'), cfg.ListOpt('default_log_levels', - default=[ - 'amqp=WARN', - 'amqplib=WARN', - 'boto=WARN', - 'qpid=WARN', - 'sqlalchemy=WARN', - 'suds=INFO', - 'oslo.messaging=INFO', - 'iso8601=WARN', - 'requests.packages.urllib3.connectionpool=WARN' - ], - help='List of logger=LEVEL pairs'), + default=DEFAULT_LOG_LEVELS, + help='List of logger=LEVEL pairs.'), cfg.BoolOpt('publish_errors', default=False, - help='Publish error events'), + help='Enables or disables publication of error events.'), cfg.BoolOpt('fatal_deprecations', default=False, - help='Make deprecations fatal'), + help='Enables or disables fatal status of deprecations.'), # NOTE(mikal): there are two options here because sometimes we are handed # a full instance (and could include more information), and other times we # are just handed a UUID for the instance. cfg.StrOpt('instance_format', default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), + help='The format for an instance that is passed with the log ' + 'message.'), cfg.StrOpt('instance_uuid_format', default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), + help='The format for an instance UUID that is passed with the ' + 'log message.'), ] CONF = cfg.CONF @@ -244,45 +224,20 @@ def _get_log_file_path(binary=None): return None -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - secret = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS: - message = re.sub(pattern, secret, message) - return message - - class BaseLoggerAdapter(logging.LoggerAdapter): def audit(self, msg, *args, **kwargs): self.log(logging.AUDIT, msg, *args, **kwargs) + def isEnabledFor(self, level): + if _PY26: + # This method was added in python 2.7 (and it does the exact + # same logic, so we need to do the exact same logic so that + # python 2.6 has this capability as well). + return self.logger.isEnabledFor(level) + else: + return super(BaseLoggerAdapter, self).isEnabledFor(level) + class LazyAdapter(BaseLoggerAdapter): def __init__(self, name='unknown', version='unknown'): @@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter): def logger(self): if not self._logger: self._logger = getLogger(self.name, self.version) + if six.PY3: + # In Python 3, the code fails because the 'manager' attribute + # cannot be found when using a LoggerAdapter as the + # underlying logger. Work around this issue. + self._logger.manager = self._logger.logger.manager return self._logger @@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter): self.warn(stdmsg, *args, **kwargs) def process(self, msg, kwargs): - # NOTE(mrodden): catch any Message/other object and - # coerce to unicode before they can get - # to the python logging and possibly - # cause string encoding trouble - if not isinstance(msg, six.string_types): + # NOTE(jecarey): If msg is not unicode, coerce it into unicode + # before it can get to the python logging and + # possibly cause string encoding trouble + if not isinstance(msg, six.text_type): msg = six.text_type(msg) if 'extra' not in kwargs: @@ -424,9 +383,7 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): def logging_excepthook(exc_type, value, tb): - extra = {} - if CONF.verbose or CONF.debug: - extra['exc_info'] = (exc_type, value, tb) + extra = {'exc_info': (exc_type, value, tb)} getLogger(product_name).critical( "".join(traceback.format_exception_only(exc_type, value)), **extra) @@ -450,8 +407,8 @@ def _load_log_config(log_config_append): try: logging.config.fileConfig(log_config_append, disable_existing_loggers=False) - except moves.configparser.Error as exc: - raise LogConfigError(log_config_append, str(exc)) + except (moves.configparser.Error, KeyError) as exc: + raise LogConfigError(log_config_append, six.text_type(exc)) def setup(product_name, version='unknown'): @@ -463,10 +420,20 @@ def setup(product_name, version='unknown'): sys.excepthook = _create_logging_excepthook(product_name) -def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) +def set_defaults(logging_context_format_string=None, + default_log_levels=None): + # Just in case the caller is not setting the + # default_log_level. This is insurance because + # we introduced the default_log_level parameter + # later in a backwards in-compatible change + if default_log_levels is not None: + cfg.set_defaults( + log_opts, + default_log_levels=default_log_levels) + if logging_context_format_string is not None: + cfg.set_defaults( + log_opts, + logging_context_format_string=logging_context_format_string) def _find_facility_from_conf(): @@ -496,10 +463,16 @@ def _find_facility_from_conf(): class RFCSysLogHandler(logging.handlers.SysLogHandler): def __init__(self, *args, **kwargs): self.binary_name = _get_binary_name() - super(RFCSysLogHandler, self).__init__(*args, **kwargs) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + logging.handlers.SysLogHandler.__init__(self, *args, **kwargs) def format(self, record): - msg = super(RFCSysLogHandler, self).format(record) + # Do not use super() unless type(logging.handlers.SysLogHandler) + # is 'type' (Python 2.7). + # Use old style calls, if the type is 'classobj' (Python 2.6) + msg = logging.handlers.SysLogHandler.format(self, record) msg = self.binary_name + ' ' + msg return msg @@ -509,18 +482,6 @@ def _setup_logging_from_conf(project, version): for handler in log_root.handlers: log_root.removeHandler(handler) - if CONF.use_syslog: - facility = _find_facility_from_conf() - # TODO(bogdando) use the format provided by RFCSysLogHandler - # after existing syslog format deprecation in J - if CONF.use_syslog_rfc_format: - syslog = RFCSysLogHandler(address='/dev/log', - facility=facility) - else: - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - logpath = _get_log_file_path() if logpath: filelog = logging.handlers.WatchedFileHandler(logpath) @@ -537,9 +498,14 @@ def _setup_logging_from_conf(project, version): log_root.addHandler(streamlog) if CONF.publish_errors: - handler = importutils.import_object( - "openstack_dashboard.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) + try: + handler = importutils.import_object( + "openstack_dashboard.openstack.common.log_handler.PublishErrorsHandler", + logging.ERROR) + except ImportError: + handler = importutils.import_object( + "oslo.messaging.notify.log_handler.PublishErrorsHandler", + logging.ERROR) log_root.addHandler(handler) datefmt = CONF.log_date_format @@ -565,9 +531,29 @@ def _setup_logging_from_conf(project, version): for pair in CONF.default_log_levels: mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) logger = logging.getLogger(mod) - logger.setLevel(level) + # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name + # to integer code. + if sys.version_info < (2, 7): + level = logging.getLevelName(level_name) + logger.setLevel(level) + else: + logger.setLevel(level_name) + + if CONF.use_syslog: + try: + facility = _find_facility_from_conf() + # TODO(bogdando) use the format provided by RFCSysLogHandler + # after existing syslog format deprecation in J + if CONF.use_syslog_rfc_format: + syslog = RFCSysLogHandler(facility=facility) + else: + syslog = logging.handlers.SysLogHandler(facility=facility) + log_root.addHandler(syslog) + except socket.error: + log_root.error('Unable to add syslog handler. Verify that syslog ' + 'is running.') + _loggers = {} @@ -637,6 +623,12 @@ class ContextFormatter(logging.Formatter): def format(self, record): """Uses contextstring if request_id is set, otherwise default.""" + # NOTE(jecarey): If msg is not unicode, coerce it into unicode + # before it can get to the python logging and + # possibly cause string encoding trouble + if not isinstance(record.msg, six.text_type): + record.msg = six.text_type(record.msg) + # store project info record.project = self.project record.version = self.version @@ -656,14 +648,19 @@ class ContextFormatter(logging.Formatter): record.__dict__[key] = '' if record.__dict__.get('request_id'): - self._fmt = CONF.logging_context_format_string + fmt = CONF.logging_context_format_string else: - self._fmt = CONF.logging_default_format_string + fmt = CONF.logging_default_format_string if (record.levelno == logging.DEBUG and CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix + fmt += " " + CONF.logging_debug_format_suffix + if sys.version_info < (3, 2): + self._fmt = fmt + else: + self._style = logging.PercentStyle(fmt) + self._fmt = self._style._fmt # Cache this on the record, Logger will respect our formatted copy if record.exc_info: record.exc_text = self.formatException(record.exc_info, record) diff --git a/openstack_dashboard/openstack/common/policy.py b/openstack_dashboard/openstack/common/policy.py index d5062a80a..1be6474a3 100644 --- a/openstack_dashboard/openstack/common/policy.py +++ b/openstack_dashboard/openstack/common/policy.py @@ -77,26 +77,32 @@ as it allows particular rules to be explicitly disabled. import abc import ast +import os import re from oslo.config import cfg +from oslo.serialization import jsonutils import six import six.moves.urllib.parse as urlparse import six.moves.urllib.request as urlrequest from openstack_dashboard.openstack.common import fileutils -from openstack_dashboard.openstack.common.gettextutils import _, _LE -from openstack_dashboard.openstack.common import jsonutils +from openstack_dashboard.openstack.common._i18n import _, _LE, _LW from openstack_dashboard.openstack.common import log as logging policy_opts = [ cfg.StrOpt('policy_file', default='policy.json', - help=_('JSON file containing policy')), + help=_('The JSON file that defines policies.')), cfg.StrOpt('policy_default_rule', default='default', - help=_('Rule enforced when requested rule is not found')), + help=_('Default rule. Enforced when a requested rule is not ' + 'found.')), + cfg.MultiStrOpt('policy_dirs', + default=['policy.d'], + help=_('The directories of policy configuration files is ' + 'stored')), ] CONF = cfg.CONF @@ -215,6 +221,7 @@ class Enforcer(object): def clear(self): """Clears Enforcer rules, policy's cache and policy's path.""" self.set_rules({}) + fileutils.delete_cached_file(self.policy_path) self.default_rule = None self.policy_path = None @@ -231,31 +238,53 @@ class Enforcer(object): if self.use_conf: if not self.policy_path: - self.policy_path = self._get_policy_path() + self.policy_path = self._get_policy_path(self.policy_file) + self._load_policy_file(self.policy_path, force_reload) + for path in CONF.policy_dirs: + try: + path = self._get_policy_path(path) + except cfg.ConfigFilesNotFoundError: + LOG.warn(_LW("Can not find policy directories %s"), path) + continue + self._walk_through_policy_directory(path, + self._load_policy_file, + force_reload, False) + + def _walk_through_policy_directory(self, path, func, *args): + # We do not iterate over sub-directories. + policy_files = next(os.walk(path))[2] + policy_files.sort() + for policy_file in [p for p in policy_files if not p.startswith('.')]: + func(os.path.join(path, policy_file), *args) + + def _load_policy_file(self, path, force_reload, overwrite=True): reloaded, data = fileutils.read_cached_file( - self.policy_path, force_reload=force_reload) + path, force_reload=force_reload) if reloaded or not self.rules: rules = Rules.load_json(data, self.default_rule) - self.set_rules(rules) + self.set_rules(rules, overwrite) LOG.debug("Rules successfully reloaded") - def _get_policy_path(self): - """Locate the policy json data file. + def _get_policy_path(self, path): + """Locate the policy json data file/path. - :param policy_file: Custom policy file to locate. + :param path: It's value can be a full path or related path. When + full path specified, this function just returns the full + path. When related path specified, this function will + search configuration directories to find one that exists. :returns: The policy path - :raises: ConfigFilesNotFoundError if the file couldn't + :raises: ConfigFilesNotFoundError if the file/path couldn't be located. """ - policy_file = CONF.find_file(self.policy_file) + policy_path = CONF.find_file(path) - if policy_file: - return policy_file + if policy_path: + return policy_path - raise cfg.ConfigFilesNotFoundError((self.policy_file,)) + raise cfg.ConfigFilesNotFoundError((path,)) def enforce(self, rule, target, creds, do_raise=False, exc=None, *args, **kwargs): @@ -282,10 +311,6 @@ class Enforcer(object): from the expression. """ - # NOTE(flaper87): Not logging target or creds to avoid - # potential security issues. - LOG.debug("Rule %s will be now enforced" % rule) - self.load_rules() # Allow the rule to be a Check tree @@ -787,7 +812,7 @@ def _parse_text_rule(rule): return state.result except ValueError: # Couldn't parse the rule - LOG.exception(_LE("Failed to understand rule %r") % rule) + LOG.exception(_LE("Failed to understand rule %s") % rule) # Fail closed return FalseCheck() @@ -878,7 +903,6 @@ class GenericCheck(Check): 'Member':%(role.name)s """ - # TODO(termie): do dict inspection via dot syntax try: match = self.match % target except KeyError: @@ -891,7 +915,10 @@ class GenericCheck(Check): leftval = ast.literal_eval(self.kind) except ValueError: try: - leftval = creds[self.kind] + kind_parts = self.kind.split('.') + leftval = creds + for kind_part in kind_parts: + leftval = leftval[kind_part] except KeyError: return False return match == six.text_type(leftval) diff --git a/openstack_dashboard/policy.py b/openstack_dashboard/policy.py index 74310eec3..f0ddb56eb 100644 --- a/openstack_dashboard/policy.py +++ b/openstack_dashboard/policy.py @@ -28,6 +28,11 @@ from openstack_dashboard.openstack.common import policy LOG = logging.getLogger(__name__) CONF = cfg.CONF +# Policy Enforcer has been updated to take in a policy directory +# as a config option. However, the default value in is set to +# ['policy.d'] which causes the code to break. Set the default +# value to empty list for now. +CONF.policy_dirs = [] _ENFORCER = None _BASE_PATH = getattr(settings, 'POLICY_FILES_PATH', '')