From f4275c508a6eb86c702af49c04283ba4e9801c1e Mon Sep 17 00:00:00 2001 From: David Stanek Date: Fri, 2 May 2014 13:31:26 +0000 Subject: [PATCH] Cleanup openstack-common.conf and sync from olso An upcoming change[1] in oslo's update.py adds new warnings for style issues in the config. To compensate this commit: - orders the modules alphabetically - uses the script directive for scripts Sync with oslo-incubator caed79d This syncs Keystone with oslo-incubator commit hash caed79d8239679cb74476bb0d9e5011b4fcc39da. In keystone: $ rm -r keystone/openstack/common In oslo-incubator: $ python update.py ../keystone Commits ------- 1173e46 Remove ValueError when accessing sys.modules 17c4e21 Fix docstring indentation in systemd 90ae24b Remove redundant default=None for config options 6c7407b fileutils: port to Python 3 18f2bc1 Enforce unicode json output for jsonutils.load[s]() af36c2a Fix logging setup for Python 3.4 5ae792b Document config generator variables ea9ead8 Allow file exclusion in config generator cdcc19c Mask passwords that are included in commands 9e5a393 jsonutils.load() accepts file pointer, not string a6b2aec On Python <= 2.6, use simplejson if available fe3389e Improve help strings 8a0f567 Remove str() from LOG.* and exceptions f3f14c9 Fixed several typos 0f495ee Emit a log statement when releasing internal lock 9912e5d Add API for creating translation functions 1. https://review.openstack.org/76901 Change-Id: I89de9ae9e3c574095967a2a67e842a90b5527ffa --- keystone/openstack/common/fileutils.py | 2 +- keystone/openstack/common/gettextutils.py | 6 +- keystone/openstack/common/importutils.py | 4 +- keystone/openstack/common/jsonutils.py | 22 ++++++-- keystone/openstack/common/lockutils.py | 4 +- keystone/openstack/common/log.py | 68 ++++++++++++----------- keystone/openstack/common/policy.py | 5 +- keystone/openstack/common/strutils.py | 2 +- keystone/openstack/common/systemd.py | 6 +- openstack-common.conf | 8 +-- tools/config/generate_sample.sh | 14 +++++ tools/install_venv_common.py | 2 +- 12 files changed, 89 insertions(+), 54 deletions(-) diff --git a/keystone/openstack/common/fileutils.py b/keystone/openstack/common/fileutils.py index a51a6798e2..2ea5e6f34c 100644 --- a/keystone/openstack/common/fileutils.py +++ b/keystone/openstack/common/fileutils.py @@ -105,7 +105,7 @@ def file_open(*args, **kwargs): be able to provide a stub module that doesn't alter system state at all (for unit tests) """ - return file(*args, **kwargs) + return open(*args, **kwargs) def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): diff --git a/keystone/openstack/common/gettextutils.py b/keystone/openstack/common/gettextutils.py index 4f2d313bc6..dc3ea1b58e 100644 --- a/keystone/openstack/common/gettextutils.py +++ b/keystone/openstack/common/gettextutils.py @@ -34,7 +34,7 @@ import six _AVAILABLE_LANGUAGES = {} -# FIXME(dhellmann): Remove this when moving to keystone.i18n. +# FIXME(dhellmann): Remove this when moving to oslo.i18n. USE_LAZY = False @@ -116,7 +116,7 @@ class TranslatorFactory(object): # NOTE(dhellmann): When this module moves out of the incubator into -# keystone.i18n, these global variables can be moved to an integration +# oslo.i18n, these global variables can be moved to an integration # module within each application. # Create the global translation functions. @@ -147,7 +147,7 @@ def enable_lazy(): your project is importing _ directly instead of using the gettextutils.install() way of importing the _ function. """ - # FIXME(dhellmann): This function will be removed in keystone.i18n, + # FIXME(dhellmann): This function will be removed in oslo.i18n, # because the TranslatorFactory makes it superfluous. global _, _LI, _LW, _LE, _LC, USE_LAZY tf = TranslatorFactory('keystone', lazy=True) diff --git a/keystone/openstack/common/importutils.py b/keystone/openstack/common/importutils.py index 59dbcd2599..a9fecc7488 100644 --- a/keystone/openstack/common/importutils.py +++ b/keystone/openstack/common/importutils.py @@ -24,10 +24,10 @@ import traceback def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') + __import__(mod_str) try: - __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): + except AttributeError: raise ImportError('Class %s cannot be found (%s)' % (class_str, traceback.format_exception(*sys.exc_info()))) diff --git a/keystone/openstack/common/jsonutils.py b/keystone/openstack/common/jsonutils.py index 45957e6457..36d60d4fbf 100644 --- a/keystone/openstack/common/jsonutils.py +++ b/keystone/openstack/common/jsonutils.py @@ -31,17 +31,29 @@ This module provides a few things: ''' +import codecs import datetime import functools import inspect import itertools -import json +import sys + +if sys.version_info < (2, 7): + # On Python <= 2.6, json module is not C boosted, so try to use + # simplejson module if available + try: + import simplejson as json + except ImportError: + import json +else: + import json import six import six.moves.xmlrpc_client as xmlrpclib from keystone.openstack.common import gettextutils from keystone.openstack.common import importutils +from keystone.openstack.common import strutils from keystone.openstack.common import timeutils netaddr = importutils.try_import("netaddr") @@ -156,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) -def loads(s): - return json.loads(s) +def loads(s, encoding='utf-8'): + return json.loads(strutils.safe_decode(s, encoding)) -def load(s): - return json.load(s) +def load(fp, encoding='utf-8'): + return json.load(codecs.getreader(encoding)(fp)) try: diff --git a/keystone/openstack/common/lockutils.py b/keystone/openstack/common/lockutils.py index e3eefe6e58..7ef6814fd6 100644 --- a/keystone/openstack/common/lockutils.py +++ b/keystone/openstack/common/lockutils.py @@ -38,7 +38,7 @@ LOG = logging.getLogger(__name__) util_opts = [ cfg.BoolOpt('disable_process_locking', default=False, - help='Whether to disable inter-process locks'), + help='Enables or disables inter-process locks.'), cfg.StrOpt('lock_path', default=os.environ.get("KEYSTONE_LOCK_PATH"), help='Directory to use for lock files.') @@ -276,7 +276,7 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None): :param external: The external keyword argument denotes whether this lock should work across multiple processes. This means that if two different - workers both run a a method decorated with @synchronized('mylock', + workers both run a method decorated with @synchronized('mylock', external=True), only one of them will execute at a time. """ int_lock = internal_lock(name) diff --git a/keystone/openstack/common/log.py b/keystone/openstack/common/log.py index 5e4206adcc..be580e7bbc 100644 --- a/keystone/openstack/common/log.py +++ b/keystone/openstack/common/log.py @@ -59,7 +59,10 @@ _SANITIZE_PATTERNS = [] _FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', r'(<%(key)s>).*?()', r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', + r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' + '.*?([\'"])', + r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] for key in _SANITIZE_KEYS: for pattern in _FORMAT_PATTERNS: @@ -84,14 +87,11 @@ logging_cli_opts = [ cfg.StrOpt('log-config-append', metavar='PATH', deprecated_name='log-config', - help='The name of logging configuration file. It does not ' - 'disable existing loggers, but just appends specified ' - 'logging configuration to any other existing logging ' - 'options. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), + help='The name of a logging configuration file. This file ' + 'is appended to any existing logging configuration ' + 'files. For details about logging configuration files, ' + 'see the Python logging module documentation.'), cfg.StrOpt('log-format', - default=None, metavar='FORMAT', help='DEPRECATED. ' 'A logging.Formatter log message format string which may ' @@ -103,7 +103,7 @@ logging_cli_opts = [ default=_DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s'), + 'Default: %(default)s .'), cfg.StrOpt('log-file', metavar='PATH', deprecated_name='logfile', @@ -112,30 +112,30 @@ logging_cli_opts = [ cfg.StrOpt('log-dir', deprecated_name='logdir', help='(Optional) The base directory used for relative ' - '--log-file paths'), + '--log-file paths.'), cfg.BoolOpt('use-syslog', default=False, help='Use syslog for logging. ' 'Existing syslog format is DEPRECATED during I, ' - 'and then will be changed in J to honor RFC5424'), + 'and will chang in J to honor RFC5424.'), cfg.BoolOpt('use-syslog-rfc-format', # TODO(bogdando) remove or use True after existing # syslog format deprecation in J default=False, - help='(Optional) Use syslog rfc5424 format for logging. ' - 'If enabled, will add APP-NAME (RFC5424) before the ' - 'MSG part of the syslog message. The old format ' - 'without APP-NAME is deprecated in I, ' + help='(Optional) Enables or disables syslog rfc5424 format ' + 'for logging. If enabled, prefixes the MSG part of the ' + 'syslog message with APP-NAME (RFC5424). The ' + 'format without the APP-NAME is deprecated in I, ' 'and will be removed in J.'), cfg.StrOpt('syslog-log-facility', default='LOG_USER', - help='Syslog facility to receive log lines') + help='Syslog facility to receive log lines.') ] generic_log_opts = [ cfg.BoolOpt('use_stderr', default=True, - help='Log output to standard error') + help='Log output to standard error.') ] log_opts = [ @@ -143,18 +143,18 @@ log_opts = [ default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', - help='Format string to use for log messages with context'), + help='Format string to use for log messages with context.'), cfg.StrOpt('logging_default_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [-] %(instance)s%(message)s', - help='Format string to use for log messages without context'), + help='Format string to use for log messages without context.'), cfg.StrOpt('logging_debug_format_suffix', default='%(funcName)s %(pathname)s:%(lineno)d', - help='Data to append to log format when level is DEBUG'), + help='Data to append to log format when level is DEBUG.'), cfg.StrOpt('logging_exception_prefix', default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' '%(instance)s', - help='Prefix each line of exception output with this format'), + help='Prefix each line of exception output with this format.'), cfg.ListOpt('default_log_levels', default=[ 'amqp=WARN', @@ -167,25 +167,25 @@ log_opts = [ 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN' ], - help='List of logger=LEVEL pairs'), + help='List of logger=LEVEL pairs.'), cfg.BoolOpt('publish_errors', default=False, - help='Publish error events'), + help='Enables or disables publication of error events.'), cfg.BoolOpt('fatal_deprecations', default=False, - help='Make deprecations fatal'), + help='Enables or disables fatal status of deprecations.'), # NOTE(mikal): there are two options here because sometimes we are handed # a full instance (and could include more information), and other times we # are just handed a UUID for the instance. cfg.StrOpt('instance_format', default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), + help='The format for an instance that is passed with the log ' + 'message. '), cfg.StrOpt('instance_uuid_format', default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), + help='The format for an instance UUID that is passed with the ' + 'log message. '), ] CONF = cfg.CONF @@ -451,7 +451,7 @@ def _load_log_config(log_config_append): logging.config.fileConfig(log_config_append, disable_existing_loggers=False) except moves.configparser.Error as exc: - raise LogConfigError(log_config_append, str(exc)) + raise LogConfigError(log_config_append, six.text_type(exc)) def setup(product_name, version='unknown'): @@ -571,9 +571,15 @@ def _setup_logging_from_conf(project, version): for pair in CONF.default_log_levels: mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) logger = logging.getLogger(mod) - logger.setLevel(level) + # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name + # to integer code. + if sys.version_info < (2, 7): + level = logging.getLevelName(level_name) + logger.setLevel(level) + else: + logger.setLevel(level_name) + _loggers = {} diff --git a/keystone/openstack/common/policy.py b/keystone/openstack/common/policy.py index 38bb227117..e04c5ec712 100644 --- a/keystone/openstack/common/policy.py +++ b/keystone/openstack/common/policy.py @@ -93,10 +93,11 @@ from keystone.openstack.common import log as logging policy_opts = [ cfg.StrOpt('policy_file', default='policy.json', - help=_('JSON file containing policy')), + help=_('The JSON file that defines policies.')), cfg.StrOpt('policy_default_rule', default='default', - help=_('Rule enforced when requested rule is not found')), + help=_('Default rule. Enforced when a requested rule is not ' + 'found.')), ] CONF = cfg.CONF diff --git a/keystone/openstack/common/strutils.py b/keystone/openstack/common/strutils.py index 4a963feaa5..51cf620e72 100644 --- a/keystone/openstack/common/strutils.py +++ b/keystone/openstack/common/strutils.py @@ -78,7 +78,7 @@ def bool_from_string(subject, strict=False, default=False): Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ if not isinstance(subject, six.string_types): - subject = str(subject) + subject = six.text_type(subject) lowered = subject.strip().lower() diff --git a/keystone/openstack/common/systemd.py b/keystone/openstack/common/systemd.py index d08548b1a5..49970e3696 100644 --- a/keystone/openstack/common/systemd.py +++ b/keystone/openstack/common/systemd.py @@ -50,14 +50,16 @@ def _sd_notify(unset_env, msg): def notify(): """Send notification to Systemd that service is ready. + For details see - http://www.freedesktop.org/software/systemd/man/sd_notify.html + http://www.freedesktop.org/software/systemd/man/sd_notify.html """ _sd_notify(False, 'READY=1') def notify_once(): """Send notification once to Systemd that service is ready. + Systemd sets NOTIFY_SOCKET environment variable with the name of the socket listening for notifications from services. This method removes the NOTIFY_SOCKET environment variable to ensure @@ -75,7 +77,7 @@ def onready(notify_socket, timeout): :type timeout: float :returns: 0 service ready 1 service not ready - 2 timeout occured + 2 timeout occurred """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) sock.settimeout(timeout) diff --git a/openstack-common.conf b/openstack-common.conf index 9f24bc3404..f7f9e4d5bc 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,14 +1,11 @@ [DEFAULT] -# The list of modules to copy from openstack-common +module=config module=db module=db.sqlalchemy -module=config -module=colorizer module=fixture module=gettextutils module=importutils -module=install_venv_common module=jsonutils module=log module=policy @@ -17,5 +14,8 @@ module=systemd module=timeutils module=versionutils +script=tools/colorizer.py +script=tools/install_venv_common.py + # The base module to hold the copy of openstack.common base=keystone diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index 73fe0e3b53..a09ee82544 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +# Generate sample configuration for your project. +# +# Aside from the command line flags, it also respects a config file which +# should be named oslo.config.generator.rc and be placed in the same directory. +# +# You can then export the following variables: +# KEYSTONE_CONFIG_GENERATOR_EXTRA_MODULES: list of modules to interrogate for options. +# KEYSTONE_CONFIG_GENERATOR_EXTRA_LIBRARIES: list of libraries to discover. +# KEYSTONE_CONFIG_GENERATOR_EXCLUDED_FILES: list of files to remove from automatic listing. + print_hint() { echo "Try \`${0##*/} --help' for more information." >&2 } @@ -95,6 +105,10 @@ then source "$RC_FILE" fi +for filename in ${KEYSTONE_CONFIG_GENERATOR_EXCLUDED_FILES}; do + FILES="${FILES[@]/$filename/}" +done + for mod in ${KEYSTONE_CONFIG_GENERATOR_EXTRA_MODULES}; do MODULES="$MODULES -m $mod" done diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py index 46822e3293..e279159abb 100644 --- a/tools/install_venv_common.py +++ b/tools/install_venv_common.py @@ -125,7 +125,7 @@ class InstallVenv(object): parser.add_option('-n', '--no-site-packages', action='store_true', help="Do not inherit packages from global Python " - "install") + "install.") return parser.parse_args(argv[1:])[0]