diff --git a/keystone/openstack/common/config/generator.py b/keystone/openstack/common/config/generator.py index 39707b34b9..7b17384ea6 100644 --- a/keystone/openstack/common/config/generator.py +++ b/keystone/openstack/common/config/generator.py @@ -150,7 +150,7 @@ def _import_module(mod_str): def _is_in_group(opt, group): - "Check if opt is in group." + """Check if opt is in group.""" for value in group._opts.values(): # NOTE(llu): Temporary workaround for bug #1262148, wait until # newly released oslo.config support '==' operator. @@ -159,7 +159,7 @@ def _is_in_group(opt, group): return False -def _guess_groups(opt, mod_obj): +def _guess_groups(opt): # is it in the DEFAULT group? if _is_in_group(opt, cfg.CONF): return 'DEFAULT' @@ -193,7 +193,7 @@ def _list_opts(obj): ret = {} for opt in opts: - ret.setdefault(_guess_groups(opt, obj), []).append(opt) + ret.setdefault(_guess_groups(opt), []).append(opt) return ret.items() @@ -253,7 +253,6 @@ def _print_opt(opt): if not opt_help: sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) opt_help = "" - opt_type = None try: opt_type = OPTION_REGEX.search(str(type(opt))).group(0) except (ValueError, AttributeError) as err: diff --git a/keystone/openstack/common/fileutils.py b/keystone/openstack/common/fileutils.py index 2ea5e6f34c..b454081b3c 100644 --- a/keystone/openstack/common/fileutils.py +++ b/keystone/openstack/common/fileutils.py @@ -99,7 +99,7 @@ def remove_path_on_error(path, remove=delete_if_exists): def file_open(*args, **kwargs): """Open file - see built-in file() documentation for more details + see built-in open() documentation for more details Note: The reason this is kept in a separate module is to easily be able to provide a stub module that doesn't alter system diff --git a/keystone/openstack/common/fixture/mockpatch.py b/keystone/openstack/common/fixture/mockpatch.py index 4ca6bea6e7..ab2dc02163 100644 --- a/keystone/openstack/common/fixture/mockpatch.py +++ b/keystone/openstack/common/fixture/mockpatch.py @@ -17,12 +17,12 @@ ############################################################################## ############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the oslotest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## +# +# DO NOT MODIFY THIS FILE +# +# This file is being graduated to the oslotest library. Please make all +# changes there, and only backport critical fixes here. - dhellmann +# ############################################################################## ############################################################################## diff --git a/keystone/openstack/common/fixture/moxstubout.py b/keystone/openstack/common/fixture/moxstubout.py index c76c4bdeb5..d92c9626db 100644 --- a/keystone/openstack/common/fixture/moxstubout.py +++ b/keystone/openstack/common/fixture/moxstubout.py @@ -17,12 +17,12 @@ ############################################################################## ############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the oslotest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## +# +# DO NOT MODIFY THIS FILE +# +# This file is being graduated to the oslotest library. Please make all +# changes there, and only backport critical fixes here. - dhellmann +# ############################################################################## ############################################################################## diff --git a/keystone/openstack/common/gettextutils.py b/keystone/openstack/common/gettextutils.py index dc3ea1b58e..55a6c06ba6 100644 --- a/keystone/openstack/common/gettextutils.py +++ b/keystone/openstack/common/gettextutils.py @@ -373,8 +373,8 @@ def get_available_languages(domain): 'zh_Hant_HK': 'zh_HK', 'zh_Hant': 'zh_TW', 'fil': 'tl_PH'} - for (locale, alias) in six.iteritems(aliases): - if locale in language_list and alias not in language_list: + for (locale_, alias) in six.iteritems(aliases): + if locale_ in language_list and alias not in language_list: language_list.append(alias) _AVAILABLE_LANGUAGES[domain] = language_list diff --git a/keystone/openstack/common/jsonutils.py b/keystone/openstack/common/jsonutils.py index 36d60d4fbf..78e6c100be 100644 --- a/keystone/openstack/common/jsonutils.py +++ b/keystone/openstack/common/jsonutils.py @@ -168,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) -def loads(s, encoding='utf-8'): - return json.loads(strutils.safe_decode(s, encoding)) +def loads(s, encoding='utf-8', **kwargs): + return json.loads(strutils.safe_decode(s, encoding), **kwargs) -def load(fp, encoding='utf-8'): - return json.load(codecs.getreader(encoding)(fp)) +def load(fp, encoding='utf-8', **kwargs): + return json.load(codecs.getreader(encoding)(fp), **kwargs) try: diff --git a/keystone/openstack/common/lockutils.py b/keystone/openstack/common/lockutils.py index 7ef6814fd6..080c55f570 100644 --- a/keystone/openstack/common/lockutils.py +++ b/keystone/openstack/common/lockutils.py @@ -196,6 +196,7 @@ if os.name == 'nt': else: import base64 import hashlib + import posix_ipc InterProcessLock = _PosixLock FileLock = _FcntlLock @@ -239,7 +240,7 @@ def external_lock(name, lock_file_prefix=None, lock_path=None): def remove_external_lock_file(name, lock_file_prefix=None): - """Remove a external lock file when it's not used anymore + """Remove an external lock file when it's not used anymore This will be helpful when we have a lot of lock files """ with internal_lock(name): diff --git a/keystone/openstack/common/log.py b/keystone/openstack/common/log.py index be580e7bbc..2fb08704b6 100644 --- a/keystone/openstack/common/log.py +++ b/keystone/openstack/common/log.py @@ -62,7 +62,7 @@ _FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' '.*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] + r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] for key in _SANITIZE_KEYS: for pattern in _FORMAT_PATTERNS: @@ -117,7 +117,7 @@ logging_cli_opts = [ default=False, help='Use syslog for logging. ' 'Existing syslog format is DEPRECATED during I, ' - 'and will chang in J to honor RFC5424.'), + 'and will change in J to honor RFC5424.'), cfg.BoolOpt('use-syslog-rfc-format', # TODO(bogdando) remove or use True after existing # syslog format deprecation in J @@ -424,9 +424,7 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): def logging_excepthook(exc_type, value, tb): - extra = {} - if CONF.verbose or CONF.debug: - extra['exc_info'] = (exc_type, value, tb) + extra = {'exc_info': (exc_type, value, tb)} getLogger(product_name).critical( "".join(traceback.format_exception_only(exc_type, value)), **extra) @@ -464,9 +462,8 @@ def setup(product_name, version='unknown'): def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) + cfg.set_defaults( + log_opts, logging_context_format_string=logging_context_format_string) def _find_facility_from_conf(): @@ -543,9 +540,14 @@ def _setup_logging_from_conf(project, version): log_root.addHandler(streamlog) if CONF.publish_errors: - handler = importutils.import_object( - "keystone.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) + try: + handler = importutils.import_object( + "keystone.openstack.common.log_handler.PublishErrorsHandler", + logging.ERROR) + except ImportError: + handler = importutils.import_object( + "oslo.messaging.notify.log_handler.PublishErrorsHandler", + logging.ERROR) log_root.addHandler(handler) datefmt = CONF.log_date_format @@ -668,14 +670,19 @@ class ContextFormatter(logging.Formatter): record.__dict__[key] = '' if record.__dict__.get('request_id'): - self._fmt = CONF.logging_context_format_string + fmt = CONF.logging_context_format_string else: - self._fmt = CONF.logging_default_format_string + fmt = CONF.logging_default_format_string if (record.levelno == logging.DEBUG and CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix + fmt += " " + CONF.logging_debug_format_suffix + if sys.version_info < (3, 2): + self._fmt = fmt + else: + self._style = logging.PercentStyle(fmt) + self._fmt = self._style._fmt # Cache this on the record, Logger will respect our formatted copy if record.exc_info: record.exc_text = self.formatException(record.exc_info, record) diff --git a/keystone/openstack/common/loopingcall.py b/keystone/openstack/common/loopingcall.py index 6e36a47d28..2ae94590fa 100644 --- a/keystone/openstack/common/loopingcall.py +++ b/keystone/openstack/common/loopingcall.py @@ -79,8 +79,9 @@ class FixedIntervalLoopingCall(LoopingCallBase): break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: - LOG.warn(_LW('task run outlasted interval by %s sec') % - -delay) + LOG.warn(_LW('task %(func_name)s run outlasted ' + 'interval by %(delay)s sec'), + {'func_name': repr(self.f), 'delay': -delay}) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() @@ -121,8 +122,9 @@ class DynamicLoopingCall(LoopingCallBase): if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call sleeping for %.02f ' - 'seconds', idle) + LOG.debug('Dynamic looping call %(func_name)s sleeping ' + 'for %(idle).02f seconds', + {'func_name': repr(self.f), 'idle': idle}) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() diff --git a/keystone/openstack/common/policy.py b/keystone/openstack/common/policy.py index e04c5ec712..fb10df7ebb 100644 --- a/keystone/openstack/common/policy.py +++ b/keystone/openstack/common/policy.py @@ -283,10 +283,6 @@ class Enforcer(object): from the expression. """ - # NOTE(flaper87): Not logging target or creds to avoid - # potential security issues. - LOG.debug("Rule %s will be now enforced" % rule) - self.load_rules() # Allow the rule to be a Check tree diff --git a/keystone/openstack/common/service.py b/keystone/openstack/common/service.py index 1b70ff200d..5d1664a7a9 100644 --- a/keystone/openstack/common/service.py +++ b/keystone/openstack/common/service.py @@ -391,9 +391,12 @@ class ProcessLauncher(object): while True: self.handle_signal() self._respawn_children() - if self.sigcaught: - signame = _signo_to_signame(self.sigcaught) - LOG.info(_LI('Caught %s, stopping children'), signame) + # No signal means that stop was called. Don't clean up here. + if not self.sigcaught: + return + + signame = _signo_to_signame(self.sigcaught) + LOG.info(_LI('Caught %s, stopping children'), signame) if not _is_sighup_and_daemon(self.sigcaught): break @@ -404,6 +407,11 @@ class ProcessLauncher(object): except eventlet.greenlet.GreenletExit: LOG.info(_LI("Wait called after thread killed. Cleaning up.")) + self.stop() + + def stop(self): + """Terminate child processes and wait on each.""" + self.running = False for pid in self.children: try: os.kill(pid, signal.SIGTERM) diff --git a/keystone/openstack/common/timeutils.py b/keystone/openstack/common/timeutils.py index 52688a0268..c48da95f12 100644 --- a/keystone/openstack/common/timeutils.py +++ b/keystone/openstack/common/timeutils.py @@ -114,7 +114,7 @@ def utcnow(): def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formatted date from timestamp.""" + """Returns an iso8601 formatted date from timestamp.""" return isotime(datetime.datetime.utcfromtimestamp(timestamp)) @@ -134,7 +134,7 @@ def set_time_override(override_time=None): def advance_time_delta(timedelta): """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) + assert utcnow.override_time is not None try: for dt in utcnow.override_time: dt += timedelta diff --git a/keystone/openstack/common/versionutils.py b/keystone/openstack/common/versionutils.py index 8f22968556..78534409c7 100644 --- a/keystone/openstack/common/versionutils.py +++ b/keystone/openstack/common/versionutils.py @@ -18,6 +18,7 @@ Helpers for comparing version strings. """ import functools + import pkg_resources from keystone.openstack.common.gettextutils import _ diff --git a/tools/colorizer.py b/tools/colorizer.py index c9f7d630a1..a16c620ca6 100755 --- a/tools/colorizer.py +++ b/tools/colorizer.py @@ -42,11 +42,11 @@ """Display a subunit stream through a colorized unittest test runner.""" import heapq -import subunit import sys import unittest import six +import subunit import testtools diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh index e35dbefa5c..d9ff684e2e 100755 --- a/tools/config/check_uptodate.sh +++ b/tools/config/check_uptodate.sh @@ -16,6 +16,10 @@ TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` trap "rm -rf $TEMPDIR" EXIT tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} +if [ $? != 0 ] +then + exit 1 +fi if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} then diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index a09ee82544..c5773d127c 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -125,6 +125,11 @@ DEFAULT_MODULEPATH=keystone.openstack.common.config.generator MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE +if [ $? != 0 ] +then + echo "Can not generate $OUTPUTFILE" + exit 1 +fi # Hook to allow projects to append custom config file snippets CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)