Sync with oslo-incubator e9bb0b59

This syncs Keystone with oslo-incubator commit hash
e9bb0b596650540f336afb070a1f8c7de099721f .

In keystone:

 $ rm -r keystone/openstack/common

In oslo-incubator:

 $ git checkout e9bb0b596650540f336afb070a1f8c7de099721f
 $ python update.py ../keystone

Commits
-------
bc48099 Log the function name of looping call
109e325 Use oslo.messaging to publish log errors
de4adbc pep8: fixed multiple violations
9651085 generator: remove unused param, small clean up
eac71f5 Fix common.log.ContextFormatter for Python 3
238e601 Make policy debug logging less verbose
250cd88 Fixed a new pep8 error and a small typo
5e3d3a5 Mask passwords included without quotes at the ends of commands
e7c8a45 Improve error handling for conf sample generation
9e88af1 fixed typos found by RETF rules
5f12bba Implement stop method in ProcessLauncher
0d7296f Add kwargs to jsonutils.load(s) functions
9c88dc3 file_open: fixed docstring to refer to open() instead of file()
d78b633 Fixes a simple spelling mistake
621d831 always log a traceback in the sys.excepthook

Change-Id: I4ef56ce08e1f3bbc0ddb82a495da153e09a14c50

keystone/openstack/common/config/generator.py

@@ -150,7 +150,7 @@ def _import_module(mod_str):
 
 
 def _is_in_group(opt, group):
-    "Check if opt is in group."
+    """Check if opt is in group."""
     for value in group._opts.values():
         # NOTE(llu): Temporary workaround for bug #1262148, wait until
         # newly released oslo.config support '==' operator.
@@ -159,7 +159,7 @@ def _is_in_group(opt, group):
     return False
 
 
-def _guess_groups(opt, mod_obj):
+def _guess_groups(opt):
     # is it in the DEFAULT group?
     if _is_in_group(opt, cfg.CONF):
         return 'DEFAULT'
@@ -193,7 +193,7 @@ def _list_opts(obj):
 
     ret = {}
     for opt in opts:
-        ret.setdefault(_guess_groups(opt, obj), []).append(opt)
+        ret.setdefault(_guess_groups(opt), []).append(opt)
     return ret.items()
 
 
@@ -253,7 +253,6 @@ def _print_opt(opt):
     if not opt_help:
         sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
         opt_help = ""
-    opt_type = None
     try:
         opt_type = OPTION_REGEX.search(str(type(opt))).group(0)
     except (ValueError, AttributeError) as err:

keystone/openstack/common/fileutils.py

@@ -99,7 +99,7 @@ def remove_path_on_error(path, remove=delete_if_exists):
 def file_open(*args, **kwargs):
     """Open file
 
-    see built-in file() documentation for more details
+    see built-in open() documentation for more details
 
     Note: The reason this is kept in a separate module is to easily
     be able to provide a stub module that doesn't alter system

keystone/openstack/common/fixture/mockpatch.py

@@ -17,12 +17,12 @@
 
 ##############################################################################
 ##############################################################################
-##
-## DO NOT MODIFY THIS FILE
-##
-## This file is being graduated to the oslotest library. Please make all
-## changes there, and only backport critical fixes here. - dhellmann
-##
+#
+# DO NOT MODIFY THIS FILE
+#
+# This file is being graduated to the oslotest library. Please make all
+# changes there, and only backport critical fixes here. - dhellmann
+#
 ##############################################################################
 ##############################################################################
 

keystone/openstack/common/fixture/moxstubout.py

@@ -17,12 +17,12 @@
 
 ##############################################################################
 ##############################################################################
-##
-## DO NOT MODIFY THIS FILE
-##
-## This file is being graduated to the oslotest library. Please make all
-## changes there, and only backport critical fixes here. - dhellmann
-##
+#
+# DO NOT MODIFY THIS FILE
+#
+# This file is being graduated to the oslotest library. Please make all
+# changes there, and only backport critical fixes here. - dhellmann
+#
 ##############################################################################
 ##############################################################################
 

keystone/openstack/common/gettextutils.py

@@ -373,8 +373,8 @@ def get_available_languages(domain):
                'zh_Hant_HK': 'zh_HK',
                'zh_Hant': 'zh_TW',
                'fil': 'tl_PH'}
-    for (locale, alias) in six.iteritems(aliases):
-        if locale in language_list and alias not in language_list:
+    for (locale_, alias) in six.iteritems(aliases):
+        if locale_ in language_list and alias not in language_list:
             language_list.append(alias)
 
     _AVAILABLE_LANGUAGES[domain] = language_list

keystone/openstack/common/jsonutils.py

@@ -168,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs):
     return json.dumps(value, default=default, **kwargs)
 
 
-def loads(s, encoding='utf-8'):
-    return json.loads(strutils.safe_decode(s, encoding))
+def loads(s, encoding='utf-8', **kwargs):
+    return json.loads(strutils.safe_decode(s, encoding), **kwargs)
 
 
-def load(fp, encoding='utf-8'):
-    return json.load(codecs.getreader(encoding)(fp))
+def load(fp, encoding='utf-8', **kwargs):
+    return json.load(codecs.getreader(encoding)(fp), **kwargs)
 
 
 try:

keystone/openstack/common/lockutils.py

@@ -196,6 +196,7 @@ if os.name == 'nt':
 else:
     import base64
     import hashlib
+
     import posix_ipc
     InterProcessLock = _PosixLock
     FileLock = _FcntlLock
@@ -239,7 +240,7 @@ def external_lock(name, lock_file_prefix=None, lock_path=None):
 
 
 def remove_external_lock_file(name, lock_file_prefix=None):
-    """Remove a external lock file when it's not used anymore
+    """Remove an external lock file when it's not used anymore
     This will be helpful when we have a lot of lock files
     """
     with internal_lock(name):

keystone/openstack/common/log.py

@@ -62,7 +62,7 @@ _FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
                     r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
                     r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
                     '.*?([\'"])',
-                    r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
+                    r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
 
 for key in _SANITIZE_KEYS:
     for pattern in _FORMAT_PATTERNS:
@@ -117,7 +117,7 @@ logging_cli_opts = [
                 default=False,
                 help='Use syslog for logging. '
                      'Existing syslog format is DEPRECATED during I, '
-                     'and will chang in J to honor RFC5424.'),
+                     'and will change in J to honor RFC5424.'),
     cfg.BoolOpt('use-syslog-rfc-format',
                 # TODO(bogdando) remove or use True after existing
                 #    syslog format deprecation in J
@@ -424,9 +424,7 @@ class JSONFormatter(logging.Formatter):
 
 def _create_logging_excepthook(product_name):
     def logging_excepthook(exc_type, value, tb):
-        extra = {}
-        if CONF.verbose or CONF.debug:
-            extra['exc_info'] = (exc_type, value, tb)
+        extra = {'exc_info': (exc_type, value, tb)}
         getLogger(product_name).critical(
             "".join(traceback.format_exception_only(exc_type, value)),
             **extra)
@@ -464,9 +462,8 @@ def setup(product_name, version='unknown'):
 
 
 def set_defaults(logging_context_format_string):
-    cfg.set_defaults(log_opts,
-                     logging_context_format_string=
-                     logging_context_format_string)
+    cfg.set_defaults(
+        log_opts, logging_context_format_string=logging_context_format_string)
 
 
 def _find_facility_from_conf():
@@ -543,9 +540,14 @@ def _setup_logging_from_conf(project, version):
         log_root.addHandler(streamlog)
 
     if CONF.publish_errors:
-        handler = importutils.import_object(
-            "keystone.openstack.common.log_handler.PublishErrorsHandler",
-            logging.ERROR)
+        try:
+            handler = importutils.import_object(
+                "keystone.openstack.common.log_handler.PublishErrorsHandler",
+                logging.ERROR)
+        except ImportError:
+            handler = importutils.import_object(
+                "oslo.messaging.notify.log_handler.PublishErrorsHandler",
+                logging.ERROR)
         log_root.addHandler(handler)
 
     datefmt = CONF.log_date_format
@@ -668,14 +670,19 @@ class ContextFormatter(logging.Formatter):
                 record.__dict__[key] = ''
 
         if record.__dict__.get('request_id'):
-            self._fmt = CONF.logging_context_format_string
+            fmt = CONF.logging_context_format_string
         else:
-            self._fmt = CONF.logging_default_format_string
+            fmt = CONF.logging_default_format_string
 
         if (record.levelno == logging.DEBUG and
                 CONF.logging_debug_format_suffix):
-            self._fmt += " " + CONF.logging_debug_format_suffix
+            fmt += " " + CONF.logging_debug_format_suffix
 
+        if sys.version_info < (3, 2):
+            self._fmt = fmt
+        else:
+            self._style = logging.PercentStyle(fmt)
+            self._fmt = self._style._fmt
         # Cache this on the record, Logger will respect our formatted copy
         if record.exc_info:
             record.exc_text = self.formatException(record.exc_info, record)

keystone/openstack/common/loopingcall.py

@@ -79,8 +79,9 @@ class FixedIntervalLoopingCall(LoopingCallBase):
                         break
                     delay = interval - timeutils.delta_seconds(start, end)
                     if delay <= 0:
-                        LOG.warn(_LW('task run outlasted interval by %s sec') %
-                                 -delay)
+                        LOG.warn(_LW('task %(func_name)s run outlasted '
+                                     'interval by %(delay)s sec'),
+                                 {'func_name': repr(self.f), 'delay': -delay})
                     greenthread.sleep(delay if delay > 0 else 0)
             except LoopingCallDone as e:
                 self.stop()
@@ -121,8 +122,9 @@ class DynamicLoopingCall(LoopingCallBase):
 
                     if periodic_interval_max is not None:
                         idle = min(idle, periodic_interval_max)
-                    LOG.debug('Dynamic looping call sleeping for %.02f '
-                              'seconds', idle)
+                    LOG.debug('Dynamic looping call %(func_name)s sleeping '
+                              'for %(idle).02f seconds',
+                              {'func_name': repr(self.f), 'idle': idle})
                     greenthread.sleep(idle)
             except LoopingCallDone as e:
                 self.stop()

keystone/openstack/common/policy.py

@@ -283,10 +283,6 @@ class Enforcer(object):
                 from the expression.
         """
 
-        # NOTE(flaper87): Not logging target or creds to avoid
-        # potential security issues.
-        LOG.debug("Rule %s will be now enforced" % rule)
-
         self.load_rules()
 
         # Allow the rule to be a Check tree

keystone/openstack/common/service.py

@@ -391,9 +391,12 @@ class ProcessLauncher(object):
             while True:
                 self.handle_signal()
                 self._respawn_children()
-                if self.sigcaught:
-                    signame = _signo_to_signame(self.sigcaught)
-                    LOG.info(_LI('Caught %s, stopping children'), signame)
+                # No signal means that stop was called.  Don't clean up here.
+                if not self.sigcaught:
+                    return
+
+                signame = _signo_to_signame(self.sigcaught)
+                LOG.info(_LI('Caught %s, stopping children'), signame)
                 if not _is_sighup_and_daemon(self.sigcaught):
                     break
 
@@ -404,6 +407,11 @@ class ProcessLauncher(object):
         except eventlet.greenlet.GreenletExit:
             LOG.info(_LI("Wait called after thread killed.  Cleaning up."))
 
+        self.stop()
+
+    def stop(self):
+        """Terminate child processes and wait on each."""
+        self.running = False
         for pid in self.children:
             try:
                 os.kill(pid, signal.SIGTERM)

keystone/openstack/common/timeutils.py

@@ -114,7 +114,7 @@ def utcnow():
 
 
 def iso8601_from_timestamp(timestamp):
-    """Returns a iso8601 formatted date from timestamp."""
+    """Returns an iso8601 formatted date from timestamp."""
     return isotime(datetime.datetime.utcfromtimestamp(timestamp))
 
 
@@ -134,7 +134,7 @@ def set_time_override(override_time=None):
 
 def advance_time_delta(timedelta):
     """Advance overridden time using a datetime.timedelta."""
-    assert(not utcnow.override_time is None)
+    assert utcnow.override_time is not None
     try:
         for dt in utcnow.override_time:
             dt += timedelta

keystone/openstack/common/versionutils.py

@@ -18,6 +18,7 @@ Helpers for comparing version strings.
 """
 
 import functools
+
 import pkg_resources
 
 from keystone.openstack.common.gettextutils import _

tools/colorizer.py

@@ -42,11 +42,11 @@
 """Display a subunit stream through a colorized unittest test runner."""
 
 import heapq
-import subunit
 import sys
 import unittest
 
 import six
+import subunit
 import testtools
 
 

tools/config/check_uptodate.sh

@@ -16,6 +16,10 @@ TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
 trap "rm -rf $TEMPDIR" EXIT
 
 tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
+if [ $? != 0 ]
+then
+    exit 1
+fi
 
 if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
 then

tools/config/generate_sample.sh

@@ -125,6 +125,11 @@ DEFAULT_MODULEPATH=keystone.openstack.common.config.generator
 MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
 python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
+if [ $? != 0 ]
+then
+    echo "Can not generate $OUTPUTFILE"
+    exit 1
+fi
 
 # Hook to allow projects to append custom config file snippets
 CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)