Sync latest from oslo-incubator

Changes include:
1523f00 threadgroup: don't log GreenletExit
a99c5f1 Report: Support Sequences and Mappings as Data
4ca5091 Fixes nits in module policy
411ba35 Simplify is_user_context method
5d43644 cliutils restore python3 string decode
a3220c5 add list_opts to all modules with configuration options
df774ff Import PublishErrorsHandler from oslo.messaging
d0a1fde Switch fileutils to use python logging
b2d35ee Use list.pop(0) to keep the code simpler
262fc82 Correct default rule name for policy.Enforcer
046bc2a Mask secrets when output conf

Removed fixtures that are no longer in oslo-incubator as the
previous reviews in this series switched to using the equivalent
fixtures in oslo libraries.

Cleaned up openstack-common.conf to include only the modules
that are still in oslo-incubator.

Change-Id: Ifd2d5bd506aeb2a9400c5075c3ca24b3c02d1cb6

nova/openstack/common/cliutils.py

@@ -180,6 +180,9 @@ def print_list(objs, fields, formatters=None, sortby_index=0,
                 row.append(data)
         pt.add_row(row)
 
+    if six.PY3:
+        print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode())
+    else:
         print(encodeutils.safe_encode(pt.get_string(**kwargs)))
 
 
@@ -208,6 +211,10 @@ def print_dict(dct, dict_property="Property", wrap=0):
                 col1 = ''
         else:
             pt.add_row([k, v])
+
+    if six.PY3:
+        print(encodeutils.safe_encode(pt.get_string()).decode())
+    else:
         print(encodeutils.safe_encode(pt.get_string()))
 
 

nova/openstack/common/context.py

@@ -117,10 +117,6 @@ def get_context_from_function_and_args(function, args, kwargs):
 
 def is_user_context(context):
     """Indicates if the request context is a normal user."""
-    if not context:
+    if not context or context.is_admin:
         return False
-    if context.is_admin:
+    return context.user_id and context.project_id
-        return False
-    if not context.user_id or not context.project_id:
-        return False
-    return True

nova/openstack/common/eventlet_backdoor.py

@@ -16,6 +16,7 @@
 
 from __future__ import print_function
 
+import copy
 import errno
 import gc
 import os
@@ -49,6 +50,12 @@ CONF.register_opts(eventlet_backdoor_opts)
 LOG = logging.getLogger(__name__)
 
 
+def list_opts():
+    """Entry point for oslo.config-generator.
+    """
+    return [(None, copy.deepcopy(eventlet_backdoor_opts))]
+
+
 class EventletBackdoorConfigValueError(Exception):
     def __init__(self, port_range, help_msg, ex):
         msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '

nova/openstack/common/fileutils.py

@@ -15,13 +15,12 @@
 
 import contextlib
 import errno
+import logging
 import os
 import tempfile
 
 from oslo.utils import excutils
 
-from nova.openstack.common import log as logging
-
 LOG = logging.getLogger(__name__)
 
 _FILE_CACHE = {}

nova/openstack/common/imageutils.py

@@ -101,10 +101,9 @@ class QemuImgInfo(object):
             real_details = real_details.strip().lower()
         elif root_cmd == 'snapshot_list':
             # Next line should be a header, starting with 'ID'
-            if not lines_after or not lines_after[0].startswith("ID"):
+            if not lines_after or not lines_after.pop(0).startswith("ID"):
                 msg = _("Snapshot list encountered but no header found!")
                 raise ValueError(msg)
-            del lines_after[0]
             real_details = []
             # This is the sprintf pattern we will try to match
             # "%-10s%-20s%7s%20s%15s"
@@ -119,6 +118,7 @@ class QemuImgInfo(object):
                 date_pieces = line_pieces[5].split(":")
                 if len(date_pieces) != 3:
                     break
+                lines_after.pop(0)
                 real_details.append({
                     'id': line_pieces[0],
                     'tag': line_pieces[1],
@@ -126,7 +126,6 @@ class QemuImgInfo(object):
                     'date': line_pieces[3],
                     'vm_clock': line_pieces[4] + " " + line_pieces[5],
                 })
-                del lines_after[0]
         return real_details
 
     def _parse(self, cmd_output):

nova/openstack/common/log.py

@@ -27,6 +27,7 @@ It also allows setting of formatting information through conf.
 
 """
 
+import copy
 import inspect
 import itertools
 import logging
@@ -174,6 +175,16 @@ CONF.register_cli_opts(logging_cli_opts)
 CONF.register_opts(generic_log_opts)
 CONF.register_opts(log_opts)
 
+
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(common_cli_opts)),
+            (None, copy.deepcopy(logging_cli_opts)),
+            (None, copy.deepcopy(generic_log_opts)),
+            (None, copy.deepcopy(log_opts)),
+            ]
+
+
 # our new audit level
 # NOTE(jkoelker) Since we synthesized an audit level, make the logging
 #                module aware of it so it acts like other levels.
@@ -498,11 +509,6 @@ def _setup_logging_from_conf(project, version):
         log_root.addHandler(streamlog)
 
     if CONF.publish_errors:
-        try:
-            handler = importutils.import_object(
-                "nova.openstack.common.log_handler.PublishErrorsHandler",
-                logging.ERROR)
-        except ImportError:
         handler = importutils.import_object(
             "oslo.messaging.notify.log_handler.PublishErrorsHandler",
             logging.ERROR)

nova/openstack/common/memorycache.py

@@ -16,6 +16,8 @@
 
 """Super simple fake memcache client."""
 
+import copy
+
 from oslo.config import cfg
 from oslo.utils import timeutils
 
@@ -28,6 +30,11 @@ CONF = cfg.CONF
 CONF.register_opts(memcache_opts)
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(memcache_opts))]
+
+
 def get_client(memcached_servers=None):
     client_cls = Client
 

nova/openstack/common/periodic_task.py

@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import random
 import time
 
@@ -36,6 +37,11 @@ LOG = logging.getLogger(__name__)
 DEFAULT_INTERVAL = 60.0
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(periodic_opts))]
+
+
 class InvalidPeriodicTaskArg(Exception):
     message = _("Unexpected argument for periodic task creation: %(arg)s.")
 

nova/openstack/common/policy.py

@@ -77,6 +77,7 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import ast
+import copy
 import os
 import re
 
@@ -101,8 +102,8 @@ policy_opts = [
                       'found.')),
     cfg.MultiStrOpt('policy_dirs',
                     default=['policy.d'],
-                    help=_('The directories of policy configuration files is '
+                    help=_('Directories where policy configuration files are '
-                           'stored')),
+                           'stored.')),
 ]
 
 CONF = cfg.CONF
@@ -113,6 +114,11 @@ LOG = logging.getLogger(__name__)
 _checks = {}
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(policy_opts))]
+
+
 class PolicyNotAuthorized(Exception):
 
     def __init__(self, rule):
@@ -193,8 +199,8 @@ class Enforcer(object):
 
     def __init__(self, policy_file=None, rules=None,
                  default_rule=None, use_conf=True):
-        self.rules = Rules(rules, default_rule)
         self.default_rule = default_rule or CONF.policy_default_rule
+        self.rules = Rules(rules, self.default_rule)
 
         self.policy_path = None
         self.policy_file = policy_file or CONF.policy_file
@@ -245,7 +251,7 @@ class Enforcer(object):
                 try:
                     path = self._get_policy_path(path)
                 except cfg.ConfigFilesNotFoundError:
-                    LOG.warn(_LW("Can not find policy directories %s"), path)
+                    LOG.warn(_LW("Can not find policy directory: %s"), path)
                     continue
                 self._walk_through_policy_directory(path,
                                                     self._load_policy_file,
@@ -299,7 +305,7 @@ class Enforcer(object):
         :param do_raise: Whether to raise an exception or not if check
                         fails.
         :param exc: Class of the exception to raise if the check fails.
-                    Any remaining arguments passed to check() (both
+                    Any remaining arguments passed to enforce() (both
                     positional and keyword arguments) will be passed to
                     the exception class. If not specified, PolicyNotAuthorized
                     will be used.

nova/openstack/common/report/generators/threading.py

@@ -44,8 +44,7 @@ class ThreadReportGenerator(object):
             for thread_id, stack in sys._current_frames().items()
         ]
 
-        thread_pairs = dict(zip(range(len(threadModels)), threadModels))
+        return mwdv.ModelWithDefaultViews(threadModels,
-        return mwdv.ModelWithDefaultViews(thread_pairs,
                                           text_view=text_views.MultiView())
 
 
@@ -68,6 +67,5 @@ class GreenThreadReportGenerator(object):
             for gr in rutils._find_objects(greenlet.greenlet)
         ]
 
-        thread_pairs = dict(zip(range(len(threadModels)), threadModels))
+        return mwdv.ModelWithDefaultViews(threadModels,
-        return mwdv.ModelWithDefaultViews(thread_pairs,
                                           text_view=text_views.MultiView())

nova/openstack/common/report/models/base.py

@@ -37,13 +37,29 @@ class ReportModel(col.MutableMapping):
     model.  An appropriate object for a view is callable with
     a single parameter: the model to be serialized.
 
-    :param data: a dictionary of data to initially associate with the model
+    If present, the object passed in as data will be transformed
+    into a standard python dict.  For mappings, this is fairly
+    straightforward.  For sequences, the indices become keys
+    and the items become values.
+
+    :param data: a sequence or mapping of data to associate with the model
     :param attached_view: a view object to attach to this model
     """
 
     def __init__(self, data=None, attached_view=None):
         self.attached_view = attached_view
-        self.data = data or {}
+
+        if data is not None:
+            if isinstance(data, col.Mapping):
+                self.data = dict(data)
+            elif isinstance(data, col.Sequence):
+                # convert a list [a, b, c] to a dict {0: a, 1: b, 2: c}
+                self.data = dict(enumerate(data))
+            else:
+                raise TypeError('Data for the model must be a sequence '
+                                'or mapping.')
+        else:
+            self.data = {}
 
     def __str__(self):
         self_cpy = copy.deepcopy(self)
@@ -89,6 +105,8 @@ class ReportModel(col.MutableMapping):
         try:
             return self.data[attrname]
         except KeyError:
+            # we don't have that key in data, and the
+            # model class doesn't have that attribute
             raise AttributeError(
                 "'{cl}' object has no attribute '{an}'".format(
                     cl=type(self).__name__, an=attrname

nova/openstack/common/report/models/conf.py

@@ -41,8 +41,15 @@ class ConfigModel(mwdv.ModelWithDefaultViews):
         def opt_title(optname, co):
             return co._opts[optname]['opt'].name
 
+        def opt_value(opt_obj, value):
+            if opt_obj['opt'].secret:
+                return '*******'
+            else:
+                return value
+
         self['default'] = dict(
-            (opt_title(optname, conf_obj), conf_obj[optname])
+            (opt_title(optname, conf_obj),
+             opt_value(conf_obj._opts[optname], conf_obj[optname]))
             for optname in conf_obj._opts
         )
 
@@ -50,9 +57,10 @@ class ConfigModel(mwdv.ModelWithDefaultViews):
         for groupname in conf_obj._groups:
             group_obj = conf_obj._groups[groupname]
             curr_group_opts = dict(
-                (opt_title(optname, group_obj), conf_obj[groupname][optname])
+                (opt_title(optname, group_obj),
-                for optname in group_obj._opts
+                 opt_value(group_obj._opts[optname],
-            )
+                           conf_obj[groupname][optname]))
+                for optname in group_obj._opts)
             groups[group_obj.name] = curr_group_opts
 
         self.update(groups)

nova/openstack/common/sslutils.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import os
 import ssl
 
@@ -32,9 +33,14 @@ ssl_opts = [
                     "the server securely."),
 ]
 
-
 CONF = cfg.CONF
-CONF.register_opts(ssl_opts, "ssl")
+config_section = 'ssl'
+CONF.register_opts(ssl_opts, config_section)
+
+
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(config_section, copy.deepcopy(ssl_opts))]
 
 
 def is_enabled():

nova/openstack/common/threadgroup.py

@@ -96,6 +96,8 @@ class ThreadGroup(object):
                 continue
             try:
                 x.stop()
+            except eventlet.greenlet.GreenletExit:
+                pass
             except Exception as ex:
                 LOG.exception(ex)
 

openstack-common.conf

@@ -2,22 +2,15 @@
 
 # The list of modules to copy from oslo-incubator
 module=cliutils
-module=config
 module=context
 module=eventlet_backdoor
 module=fileutils
-module=fixture.config
 module=fixture.logging
-module=fixture.mockpatch
-module=fixture.moxstubout
 module=imageutils
-module=install_venv_common
 module=local
 module=log
 module=loopingcall
 module=memorycache
-module=middleware.base
-module=middleware.request_id
 module=periodic_task
 module=policy
 module=report