Merge "Sync latest config file generator from oslo-incubator"

nova/openstack/common/config/generator.py

@@ -18,6 +18,7 @@
 
 from __future__ import print_function
 
+import argparse
 import imp
 import os
 import re
@@ -27,6 +28,7 @@ import textwrap
 
 from oslo.config import cfg
 import six
+import stevedore.named
 
 from nova.openstack.common import gettextutils
 from nova.openstack.common import importutils
@@ -59,34 +61,55 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
 WORDWRAP_WIDTH = 60
 
 
-def generate(srcfiles):
+def generate(argv):
+    parser = argparse.ArgumentParser(
+        description='generate sample configuration file',
+    )
+    parser.add_argument('-m', dest='modules', action='append')
+    parser.add_argument('-l', dest='libraries', action='append')
+    parser.add_argument('srcfiles', nargs='*')
+    parsed_args = parser.parse_args(argv)
+
     mods_by_pkg = dict()
-    for filepath in srcfiles:
+    for filepath in parsed_args.srcfiles:
         pkg_name = filepath.split(os.sep)[1]
         mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
                             os.path.basename(filepath).split('.')[0]])
         mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
     # NOTE(lzyeval): place top level modules before packages
-    pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
+    pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
-    pkg_names.sort()
+    ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
-    ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
-    ext_names.sort()
     pkg_names.extend(ext_names)
 
     # opts_by_group is a mapping of group name to an options list
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
-    extra_modules = os.getenv("NOVA_CONFIG_GENERATOR_EXTRA_MODULES", "")
+    if parsed_args.modules:
-    if extra_modules:
+        for module_name in parsed_args.modules:
-        for module_name in extra_modules.split(','):
-            module_name = module_name.strip()
             module = _import_module(module_name)
             if module:
                 for group, opts in _list_opts(module):
                     opts_by_group.setdefault(group, []).append((module_name,
                                                                 opts))
 
+    # Look for entry points defined in libraries (or applications) for
+    # option discovery, and include their return values in the output.
+    #
+    # Each entry point should be a function returning an iterable
+    # of pairs with the group name (or None for the default group)
+    # and the list of Opt instances for that group.
+    if parsed_args.libraries:
+        loader = stevedore.named.NamedExtensionManager(
+            'oslo.config.opts',
+            names=list(set(parsed_args.libraries)),
+            invoke_on_load=False,
+        )
+        for ext in loader:
+            for group, opts in ext.plugin():
+                opt_list = opts_by_group.setdefault(group or 'DEFAULT', [])
+                opt_list.append((ext.name, opts))
+
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
         mods.sort()
@@ -120,7 +143,7 @@ def _import_module(mod_str):
 
 def _is_in_group(opt, group):
     "Check if opt is in group."
-    for key, value in group._opts.items():
+    for value in group._opts.values():
         # NOTE(llu): Temporary workaround for bug #1262148, wait until
         # newly released oslo.config support '==' operator.
         if not(value['opt'] != opt):
@@ -134,7 +157,7 @@ def _guess_groups(opt, mod_obj):
         return 'DEFAULT'
 
     # what other groups is it in?
-    for key, value in cfg.CONF.items():
+    for value in cfg.CONF.values():
         if isinstance(value, cfg.CONF.GroupAttr):
             if _is_in_group(opt, value._group):
                 return value._group.name

tools/config/check_uptodate.sh

@@ -1,9 +1,25 @@
-#!/bin/sh
+#!/usr/bin/env bash
-TEMPDIR=`mktemp -d`
+
-CFGFILE=nova.conf.sample
+PROJECT_NAME=${PROJECT_NAME:-nova}
-tools/config/generate_sample.sh -b ./ -p nova -o $TEMPDIR
+CFGFILE_NAME=${PROJECT_NAME}.conf.sample
-if ! diff $TEMPDIR/$CFGFILE etc/nova/$CFGFILE
+
-then
+if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
-    echo "E: nova.conf.sample is not up to date, please run tools/config/generate_sample.sh"
+    CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
-    exit 42
+elif [ -e etc/${CFGFILE_NAME} ]; then
+    CFGFILE=etc/${CFGFILE_NAME}
+else
+    echo "${0##*/}: can not find config file"
+    exit 1
+fi
+
+TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
+trap "rm -rf $TEMPDIR" EXIT
+
+tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
+
+if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
+then
+   echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
+   echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
+   exit 1
 fi

tools/config/generate_sample.sh

@@ -4,8 +4,8 @@ print_hint() {
     echo "Try \`${0##*/} --help' for more information." >&2
 }
 
-PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \
+PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
-                 --long help,base-dir:,package-name:,output-dir: -- "$@")
+                 --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
 
 if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
 
@@ -21,6 +21,8 @@ while true; do
             echo "-b, --base-dir=DIR        project base directory"
             echo "-p, --package-name=NAME   project package name"
             echo "-o, --output-dir=DIR      file output directory"
+            echo "-m, --module=MOD          extra python module to interrogate for options"
+            echo "-l, --library=LIB         extra library that registers options for discovery"
             exit 0
             ;;
         -b|--base-dir)
@@ -38,6 +40,16 @@ while true; do
             OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
             shift
             ;;
+        -m|--module)
+            shift
+            MODULES="$MODULES -m $1"
+            shift
+            ;;
+        -l|--library)
+            shift
+            LIBRARIES="$LIBRARIES -l $1"
+            shift
+            ;;
         --)
             break
             ;;
@@ -77,12 +89,20 @@ find $TARGETDIR -type f -name "*.pyc" -delete
 FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
         -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
 
-EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc"
+RC_FILE="`dirname $0`/oslo.config.generator.rc"
-if test -r "$EXTRA_MODULES_FILE"
+if test -r "$RC_FILE"
 then
-    source "$EXTRA_MODULES_FILE"
+    source "$RC_FILE"
 fi
 
+for mod in ${NOVA_CONFIG_GENERATOR_EXTRA_MODULES}; do
+    MODULES="$MODULES -m $mod"
+done
+
+for lib in ${NOVA_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
+    LIBRARIES="$LIBRARIES -l $lib"
+done
+
 export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
@@ -90,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 DEFAULT_MODULEPATH=nova.openstack.common.config.generator
 MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
-python -m $MODULEPATH $FILES > $OUTPUTFILE
+python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
 
 # Hook to allow projects to append custom config file snippets
 CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)

tools/config/oslo.config.generator.rc

@@ -1 +1 @@
-export NOVA_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token
+NOVA_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token