Merge "Sync config file generator from oslo"

openstack-common.conf

@@ -1,6 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from openstack-common
+module=config
 module=install_venv_common
 module=lockutils
 module=log

tempest/openstack/common/config/generator.py

@@ -0,0 +1,268 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 SINA Corporation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""Extracts OpenStack config option info from module(s)."""
+
+from __future__ import print_function
+
+import imp
+import os
+import re
+import socket
+import sys
+import textwrap
+
+from oslo.config import cfg
+
+from tempest.openstack.common import gettextutils
+from tempest.openstack.common import importutils
+
+gettextutils.install('tempest')
+
+STROPT = "StrOpt"
+BOOLOPT = "BoolOpt"
+INTOPT = "IntOpt"
+FLOATOPT = "FloatOpt"
+LISTOPT = "ListOpt"
+MULTISTROPT = "MultiStrOpt"
+
+OPT_TYPES = {
+    STROPT: 'string value',
+    BOOLOPT: 'boolean value',
+    INTOPT: 'integer value',
+    FLOATOPT: 'floating point value',
+    LISTOPT: 'list value',
+    MULTISTROPT: 'multi valued',
+}
+
+OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
+                                              FLOATOPT, LISTOPT,
+                                              MULTISTROPT]))
+
+PY_EXT = ".py"
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                       "../../../../"))
+WORDWRAP_WIDTH = 60
+
+
+def generate(srcfiles):
+    mods_by_pkg = dict()
+    for filepath in srcfiles:
+        pkg_name = filepath.split(os.sep)[1]
+        mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
+                            os.path.basename(filepath).split('.')[0]])
+        mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
+    # NOTE(lzyeval): place top level modules before packages
+    pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
+    pkg_names.sort()
+    ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
+    ext_names.sort()
+    pkg_names.extend(ext_names)
+
+    # opts_by_group is a mapping of group name to an options list
+    # The options list is a list of (module, options) tuples
+    opts_by_group = {'DEFAULT': []}
+
+    for module_name in os.getenv(
+            "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','):
+        module = _import_module(module_name)
+        if module:
+            for group, opts in _list_opts(module):
+                opts_by_group.setdefault(group, []).append((module_name, opts))
+
+    for pkg_name in pkg_names:
+        mods = mods_by_pkg.get(pkg_name)
+        mods.sort()
+        for mod_str in mods:
+            if mod_str.endswith('.__init__'):
+                mod_str = mod_str[:mod_str.rfind(".")]
+
+            mod_obj = _import_module(mod_str)
+            if not mod_obj:
+                raise RuntimeError("Unable to import module %s" % mod_str)
+
+            for group, opts in _list_opts(mod_obj):
+                opts_by_group.setdefault(group, []).append((mod_str, opts))
+
+    print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
+    for group, opts in opts_by_group.items():
+        print_group_opts(group, opts)
+
+
+def _import_module(mod_str):
+    try:
+        if mod_str.startswith('bin.'):
+            imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:]))
+            return sys.modules[mod_str[4:]]
+        else:
+            return importutils.import_module(mod_str)
+    except ImportError as ie:
+        sys.stderr.write("%s\n" % str(ie))
+        return None
+    except Exception:
+        return None
+
+
+def _is_in_group(opt, group):
+    "Check if opt is in group."
+    for key, value in group._opts.items():
+        if value['opt'] == opt:
+            return True
+    return False
+
+
+def _guess_groups(opt, mod_obj):
+    # is it in the DEFAULT group?
+    if _is_in_group(opt, cfg.CONF):
+        return 'DEFAULT'
+
+    # what other groups is it in?
+    for key, value in cfg.CONF.items():
+        if isinstance(value, cfg.CONF.GroupAttr):
+            if _is_in_group(opt, value._group):
+                return value._group.name
+
+    raise RuntimeError(
+        "Unable to find group for option %s, "
+        "maybe it's defined twice in the same group?"
+        % opt.name
+    )
+
+
+def _list_opts(obj):
+    def is_opt(o):
+        return (isinstance(o, cfg.Opt) and
+                not isinstance(o, cfg.SubCommandOpt))
+
+    opts = list()
+    for attr_str in dir(obj):
+        attr_obj = getattr(obj, attr_str)
+        if is_opt(attr_obj):
+            opts.append(attr_obj)
+        elif (isinstance(attr_obj, list) and
+              all(map(lambda x: is_opt(x), attr_obj))):
+            opts.extend(attr_obj)
+
+    ret = {}
+    for opt in opts:
+        ret.setdefault(_guess_groups(opt, obj), []).append(opt)
+    return ret.items()
+
+
+def print_group_opts(group, opts_by_module):
+    print("[%s]" % group)
+    print('')
+    for mod, opts in opts_by_module:
+        print('#')
+        print('# Options defined in %s' % mod)
+        print('#')
+        print('')
+        for opt in opts:
+            _print_opt(opt)
+        print('')
+
+
+def _get_my_ip():
+    try:
+        csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        csock.connect(('8.8.8.8', 80))
+        (addr, port) = csock.getsockname()
+        csock.close()
+        return addr
+    except socket.error:
+        return None
+
+
+def _sanitize_default(name, value):
+    """Set up a reasonably sensible default for pybasedir, my_ip and host."""
+    if value.startswith(sys.prefix):
+        # NOTE(jd) Don't use os.path.join, because it is likely to think the
+        # second part is an absolute pathname and therefore drop the first
+        # part.
+        value = os.path.normpath("/usr/" + value[len(sys.prefix):])
+    elif value.startswith(BASEDIR):
+        return value.replace(BASEDIR, '/usr/lib/python/site-packages')
+    elif BASEDIR in value:
+        return value.replace(BASEDIR, '')
+    elif value == _get_my_ip():
+        return '10.0.0.1'
+    elif value == socket.gethostname() and 'host' in name:
+        return 'tempest'
+    elif value.strip() != value:
+        return '"%s"' % value
+    return value
+
+
+def _print_opt(opt):
+    opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help
+    if not opt_help:
+        sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
+        opt_help = ""
+    opt_type = None
+    try:
+        opt_type = OPTION_REGEX.search(str(type(opt))).group(0)
+    except (ValueError, AttributeError) as err:
+        sys.stderr.write("%s\n" % str(err))
+        sys.exit(1)
+    opt_help += ' (' + OPT_TYPES[opt_type] + ')'
+    print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
+    if opt.deprecated_opts:
+        for deprecated_opt in opt.deprecated_opts:
+            if deprecated_opt.name:
+                deprecated_group = (deprecated_opt.group if
+                                    deprecated_opt.group else "DEFAULT")
+                print('# Deprecated group/name - [%s]/%s' %
+                      (deprecated_group,
+                       deprecated_opt.name))
+    try:
+        if opt_default is None:
+            print('#%s=<None>' % opt_name)
+        elif opt_type == STROPT:
+            assert(isinstance(opt_default, basestring))
+            print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
+                                                          opt_default)))
+        elif opt_type == BOOLOPT:
+            assert(isinstance(opt_default, bool))
+            print('#%s=%s' % (opt_name, str(opt_default).lower()))
+        elif opt_type == INTOPT:
+            assert(isinstance(opt_default, int) and
+                   not isinstance(opt_default, bool))
+            print('#%s=%s' % (opt_name, opt_default))
+        elif opt_type == FLOATOPT:
+            assert(isinstance(opt_default, float))
+            print('#%s=%s' % (opt_name, opt_default))
+        elif opt_type == LISTOPT:
+            assert(isinstance(opt_default, list))
+            print('#%s=%s' % (opt_name, ','.join(opt_default)))
+        elif opt_type == MULTISTROPT:
+            assert(isinstance(opt_default, list))
+            if not opt_default:
+                opt_default = ['']
+            for default in opt_default:
+                print('#%s=%s' % (opt_name, default))
+        print('')
+    except Exception:
+        sys.stderr.write('Error in option "%s"\n' % opt_name)
+        sys.exit(1)
+
+
+def main():
+    generate(sys.argv[1:])
+
+if __name__ == '__main__':
+    main()

tools/config/generate_sample.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+print_hint() {
+    echo "Try \`${0##*/} --help' for more information." >&2
+}
+
+PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \
+                 --long help,base-dir:,package-name:,output-dir: -- "$@")
+
+if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
+
+eval set -- "$PARSED_OPTIONS"
+
+while true; do
+    case "$1" in
+        -h|--help)
+            echo "${0##*/} [options]"
+            echo ""
+            echo "options:"
+            echo "-h, --help                show brief help"
+            echo "-b, --base-dir=DIR        project base directory"
+            echo "-p, --package-name=NAME   project package name"
+            echo "-o, --output-dir=DIR      file output directory"
+            exit 0
+            ;;
+        -b|--base-dir)
+            shift
+            BASEDIR=`echo $1 | sed -e 's/\/*$//g'`
+            shift
+            ;;
+        -p|--package-name)
+            shift
+            PACKAGENAME=`echo $1`
+            shift
+            ;;
+        -o|--output-dir)
+            shift
+            OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
+            shift
+            ;;
+        --)
+            break
+            ;;
+    esac
+done
+
+BASEDIR=${BASEDIR:-`pwd`}
+if ! [ -d $BASEDIR ]
+then
+    echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1
+elif [[ $BASEDIR != /* ]]
+then
+    BASEDIR=$(cd "$BASEDIR" && pwd)
+fi
+
+PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}}
+TARGETDIR=$BASEDIR/$PACKAGENAME
+if ! [ -d $TARGETDIR ]
+then
+    echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1
+fi
+
+OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc}
+# NOTE(bnemec): Some projects put their sample config in etc/,
+#               some in etc/$PACKAGENAME/
+if [ -d $OUTPUTDIR/$PACKAGENAME ]
+then
+    OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME
+elif ! [ -d $OUTPUTDIR ]
+then
+    echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
+    exit 1
+fi
+
+BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
+find $TARGETDIR -type f -name "*.pyc" -delete
+FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
+        -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
+
+EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc"
+if test -r "$EXTRA_MODULES_FILE"
+then
+    source "$EXTRA_MODULES_FILE"
+fi
+
+export EVENTLET_NO_GREENDNS=yes
+
+OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
+[ "$OS_VARS" ] && eval "unset \$OS_VARS"
+DEFAULT_MODULEPATH=tempest.openstack.common.config.generator
+MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
+OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
+python -m $MODULEPATH $FILES > $OUTPUTFILE