oslo: update config generator

Change-Id: I5a456fe8a898bc042e7580386b6602b75fa7e1c0

etc/heat/heat.conf.sample

@@ -217,10 +217,12 @@
 
 # (Optional) Name of log file to output to. If no default is
 # set, logging will go to stdout. (string value)
+# Deprecated group/name - [DEFAULT]/logfile
 #log_file=<None>
 
 # (Optional) The base directory used for relative --log-file
 # paths (string value)
+# Deprecated group/name - [DEFAULT]/logdir
 #log_dir=<None>
 
 # Use syslog for logging. (boolean value)
@@ -314,6 +316,7 @@
 #
 
 # Use durable queues in amqp. (boolean value)
+# Deprecated group/name - [DEFAULT]/rabbit_durable_queues
 #amqp_durable_queues=false
 
 # Auto-delete queues in amqp. (boolean value)
@@ -528,10 +531,12 @@
 #
 
 # The backend to use for db (string value)
+# Deprecated group/name - [DEFAULT]/db_backend
 #backend=sqlalchemy
 
 # Enable the experimental use of thread pooling for all DB API
 # calls (boolean value)
+# Deprecated group/name - [DEFAULT]/dbapi_use_tpool
 #use_tpool=false
 
 
@@ -541,6 +546,9 @@
 
 # The SQLAlchemy connection string used to connect to the
 # database (string value)
+# Deprecated group/name - [DEFAULT]/sql_connection
+# Deprecated group/name - [DATABASE]/sql_connection
+# Deprecated group/name - [sql]/connection
 #connection=sqlite:////heat/openstack/common/db/$sqlite_db
 
 # The SQLAlchemy connection string used to connect to the
@@ -549,38 +557,53 @@
 
 # timeout before idle sql connections are reaped (integer
 # value)
+# Deprecated group/name - [DEFAULT]/sql_idle_timeout
+# Deprecated group/name - [DATABASE]/sql_idle_timeout
 #idle_timeout=3600
 
 # Minimum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_min_pool_size
+# Deprecated group/name - [DATABASE]/sql_min_pool_size
 #min_pool_size=1
 
 # Maximum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_pool_size
+# Deprecated group/name - [DATABASE]/sql_max_pool_size
 #max_pool_size=<None>
 
 # maximum db connection retries during startup. (setting -1
 # implies an infinite retry count) (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_retries
+# Deprecated group/name - [DATABASE]/sql_max_retries
 #max_retries=10
 
 # interval between retries of opening a sql connection
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_retry_interval
+# Deprecated group/name - [DATABASE]/reconnect_interval
 #retry_interval=10
 
 # If set, use this value for max_overflow with sqlalchemy
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_overflow
+# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow
 #max_overflow=<None>
 
 # Verbosity of SQL debugging information. 0=None,
 # 100=Everything (integer value)
+# Deprecated group/name - [DEFAULT]/sql_connection_debug
 #connection_debug=0
 
 # Add python stack traces to SQL as comment strings (boolean
 # value)
+# Deprecated group/name - [DEFAULT]/sql_connection_trace
 #connection_trace=false
 
 # If set, use this value for pool_timeout with sqlalchemy
 # (integer value)
+# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout
 #pool_timeout=<None>
 
 
@@ -670,6 +693,7 @@
 #
 
 # Matchmaker ring file (JSON) (string value)
+# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
 #ringfile=/etc/oslo/matchmaker_ring.json
 
 
@@ -896,6 +920,7 @@
 
 # If defined, the memcache server(s) to use for caching (list
 # value)
+# Deprecated group/name - [DEFAULT]/memcache_servers
 #memcached_servers=<None>
 
 # In order to prevent excessive requests and validations, the

heat/openstack/common/config/generator.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 SINA Corporation
 # All Rights Reserved.
 #
@@ -28,6 +26,7 @@ import sys
 import textwrap
 
 from oslo.config import cfg
+import six
 
 from heat.openstack.common import gettextutils
 from heat.openstack.common import importutils
@@ -78,12 +77,15 @@ def generate(srcfiles):
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
-    for module_name in os.getenv(
+    extra_modules = os.getenv("HEAT_CONFIG_GENERATOR_EXTRA_MODULES", "")
-            "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','):
+    if extra_modules:
-        module = _import_module(module_name)
+        for module_name in extra_modules.split(','):
-        if module:
+            module_name = module_name.strip()
-            for group, opts in _list_opts(module):
+            module = _import_module(module_name)
-                opts_by_group.setdefault(group, []).append((module_name, opts))
+            if module:
+                for group, opts in _list_opts(module):
+                    opts_by_group.setdefault(group, []).append((module_name,
+                                                                opts))
 
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
@@ -94,7 +96,7 @@ def generate(srcfiles):
 
             mod_obj = _import_module(mod_str)
             if not mod_obj:
-                continue
+                raise RuntimeError("Unable to import module %s" % mod_str)
 
             for group, opts in _list_opts(mod_obj):
                 opts_by_group.setdefault(group, []).append((mod_str, opts))
@@ -111,10 +113,8 @@ def _import_module(mod_str):
             return sys.modules[mod_str[4:]]
         else:
             return importutils.import_module(mod_str)
-    except ImportError as ie:
+    except Exception as e:
-        sys.stderr.write("%s\n" % str(ie))
+        sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e)))
-        return None
-    except Exception:
         return None
 
 
@@ -221,11 +221,19 @@ def _print_opt(opt):
         sys.exit(1)
     opt_help += ' (' + OPT_TYPES[opt_type] + ')'
     print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
+    if opt.deprecated_opts:
+        for deprecated_opt in opt.deprecated_opts:
+            if deprecated_opt.name:
+                deprecated_group = (deprecated_opt.group if
+                                    deprecated_opt.group else "DEFAULT")
+                print('# Deprecated group/name - [%s]/%s' %
+                      (deprecated_group,
+                       deprecated_opt.name))
     try:
         if opt_default is None:
             print('#%s=<None>' % opt_name)
         elif opt_type == STROPT:
-            assert(isinstance(opt_default, basestring))
+            assert(isinstance(opt_default, six.string_types))
             print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
                                                           opt_default)))
         elif opt_type == BOOLOPT:

tools/config/generate_sample.sh

@@ -73,6 +73,7 @@ then
 fi
 
 BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
+find $TARGETDIR -type f -name "*.pyc" -delete
 FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
         -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
 
@@ -86,7 +87,13 @@ export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 [ "$OS_VARS" ] && eval "unset \$OS_VARS"
-
+DEFAULT_MODULEPATH=heat.openstack.common.config.generator
-MODULEPATH=heat.openstack.common.config.generator
+MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
 python -m $MODULEPATH $FILES > $OUTPUTFILE
+
+# Hook to allow projects to append custom config file snippets
+CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
+for CONCAT_FILE in $CONCAT_FILES; do
+    cat $CONCAT_FILE >> $OUTPUTFILE
+done

tools/config/oslo.config.generator.rc

@@ -1 +1 @@
-export OSLO_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token
+export HEAT_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token