Autogenerate config options and sample config file

This patch adds to "docs" tox job ability to automatically
generate sample config file and config reference based on
config options defined in code.

Change-Id: Id960cc28d1b13ef3ed15e911f59620afb83f45b9

.gitignore

@@ -22,11 +22,13 @@ ChangeLog
 cover/
 doc/build/*
 dist/
-etc/
+etc/*.sample
 zuul/versioninfo
 
 # Files created by releasenotes build
 releasenotes/build
 
+# Docs related files
+doc/source/_static/config-samples/*.sample
+
 Pipfile.lock
-tobiko.conf

doc/source/conf.py

@@ -59,6 +59,8 @@ extensions = [
     'sphinx.ext.ifconfig',
     'sphinx.ext.graphviz',
     'sphinx.ext.todo',
+    'oslo_config.sphinxext',
+    'oslo_config.sphinxconfiggen',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -111,3 +113,23 @@ html_theme_options = {
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+
+# -- Options for oslo_config.sphinxconfiggen ---------------------------------
+
+_config_generator_config_files = [
+    'tobiko.conf',
+]
+
+
+def _get_config_generator_config_definition(conf):
+    config_file_path = '../../etc/oslo-config-generator/%s' % conf
+    # oslo_config.sphinxconfiggen appends '.conf.sample' to the filename,
+    # strip file extentension (.conf or .ini).
+    output_file_path = '_static/config-samples/%s' % conf.rsplit('.', 1)[0]
+    return (config_file_path, output_file_path)
+
+
+config_generator_config_file = [
+    _get_config_generator_config_definition(conf)
+    for conf in _config_generator_config_files
+]

doc/source/configuration/index.rst

@@ -0,0 +1,25 @@
+.. _configuring:
+
+=============================
+Tobiko Configuration Options
+=============================
+
+This section provides a list of all configuration options for Tobiko.
+These are auto-generated from Tobiko code when this documentation is
+built.
+
+Configuration Reference
+-----------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   tobiko.rst
+
+Sample Configuration Files
+--------------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   samples/tobiko.rst

doc/source/configuration/samples/tobiko.rst

@@ -0,0 +1,8 @@
+===================
+Sample tobiko.conf
+===================
+
+This sample configuration can also be viewed in `the raw format
+<../../_static/config-samples/tobiko.conf.sample>`_.
+
+.. literalinclude:: ../../_static/config-samples/tobiko.conf.sample

doc/source/configuration/tobiko.rst

@@ -0,0 +1,6 @@
+===========
+tobiko.conf
+===========
+
+.. show-options::
+   :config-file: etc/oslo-config-generator/tobiko.conf

doc/source/contents.rst

@@ -11,3 +11,10 @@ Tobiko Documentation Contents
    :maxdepth: 1
 
    reference/index
+
+Configuration Reference
+-----------------------
+.. toctree::
+   :maxdepth: 2
+
+   configuration/index

etc/oslo-config-generator/tobiko.conf

@@ -0,0 +1,5 @@
+[DEFAULT]
+output_file = etc/tobiko.conf.sample
+wrap_width = 79
+
+namespace = tobiko

setup.cfg

@@ -34,6 +34,8 @@ console_scripts =
     tobiko-list = tobiko.cmd.list:main
     tobiko-fault = tobiko.cmd.fault:main
     tobiko = tobiko.cmd.run:main
+oslo.config.opts =
+    tobiko = tobiko.config:list_tobiko_options
 
 [global]
 setup-hooks =

tobiko/config.py

@@ -14,6 +14,7 @@
 from __future__ import absolute_import
 
 import importlib
+import itertools
 import logging
 import os
 
@@ -36,6 +37,16 @@ CONFIG_DIRS = [os.getcwd(),
                os.path.expanduser("~/.tobiko"),
                '/etc/tobiko']
 
+HTTP_CONF_GROUP_NAME = "http"
+
+HTTP_OPTIONS = [
+    cfg.StrOpt('http_proxy',
+               help="HTTP proxy URL for Rest APIs"),
+    cfg.StrOpt('https_proxy',
+               help="HTTPS proxy URL for Rest APIs"),
+    cfg.StrOpt('no_proxy',
+               help="Don't use proxy server to connect to listed hosts")]
+
 
 class GlobalConfig(object):
 
@@ -93,14 +104,7 @@ def init_tobiko_config(default_config_dirs=None, product_name='tobiko',
 def register_tobiko_options(conf):
 
     conf.register_opts(
-        group=cfg.OptGroup('http'),
-        opts=[cfg.StrOpt('http_proxy',
-                         help="HTTP proxy URL for Rest APIs"),
-              cfg.StrOpt('https_proxy',
-                         help="HTTPS proxy URL for Rest APIs"),
-              cfg.StrOpt('no_proxy',
-                         help="Don't use proxy server to connect to listed "
-                         "hosts")])
+        group=cfg.OptGroup(HTTP_CONF_GROUP_NAME), opts=HTTP_OPTIONS)
 
     for module_name in CONFIG_MODULES:
         module = importlib.import_module(module_name)
@@ -108,6 +112,22 @@ def register_tobiko_options(conf):
             module.register_tobiko_options(conf=conf)
 
 
+def list_http_options():
+    return [
+        (HTTP_CONF_GROUP_NAME, itertools.chain(HTTP_OPTIONS))
+    ]
+
+
+def list_tobiko_options():
+    all_options = list_http_options()
+
+    for module_name in CONFIG_MODULES:
+        module = importlib.import_module(module_name)
+        if hasattr(module, 'list_options'):
+            all_options += module.list_options()
+    return all_options
+
+
 def setup_tobiko_config(conf):
     # Redirect all warnings to logging library
     logging.captureWarnings(True)

tobiko/openstack/glance/config.py

@@ -13,40 +13,61 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
 
 CIRROS_IMAGE_URL = \
     'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img'
 
+GROUP_NAME = 'glance'
+OPTIONS = [
+    cfg.StrOpt('image_dir',
+               default='~/.tobiko/cache/glance/images',
+               help=("Default directory where to look for image "
+                     "files")),
+]
+
 
 GLANCE_IMAGE_NAMES = ['cirros',
                       'ubuntu']
 
 
 def register_tobiko_options(conf):
-    conf.register_opts(
-        group=cfg.OptGroup('glance'),
-        opts=[cfg.StrOpt('image_dir',
-                         default='~/.tobiko/cache/glance/images',
-                         help=("Default directory where to look for image "
-                               "files")), ])
+    conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
+
+    for image_options in get_images_options():
+        conf.register_opts(group=image_options[0], opts=image_options[1])
+
 
+def get_images_options():
+    options = []
     for name in GLANCE_IMAGE_NAMES:
         group_name = name.lower()
-        conf.register_opts(
-            group=cfg.OptGroup(group_name),
-            opts=[cfg.StrOpt('image_name',
-                             help="Default " + name + " image name"),
-                  cfg.StrOpt('image_url',
-                             help="Default " + name + " image URL"),
-                  cfg.StrOpt('image_file',
-                             help="Default " + name + " image filename"),
-                  cfg.StrOpt('container_format',
-                             help="Default " + name + " container format"),
-                  cfg.StrOpt('disk_format',
-                             help="Default " + name + " disk format"),
-                  cfg.StrOpt('username',
-                             help="Default " + name + " username"),
-                  cfg.StrOpt('password',
-                             help="Default " + name + " password"), ])
+        options += [(
+            group_name,
+            [cfg.StrOpt('image_name',
+                        help="Default " + name + " image name"),
+             cfg.StrOpt('image_url',
+                        help="Default " + name + " image URL"),
+             cfg.StrOpt('image_file',
+                        help="Default " + name + " image filename"),
+             cfg.StrOpt('container_format',
+                        help="Default " + name + " container format"),
+             cfg.StrOpt('disk_format',
+                        help="Default " + name + " disk format"),
+             cfg.StrOpt('username',
+                        help="Default " + name + " username"),
+             cfg.StrOpt('password',
+                        help="Default " + name + " password")])]
+
+    return options
+
+
+def list_options():
+    options = [(GROUP_NAME, itertools.chain(OPTIONS))]
+    for image_options in get_images_options():
+        options += [
+            (image_options[0], itertools.chain(image_options[1]))]
+    return options

tobiko/openstack/keystone/config.py

@@ -13,40 +13,48 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
+GROUP_NAME = 'keystone'
+OPTIONS = [
+    cfg.IntOpt('api_version',
+               default=None,
+               help="Identity API version"),
+    cfg.StrOpt('auth_url',
+               default=None,
+               help="Identity service URL"),
+    cfg.StrOpt('username',
+               default=None,
+               help="Username"),
+    cfg.StrOpt('project_name',
+               default=None,
+               help="Project name"),
+    cfg.StrOpt('password',
+               default=None,
+               help="Password"),
+    cfg.StrOpt('domain_name',
+               default=None,
+               help="Domain name"),
+    cfg.StrOpt('user_domain_name',
+               default=None,
+               help="User domain name"),
+    cfg.StrOpt('project_domain_name',
+               default=None,
+               help="Project domain name"),
+    cfg.StrOpt('project_domain_id',
+               default=None,
+               help="Project domain ID"),
+    cfg.StrOpt('trust_id',
+               default=None,
+               help="Trust ID for trust scoping.")]
+
 
 def register_tobiko_options(conf):
 
-    conf.register_opts(
-        group=cfg.OptGroup('keystone'),
-        opts=[cfg.IntOpt('api_version',
-                         default=None,
-                         help="Identity API version"),
-              cfg.StrOpt('auth_url',
-                         default=None,
-                         help="Identity service URL"),
-              cfg.StrOpt('username',
-                         default=None,
-                         help="Username"),
-              cfg.StrOpt('project_name',
-                         default=None,
-                         help="Project name"),
-              cfg.StrOpt('password',
-                         default=None,
-                         help="Password"),
-              cfg.StrOpt('domain_name',
-                         default=None,
-                         help="Domain name"),
-              cfg.StrOpt('user_domain_name',
-                         default=None,
-                         help="User domain name"),
-              cfg.StrOpt('project_domain_name',
-                         default=None,
-                         help="Project domain name"),
-              cfg.StrOpt('project_domain_id',
-                         default=None,
-                         help="Project domain ID"),
-              cfg.StrOpt('trust_id',
-                         default=None,
-                         help="Trust ID for trust scoping.")])
+    conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
+
+
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/openstack/neutron/config.py

@@ -13,33 +13,41 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
+GROUP_NAME = 'neutron'
+OPTIONS = [
+    cfg.StrOpt('floating_network',
+               help="Network for creating floating IPs"),
+    cfg.StrOpt('ipv4_cidr',
+               default='10.100.0.0/16',
+               help="The CIDR block to allocate IPv4 subnets from"),
+    cfg.IntOpt('ipv4_prefixlen',
+               default=24,
+               help="The mask bits for IPv4 subnets"),
+    cfg.StrOpt('ipv6_cidr',
+               default='2003::/48',
+               help="The CIDR block to allocate IPv6 subnets from"),
+    cfg.IntOpt('ipv6_prefixlen',
+               default=64,
+               help="The mask bits for IPv6 subnets"),
+    cfg.IntOpt('custom_mtu_size',
+               default=1400,
+               help=("Customized maximum transfer unit size\n"
+                     "Notes:\n"
+                     " - MTU values as small as 1000 has been seen "
+                     "breaking networking binding due to an "
+                     "unknown cause.\n"
+                     " - Too big MTU values (like greater than 1400)"
+                     " may be refused during network creation")),
+]
+
 
 def register_tobiko_options(conf):
-    conf.register_opts(
-        group=cfg.OptGroup('neutron'),
-        opts=[cfg.StrOpt('floating_network',
-                         help="Network for creating floating IPs"),
-              cfg.StrOpt('ipv4_cidr',
-                         default='10.100.0.0/16',
-                         help="The CIDR block to allocate IPv4 subnets from"),
-              cfg.IntOpt('ipv4_prefixlen',
-                         default=24,
-                         help="The mask bits for IPv4 subnets"),
-              cfg.StrOpt('ipv6_cidr',
-                         default='2003::/48',
-                         help="The CIDR block to allocate IPv6 subnets from"),
-              cfg.IntOpt('ipv6_prefixlen',
-                         default=64,
-                         help="The mask bits for IPv6 subnets"),
-              cfg.IntOpt('custom_mtu_size',
-                         default=1400,
-                         help=("Customized maximum transfer unit size\n"
-                               "Notes:\n"
-                               " - MTU values as small as 1000 has been seen "
-                               "breaking networking binding due to an "
-                               "unknown cause.\n"
-                               " - Too big MTU values (like greater than 1400)"
-                               " may be refused during network creation")),
-              ])
+    conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
+
+
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/openstack/nova/config.py

@@ -13,14 +13,22 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
+GROUP_NAME = "nova"
+OPTIONS = [
+    cfg.StrOpt('flavor',
+               help="Default flavor for new server instances"),
+    cfg.StrOpt('key_file', default='~/.ssh/id_rsa',
+               help="Default SSH key to login to server instances"),
+]
+
 
 def register_tobiko_options(conf):
-    conf.register_opts(
-        group=cfg.OptGroup('nova'),
-        opts=[cfg.StrOpt('flavor',
-                         help="Default flavor for new server instances"),
-              cfg.StrOpt('key_file', default='~/.ssh/id_rsa',
-                         help="Default SSH key to login to server instances"),
-              ])
+    conf.register_opts(group=cfg.OptGroup('nova'), opts=OPTIONS)
+
+
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/shell/ping/config.py

@@ -13,35 +13,42 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
+GROUP_NAME = "ping"
+OPTIONS = [
+    cfg.IntOpt('count',
+               default=1,
+               help="Number of ICMP messages to wait before ending "
+                    "ping command execution"),
+    cfg.IntOpt('deadline',
+               default=5,
+               help="Max seconds waited from ping command before "
+                    "self terminating himself"),
+    cfg.StrOpt('fragmentation',
+               default=True,
+               help="If disable it will not allow ICMP messages to "
+                    "be delivered in smaller fragments"),
+    cfg.StrOpt('interval',
+               default=1,
+               help="Seconds of time interval between "
+                    "consecutive before ICMP messages"),
+    cfg.IntOpt('packet_size',
+               default=None,
+               help="Size in bytes of ICMP messages (including "
+                    "headers and payload)"),
+    cfg.IntOpt('timeout',
+               default=90.,
+               help="Maximum time in seconds a sequence of ICMP "
+                    "messages is sent to a destination host before "
+                    "reporting as a failure")]
+
 
 def register_tobiko_options(conf):
+    conf.register_opts(group=cfg.OptGroup('ping'), opts=OPTIONS)
+
 
-    conf.register_opts(
-        group=cfg.OptGroup('ping'),
-        opts=[cfg.IntOpt('count',
-                         default=1,
-                         help="Number of ICMP messages to wait before ending "
-                              "ping command execution"),
-              cfg.IntOpt('deadline',
-                         default=5,
-                         help="Max seconds waited from ping command before "
-                              "self terminating himself"),
-              cfg.StrOpt('fragmentation',
-                         default=True,
-                         help="If disable it will not allow ICMP messages to "
-                              "be delivered in smaller fragments"),
-              cfg.StrOpt('interval',
-                         default=1,
-                         help="Seconds of time interval between "
-                              "consecutive before ICMP messages"),
-              cfg.IntOpt('packet_size',
-                         default=None,
-                         help="Size in bytes of ICMP messages (including "
-                              "headers and payload)"),
-              cfg.IntOpt('timeout',
-                         default=90.,
-                         help="Maximum time in seconds a sequence of ICMP "
-                              "messages is sent to a destination host before "
-                              "reporting as a failure")])
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/shell/sh/config.py

@@ -15,14 +15,22 @@
 #    under the License.
 from __future__ import absolute_import
 
+import itertools
+
 from oslo_config import cfg
 
+GROUP_NAME = 'shell'
+OPTIONS = [
+    cfg.StrOpt('command',
+               default='/bin/sh -c',
+               help="Default shell command used for executing "
+                    "local commands")
+]
+
 
 def register_tobiko_options(conf):
+    conf.register_opts(group=cfg.OptGroup('shell'), opts=OPTIONS)
+
 
-    conf.register_opts(
-        group=cfg.OptGroup('shell'),
-        opts=[cfg.StrOpt('command',
-                         default='/bin/sh -c',
-                         help="Default shell command used for executing "
-                              "local commands")])
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/shell/ssh/config.py

@@ -14,58 +14,65 @@
 from __future__ import absolute_import
 
 import getpass
+import itertools
 
 from oslo_config import cfg
 from oslo_log import log
 
+GROUP_NAME = 'ssh'
+OPTIONS = [
+    cfg.BoolOpt('debug',
+                default=False,
+                help=('Logout debugging messages of paramiko '
+                      'library')),
+    cfg.StrOpt('command',
+               default='/usr/bin/ssh',
+               help=('Default SSH client command')),
+    cfg.StrOpt('port',
+               default=22,
+               help=('Default SSH port')),
+    cfg.StrOpt('username',
+               default=getpass.getuser(),
+               help=('Default SSH username')),
+    cfg.ListOpt('config_files',
+                default=['/etc/ssh/ssh_config', '~/.ssh/config'],
+                help="Default user SSH configuration files"),
+    cfg.StrOpt('key_file',
+               default='~/.ssh/id_rsa',
+               help="Default SSH private key file"),
+    cfg.BoolOpt('allow_agent',
+                default=False,
+                help=("Set to False to disable connecting to the "
+                      "SSH agent")),
+    cfg.BoolOpt('compress',
+                default=False,
+                help="Set to True to turn on compression"),
+    cfg.FloatOpt('timeout',
+                 default=5.,
+                 help="SSH connect timeout in seconds"),
+    cfg.IntOpt('connection_attempts',
+               default=60,
+               help=("Incremental seconds to wait after every "
+                     "failed SSH connection attempt")),
+    cfg.FloatOpt('connection_interval',
+                 default=5.,
+                 help=("Minimal seconds to wait between every "
+                       "failed SSH connection attempt")),
+    cfg.StrOpt('proxy_jump',
+               default=None,
+               help="Default SSH proxy server"),
+    cfg.StrOpt('proxy_command',
+               default=None,
+               help="Default proxy command"),
+]
+
 
 def register_tobiko_options(conf):
-    conf.register_opts(
-        group=cfg.OptGroup('ssh'),
-        opts=[cfg.BoolOpt('debug',
-                          default=False,
-                          help=('Logout debugging messages of paramiko '
-                                'library')),
-              cfg.StrOpt('command',
-                         default='/usr/bin/ssh',
-                         help=('Default SSH client command')),
-              cfg.StrOpt('port',
-                         default=22,
-                         help=('Default SSH port')),
-              cfg.StrOpt('username',
-                         default=getpass.getuser(),
-                         help=('Default SSH username')),
-              cfg.ListOpt('config_files',
-                          default=['/etc/ssh/ssh_config', '~/.ssh/config'],
-                          help="Default user SSH configuration files"),
-              cfg.StrOpt('key_file',
-                         default='~/.ssh/id_rsa',
-                         help="Default SSH private key file"),
-              cfg.BoolOpt('allow_agent',
-                          default=False,
-                          help=("Set to False to disable connecting to the "
-                                "SSH agent")),
-              cfg.BoolOpt('compress',
-                          default=False,
-                          help="Set to True to turn on compression"),
-              cfg.FloatOpt('timeout',
-                           default=5.,
-                           help="SSH connect timeout in seconds"),
-              cfg.IntOpt('connection_attempts',
-                         default=60,
-                         help=("Incremental seconds to wait after every "
-                               "failed SSH connection attempt")),
-              cfg.FloatOpt('connection_interval',
-                           default=5.,
-                           help=("Minimal seconds to wait between every "
-                                 "failed SSH connection attempt")),
-              cfg.StrOpt('proxy_jump',
-                         default=None,
-                         help="Default SSH proxy server"),
-              cfg.StrOpt('proxy_command',
-                         default=None,
-                         help="Default proxy command"),
-              ])
+    conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
+
+
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]
 
 
 def setup_tobiko_config(conf):