x/deb-oslo.config
Code Issues Proposed changes

Add oslo-config-generator

Browse Source 
Add a sample config file generator utility. This is a dramatically
different approach from the generator in oslo-incubator, whereby we move
away from magic towards much more explicit control over the generator.

blueprint: oslo-config-generator
Closes-Bug: #1300546
Change-Id: I15686708fc9460948a58cfea3d18dae40ba1fda9
This commit is contained in:
Mark McLoughlin 2014-06-18 16:24:33 +01:00
parent f18797ab7d
commit 97c6352008
6 changed files with 864 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/source/generator.rst Normal file
View File

@ -0,0 +1,11 @@
---------------------
oslo-config-generator
---------------------
.. automodule:: oslo.config.generator
.. currentmodule:: oslo.config.generator
.. autofunction:: main
.. autofunction:: generate
.. autofunction:: register_cli_opts

1
doc/source/index.rst
View File

@ -18,6 +18,7 @@ Contents
parser
exceptions
styleguide
generator
Release Notes
=============

340
oslo/config/generator.py Normal file
View File

@ -0,0 +1,340 @@
# Copyright 2012 SINA Corporation
# Copyright 2014 Cisco Systems, Inc.
# All Rights Reserved.
# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
r"""
A sample configuration file generator.
oslo-config-generator is a utility for generating sample config files. For
example, to generate a sample config file for oslo.messaging you would run::
$> oslo-config-generator --namespace oslo.messaging > oslo.messaging.conf
This generated sample lists all of the available options, along with their help
string, type, deprecated aliases and defaults.
The --namespace option specifies an entry point name registered under the
'oslo.config.opts' entry point namespace. For example, in oslo.messaging's
setup.cfg we have::
[entry_points]
oslo.config.opts =
oslo.messaging = oslo.messaging.opts:list_opts
The callable referenced by the entry point should take no arguments and return
a list of (group_name, [opt_1, opt_2]) tuples. For example::
opts = [
cfg.StrOpt('foo'),
cfg.StrOpt('bar'),
]
cfg.CONF.register_opts(opts, group='blaa')
def list_opts():
return [('blaa', opts)]
You might choose to return a copy of the options so that the return value can't
be modified for nefarious purposes::
def list_opts():
return [('blaa', copy.deepcopy(opts))]
A single codebase might have multiple programs, each of which use a subset of
the total set of options registered by the codebase. In that case, you can
register multiple entry points::
[entry_points]
oslo.config.opts =
nova.common = nova.config:list_common_opts
nova.api = nova.config:list_api_opts
nova.compute = nova.config:list_compute_opts
and generate a config file specific to each program::
$> oslo-config-generator --namespace oslo.messaging \
--namespace nova.common \
--namespace nova.api > nova-api.conf
$> oslo-config-generator --namespace oslo.messaging \
--namespace nova.common \
--namespace nova.compute > nova-compute.conf
To make this more convenient, you can use config files to describe your config
files::
$> cat > config-generator/api.conf <<EOF
[DEFAULT]
output_file = etc/nova/nova-api.conf
namespace = oslo.messaging
namespace = nova.common
namespace = nova.api
EOF
$> cat > config-generator/compute.conf <<EOF
[DEFAULT]
output_file = etc/nova/nova-compute.conf
namespace = oslo.messaging
namespace = nova.compute
namespace = nova.compute
EOF
$> oslo-config-generator --config-file config-generator/api.conf
$> oslo-config-generator --config-file config-generator/compute.conf
The default runtime values of configuration options are not always the most
suitable values to include in sample config files - for example, rather than
including the IP address or hostname of the machine where the config file
was generated, you might want to include something like '10.0.0.1'. To
facilitate this, applications can supply their own 'sanitizer' function via
the 'oslo.config.sanitizer' entry point namespace. For example::
def sanitize_default(self, opt, default_str):
if opt.dest == 'base_dir' and default_str == os.getcwd():
return '.'
else:
return default_str
the callable is registered as a entry point::
[entry_points]
oslo.config.opts =
myapp = myapp.opts:list_opts
oslo.config.sanitizer
myapp = myapp.opts:sanitize_default
before being passed via the --sanitizer command line option:
$> oslo-config-generator --namespace myapp \
--sanitizer myapp > myapp.conf
"""
import logging
import operator
import sys
import textwrap
from oslo.config import cfg
import stevedore.named
LOG = logging.getLogger(__name__)
_generator_opts = [
cfg.StrOpt('output-file',
help='Path of the file to write to. Defaults to stdout.'),
cfg.IntOpt('wrap-width',
default=70,
help='The maximum length of help lines.'),
cfg.MultiStrOpt('namespace',
help='Option namespace under "oslo.config.opts" in which '
'to query for options.'),
cfg.StrOpt('sanitizer',
help='An entry point name under "oslo.config.sanitizer" for a '
'sanitize_default(opt, default_str) callable which will '
'sanitize the stringified default value of an option '
'before outputting it.'),
]
def register_cli_opts(conf):
"""Register the formatter's CLI options with a ConfigOpts instance.
Note, this must be done before the ConfigOpts instance is called to parse
the configuration.
:param conf: a ConfigOpts instance
:raises: DuplicateOptError, ArgsAlreadyParsedError
"""
conf.register_cli_opts(_generator_opts)
class _OptFormatter(object):
"""Format configuration option descriptions to a file."""
_TYPE_DESCRIPTIONS = {
cfg.StrOpt: 'string value',
cfg.BoolOpt: 'boolean value',
cfg.IntOpt: 'integer value',
cfg.FloatOpt: 'floating point value',
cfg.ListOpt: 'list value',
cfg.DictOpt: 'dict value',
cfg.MultiStrOpt: 'multi valued',
}
def __init__(self, output_file=None, sanitize_default=None, wrap_width=70):
"""Construct an OptFormatter object.
:param output_file: a writeable file object
:param sanitize_default: a sanitize_default(opt, default_str) callable
:param wrap_width: The maximum length of help lines, 0 to not wrap
"""
self.output_file = output_file or sys.stdout
self.sanitize_default = sanitize_default or (lambda o, d: d)
self.wrap_width = wrap_width
def format(self, opt):
"""Format a description of an option to the output file.
:param opt: a cfg.Opt instance
"""
if not opt.help:
LOG.warning('"%s" is missing a help string', opt.dest)
opt_type = self._TYPE_DESCRIPTIONS.get(type(opt), 'unknown type')
help_text = u'%s(%s)' % (opt.help + ' ' if opt.help else '', opt_type)
if self.wrap_width is not None and self.wrap_width > 0:
lines = [textwrap.fill(help_text,
self.wrap_width,
initial_indent='# ',
subsequent_indent='# ') + '\n']
else:
lines = ['# ' + help_text + '\n']
for d in opt.deprecated_opts:
lines.append('# Deprecated group/name - [%s]/%s\n' %
(d.group or 'DEFAULT', d.name or opt.dest))
if opt.default is None:
default_str = '<None>'
elif isinstance(opt, cfg.StrOpt):
default_str = opt.default
elif isinstance(opt, cfg.BoolOpt):
default_str = str(opt.default).lower()
elif (isinstance(opt, cfg.IntOpt) or
isinstance(opt, cfg.FloatOpt)):
default_str = str(opt.default)
elif isinstance(opt, cfg.ListOpt):
default_str = ','.join(opt.default)
elif isinstance(opt, cfg.DictOpt):
sorted_items = sorted(opt.default.items(),
key=operator.itemgetter(0))
default_str = ','.join(['%s:%s' % i for i in sorted_items])
elif isinstance(opt, cfg.MultiStrOpt):
default_str = str(opt.default)
else:
LOG.warning('Unknown option type: %s', repr(opt))
default_str = str(opt.default)
defaults = [default_str]
if isinstance(opt, cfg.MultiStrOpt) and opt.default:
defaults = opt.default
for default_str in defaults:
default_str = self.sanitize_default(opt, default_str)
if default_str.strip() != default_str:
default_str = '"%s"' % default_str
lines.append('#%s = %s\n' % (opt.dest, default_str))
self.writelines(lines)
def write(self, s):
"""Write an arbitrary string to the output file.
:param s: an arbitrary string
"""
self.output_file.write(s)
def writelines(self, l):
"""Write an arbitrary sequence of strings to the output file.
:param l: a list of arbitrary strings
"""
self.output_file.writelines(l)
def _get_sanitizer(name):
"""Look up a sanitizer entry point name.
Look up the supplied name under the 'oslo.config.sanitizer' entry point
namespace and return the callable found there.
:param name: the entry point name, or None
:returns: the callable found, or None
"""
if name is None:
return None
return stevedore.driver.DriverManager('oslo.config.sanitizer',
name=name).driver
def _list_opts(namespaces):
"""List the options available via the given namespaces.
:param namespaces: a list of namespaces registered under 'oslo.config.opts'
:returns: a list of (namespace, [(group, [opt_1, opt_2])]) tuples
"""
mgr = stevedore.named.NamedExtensionManager('oslo.config.opts',
names=namespaces,
invoke_on_load=True)
return [(ep.name, ep.obj) for ep in mgr]
def generate(conf):
"""Generate a sample config file.
List all of the options available via the namespaces specified in the given
configuration and write a description of them to the specified output file.
:param conf: a ConfigOpts instance containing the generator's configuration
"""
conf.register_opts(_generator_opts)
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
sanitizer = _get_sanitizer(conf.sanitizer)
formatter = _OptFormatter(output_file=output_file,
sanitize_default=sanitizer,
wrap_width=conf.wrap_width)
groups = {'DEFAULT': []}
for namespace, listing in _list_opts(conf.namespace):
for group, opts in listing:
if not opts:
continue
namespaces = groups.setdefault(group or 'DEFAULT', [])
namespaces.append((namespace,
dict((opt.dest, opt) for opt in opts)))
def _output_opts(f, group, namespaces):
f.write('[%s]\n' % group)
for (namespace, opts_by_dest) in sorted(namespaces,
key=operator.itemgetter(0)):
f.write('\n#\n# From %s\n#\n' % namespace)
for opt in sorted(opts_by_dest.values(),
key=operator.attrgetter('dest')):
f.write('\n')
f.format(opt)
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'))
for group, namespaces in sorted(groups.items(),
key=operator.itemgetter(0)):
formatter.write('\n\n')
_output_opts(formatter, group, namespaces)
def main(args=None):
"""The main function of oslo-config-generator."""
logging.basicConfig(level=logging.WARN)
conf = cfg.ConfigOpts()
register_cli_opts(conf)
conf(args)
generate(conf)
if __name__ == '__main__':
main()

1
requirements.txt
View File

@ -1,2 +1,3 @@
argparse
six>=1.7.0
stevedore>=0.14

4
setup.cfg
View File

@ -31,6 +31,10 @@ namespace_packages =
setup-hooks =
pbr.hooks.setup_hook
[entry_points]
console_scripts =
oslo-config-generator = oslo.config.generator:main
[build_sphinx]
source-dir = doc/source
build-dir = doc/build

507
tests/test_generator.py Normal file
View File

@ -0,0 +1,507 @@
# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import fixtures
import mock
from oslotest import base
from six import moves
import testscenarios
from oslo.config import cfg
from oslo.config import fixture as config_fixture
from oslo.config import generator
load_tests = testscenarios.load_tests_apply_scenarios
class GeneratorTestCase(base.BaseTestCase):
opts = {
'foo': cfg.StrOpt('foo', help='foo option'),
'bar': cfg.StrOpt('bar', help='bar option'),
'foo-bar': cfg.StrOpt('foo-bar', help='foobar'),
'no_help': cfg.StrOpt('no_help'),
'long_help': cfg.StrOpt('long_help',
help='Lorem ipsum dolor sit amet, consectetur '
'adipisicing elit, sed do eiusmod tempor '
'incididunt ut labore et dolore magna '
'aliqua. Ut enim ad minim veniam, quis '
'nostrud exercitation ullamco laboris '
'nisi ut aliquip ex ea commodo '
'consequat. Duis aute irure dolor in '
'reprehenderit in voluptate velit esse '
'cillum dolore eu fugiat nulla '
'pariatur. Excepteur sint occaecat '
'cupidatat non proident, sunt in culpa '
'qui officia deserunt mollit anim id est '
'laborum.'),
'deprecated_opt': cfg.StrOpt('bar',
deprecated_name='foobar',
help='deprecated'),
'deprecated_group': cfg.StrOpt('bar',
deprecated_group='group1',
deprecated_name='foobar',
help='deprecated'),
'unknown_type': cfg.Opt('unknown_opt',
default=123,
help='unknown'),
'str_opt': cfg.StrOpt('str_opt',
default='foo bar',
help='a string'),
'str_opt_with_space': cfg.StrOpt('str_opt',
default=' foo bar ',
help='a string with spaces'),
'bool_opt': cfg.BoolOpt('bool_opt',
default=False,
help='a boolean'),
'int_opt': cfg.IntOpt('int_opt',
default=10,
help='an integer'),
'float_opt': cfg.FloatOpt('float_opt',
default=0.1,
help='a float'),
'list_opt': cfg.ListOpt('list_opt',
default=['1', '2', '3'],
help='a list'),
'dict_opt': cfg.DictOpt('dict_opt',
default={'1': 'yes', '2': 'no'},
help='a dict'),
'multi_opt': cfg.MultiStrOpt('multi_opt',
default=['1', '2', '3'],
help='multiple strings'),
}
content_scenarios = [
('empty',
dict(opts=[], expected='''[DEFAULT]
''')),
('single_namespace',
dict(opts=[('test', [(None, [opts['foo']])])],
expected='''[DEFAULT]
#
# From test
#
# foo option (string value)
#foo = <None>
''')),
('multiple_namespaces',
dict(opts=[('test', [(None, [opts['foo']])]),
('other', [(None, [opts['bar']])])],
expected='''[DEFAULT]
#
# From other
#
# bar option (string value)
#bar = <None>
#
# From test
#
# foo option (string value)
#foo = <None>
''')),
('group',
dict(opts=[('test', [('group1', [opts['foo']])])],
expected='''[DEFAULT]
[group1]
#
# From test
#
# foo option (string value)
#foo = <None>
''')),
('empty_group',
dict(opts=[('test', [('group1', [])])],
expected='''[DEFAULT]
''')),
('multiple_groups',
dict(opts=[('test', [('group1', [opts['foo']]),
('group2', [opts['bar']])])],
expected='''[DEFAULT]
[group1]
#
# From test
#
# foo option (string value)
#foo = <None>
[group2]
#
# From test
#
# bar option (string value)
#bar = <None>
''')),
('group_in_multiple_namespaces',
dict(opts=[('test', [('group1', [opts['foo']])]),
('other', [('group1', [opts['bar']])])],
expected='''[DEFAULT]
[group1]
#
# From other
#
# bar option (string value)
#bar = <None>
#
# From test
#
# foo option (string value)
#foo = <None>
''')),
('hyphenated_name',
dict(opts=[('test', [(None, [opts['foo-bar']])])],
expected='''[DEFAULT]
#
# From test
#
# foobar (string value)
#foo_bar = <None>
''')),
('no_help',
dict(opts=[('test', [(None, [opts['no_help']])])],
log_warning=('"%s" is missing a help string', 'no_help'),
expected='''[DEFAULT]
#
# From test
#
# (string value)
#no_help = <None>
''')),
('long_help',
dict(opts=[('test', [(None, [opts['long_help']])])],
expected='''[DEFAULT]
#
# From test
#
# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do
# eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim
# ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut
# aliquip ex ea commodo consequat. Duis aute irure dolor in
# reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
# pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
# culpa qui officia deserunt mollit anim id est laborum. (string
# value)
#long_help = <None>
''')),
('long_help_wrap_at_40',
dict(opts=[('test', [(None, [opts['long_help']])])],
wrap_width=40,
expected='''[DEFAULT]
#
# From test
#
# Lorem ipsum dolor sit amet,
# consectetur adipisicing elit, sed do
# eiusmod tempor incididunt ut labore et
# dolore magna aliqua. Ut enim ad minim
# veniam, quis nostrud exercitation
# ullamco laboris nisi ut aliquip ex ea
# commodo consequat. Duis aute irure
# dolor in reprehenderit in voluptate
# velit esse cillum dolore eu fugiat
# nulla pariatur. Excepteur sint
# occaecat cupidatat non proident, sunt
# in culpa qui officia deserunt mollit
# anim id est laborum. (string value)
#long_help = <None>
''')),
('long_help_no_wrapping',
dict(opts=[('test', [(None, [opts['long_help']])])],
wrap_width=0,
expected='''[DEFAULT]
#
# From test
#
''' # noqa
'# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod '
'tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, '
'quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo '
'consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse '
'cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat '
'non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. '
'(string value)'
'''
#long_help = <None>
''')),
('deprecated',
dict(opts=[('test', [('foo', [opts['deprecated_opt']])])],
expected='''[DEFAULT]
[foo]
#
# From test
#
# deprecated (string value)
# Deprecated group/name - [DEFAULT]/foobar
#bar = <None>
''')),
('deprecated_group',
dict(opts=[('test', [('foo', [opts['deprecated_group']])])],
expected='''[DEFAULT]
[foo]
#
# From test
#
# deprecated (string value)
# Deprecated group/name - [group1]/foobar
#bar = <None>
''')),
('unknown_type',
dict(opts=[('test', [(None, [opts['unknown_type']])])],
log_warning=('Unknown option type: %s',
repr(opts['unknown_type'])),
expected='''[DEFAULT]
#
# From test
#
# unknown (unknown type)
#unknown_opt = 123
''')),
('str_opt',
dict(opts=[('test', [(None, [opts['str_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# a string (string value)
#str_opt = foo bar
''')),
('str_opt_with_space',
dict(opts=[('test', [(None, [opts['str_opt_with_space']])])],
expected='''[DEFAULT]
#
# From test
#
# a string with spaces (string value)
#str_opt = " foo bar "
''')),
('bool_opt',
dict(opts=[('test', [(None, [opts['bool_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# a boolean (boolean value)
#bool_opt = false
''')),
('int_opt',
dict(opts=[('test', [(None, [opts['int_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# an integer (integer value)
#int_opt = 10
''')),
('float_opt',
dict(opts=[('test', [(None, [opts['float_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# a float (floating point value)
#float_opt = 0.1
''')),
('list_opt',
dict(opts=[('test', [(None, [opts['list_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# a list (list value)
#list_opt = 1,2,3
''')),
('dict_opt',
dict(opts=[('test', [(None, [opts['dict_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# a dict (dict value)
#dict_opt = 1:yes,2:no
''')),
('multi_opt',
dict(opts=[('test', [(None, [opts['multi_opt']])])],
expected='''[DEFAULT]
#
# From test
#
# multiple strings (multi valued)
#multi_opt = 1
#multi_opt = 2
#multi_opt = 3
''')),
('sanitizer',
dict(opts=[('test', [(None, [opts['str_opt']])])],
sanitizer=lambda o, s: s.replace(' ', 'ish'),
expected='''[DEFAULT]
#
# From test
#
# a string (string value)
#str_opt = fooishbar
''')),
]
output_file_scenarios = [
('stdout',
dict(stdout=True, output_file=None)),
('output_file',
dict(output_file='sample.conf', stdout=False)),
]
@classmethod
def generate_scenarios(cls):
cls.scenarios = testscenarios.multiply_scenarios(
cls.content_scenarios,
cls.output_file_scenarios)
def setUp(self):
super(GeneratorTestCase, self).setUp()
self.conf = cfg.ConfigOpts()
self.config_fixture = config_fixture.Config(self.conf)
self.config = self.config_fixture.config
self.useFixture(self.config_fixture)
self.tempdir = self.useFixture(fixtures.TempDir())
def _capture_stream(self, stream_name):
self.useFixture(fixtures.MonkeyPatch("sys.%s" % stream_name,
moves.StringIO()))
return getattr(sys, stream_name)
def _capture_stdout(self):
return self._capture_stream('stdout')
@mock.patch('stevedore.named.NamedExtensionManager')
@mock.patch('stevedore.driver.DriverManager')
@mock.patch.object(generator, 'LOG')
def test_generate(self, mock_log, driver_mgr, named_mgr):
generator.register_cli_opts(self.conf)
namespaces = [i[0] for i in self.opts]
self.config(namespace=namespaces)
wrap_width = getattr(self, 'wrap_width', None)
if wrap_width is not None:
self.config(wrap_width=wrap_width)
if self.stdout:
stdout = self._capture_stdout()
else:
output_file = self.tempdir.join(self.output_file)
self.config(output_file=output_file)
mock_eps = []
for name, opts in self.opts:
mock_ep = mock.Mock()
mock_ep.configure_mock(name=name, obj=opts)
mock_eps.append(mock_ep)
named_mgr.return_value = mock_eps
sanitizer = getattr(self, 'sanitizer', None)
if sanitizer is not None:
self.config(sanitizer='test_sanitizer')
driver = mock.Mock(driver=sanitizer)
driver_mgr.return_value = driver
generator.generate(self.conf)
if self.stdout:
self.assertEqual(self.expected, stdout.getvalue())
else:
content = open(output_file).read()
self.assertEqual(self.expected, content)
named_mgr.assert_called_once_with('oslo.config.opts',
names=namespaces,
invoke_on_load=True)
if sanitizer is not None:
driver_mgr.assert_called_once_with('oslo.config.sanitizer',
name='test_sanitizer')
pass
else:
self.assertFalse(driver_mgr.called)
log_warning = getattr(self, 'log_warning', None)
if log_warning is not None:
mock_log.warning.assert_called_once_with(*log_warning)
else:
self.assertFalse(mock_log.warning.called)
GeneratorTestCase.generate_scenarios()