Machine Readable Sample Config
Adds the ability for the sample config generator to output the config data in the machine readable formats yaml and json. bp machine-readable-sample-config Change-Id: I236918f0c1da27358aace66914aae5c34afef301 Co-Authored-By: Stephen Finucane <sfinucan@redhat.com>
This commit is contained in:
Ben Nemec
parent
f0a915f596
commit
a29c084cb1
@ -24,13 +24,16 @@ Tool for generating a sample configuration file. See
|
||||
"""
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import logging
|
||||
import operator
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import json
|
||||
import pkg_resources
|
||||
import six
|
||||
import yaml
|
||||
|
||||
|
||||
from oslo_config import cfg
|
||||
@ -61,6 +64,15 @@ _generator_opts = [
|
||||
default=False,
|
||||
help='Only output summaries of help text to config files. Retain '
|
||||
'longer help text for Sphinx documents.'),
|
||||
cfg.StrOpt(
|
||||
'format',
|
||||
help='Desired format for the output. "ini" is the only one which can '
|
||||
'be used directly with oslo.config. "json" and "yaml" are '
|
||||
'intended for third-party tools that want to write config files '
|
||||
'based on the sample config data.',
|
||||
default='ini',
|
||||
choices=['ini', 'json', 'yaml'],
|
||||
dest='format_'),
|
||||
]
|
||||
|
||||
|
||||
@ -491,6 +503,108 @@ def _get_groups(conf_ns):
|
||||
return groups
|
||||
|
||||
|
||||
def _build_entry(opt, group, namespace, conf):
|
||||
"""Return a dict representing the passed in opt
|
||||
|
||||
The dict will contain all public attributes of opt, as well as additional
|
||||
entries for namespace, choices, min, and max. Any DeprecatedOpts
|
||||
contained in the deprecated_opts member will be converted to a dict with
|
||||
the format: {'group': <deprecated group>, 'name': <deprecated name>}
|
||||
|
||||
:param opt: The Opt object to represent as a dict.
|
||||
:param group: The name of the group containing opt.
|
||||
:param namespace: The name of the namespace containing opt.
|
||||
:param conf: The ConfigOpts object containing the options for the
|
||||
generator tool
|
||||
"""
|
||||
entry = {key: value for key, value in opt.__dict__.items()
|
||||
if not key.startswith('_')}
|
||||
entry['namespace'] = namespace
|
||||
# In some types, choices is explicitly set to None. Force it to [] so it
|
||||
# is always an iterable type.
|
||||
entry['choices'] = getattr(entry['type'], 'choices', []) or []
|
||||
entry['min'] = getattr(entry['type'], 'min', None)
|
||||
entry['max'] = getattr(entry['type'], 'max', None)
|
||||
entry['type'] = _format_type_name(entry['type'])
|
||||
deprecated_opts = []
|
||||
for deprecated_opt in entry['deprecated_opts']:
|
||||
# NOTE(bnemec): opt names with a - are not valid in a config file,
|
||||
# but it is possible to add a DeprecatedOpt with a - name. We
|
||||
# want to ignore those as they won't work anyway.
|
||||
if not deprecated_opt.name or '-' not in deprecated_opt.name:
|
||||
deprecated_opts.append(
|
||||
{'group': deprecated_opt.group or group,
|
||||
'name': deprecated_opt.name or entry['name'],
|
||||
})
|
||||
entry['deprecated_opts'] = deprecated_opts
|
||||
return entry
|
||||
|
||||
|
||||
def _generate_machine_readable_data(groups, conf):
|
||||
"""Create data structure for machine readable sample config
|
||||
|
||||
Returns a dictionary with the top-level keys 'options',
|
||||
'deprecated_options', and 'generator_options'.
|
||||
|
||||
'options' contains a dict mapping group names to a list of options in
|
||||
that group. Each option is represented by the result of a call to
|
||||
_build_entry. Only non-deprecated options are included in this list.
|
||||
|
||||
'deprecated_options' contains a dict mapping groups names to a list of
|
||||
opts from that group which were deprecated.
|
||||
|
||||
'generator_options' is a dict mapping the options for the sample config
|
||||
generator itself to their values.
|
||||
|
||||
:param groups: A dict of groups as returned by _get_groups.
|
||||
:param conf: The ConfigOpts object containing the options for the
|
||||
generator tool
|
||||
"""
|
||||
output_data = {'options': {},
|
||||
'deprecated_options': {},
|
||||
'generator_options': {}}
|
||||
# See _get_groups for details on the structure of group_data
|
||||
for group_name, group_data in groups.items():
|
||||
output_data['options'][group_name] = {'opts': [], 'help': ''}
|
||||
for namespace in group_data['namespaces']:
|
||||
for opt in namespace[1]:
|
||||
if group_data['object']:
|
||||
output_group = output_data['options'][group_name]
|
||||
output_group['help'] = group_data['object'].help
|
||||
entry = _build_entry(opt, group_name, namespace[0], conf)
|
||||
output_data['options'][group_name]['opts'].append(entry)
|
||||
# Need copies of the opts because we modify them
|
||||
for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
|
||||
group = deprecated_opt.pop('group')
|
||||
deprecated_options = output_data['deprecated_options']
|
||||
deprecated_options.setdefault(group, [])
|
||||
deprecated_opt['replacement_name'] = entry['name']
|
||||
deprecated_opt['replacement_group'] = group_name
|
||||
deprecated_options[group].append(deprecated_opt)
|
||||
output_data['generator_options'] = conf
|
||||
return output_data
|
||||
|
||||
|
||||
def _output_machine_readable(groups, output_file, conf):
|
||||
"""Write a machine readable sample config file
|
||||
|
||||
Take the data returned by _generate_machine_readable_data and write it in
|
||||
the format specified by the format_ attribute of conf.
|
||||
|
||||
:param groups: A dict of groups as returned by _get_groups.
|
||||
:param output_file: A file-like object to which the data should be written.
|
||||
:param conf: The ConfigOpts object containing the options for the
|
||||
generator tool
|
||||
"""
|
||||
output_data = _generate_machine_readable_data(groups, conf)
|
||||
if conf.format_ == 'yaml':
|
||||
output_file.write(yaml.safe_dump(output_data,
|
||||
default_flow_style=False))
|
||||
else:
|
||||
output_file.write(json.dumps(output_data, sort_keys=True))
|
||||
output_file.write('\n')
|
||||
|
||||
|
||||
def generate(conf):
|
||||
"""Generate a sample config file.
|
||||
|
||||
@ -504,21 +618,26 @@ def generate(conf):
|
||||
output_file = (open(conf.output_file, 'w')
|
||||
if conf.output_file else sys.stdout)
|
||||
|
||||
formatter = _OptFormatter(output_file=output_file,
|
||||
wrap_width=conf.wrap_width)
|
||||
|
||||
groups = _get_groups(_list_opts(conf.namespace))
|
||||
|
||||
# Output the "DEFAULT" section as the very first section
|
||||
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
|
||||
conf.summarize)
|
||||
if conf.format_ == 'ini':
|
||||
formatter = _OptFormatter(output_file=output_file,
|
||||
wrap_width=conf.wrap_width)
|
||||
|
||||
# output all other config sections with groups in alphabetical order
|
||||
for group, group_data in sorted(groups.items()):
|
||||
formatter.write('\n\n')
|
||||
_output_opts(formatter, group, group_data, conf.minimal,
|
||||
# Output the "DEFAULT" section as the very first section
|
||||
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
|
||||
conf.summarize)
|
||||
|
||||
# output all other config sections with groups in alphabetical order
|
||||
for group, group_data in sorted(groups.items()):
|
||||
formatter.write('\n\n')
|
||||
_output_opts(formatter, group, group_data, conf.minimal,
|
||||
conf.summarize)
|
||||
else:
|
||||
_output_machine_readable(groups,
|
||||
output_file=output_file,
|
||||
conf=conf)
|
||||
|
||||
|
||||
def main(args=None):
|
||||
"""The main function of oslo-config-generator."""
|
||||
|
||||
@ -954,6 +954,265 @@ class GeneratorTestCase(base.BaseTestCase):
|
||||
self.assertFalse(mock_log.warning.called)
|
||||
|
||||
|
||||
GENERATOR_OPTS = {'format_': 'yaml',
|
||||
'minimal': False,
|
||||
'namespace': ['test'],
|
||||
'output_file': None,
|
||||
'summarize': False,
|
||||
'wrap_width': 70}
|
||||
|
||||
|
||||
class MachineReadableGeneratorTestCase(base.BaseTestCase):
|
||||
all_opts = GeneratorTestCase.opts
|
||||
all_groups = GeneratorTestCase.groups
|
||||
content_scenarios = [
|
||||
('single_namespace',
|
||||
dict(opts=[('test', [(None, [all_opts['foo']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{'advanced': False,
|
||||
'choices': [],
|
||||
'default': None,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'foo',
|
||||
'help': 'foo option',
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'foo',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'string value'}]}}})),
|
||||
('long_help',
|
||||
dict(opts=[('test', [(None, [all_opts['long_help']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{'advanced': False,
|
||||
'choices': [],
|
||||
'default': None,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'long_help',
|
||||
'help': all_opts['long_help'].help,
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'long_help',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'string value'}]}}})),
|
||||
('long_help_pre',
|
||||
dict(opts=[('test', [(None, [all_opts['long_help_pre']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{'advanced': False,
|
||||
'choices': [],
|
||||
'default': None,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'long_help_pre',
|
||||
'help':
|
||||
all_opts['long_help_pre'].help,
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'long_help_pre',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'string value'}]}}})),
|
||||
('opt_with_DeprecatedOpt',
|
||||
dict(opts=[('test', [(None, [all_opts['opt_with_DeprecatedOpt']])])],
|
||||
expected={
|
||||
'deprecated_options': {
|
||||
'deprecated': [{'name': 'foo_bar',
|
||||
'replacement_group': 'DEFAULT',
|
||||
'replacement_name': 'foo-bar'}]},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{
|
||||
'advanced': False,
|
||||
'choices': [],
|
||||
'default': None,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [{'group': 'deprecated',
|
||||
'name': 'foo_bar'}],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'foo_bar',
|
||||
'help':
|
||||
all_opts['opt_with_DeprecatedOpt'].help,
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'foo-bar',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'boolean value'}]}}})),
|
||||
('choices_opt',
|
||||
dict(opts=[('test', [(None, [all_opts['choices_opt']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{'advanced': False,
|
||||
'choices': (None, '', 'a', 'b', 'c'),
|
||||
'default': 'a',
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'choices_opt',
|
||||
'help': all_opts['choices_opt'].help,
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'choices_opt',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'string value'}]}}})),
|
||||
('int_opt',
|
||||
dict(opts=[('test', [(None, [all_opts['int_opt']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': [{'advanced': False,
|
||||
'choices': [],
|
||||
'default': 10,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'int_opt',
|
||||
'help': all_opts['int_opt'].help,
|
||||
'max': 20,
|
||||
'metavar': None,
|
||||
'min': 1,
|
||||
'mutable': False,
|
||||
'name': 'int_opt',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'integer value'}]}}})),
|
||||
('group_help',
|
||||
dict(opts=[('test', [(all_groups['group1'], [all_opts['foo']])])],
|
||||
expected={'deprecated_options': {},
|
||||
'generator_options': GENERATOR_OPTS,
|
||||
'options': {
|
||||
'DEFAULT': {
|
||||
'help': '',
|
||||
'opts': []
|
||||
},
|
||||
'group1': {
|
||||
'help': all_groups['group1'].help,
|
||||
'opts': [{'advanced': False,
|
||||
'choices': [],
|
||||
'default': None,
|
||||
'deprecated_for_removal': False,
|
||||
'deprecated_opts': [],
|
||||
'deprecated_reason': None,
|
||||
'deprecated_since': None,
|
||||
'dest': 'foo',
|
||||
'help': all_opts['foo'].help,
|
||||
'max': None,
|
||||
'metavar': None,
|
||||
'min': None,
|
||||
'mutable': False,
|
||||
'name': 'foo',
|
||||
'namespace': 'test',
|
||||
'positional': False,
|
||||
'required': False,
|
||||
'sample_default': None,
|
||||
'secret': False,
|
||||
'short': None,
|
||||
'type': 'string value'}]}}})),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
super(MachineReadableGeneratorTestCase, self).setUp()
|
||||
|
||||
self.conf = cfg.ConfigOpts()
|
||||
self.config_fixture = config_fixture.Config(self.conf)
|
||||
self.config = self.config_fixture.config
|
||||
self.useFixture(self.config_fixture)
|
||||
|
||||
@classmethod
|
||||
def generate_scenarios(cls):
|
||||
cls.scenarios = testscenarios.multiply_scenarios(
|
||||
cls.content_scenarios)
|
||||
|
||||
@mock.patch.object(generator, '_get_raw_opts_loaders')
|
||||
def test_generate(self, raw_opts_loader):
|
||||
generator.register_cli_opts(self.conf)
|
||||
namespaces = [i[0] for i in self.opts]
|
||||
self.config(namespace=namespaces, format_='yaml')
|
||||
|
||||
# We have a static data structure matching what should be
|
||||
# returned by _list_opts() but we're mocking out a lower level
|
||||
# function that needs to return a namespace and a callable to
|
||||
# return options from that namespace. We have to pass opts to
|
||||
# the lambda to cache a reference to the name because the list
|
||||
# comprehension changes the thing pointed to by the name each
|
||||
# time through the loop.
|
||||
raw_opts_loader.return_value = [
|
||||
(ns, lambda opts=opts: opts)
|
||||
for ns, opts in self.opts
|
||||
]
|
||||
test_groups = generator._get_groups(
|
||||
generator._list_opts(self.conf.namespace))
|
||||
self.assertEqual(self.expected,
|
||||
generator._generate_machine_readable_data(test_groups,
|
||||
self.conf))
|
||||
|
||||
|
||||
class IgnoreDoublesTestCase(base.BaseTestCase):
|
||||
|
||||
opts = [cfg.StrOpt('foo', help='foo option'),
|
||||
@ -1377,3 +1636,4 @@ class AdvancedOptionsTestCase(base.BaseTestCase):
|
||||
|
||||
|
||||
GeneratorTestCase.generate_scenarios()
|
||||
MachineReadableGeneratorTestCase.generate_scenarios()
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
The sample config generator can now generate machine-readable formats of
|
||||
the sample config data. This can be consumed by deployment tools to
|
||||
automatically generate configuration files that contain all of the
|
||||
information in the traditional sample configs.
|
||||
Loading…
Reference in New Issue
Block a user