Machine Readable Sample Config
Adds the ability for the sample config generator to output the config data in the machine readable formats yaml and json. bp machine-readable-sample-config Change-Id: I236918f0c1da27358aace66914aae5c34afef301 Co-Authored-By: Stephen Finucane <sfinucan@redhat.com>
This commit is contained in:
Ben Nemec
parent
f0a915f596
commit
a29c084cb1
@ -24,13 +24,16 @@ Tool for generating a sample configuration file. See
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import collections
|
import collections
|
||||||
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import operator
|
import operator
|
||||||
import sys
|
import sys
|
||||||
import textwrap
|
import textwrap
|
||||||
|
|
||||||
|
import json
|
||||||
import pkg_resources
|
import pkg_resources
|
||||||
import six
|
import six
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
@ -61,6 +64,15 @@ _generator_opts = [
|
|||||||
default=False,
|
default=False,
|
||||||
help='Only output summaries of help text to config files. Retain '
|
help='Only output summaries of help text to config files. Retain '
|
||||||
'longer help text for Sphinx documents.'),
|
'longer help text for Sphinx documents.'),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'format',
|
||||||
|
help='Desired format for the output. "ini" is the only one which can '
|
||||||
|
'be used directly with oslo.config. "json" and "yaml" are '
|
||||||
|
'intended for third-party tools that want to write config files '
|
||||||
|
'based on the sample config data.',
|
||||||
|
default='ini',
|
||||||
|
choices=['ini', 'json', 'yaml'],
|
||||||
|
dest='format_'),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@ -491,6 +503,108 @@ def _get_groups(conf_ns):
|
|||||||
return groups
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
def _build_entry(opt, group, namespace, conf):
|
||||||
|
"""Return a dict representing the passed in opt
|
||||||
|
|
||||||
|
The dict will contain all public attributes of opt, as well as additional
|
||||||
|
entries for namespace, choices, min, and max. Any DeprecatedOpts
|
||||||
|
contained in the deprecated_opts member will be converted to a dict with
|
||||||
|
the format: {'group': <deprecated group>, 'name': <deprecated name>}
|
||||||
|
|
||||||
|
:param opt: The Opt object to represent as a dict.
|
||||||
|
:param group: The name of the group containing opt.
|
||||||
|
:param namespace: The name of the namespace containing opt.
|
||||||
|
:param conf: The ConfigOpts object containing the options for the
|
||||||
|
generator tool
|
||||||
|
"""
|
||||||
|
entry = {key: value for key, value in opt.__dict__.items()
|
||||||
|
if not key.startswith('_')}
|
||||||
|
entry['namespace'] = namespace
|
||||||
|
# In some types, choices is explicitly set to None. Force it to [] so it
|
||||||
|
# is always an iterable type.
|
||||||
|
entry['choices'] = getattr(entry['type'], 'choices', []) or []
|
||||||
|
entry['min'] = getattr(entry['type'], 'min', None)
|
||||||
|
entry['max'] = getattr(entry['type'], 'max', None)
|
||||||
|
entry['type'] = _format_type_name(entry['type'])
|
||||||
|
deprecated_opts = []
|
||||||
|
for deprecated_opt in entry['deprecated_opts']:
|
||||||
|
# NOTE(bnemec): opt names with a - are not valid in a config file,
|
||||||
|
# but it is possible to add a DeprecatedOpt with a - name. We
|
||||||
|
# want to ignore those as they won't work anyway.
|
||||||
|
if not deprecated_opt.name or '-' not in deprecated_opt.name:
|
||||||
|
deprecated_opts.append(
|
||||||
|
{'group': deprecated_opt.group or group,
|
||||||
|
'name': deprecated_opt.name or entry['name'],
|
||||||
|
})
|
||||||
|
entry['deprecated_opts'] = deprecated_opts
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
def _generate_machine_readable_data(groups, conf):
|
||||||
|
"""Create data structure for machine readable sample config
|
||||||
|
|
||||||
|
Returns a dictionary with the top-level keys 'options',
|
||||||
|
'deprecated_options', and 'generator_options'.
|
||||||
|
|
||||||
|
'options' contains a dict mapping group names to a list of options in
|
||||||
|
that group. Each option is represented by the result of a call to
|
||||||
|
_build_entry. Only non-deprecated options are included in this list.
|
||||||
|
|
||||||
|
'deprecated_options' contains a dict mapping groups names to a list of
|
||||||
|
opts from that group which were deprecated.
|
||||||
|
|
||||||
|
'generator_options' is a dict mapping the options for the sample config
|
||||||
|
generator itself to their values.
|
||||||
|
|
||||||
|
:param groups: A dict of groups as returned by _get_groups.
|
||||||
|
:param conf: The ConfigOpts object containing the options for the
|
||||||
|
generator tool
|
||||||
|
"""
|
||||||
|
output_data = {'options': {},
|
||||||
|
'deprecated_options': {},
|
||||||
|
'generator_options': {}}
|
||||||
|
# See _get_groups for details on the structure of group_data
|
||||||
|
for group_name, group_data in groups.items():
|
||||||
|
output_data['options'][group_name] = {'opts': [], 'help': ''}
|
||||||
|
for namespace in group_data['namespaces']:
|
||||||
|
for opt in namespace[1]:
|
||||||
|
if group_data['object']:
|
||||||
|
output_group = output_data['options'][group_name]
|
||||||
|
output_group['help'] = group_data['object'].help
|
||||||
|
entry = _build_entry(opt, group_name, namespace[0], conf)
|
||||||
|
output_data['options'][group_name]['opts'].append(entry)
|
||||||
|
# Need copies of the opts because we modify them
|
||||||
|
for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
|
||||||
|
group = deprecated_opt.pop('group')
|
||||||
|
deprecated_options = output_data['deprecated_options']
|
||||||
|
deprecated_options.setdefault(group, [])
|
||||||
|
deprecated_opt['replacement_name'] = entry['name']
|
||||||
|
deprecated_opt['replacement_group'] = group_name
|
||||||
|
deprecated_options[group].append(deprecated_opt)
|
||||||
|
output_data['generator_options'] = conf
|
||||||
|
return output_data
|
||||||
|
|
||||||
|
|
||||||
|
def _output_machine_readable(groups, output_file, conf):
|
||||||
|
"""Write a machine readable sample config file
|
||||||
|
|
||||||
|
Take the data returned by _generate_machine_readable_data and write it in
|
||||||
|
the format specified by the format_ attribute of conf.
|
||||||
|
|
||||||
|
:param groups: A dict of groups as returned by _get_groups.
|
||||||
|
:param output_file: A file-like object to which the data should be written.
|
||||||
|
:param conf: The ConfigOpts object containing the options for the
|
||||||
|
generator tool
|
||||||
|
"""
|
||||||
|
output_data = _generate_machine_readable_data(groups, conf)
|
||||||
|
if conf.format_ == 'yaml':
|
||||||
|
output_file.write(yaml.safe_dump(output_data,
|
||||||
|
default_flow_style=False))
|
||||||
|
else:
|
||||||
|
output_file.write(json.dumps(output_data, sort_keys=True))
|
||||||
|
output_file.write('\n')
|
||||||
|
|
||||||
|
|
||||||
def generate(conf):
|
def generate(conf):
|
||||||
"""Generate a sample config file.
|
"""Generate a sample config file.
|
||||||
|
|
||||||
@ -504,21 +618,26 @@ def generate(conf):
|
|||||||
output_file = (open(conf.output_file, 'w')
|
output_file = (open(conf.output_file, 'w')
|
||||||
if conf.output_file else sys.stdout)
|
if conf.output_file else sys.stdout)
|
||||||
|
|
||||||
formatter = _OptFormatter(output_file=output_file,
|
|
||||||
wrap_width=conf.wrap_width)
|
|
||||||
|
|
||||||
groups = _get_groups(_list_opts(conf.namespace))
|
groups = _get_groups(_list_opts(conf.namespace))
|
||||||
|
|
||||||
# Output the "DEFAULT" section as the very first section
|
if conf.format_ == 'ini':
|
||||||
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
|
formatter = _OptFormatter(output_file=output_file,
|
||||||
conf.summarize)
|
wrap_width=conf.wrap_width)
|
||||||
|
|
||||||
# output all other config sections with groups in alphabetical order
|
# Output the "DEFAULT" section as the very first section
|
||||||
for group, group_data in sorted(groups.items()):
|
_output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
|
||||||
formatter.write('\n\n')
|
|
||||||
_output_opts(formatter, group, group_data, conf.minimal,
|
|
||||||
conf.summarize)
|
conf.summarize)
|
||||||
|
|
||||||
|
# output all other config sections with groups in alphabetical order
|
||||||
|
for group, group_data in sorted(groups.items()):
|
||||||
|
formatter.write('\n\n')
|
||||||
|
_output_opts(formatter, group, group_data, conf.minimal,
|
||||||
|
conf.summarize)
|
||||||
|
else:
|
||||||
|
_output_machine_readable(groups,
|
||||||
|
output_file=output_file,
|
||||||
|
conf=conf)
|
||||||
|
|
||||||
|
|
||||||
def main(args=None):
|
def main(args=None):
|
||||||
"""The main function of oslo-config-generator."""
|
"""The main function of oslo-config-generator."""
|
||||||
|
|||||||
@ -954,6 +954,265 @@ class GeneratorTestCase(base.BaseTestCase):
|
|||||||
self.assertFalse(mock_log.warning.called)
|
self.assertFalse(mock_log.warning.called)
|
||||||
|
|
||||||
|
|
||||||
|
GENERATOR_OPTS = {'format_': 'yaml',
|
||||||
|
'minimal': False,
|
||||||
|
'namespace': ['test'],
|
||||||
|
'output_file': None,
|
||||||
|
'summarize': False,
|
||||||
|
'wrap_width': 70}
|
||||||
|
|
||||||
|
|
||||||
|
class MachineReadableGeneratorTestCase(base.BaseTestCase):
|
||||||
|
all_opts = GeneratorTestCase.opts
|
||||||
|
all_groups = GeneratorTestCase.groups
|
||||||
|
content_scenarios = [
|
||||||
|
('single_namespace',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['foo']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': None,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'foo',
|
||||||
|
'help': 'foo option',
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'foo',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'string value'}]}}})),
|
||||||
|
('long_help',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['long_help']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': None,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'long_help',
|
||||||
|
'help': all_opts['long_help'].help,
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'long_help',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'string value'}]}}})),
|
||||||
|
('long_help_pre',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['long_help_pre']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': None,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'long_help_pre',
|
||||||
|
'help':
|
||||||
|
all_opts['long_help_pre'].help,
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'long_help_pre',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'string value'}]}}})),
|
||||||
|
('opt_with_DeprecatedOpt',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['opt_with_DeprecatedOpt']])])],
|
||||||
|
expected={
|
||||||
|
'deprecated_options': {
|
||||||
|
'deprecated': [{'name': 'foo_bar',
|
||||||
|
'replacement_group': 'DEFAULT',
|
||||||
|
'replacement_name': 'foo-bar'}]},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{
|
||||||
|
'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': None,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [{'group': 'deprecated',
|
||||||
|
'name': 'foo_bar'}],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'foo_bar',
|
||||||
|
'help':
|
||||||
|
all_opts['opt_with_DeprecatedOpt'].help,
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'foo-bar',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'boolean value'}]}}})),
|
||||||
|
('choices_opt',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['choices_opt']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': (None, '', 'a', 'b', 'c'),
|
||||||
|
'default': 'a',
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'choices_opt',
|
||||||
|
'help': all_opts['choices_opt'].help,
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'choices_opt',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'string value'}]}}})),
|
||||||
|
('int_opt',
|
||||||
|
dict(opts=[('test', [(None, [all_opts['int_opt']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': 10,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'int_opt',
|
||||||
|
'help': all_opts['int_opt'].help,
|
||||||
|
'max': 20,
|
||||||
|
'metavar': None,
|
||||||
|
'min': 1,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'int_opt',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'integer value'}]}}})),
|
||||||
|
('group_help',
|
||||||
|
dict(opts=[('test', [(all_groups['group1'], [all_opts['foo']])])],
|
||||||
|
expected={'deprecated_options': {},
|
||||||
|
'generator_options': GENERATOR_OPTS,
|
||||||
|
'options': {
|
||||||
|
'DEFAULT': {
|
||||||
|
'help': '',
|
||||||
|
'opts': []
|
||||||
|
},
|
||||||
|
'group1': {
|
||||||
|
'help': all_groups['group1'].help,
|
||||||
|
'opts': [{'advanced': False,
|
||||||
|
'choices': [],
|
||||||
|
'default': None,
|
||||||
|
'deprecated_for_removal': False,
|
||||||
|
'deprecated_opts': [],
|
||||||
|
'deprecated_reason': None,
|
||||||
|
'deprecated_since': None,
|
||||||
|
'dest': 'foo',
|
||||||
|
'help': all_opts['foo'].help,
|
||||||
|
'max': None,
|
||||||
|
'metavar': None,
|
||||||
|
'min': None,
|
||||||
|
'mutable': False,
|
||||||
|
'name': 'foo',
|
||||||
|
'namespace': 'test',
|
||||||
|
'positional': False,
|
||||||
|
'required': False,
|
||||||
|
'sample_default': None,
|
||||||
|
'secret': False,
|
||||||
|
'short': None,
|
||||||
|
'type': 'string value'}]}}})),
|
||||||
|
]
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(MachineReadableGeneratorTestCase, self).setUp()
|
||||||
|
|
||||||
|
self.conf = cfg.ConfigOpts()
|
||||||
|
self.config_fixture = config_fixture.Config(self.conf)
|
||||||
|
self.config = self.config_fixture.config
|
||||||
|
self.useFixture(self.config_fixture)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def generate_scenarios(cls):
|
||||||
|
cls.scenarios = testscenarios.multiply_scenarios(
|
||||||
|
cls.content_scenarios)
|
||||||
|
|
||||||
|
@mock.patch.object(generator, '_get_raw_opts_loaders')
|
||||||
|
def test_generate(self, raw_opts_loader):
|
||||||
|
generator.register_cli_opts(self.conf)
|
||||||
|
namespaces = [i[0] for i in self.opts]
|
||||||
|
self.config(namespace=namespaces, format_='yaml')
|
||||||
|
|
||||||
|
# We have a static data structure matching what should be
|
||||||
|
# returned by _list_opts() but we're mocking out a lower level
|
||||||
|
# function that needs to return a namespace and a callable to
|
||||||
|
# return options from that namespace. We have to pass opts to
|
||||||
|
# the lambda to cache a reference to the name because the list
|
||||||
|
# comprehension changes the thing pointed to by the name each
|
||||||
|
# time through the loop.
|
||||||
|
raw_opts_loader.return_value = [
|
||||||
|
(ns, lambda opts=opts: opts)
|
||||||
|
for ns, opts in self.opts
|
||||||
|
]
|
||||||
|
test_groups = generator._get_groups(
|
||||||
|
generator._list_opts(self.conf.namespace))
|
||||||
|
self.assertEqual(self.expected,
|
||||||
|
generator._generate_machine_readable_data(test_groups,
|
||||||
|
self.conf))
|
||||||
|
|
||||||
|
|
||||||
class IgnoreDoublesTestCase(base.BaseTestCase):
|
class IgnoreDoublesTestCase(base.BaseTestCase):
|
||||||
|
|
||||||
opts = [cfg.StrOpt('foo', help='foo option'),
|
opts = [cfg.StrOpt('foo', help='foo option'),
|
||||||
@ -1377,3 +1636,4 @@ class AdvancedOptionsTestCase(base.BaseTestCase):
|
|||||||
|
|
||||||
|
|
||||||
GeneratorTestCase.generate_scenarios()
|
GeneratorTestCase.generate_scenarios()
|
||||||
|
MachineReadableGeneratorTestCase.generate_scenarios()
|
||||||
|
|||||||
@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
The sample config generator can now generate machine-readable formats of
|
||||||
|
the sample config data. This can be consumed by deployment tools to
|
||||||
|
automatically generate configuration files that contain all of the
|
||||||
|
information in the traditional sample configs.
|
||||||
Loading…
x
Reference in New Issue
Block a user