openstack/oslo.config
Code Issues Proposed changes

Merge "add to group data model to for generator"

Browse Source
This commit is contained in:
Jenkins 2017-06-28 08:21:24 +00:00 committed by Gerrit Code Review
commit a8482a323b
3 changed files with 271 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
oslo_config/cfg.py
View File

@ -231,6 +231,46 @@ group name::
--rabbit-host localhost --rabbit-port 9999
Dynamic Groups
--------------
Groups can be registered dynamically by application code. This
introduces a challenge for the sample generator, discovery mechanisms,
and validation tools, since they do not know in advance the names of
all of the groups. The ``dynamic_group_owner`` parameter to the
constructor specifies the full name of an option registered in another
group that controls repeated instances of a dynamic group. This option
is usually a MultiStrOpt.
For example, Cinder supports multiple storage backend devices and
services. To configure Cinder to communicate with multiple backends,
the ``enabled_backends`` option is set to the list of names of
backends. Each backend group includes the options for communicating
with that device or service.
Driver Groups
-------------
Groups can have dynamic sets of options, usually based on a driver
that has unique requirements. This works at runtime because the code
registers options before it uses them, but it introduces a challenge
for the sample generator, discovery mechanisms, and validation tools
because they do not know in advance the correct options for a group.
To address this issue, the driver option for a group can be named
using the ``driver_option`` parameter. Each driver option should
define its own discovery entry point namespace to return the set of
options for that driver, named using the prefix
``"oslo.config.opts."`` followed by the driver option name.
In the Cinder case described above, a ``volume_backend_name`` option
is part of the static definition of the group, so ``driver_option``
should be set to ``"volume_backend_name"``. And plugins should be
registered under ``"oslo.config.opts.volume_backend_name"`` using the
same names as the main plugin registered with
``"oslo.config.opts"``. The drivers residing within the Cinder code
base have an entry point named ``"cinder"`` registered.
Accessing Option Values In Your Code
------------------------------------
@ -1709,18 +1749,51 @@ class OptGroup(object):
the group description as displayed in --help
:param name: the group name
:type name: str
:param title: the group title for --help
:type title: str
:param help: the group description for --help
:type help: str
:param dynamic_group_owner: The name of the option that controls
repeated instances of this group.
:type dynamic_group_owner: str
:param driver_option: The name of the option within the group that
controls which driver will register options.
:type driver_option: str
"""
def __init__(self, name, title=None, help=None):
def __init__(self, name, title=None, help=None,
dynamic_group_owner='',
driver_option=''):
"""Constructs an OptGroup object."""
self.name = name
self.title = "%s options" % name if title is None else title
self.help = help
self.dynamic_group_owner = dynamic_group_owner
self.driver_option = driver_option
self._opts = {} # dict of dicts of (opt:, override:, default:)
self._argparse_group = None
self._driver_opts = {} # populated by the config generator
def _save_driver_opts(self, opts):
"""Save known driver opts.
:param opts: mapping between driver name and list of opts
:type opts: dict
"""
self._driver_opts.update(opts)
def _get_generator_data(self):
"Return a dict with data for the sample generator."
return {
'help': self.help or '',
'dynamic_group_owner': self.dynamic_group_owner,
'driver_option': self.driver_option,
'driver_opts': self._driver_opts,
}
def _register_opt(self, opt, cli=False):
"""Add an opt to this group.

100
oslo_config/generator.py
View File

@ -410,6 +410,33 @@ def _get_raw_opts_loaders(namespaces):
return [(e.name, e.plugin) for e in mgr]
def _get_driver_opts_loaders(namespaces, driver_option_name):
mgr = stevedore.named.NamedExtensionManager(
namespace='oslo.config.opts.' + driver_option_name,
names=namespaces,
on_load_failure_callback=on_load_failure_callback,
invoke_on_load=False)
return [(e.name, e.plugin) for e in mgr]
def _get_driver_opts(driver_option_name, namespaces):
"""List the options available from plugins for drivers based on the option.
:param driver_option_name: The name of the option controlling the
driver options.
:param namespaces: a list of namespaces registered under
'oslo.config.opts.' + driver_option_name
:returns: a dict mapping driver name to option list
"""
all_opts = {}
loaders = _get_driver_opts_loaders(namespaces, driver_option_name)
for plugin_name, loader in loaders:
for driver_name, option_list in loader().items():
all_opts.setdefault(driver_name, []).extend(option_list)
return all_opts
def _get_opt_default_updaters(namespaces):
mgr = stevedore.named.NamedExtensionManager(
'oslo.config.opts.defaults',
@ -441,11 +468,41 @@ def _list_opts(namespaces):
_update_defaults(namespaces)
# Ask for the option definitions. At this point any global default
# changes made by the updaters should be in effect.
opts = [
(namespace, loader())
for namespace, loader in loaders
]
return _cleanup_opts(opts)
response = []
for namespace, loader in loaders:
# The loaders return iterables for the group opts, and we need
# to extend them, so build a list.
namespace_values = []
# Look through the groups and find any that need drivers so we
# can load those extra options.
for group, group_opts in loader():
# group_opts is an iterable but we are going to extend it
# so convert it to a list.
group_opts = list(group_opts)
if isinstance(group, cfg.OptGroup):
if group.driver_option:
# Load the options for all of the known drivers.
driver_opts = _get_driver_opts(
group.driver_option,
namespaces,
)
# Save the list of names of options for each
# driver in the group for use later. Add the
# options to the group_opts list so they are
# processed along with the static options in that
# group.
driver_opt_names = {}
for driver_name, opts in sorted(driver_opts.items()):
# Multiple plugins may add values to the same
# driver name, so combine the lists we do
# find.
driver_opt_names.setdefault(driver_name, []).extend(
o.name for o in opts)
group_opts.extend(opts)
group._save_driver_opts(driver_opt_names)
namespace_values.append((group, group_opts))
response.append((namespace, namespace_values))
return _cleanup_opts(response)
def on_load_failure_callback(*args, **kwargs):
@ -565,14 +622,22 @@ def _generate_machine_readable_data(groups, conf):
'generator_options': {}}
# See _get_groups for details on the structure of group_data
for group_name, group_data in groups.items():
output_data['options'][group_name] = {'opts': [], 'help': ''}
output_group = {'opts': [], 'help': ''}
output_data['options'][group_name] = output_group
for namespace in group_data['namespaces']:
for opt in namespace[1]:
if group_data['object']:
output_group = output_data['options'][group_name]
output_group['help'] = group_data['object'].help
output_group.update(
group_data['object']._get_generator_data()
)
else:
output_group.update({
'dynamic_group_owner': '',
'driver_option': '',
'driver_opts': {},
})
entry = _build_entry(opt, group_name, namespace[0], conf)
output_data['options'][group_name]['opts'].append(entry)
output_group['opts'].append(entry)
# Need copies of the opts because we modify them
for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
group = deprecated_opt.pop('group')
@ -581,6 +646,16 @@ def _generate_machine_readable_data(groups, conf):
deprecated_opt['replacement_name'] = entry['name']
deprecated_opt['replacement_group'] = group_name
deprecated_options[group].append(deprecated_opt)
# Build the list of options in the group that are not tied to
# a driver.
non_driver_opt_names = [
o['name']
for o in output_group['opts']
if not any(o['name'] in output_group['driver_opts'][d]
for d in output_group['driver_opts'])
]
output_group['standard_opts'] = non_driver_opt_names
output_data['generator_options'] = dict(conf)
return output_data
@ -605,7 +680,7 @@ def _output_machine_readable(groups, output_file, conf):
output_file.write('\n')
def generate(conf):
def generate(conf, output_file=None):
"""Generate a sample config file.
List all of the options available via the namespaces specified in the given
@ -615,8 +690,9 @@ def generate(conf):
"""
conf.register_opts(_generator_opts)
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
if output_file is None:
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
groups = _get_groups(_list_opts(conf.namespace))

109
oslo_config/tests/test_generator.py
View File

@ -26,6 +26,9 @@ from oslo_config import fixture as config_fixture
from oslo_config import generator
from oslo_config import types
import yaml
load_tests = testscenarios.load_tests_apply_scenarios
@ -954,6 +957,80 @@ class GeneratorTestCase(base.BaseTestCase):
self.assertFalse(mock_log.warning.called)
class DriverOptionTestCase(base.BaseTestCase):
def setUp(self):
super(DriverOptionTestCase, self).setUp()
self.conf = cfg.ConfigOpts()
self.config_fixture = config_fixture.Config(self.conf)
self.config = self.config_fixture.config
self.useFixture(self.config_fixture)
@mock.patch.object(generator, '_get_driver_opts_loaders')
@mock.patch.object(generator, '_get_raw_opts_loaders')
@mock.patch.object(generator, 'LOG')
def test_driver_option(self, mock_log, raw_opts_loader,
driver_opts_loader):
group = cfg.OptGroup(
name='test_group',
title='Test Group',
driver_option='foo',
)
regular_opts = [
cfg.MultiStrOpt('foo', help='foo option'),
cfg.StrOpt('bar', help='bar option'),
]
driver_opts = {
'd1': [
cfg.StrOpt('d1-foo', help='foo option'),
],
'd2': [
cfg.StrOpt('d2-foo', help='foo option'),
],
}
# We have a static data structure matching what should be
# returned by _list_opts() but we're mocking out a lower level
# function that needs to return a namespace and a callable to
# return options from that namespace. We have to pass opts to
# the lambda to cache a reference to the name because the list
# comprehension changes the thing pointed to by the name each
# time through the loop.
raw_opts_loader.return_value = [
('testing', lambda: [(group, regular_opts)]),
]
driver_opts_loader.return_value = [
('testing', lambda: driver_opts),
]
# Initialize the generator to produce YAML output to a buffer.
generator.register_cli_opts(self.conf)
self.config(namespace=['test_generator'], format_='yaml')
stdout = moves.StringIO()
# Generate the output and parse it back to a data structure.
generator.generate(self.conf, output_file=stdout)
body = stdout.getvalue()
actual = yaml.safe_load(body)
test_section = actual['options']['test_group']
self.assertEqual('foo', test_section['driver_option'])
found_option_names = [
o['name']
for o in test_section['opts']
]
self.assertEqual(
['foo', 'bar', 'd1-foo', 'd2-foo'],
found_option_names
)
self.assertEqual(
{'d1': ['d1-foo'], 'd2': ['d2-foo']},
test_section['driver_opts'],
)
GENERATOR_OPTS = {'format_': 'yaml',
'minimal': False,
'namespace': ['test'],
@ -972,7 +1049,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['foo'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1000,7 +1081,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['long_help'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1028,7 +1113,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['long_help_pre'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1061,7 +1150,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['foo-bar'],
'opts': [{
'advanced': False,
'choices': [],
@ -1092,7 +1185,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['choices_opt'],
'opts': [{'advanced': False,
'choices': (None, '', 'a', 'b', 'c'),
'default': 'a',
@ -1120,7 +1217,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['int_opt'],
'opts': [{'advanced': False,
'choices': [],
'default': 10,
@ -1148,11 +1249,19 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
# 'driver_option': '',
# 'driver_opts': [],
# 'dynamic_group_owner': '',
'help': '',
'standard_opts': [],
'opts': []
},
'group1': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': all_groups['group1'].help,
'standard_opts': ['foo'],
'opts': [{'advanced': False,
'choices': [],
'default': None,