Merge "add to group data model to for generator"

oslo_config/cfg.py

@@ -231,6 +231,46 @@ group name::
 
     --rabbit-host localhost --rabbit-port 9999
 
+Dynamic Groups
+--------------
+
+Groups can be registered dynamically by application code. This
+introduces a challenge for the sample generator, discovery mechanisms,
+and validation tools, since they do not know in advance the names of
+all of the groups. The ``dynamic_group_owner`` parameter to the
+constructor specifies the full name of an option registered in another
+group that controls repeated instances of a dynamic group. This option
+is usually a MultiStrOpt.
+
+For example, Cinder supports multiple storage backend devices and
+services. To configure Cinder to communicate with multiple backends,
+the ``enabled_backends`` option is set to the list of names of
+backends. Each backend group includes the options for communicating
+with that device or service.
+
+Driver Groups
+-------------
+
+Groups can have dynamic sets of options, usually based on a driver
+that has unique requirements. This works at runtime because the code
+registers options before it uses them, but it introduces a challenge
+for the sample generator, discovery mechanisms, and validation tools
+because they do not know in advance the correct options for a group.
+
+To address this issue, the driver option for a group can be named
+using the ``driver_option`` parameter.  Each driver option should
+define its own discovery entry point namespace to return the set of
+options for that driver, named using the prefix
+``"oslo.config.opts."`` followed by the driver option name.
+
+In the Cinder case described above, a ``volume_backend_name`` option
+is part of the static definition of the group, so ``driver_option``
+should be set to ``"volume_backend_name"``. And plugins should be
+registered under ``"oslo.config.opts.volume_backend_name"`` using the
+same names as the main plugin registered with
+``"oslo.config.opts"``. The drivers residing within the Cinder code
+base have an entry point named ``"cinder"`` registered.
+
 Accessing Option Values In Your Code
 ------------------------------------
 
@@ -1709,18 +1749,51 @@ class OptGroup(object):
         the group description as displayed in --help
 
     :param name: the group name
+    :type name: str
     :param title: the group title for --help
+    :type title: str
     :param help: the group description for --help
+    :type help: str
+    :param dynamic_group_owner: The name of the option that controls
+                                repeated instances of this group.
+    :type dynamic_group_owner: str
+    :param driver_option: The name of the option within the group that
+                          controls which driver will register options.
+    :type driver_option: str
+
     """
 
-    def __init__(self, name, title=None, help=None):
+    def __init__(self, name, title=None, help=None,
+                 dynamic_group_owner='',
+                 driver_option=''):
         """Constructs an OptGroup object."""
         self.name = name
         self.title = "%s options" % name if title is None else title
         self.help = help
+        self.dynamic_group_owner = dynamic_group_owner
+        self.driver_option = driver_option
 
         self._opts = {}  # dict of dicts of (opt:, override:, default:)
         self._argparse_group = None
+        self._driver_opts = {}  # populated by the config generator
+
+    def _save_driver_opts(self, opts):
+        """Save known driver opts.
+
+        :param opts: mapping between driver name and list of opts
+        :type opts: dict
+
+        """
+        self._driver_opts.update(opts)
+
+    def _get_generator_data(self):
+        "Return a dict with data for the sample generator."
+        return {
+            'help': self.help or '',
+            'dynamic_group_owner': self.dynamic_group_owner,
+            'driver_option': self.driver_option,
+            'driver_opts': self._driver_opts,
+        }
 
     def _register_opt(self, opt, cli=False):
         """Add an opt to this group.

oslo_config/generator.py

@@ -410,6 +410,33 @@ def _get_raw_opts_loaders(namespaces):
     return [(e.name, e.plugin) for e in mgr]
 
 
+def _get_driver_opts_loaders(namespaces, driver_option_name):
+    mgr = stevedore.named.NamedExtensionManager(
+        namespace='oslo.config.opts.' + driver_option_name,
+        names=namespaces,
+        on_load_failure_callback=on_load_failure_callback,
+        invoke_on_load=False)
+    return [(e.name, e.plugin) for e in mgr]
+
+
+def _get_driver_opts(driver_option_name, namespaces):
+    """List the options available from plugins for drivers based on the option.
+
+    :param driver_option_name: The name of the option controlling the
+                               driver options.
+    :param namespaces: a list of namespaces registered under
+                       'oslo.config.opts.' + driver_option_name
+    :returns: a dict mapping driver name to option list
+
+    """
+    all_opts = {}
+    loaders = _get_driver_opts_loaders(namespaces, driver_option_name)
+    for plugin_name, loader in loaders:
+        for driver_name, option_list in loader().items():
+            all_opts.setdefault(driver_name, []).extend(option_list)
+    return all_opts
+
+
 def _get_opt_default_updaters(namespaces):
     mgr = stevedore.named.NamedExtensionManager(
         'oslo.config.opts.defaults',
@@ -441,11 +468,41 @@ def _list_opts(namespaces):
     _update_defaults(namespaces)
     # Ask for the option definitions. At this point any global default
     # changes made by the updaters should be in effect.
-    opts = [
+    response = []
-        (namespace, loader())
+    for namespace, loader in loaders:
-        for namespace, loader in loaders
+        # The loaders return iterables for the group opts, and we need
-    ]
+        # to extend them, so build a list.
-    return _cleanup_opts(opts)
+        namespace_values = []
+        # Look through the groups and find any that need drivers so we
+        # can load those extra options.
+        for group, group_opts in loader():
+            # group_opts is an iterable but we are going to extend it
+            # so convert it to a list.
+            group_opts = list(group_opts)
+            if isinstance(group, cfg.OptGroup):
+                if group.driver_option:
+                    # Load the options for all of the known drivers.
+                    driver_opts = _get_driver_opts(
+                        group.driver_option,
+                        namespaces,
+                    )
+                    # Save the list of names of options for each
+                    # driver in the group for use later. Add the
+                    # options to the group_opts list so they are
+                    # processed along with the static options in that
+                    # group.
+                    driver_opt_names = {}
+                    for driver_name, opts in sorted(driver_opts.items()):
+                        # Multiple plugins may add values to the same
+                        # driver name, so combine the lists we do
+                        # find.
+                        driver_opt_names.setdefault(driver_name, []).extend(
+                            o.name for o in opts)
+                        group_opts.extend(opts)
+                    group._save_driver_opts(driver_opt_names)
+            namespace_values.append((group, group_opts))
+        response.append((namespace, namespace_values))
+    return _cleanup_opts(response)
 
 
 def on_load_failure_callback(*args, **kwargs):
@@ -565,14 +622,22 @@ def _generate_machine_readable_data(groups, conf):
                    'generator_options': {}}
     # See _get_groups for details on the structure of group_data
     for group_name, group_data in groups.items():
-        output_data['options'][group_name] = {'opts': [], 'help': ''}
+        output_group = {'opts': [], 'help': ''}
+        output_data['options'][group_name] = output_group
         for namespace in group_data['namespaces']:
             for opt in namespace[1]:
                 if group_data['object']:
-                    output_group = output_data['options'][group_name]
+                    output_group.update(
-                    output_group['help'] = group_data['object'].help
+                        group_data['object']._get_generator_data()
+                    )
+                else:
+                    output_group.update({
+                        'dynamic_group_owner': '',
+                        'driver_option': '',
+                        'driver_opts': {},
+                    })
                 entry = _build_entry(opt, group_name, namespace[0], conf)
-                output_data['options'][group_name]['opts'].append(entry)
+                output_group['opts'].append(entry)
                 # Need copies of the opts because we modify them
                 for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
                     group = deprecated_opt.pop('group')
@@ -581,6 +646,16 @@ def _generate_machine_readable_data(groups, conf):
                     deprecated_opt['replacement_name'] = entry['name']
                     deprecated_opt['replacement_group'] = group_name
                     deprecated_options[group].append(deprecated_opt)
+        # Build the list of options in the group that are not tied to
+        # a driver.
+        non_driver_opt_names = [
+            o['name']
+            for o in output_group['opts']
+            if not any(o['name'] in output_group['driver_opts'][d]
+                       for d in output_group['driver_opts'])
+        ]
+        output_group['standard_opts'] = non_driver_opt_names
+
     output_data['generator_options'] = dict(conf)
     return output_data
 
@@ -605,7 +680,7 @@ def _output_machine_readable(groups, output_file, conf):
     output_file.write('\n')
 
 
-def generate(conf):
+def generate(conf, output_file=None):
     """Generate a sample config file.
 
     List all of the options available via the namespaces specified in the given
@@ -615,8 +690,9 @@ def generate(conf):
     """
     conf.register_opts(_generator_opts)
 
-    output_file = (open(conf.output_file, 'w')
+    if output_file is None:
-                   if conf.output_file else sys.stdout)
+        output_file = (open(conf.output_file, 'w')
+                       if conf.output_file else sys.stdout)
 
     groups = _get_groups(_list_opts(conf.namespace))
 

oslo_config/tests/test_generator.py

@@ -26,6 +26,9 @@ from oslo_config import fixture as config_fixture
 from oslo_config import generator
 from oslo_config import types
 
+import yaml
+
+
 load_tests = testscenarios.load_tests_apply_scenarios
 
 
@@ -954,6 +957,80 @@ class GeneratorTestCase(base.BaseTestCase):
             self.assertFalse(mock_log.warning.called)
 
 
+class DriverOptionTestCase(base.BaseTestCase):
+
+    def setUp(self):
+        super(DriverOptionTestCase, self).setUp()
+
+        self.conf = cfg.ConfigOpts()
+        self.config_fixture = config_fixture.Config(self.conf)
+        self.config = self.config_fixture.config
+        self.useFixture(self.config_fixture)
+
+    @mock.patch.object(generator, '_get_driver_opts_loaders')
+    @mock.patch.object(generator, '_get_raw_opts_loaders')
+    @mock.patch.object(generator, 'LOG')
+    def test_driver_option(self, mock_log, raw_opts_loader,
+                           driver_opts_loader):
+        group = cfg.OptGroup(
+            name='test_group',
+            title='Test Group',
+            driver_option='foo',
+        )
+        regular_opts = [
+            cfg.MultiStrOpt('foo', help='foo option'),
+            cfg.StrOpt('bar', help='bar option'),
+        ]
+        driver_opts = {
+            'd1': [
+                cfg.StrOpt('d1-foo', help='foo option'),
+            ],
+            'd2': [
+                cfg.StrOpt('d2-foo', help='foo option'),
+            ],
+        }
+
+        # We have a static data structure matching what should be
+        # returned by _list_opts() but we're mocking out a lower level
+        # function that needs to return a namespace and a callable to
+        # return options from that namespace. We have to pass opts to
+        # the lambda to cache a reference to the name because the list
+        # comprehension changes the thing pointed to by the name each
+        # time through the loop.
+        raw_opts_loader.return_value = [
+            ('testing', lambda: [(group, regular_opts)]),
+        ]
+        driver_opts_loader.return_value = [
+            ('testing', lambda: driver_opts),
+        ]
+
+        # Initialize the generator to produce YAML output to a buffer.
+        generator.register_cli_opts(self.conf)
+        self.config(namespace=['test_generator'], format_='yaml')
+        stdout = moves.StringIO()
+
+        # Generate the output and parse it back to a data structure.
+        generator.generate(self.conf, output_file=stdout)
+        body = stdout.getvalue()
+        actual = yaml.safe_load(body)
+
+        test_section = actual['options']['test_group']
+
+        self.assertEqual('foo', test_section['driver_option'])
+        found_option_names = [
+            o['name']
+            for o in test_section['opts']
+        ]
+        self.assertEqual(
+            ['foo', 'bar', 'd1-foo', 'd2-foo'],
+            found_option_names
+        )
+        self.assertEqual(
+            {'d1': ['d1-foo'], 'd2': ['d2-foo']},
+            test_section['driver_opts'],
+        )
+
+
 GENERATOR_OPTS = {'format_': 'yaml',
                   'minimal': False,
                   'namespace': ['test'],
@@ -972,7 +1049,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': ['foo'],
                                 'opts': [{'advanced': False,
                                           'choices': [],
                                           'default': None,
@@ -1000,7 +1081,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': ['long_help'],
                                 'opts': [{'advanced': False,
                                           'choices': [],
                                           'default': None,
@@ -1028,7 +1113,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': ['long_help_pre'],
                                 'opts': [{'advanced': False,
                                           'choices': [],
                                           'default': None,
@@ -1061,7 +1150,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                   'generator_options': GENERATOR_OPTS,
                   'options': {
                       'DEFAULT': {
+                          'driver_option': '',
+                          'driver_opts': {},
+                          'dynamic_group_owner': '',
                           'help': '',
+                          'standard_opts': ['foo-bar'],
                           'opts': [{
                               'advanced': False,
                               'choices': [],
@@ -1092,7 +1185,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': ['choices_opt'],
                                 'opts': [{'advanced': False,
                                           'choices': (None, '', 'a', 'b', 'c'),
                                           'default': 'a',
@@ -1120,7 +1217,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': ['int_opt'],
                                 'opts': [{'advanced': False,
                                           'choices': [],
                                           'default': 10,
@@ -1148,11 +1249,19 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
                         'generator_options': GENERATOR_OPTS,
                         'options': {
                             'DEFAULT': {
+                                # 'driver_option': '',
+                                # 'driver_opts': [],
+                                # 'dynamic_group_owner': '',
                                 'help': '',
+                                'standard_opts': [],
                                 'opts': []
                             },
                             'group1': {
+                                'driver_option': '',
+                                'driver_opts': {},
+                                'dynamic_group_owner': '',
                                 'help': all_groups['group1'].help,
+                                'standard_opts': ['foo'],
                                 'opts': [{'advanced': False,
                                           'choices': [],
                                           'default': None,