Allow limiting Connection service_types from oslo.config

Add a service_types kwarg to cloud_region.from_conf and
Connection.__init__, accepting a list/set of service types. All other
service types will be explicitly disabled, and we won't attempt to load
their configs.

Change-Id: I3d16d17caa2e8a58b7064c54e930468288aa6ff1

openstack/config/cloud_region.py

@@ -115,7 +115,7 @@ def from_session(session, name=None, region_name=None,
         app_name=app_name, app_version=app_version)
 
 
-def from_conf(conf, session=None, **kwargs):
+def from_conf(conf, session=None, service_types=None, **kwargs):
     """Create a CloudRegion from oslo.config ConfigOpts.
 
     :param oslo_config.cfg.ConfigOpts conf:
@@ -126,6 +126,16 @@ def from_conf(conf, session=None, **kwargs):
     :param keystoneauth1.session.Session session:
         An existing authenticated Session to use. This is currently required.
         TODO: Load this (and auth) from the conf.
+    :param service_types:
+        A list/set of service types for which to look for and process config
+        opts. If None, all known service types are processed. Note that we will
+        not error if a supplied service type can not be processed successfully
+        (unless you try to use the proxy, of course). This tolerates uses where
+        the consuming code has paths for a given service, but those paths are
+        not exercised for given end user setups, and we do not want to generate
+        errors for e.g. missing/invalid conf sections in those cases. We also
+        don't check to make sure your service types are spelled correctly -
+        caveat implementor.
     :param kwargs:
         Additional keyword arguments to be passed directly to the CloudRegion
         constructor.
@@ -140,6 +150,11 @@ def from_conf(conf, session=None, **kwargs):
     config_dict = kwargs.pop('config', config_defaults.get_defaults())
     stm = os_service_types.ServiceTypes()
     for st in stm.all_types_by_service_type:
+        if service_types is not None and st not in service_types:
+            _disable_service(
+                config_dict, st,
+                reason="Not in the list of requested service_types.")
+            continue
         project_name = stm.get_project_name(st)
         if project_name not in conf:
             if '-' in project_name:

openstack/connection.py

@@ -271,6 +271,7 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
                  task_manager=None,
                  rate_limit=None,
                  oslo_conf=None,
+                 service_types=None,
                  **kwargs):
         """Create a connection to a cloud.
 
@@ -322,6 +323,11 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
             An oslo.config ``CONF`` object that has been populated with
             ``keystoneauth1.loading.register_adapter_conf_options`` in
             groups named by the OpenStack service's project name.
+        :param service_types:
+            A list/set of service types this Connection should support. All
+            other service types will be disabled (will error if used).
+            **Currently only supported in conjunction with the ``oslo_conf``
+            kwarg.**
         :param kwargs: If a config is not provided, the rest of the parameters
             provided are assumed to be arguments to be passed to the
             CloudRegion constructor.
@@ -336,7 +342,7 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
             if oslo_conf:
                 self.config = cloud_region.from_conf(
                     oslo_conf, session=session, app_name=app_name,
-                    app_version=app_version)
+                    app_version=app_version, service_types=service_types)
             elif session:
                 self.config = cloud_region.from_session(
                     session=session,

openstack/tests/unit/base.py

@@ -14,12 +14,14 @@
 # under the License.
 
 import collections
+import os
 import time
 import uuid
 
 import fixtures
-import os
+from keystoneauth1 import loading as ks_loading
 import openstack.config as occ
+from oslo_config import cfg
 from requests import structures
 from requests_mock.contrib import fixture as rm_fixture
 from six.moves import urllib
@@ -118,6 +120,23 @@ class TestCase(base.TestCase):
             vendor_files=[vendor.name],
             secure_files=['non-existant'])
 
+        self.oslo_config_dict = {
+            # All defaults for nova
+            'nova': {},
+            # monasca-api not in the service catalog
+            'monasca-api': {},
+            # Overrides for heat
+            'heat': {
+                'region_name': 'SpecialRegion',
+                'interface': 'internal',
+                'endpoint_override': 'https://example.org:8888/heat/v2'
+            },
+            # test a service with dashes
+            'ironic_inspector': {
+                'endpoint_override': 'https://example.org:5050',
+            },
+        }
+
         # FIXME(notmorgan): Convert the uri_registry, discovery.json, and
         # use of keystone_v3/v2 to a proper fixtures.Fixture. For now this
         # is acceptable, but eventually this should become it's own fixture
@@ -139,6 +158,16 @@ class TestCase(base.TestCase):
         self.use_keystone_v3()
         self.__register_uris_called = False
 
+    def _load_ks_cfg_opts(self):
+        conf = cfg.ConfigOpts()
+        for group, opts in self.oslo_config_dict.items():
+            conf.register_group(cfg.OptGroup(group))
+            if opts is not None:
+                ks_loading.register_adapter_conf_options(conf, group)
+                for name, val in opts.items():
+                    conf.set_override(name, val, group=group)
+        return conf
+
     # TODO(shade) Update this to handle service type aliases
     def get_mock_url(self, service_type, interface='public', resource=None,
                      append=None, base_url_append=None,

openstack/tests/unit/config/test_from_conf.py

@@ -13,8 +13,6 @@
 import uuid
 
 from keystoneauth1 import exceptions as ks_exc
-from keystoneauth1 import loading as ks_loading
-from oslo_config import cfg
 
 from openstack.config import cloud_region
 from openstack import connection
@@ -25,40 +23,12 @@ from openstack.tests.unit import base
 
 class TestFromConf(base.TestCase):
 
-    def setUp(self):
+    def _get_conn(self, **from_conf_kwargs):
-        super(TestFromConf, self).setUp()
-        self.oslo_config_dict = {
-            # All defaults for nova
-            'nova': {},
-            # monasca-api not in the service catalog
-            'monasca-api': {},
-            # Overrides for heat
-            'heat': {
-                'region_name': 'SpecialRegion',
-                'interface': 'internal',
-                'endpoint_override': 'https://example.org:8888/heat/v2'
-            },
-            # test a service with dashes
-            'ironic_inspector': {
-                'endpoint_override': 'https://example.org:5050',
-            },
-        }
-
-    def _load_ks_cfg_opts(self):
-        conf = cfg.ConfigOpts()
-        for group, opts in self.oslo_config_dict.items():
-            conf.register_group(cfg.OptGroup(group))
-            if opts is not None:
-                ks_loading.register_adapter_conf_options(conf, group)
-                for name, val in opts.items():
-                    conf.set_override(name, val, group=group)
-        return conf
-
-    def _get_conn(self):
         oslocfg = self._load_ks_cfg_opts()
         # Throw name in here to prove **kwargs is working
         config = cloud_region.from_conf(
-            oslocfg, session=self.cloud.session, name='from_conf.example.com')
+            oslocfg, session=self.cloud.session, name='from_conf.example.com',
+            **from_conf_kwargs)
         self.assertEqual('from_conf.example.com', config.name)
 
         return connection.Connection(config=config)
@@ -161,39 +131,58 @@ class TestFromConf(base.TestCase):
 
         self.assertTrue(adap.get_introspection('abcd').is_finished)
 
-    def _test_missing_invalid_permutations(self, expected_reason):
+    def assert_service_disabled(self, service_type, expected_reason,
-        # Do special things to self.oslo_config_dict['heat'] before calling
+                                **from_conf_kwargs):
-        # this method.
+        conn = self._get_conn(**from_conf_kwargs)
-        conn = self._get_conn()
+        # The _ServiceDisabledProxyShim loads up okay...
-
+        adap = getattr(conn, service_type)
-        adap = conn.orchestration
+        # ...but freaks out if you try to use it.
         ex = self.assertRaises(
             exceptions.ServiceDisabledException, getattr, adap, 'get')
-        self.assertIn("Service 'orchestration' is disabled because its "
+        self.assertIn("Service '%s' is disabled because its configuration "
-                      "configuration could not be loaded.", ex.message)
+                      "could not be loaded." % service_type, ex.message)
         self.assertIn(expected_reason, ex.message)
 
     def test_no_such_conf_section(self):
         """No conf section (therefore no adapter opts) for service type."""
         del self.oslo_config_dict['heat']
-        self._test_missing_invalid_permutations(
+        self.assert_service_disabled(
+            'orchestration',
             "No section for project 'heat' (service type 'orchestration') was "
             "present in the config.")
 
+    def test_no_such_conf_section_ignore_service_type(self):
+        """Ignore absent conf section if service type not requested."""
+        del self.oslo_config_dict['heat']
+        self.assert_service_disabled(
+            'orchestration', "Not in the list of requested service_types.",
+            # 'orchestration' absent from this list
+            service_types=['compute'])
+
     def test_no_adapter_opts(self):
         """Conf section present, but opts for service type not registered."""
         self.oslo_config_dict['heat'] = None
-        self._test_missing_invalid_permutations(
+        self.assert_service_disabled(
+            'orchestration',
             "Encountered an exception attempting to process config for "
             "project 'heat' (service type 'orchestration'): no such option")
 
+    def test_no_adapter_opts_ignore_service_type(self):
+        """Ignore unregistered conf section if service type not requested."""
+        self.oslo_config_dict['heat'] = None
+        self.assert_service_disabled(
+            'orchestration', "Not in the list of requested service_types.",
+            # 'orchestration' absent from this list
+            service_types=['compute'])
+
     def test_invalid_adapter_opts(self):
         """Adapter opts are bogus, in exception-raising ways."""
         self.oslo_config_dict['heat'] = {
             'interface': 'public',
             'valid_interfaces': 'private',
         }
-        self._test_missing_invalid_permutations(
+        self.assert_service_disabled(
+            'orchestration',
             "Encountered an exception attempting to process config for "
             "project 'heat' (service type 'orchestration'): interface and "
             "valid_interfaces are mutually exclusive.")
@@ -209,3 +198,10 @@ class TestFromConf(base.TestCase):
         # Monasca is not in the service catalog
         self.assertRaises(ks_exc.catalog.EndpointNotFound,
                           getattr, conn, 'monitoring')
+
+    def test_no_endpoint_ignore_service_type(self):
+        """Bogus service type disabled if not in requested service_types."""
+        self.assert_service_disabled(
+            'monitoring', "Not in the list of requested service_types.",
+            # 'monitoring' absent from this list
+            service_types={'compute', 'orchestration', 'bogus'})

openstack/tests/unit/test_connection.py

@@ -18,6 +18,7 @@ import mock
 
 from openstack import connection
 import openstack.config
+from openstack import service_description
 from openstack.tests.unit import base
 from openstack.tests.unit.fake import fake_service
 
@@ -240,6 +241,31 @@ class TestConnection(base.TestCase):
         self.assertFalse(sot.session.verify)
 
 
+class TestOsloConfig(TestConnection):
+    def test_from_conf(self):
+        c1 = connection.Connection(cloud='sample-cloud')
+        conn = connection.Connection(
+            session=c1.session, oslo_conf=self._load_ks_cfg_opts())
+        # There was no config for keystone
+        self.assertIsInstance(
+            conn.identity, service_description._ServiceDisabledProxyShim)
+        # But nova was in there
+        self.assertEqual('openstack.compute.v2._proxy',
+                         conn.compute.__class__.__module__)
+
+    def test_from_conf_filter_service_types(self):
+        c1 = connection.Connection(cloud='sample-cloud')
+        conn = connection.Connection(
+            session=c1.session, oslo_conf=self._load_ks_cfg_opts(),
+            service_types={'orchestration', 'i-am-ignored'})
+        # There was no config for keystone
+        self.assertIsInstance(
+            conn.identity, service_description._ServiceDisabledProxyShim)
+        # Nova was in there, but disabled because not requested
+        self.assertIsInstance(
+            conn.compute, service_description._ServiceDisabledProxyShim)
+
+
 class TestNetworkConnection(base.TestCase):
 
     # Verify that if the catalog has the suffix we don't mess things up.