Make factory for a CloudRegion from CONF objects

This commit enables SDK consumers using oslo.config for keystoneauth1
Adapter settings by introducing a method:

openstack.config.cloud_region.from_conf

This accepts:
- An oslo.config ConfigOpts containing Adapter options in sections named
  according to project (e.g. [nova], not [compute]). Current behavior is
  to use defaults if no such section exists, which may not be what we
  want long term.
- A Session. This is currently required - if unspecified, a
  ConfigException is raised - but in the future we probably want to
  support creating one (and an auth) from the conf.
- Other kwargs to be passed to the CloudRegion constructor.

The method returns a CloudRegion that can be used to create a
Connection.

Needed-By: blueprint openstacksdk-in-nova
Co-Authored-By: Eric Fried <openstack@fried.cc>
Change-Id: I05fb4da39d2eefc91828ace02db2741b62a2cb0a

lower-constraints.txt

@@ -14,7 +14,7 @@ jmespath==0.9.0
 jsonpatch==1.16
 jsonpointer==1.13
 jsonschema==2.6.0
-keystoneauth1==3.13.0
+keystoneauth1==3.14.0
 linecache2==1.0.0
 mock==2.0.0
 mox3==0.20.0
@@ -22,6 +22,7 @@ munch==2.1.0
 netifaces==0.10.4
 os-client-config==1.28.0
 os-service-types==1.2.0
+oslo.config==6.1.0
 oslotest==3.2.0
 pbr==2.0.0
 prometheus-client==0.4.2

openstack/config/cloud_region.py

@@ -17,6 +17,7 @@ import warnings
 
 from keystoneauth1 import discover
 import keystoneauth1.exceptions.catalog
+from keystoneauth1.loading import adapter as ks_load_adap
 from keystoneauth1 import session as ks_session
 import os_service_types
 import requestsexceptions
@@ -45,6 +46,10 @@ SCOPE_KEYS = {
 }
 
 
+# Sentinel for nonexistence
+_ENOENT = object()
+
+
 def _make_key(key, service_type):
     if not service_type:
         return key
@@ -98,6 +103,56 @@ def from_session(session, name=None, region_name=None,
         app_name=app_name, app_version=app_version)
 
 
+def from_conf(conf, session=None, **kwargs):
+    """Create a CloudRegion from oslo.config ConfigOpts.
+
+    :param oslo_config.cfg.ConfigOpts conf:
+        An oslo.config ConfigOpts containing keystoneauth1.Adapter options in
+        sections named according to project (e.g. [nova], not [compute]).
+        TODO: Current behavior is to use defaults if no such section exists,
+        which may not be what we want long term.
+    :param keystoneauth1.session.Session session:
+        An existing authenticated Session to use. This is currently required.
+        TODO: Load this (and auth) from the conf.
+    :param kwargs:
+        Additional keyword arguments to be passed directly to the CloudRegion
+        constructor.
+    :raise openstack.exceptions.ConfigException:
+        If session is not specified.
+    :return:
+        An openstack.config.cloud_region.CloudRegion.
+    """
+    if not session:
+        # TODO(mordred) Fill this in - not needed for first stab with nova
+        raise exceptions.ConfigException("A Session must be supplied.")
+    config_dict = kwargs.pop('config', config_defaults.get_defaults())
+    stm = os_service_types.ServiceTypes()
+    # TODO(mordred) Think about region_name here
+    region_name = kwargs.pop('region_name', None)
+    for st in stm.all_types_by_service_type:
+        project_name = stm.get_project_name(st)
+        if project_name not in conf:
+            continue
+        opt_dict = {}
+        # Populate opt_dict with (appropriately processed) Adapter conf opts
+        try:
+            ks_load_adap.process_conf_options(conf[project_name], opt_dict)
+        except Exception:
+            # NOTE(efried): This is for oslo_config.cfg.NoSuchOptError, but we
+            # don't want to drag in oslo.config just for that.
+            continue
+        # Load them into config_dict under keys prefixed by ${service_type}_
+        for raw_name, opt_val in opt_dict.items():
+            if raw_name == 'region_name':
+                region_name = opt_val
+                continue
+            config_name = '_'.join([st, raw_name])
+            config_dict[config_name] = opt_val
+    return CloudRegion(
+        session=session, region_name=region_name, config=config_dict,
+        **kwargs)
+
+
 class CloudRegion(object):
     """The configuration for a Region of an OpenStack Cloud.
 

openstack/connection.py

@@ -112,6 +112,27 @@ is needed:
         compute_api_version='2',
         identity_interface='internal')
 
+From oslo.conf CONF object
+--------------------------
+
+For applications that have an oslo.config ``CONF`` object that has been
+populated with ``keystoneauth1.loading.register_adapter_conf_options`` in
+groups named by the OpenStack service's project name, it is possible to
+construct a Connection with the ``CONF`` object and an authenticated Session.
+
+.. note::
+
+    This is primarily intended for use by OpenStack services to talk amongst
+    themselves.
+
+.. code-block:: python
+
+    from openstack import connection
+
+    conn = connection.Connection(
+        session=session,
+        oslo_config=CONF)
+
 From existing CloudRegion
 -------------------------
 
@@ -249,6 +270,7 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
                  use_direct_get=False,
                  task_manager=None,
                  rate_limit=None,
+                 oslo_conf=None,
                  **kwargs):
         """Create a connection to a cloud.
 
@@ -295,6 +317,11 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
             keys as service-type and values as floats expressing the calls
             per second for that service. Defaults to None, which means no
             rate-limiting is performed.
+        :param oslo_conf: An oslo.config CONF object.
+        :type oslo_conf: :class:`~oslo_config.cfg.ConfigOpts`
+            An oslo.config ``CONF`` object that has been populated with
+            ``keystoneauth1.loading.register_adapter_conf_options`` in
+            groups named by the OpenStack service's project name.
         :param kwargs: If a config is not provided, the rest of the parameters
             provided are assumed to be arguments to be passed to the
             CloudRegion constructor.
@@ -306,7 +333,11 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
                 self._extra_services[service.service_type] = service
 
         if not self.config:
-            if session:
+            if oslo_conf:
+                self.config = cloud_region.from_conf(
+                    oslo_conf, session=session, app_name=app_name,
+                    app_version=app_version)
+            elif session:
                 self.config = cloud_region.from_session(
                     session=session,
                     app_name=app_name, app_version=app_version,

openstack/tests/unit/config/test_from_conf.py

@@ -0,0 +1,165 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import uuid
+
+from keystoneauth1 import exceptions as ks_exc
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+from openstack.config import cloud_region
+from openstack import connection
+from openstack import exceptions
+from openstack.tests import fakes
+from openstack.tests.unit import base
+
+
+class TestFromConf(base.TestCase):
+
+    def setUp(self):
+        super(TestFromConf, self).setUp()
+        self.oslo_config_dict = {
+            # All defaults for nova
+            'nova': {},
+            # monasca not in the service catalog
+            'monasca': {},
+            # Overrides for heat
+            'heat': {
+                'region_name': 'SpecialRegion',
+                'interface': 'internal',
+                'endpoint_override': 'https://example.org:8888/heat/v2'
+            },
+        }
+
+    def _load_ks_cfg_opts(self):
+        conf = cfg.ConfigOpts()
+        for group, opts in self.oslo_config_dict.items():
+            if opts is not None:
+                ks_loading.register_adapter_conf_options(conf, group)
+                for name, val in opts.items():
+                    conf.set_override(name, val, group=group)
+        return conf
+
+    def _get_conn(self):
+        oslocfg = self._load_ks_cfg_opts()
+        # Throw name in here to prove **kwargs is working
+        config = cloud_region.from_conf(
+            oslocfg, session=self.cloud.session, name='from_conf.example.com')
+        self.assertEqual('from_conf.example.com', config.name)
+
+        # TODO(efried): Currently region_name gets set to the last value seen
+        # in the config, which is nondeterministic and surely incorrect.
+        # Sometimes that's SpecialRegion, but some tests use the base fixtures
+        # which have no compute endpoint in SpecialRegion. Force override for
+        # now to make those tests work.
+        config.region_name = None
+
+        return connection.Connection(config=config)
+
+    def test_adapter_opts_set(self):
+        """Adapter opts specified in the conf."""
+        conn = self._get_conn()
+
+        discovery = {
+            "versions": {
+                "values": [
+                    {"status": "stable",
+                     "updated": "2019-06-01T00:00:00Z",
+                     "media-types": [{
+                         "base": "application/json",
+                         "type": "application/vnd.openstack.heat-v2+json"}],
+                     "id": "v2.0",
+                     "links": [{
+                         "href": "https://example.org:8888/heat/v2",
+                         "rel": "self"}]
+                     }]
+            }
+        }
+        self.register_uris([
+            dict(method='GET',
+                 uri='https://example.org:8888/heat/v2',
+                 json=discovery),
+            dict(method='GET',
+                 uri='https://example.org:8888/heat/v2/foo',
+                 json={'foo': {}}),
+        ])
+
+        adap = conn.orchestration
+        # TODO(efried): Fix this when region_name behaves correctly.
+        # self.assertEqual('SpecialRegion', adap.region_name)
+        self.assertEqual('orchestration', adap.service_type)
+        self.assertEqual('internal', adap.interface)
+        self.assertEqual('https://example.org:8888/heat/v2',
+                         adap.endpoint_override)
+
+        adap.get('/foo')
+        self.assert_calls()
+
+    def test_default_adapter_opts(self):
+        """Adapter opts are registered, but all defaulting in conf."""
+        conn = self._get_conn()
+
+        # Nova has empty adapter config, so these default
+        adap = conn.compute
+        self.assertIsNone(adap.region_name)
+        self.assertEqual('compute', adap.service_type)
+        self.assertEqual('public', adap.interface)
+        self.assertIsNone(adap.endpoint_override)
+
+        server_id = str(uuid.uuid4())
+        server_name = self.getUniqueString('name')
+        fake_server = fakes.make_fake_server(server_id, server_name)
+        self.register_uris([
+            self.get_nova_discovery_mock_dict(),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'compute', 'public', append=['servers', 'detail']),
+                 json={'servers': [fake_server]}),
+        ])
+        s = next(adap.servers())
+        self.assertEqual(s.id, server_id)
+        self.assertEqual(s.name, server_name)
+        self.assert_calls()
+
+    def test_no_adapter_opts(self):
+        """Adapter opts for service type not registered."""
+        del self.oslo_config_dict['heat']
+        conn = self._get_conn()
+
+        # TODO(efried): This works, even though adapter opts are not
+        # registered. Should it?
+        adap = conn.orchestration
+        self.assertIsNone(adap.region_name)
+        self.assertEqual('orchestration', adap.service_type)
+        self.assertEqual('public', adap.interface)
+        self.assertIsNone(adap.endpoint_override)
+
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'orchestration', append=['foo']),
+                 json={'foo': {}})
+        ])
+        adap.get('/foo')
+        self.assert_calls()
+
+    def test_no_session(self):
+        # TODO(efried): Currently calling without a Session is not implemented.
+        self.assertRaises(exceptions.ConfigException,
+                          cloud_region.from_conf, self._load_ks_cfg_opts())
+
+    def test_no_endpoint(self):
+        """Conf contains adapter opts, but service type not in catalog."""
+        conn = self._get_conn()
+        # Monasca is not in the service catalog
+        self.assertRaises(ks_exc.catalog.EndpointNotFound,
+                          getattr, conn, 'monitoring')

releasenotes/notes/conf-object-ctr-c0e1da0a67dad841.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Added the ability to create a ``Connection`` from an ``oslo.config``
+    ``CONF`` object. This is primarily intended to be used by OpenStack
+    services using SDK for inter-service communication.

requirements.txt

@@ -8,7 +8,7 @@ requestsexceptions>=1.2.0 # Apache-2.0
 jsonpatch!=1.20,>=1.16 # BSD
 six>=1.10.0 # MIT
 os-service-types>=1.2.0 # Apache-2.0
-keystoneauth1>=3.13.0 # Apache-2.0
+keystoneauth1>=3.14.0 # Apache-2.0
 
 munch>=2.1.0 # MIT
 decorator>=3.4.0 # BSD

test-requirements.txt

@@ -10,6 +10,7 @@ jsonschema>=2.6.0 # MIT
 mock>=2.0.0 # BSD
 prometheus-client>=0.4.2 # Apache-2.0
 python-subunit>=1.0.0 # Apache-2.0/BSD
+oslo.config>=6.1.0 # Apache-2.0
 oslotest>=3.2.0 # Apache-2.0
 requests-mock>=1.2.0 # Apache-2.0
 statsd>=3.3.0