From 86ad9debd1cb08c2a2e53fd30502b020a0c3e4fe Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 15 Mar 2019 14:50:04 +0000 Subject: [PATCH] Make factory for a CloudRegion from CONF objects This commit enables SDK consumers using oslo.config for keystoneauth1 Adapter settings by introducing a method: openstack.config.cloud_region.from_conf This accepts: - An oslo.config ConfigOpts containing Adapter options in sections named according to project (e.g. [nova], not [compute]). Current behavior is to use defaults if no such section exists, which may not be what we want long term. - A Session. This is currently required - if unspecified, a ConfigException is raised - but in the future we probably want to support creating one (and an auth) from the conf. - Other kwargs to be passed to the CloudRegion constructor. The method returns a CloudRegion that can be used to create a Connection. Needed-By: blueprint openstacksdk-in-nova Co-Authored-By: Eric Fried Change-Id: I05fb4da39d2eefc91828ace02db2741b62a2cb0a --- lower-constraints.txt | 3 +- openstack/config/cloud_region.py | 55 ++++++ openstack/connection.py | 33 +++- openstack/tests/unit/config/test_from_conf.py | 165 ++++++++++++++++++ .../conf-object-ctr-c0e1da0a67dad841.yaml | 6 + requirements.txt | 2 +- test-requirements.txt | 1 + 7 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 openstack/tests/unit/config/test_from_conf.py create mode 100644 releasenotes/notes/conf-object-ctr-c0e1da0a67dad841.yaml diff --git a/lower-constraints.txt b/lower-constraints.txt index 93758a7e4..affeed9e7 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -14,7 +14,7 @@ jmespath==0.9.0 jsonpatch==1.16 jsonpointer==1.13 jsonschema==2.6.0 -keystoneauth1==3.13.0 +keystoneauth1==3.14.0 linecache2==1.0.0 mock==2.0.0 mox3==0.20.0 @@ -22,6 +22,7 @@ munch==2.1.0 netifaces==0.10.4 os-client-config==1.28.0 os-service-types==1.2.0 +oslo.config==6.1.0 oslotest==3.2.0 pbr==2.0.0 prometheus-client==0.4.2 diff --git a/openstack/config/cloud_region.py b/openstack/config/cloud_region.py index c9d3682e8..044669fee 100644 --- a/openstack/config/cloud_region.py +++ b/openstack/config/cloud_region.py @@ -17,6 +17,7 @@ import warnings from keystoneauth1 import discover import keystoneauth1.exceptions.catalog +from keystoneauth1.loading import adapter as ks_load_adap from keystoneauth1 import session as ks_session import os_service_types import requestsexceptions @@ -45,6 +46,10 @@ SCOPE_KEYS = { } +# Sentinel for nonexistence +_ENOENT = object() + + def _make_key(key, service_type): if not service_type: return key @@ -98,6 +103,56 @@ def from_session(session, name=None, region_name=None, app_name=app_name, app_version=app_version) +def from_conf(conf, session=None, **kwargs): + """Create a CloudRegion from oslo.config ConfigOpts. + + :param oslo_config.cfg.ConfigOpts conf: + An oslo.config ConfigOpts containing keystoneauth1.Adapter options in + sections named according to project (e.g. [nova], not [compute]). + TODO: Current behavior is to use defaults if no such section exists, + which may not be what we want long term. + :param keystoneauth1.session.Session session: + An existing authenticated Session to use. This is currently required. + TODO: Load this (and auth) from the conf. + :param kwargs: + Additional keyword arguments to be passed directly to the CloudRegion + constructor. + :raise openstack.exceptions.ConfigException: + If session is not specified. + :return: + An openstack.config.cloud_region.CloudRegion. + """ + if not session: + # TODO(mordred) Fill this in - not needed for first stab with nova + raise exceptions.ConfigException("A Session must be supplied.") + config_dict = kwargs.pop('config', config_defaults.get_defaults()) + stm = os_service_types.ServiceTypes() + # TODO(mordred) Think about region_name here + region_name = kwargs.pop('region_name', None) + for st in stm.all_types_by_service_type: + project_name = stm.get_project_name(st) + if project_name not in conf: + continue + opt_dict = {} + # Populate opt_dict with (appropriately processed) Adapter conf opts + try: + ks_load_adap.process_conf_options(conf[project_name], opt_dict) + except Exception: + # NOTE(efried): This is for oslo_config.cfg.NoSuchOptError, but we + # don't want to drag in oslo.config just for that. + continue + # Load them into config_dict under keys prefixed by ${service_type}_ + for raw_name, opt_val in opt_dict.items(): + if raw_name == 'region_name': + region_name = opt_val + continue + config_name = '_'.join([st, raw_name]) + config_dict[config_name] = opt_val + return CloudRegion( + session=session, region_name=region_name, config=config_dict, + **kwargs) + + class CloudRegion(object): """The configuration for a Region of an OpenStack Cloud. diff --git a/openstack/connection.py b/openstack/connection.py index 8a2d0b5a0..f4fed899a 100644 --- a/openstack/connection.py +++ b/openstack/connection.py @@ -112,6 +112,27 @@ is needed: compute_api_version='2', identity_interface='internal') +From oslo.conf CONF object +-------------------------- + +For applications that have an oslo.config ``CONF`` object that has been +populated with ``keystoneauth1.loading.register_adapter_conf_options`` in +groups named by the OpenStack service's project name, it is possible to +construct a Connection with the ``CONF`` object and an authenticated Session. + +.. note:: + + This is primarily intended for use by OpenStack services to talk amongst + themselves. + +.. code-block:: python + + from openstack import connection + + conn = connection.Connection( + session=session, + oslo_config=CONF) + From existing CloudRegion ------------------------- @@ -249,6 +270,7 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta, use_direct_get=False, task_manager=None, rate_limit=None, + oslo_conf=None, **kwargs): """Create a connection to a cloud. @@ -295,6 +317,11 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta, keys as service-type and values as floats expressing the calls per second for that service. Defaults to None, which means no rate-limiting is performed. + :param oslo_conf: An oslo.config CONF object. + :type oslo_conf: :class:`~oslo_config.cfg.ConfigOpts` + An oslo.config ``CONF`` object that has been populated with + ``keystoneauth1.loading.register_adapter_conf_options`` in + groups named by the OpenStack service's project name. :param kwargs: If a config is not provided, the rest of the parameters provided are assumed to be arguments to be passed to the CloudRegion constructor. @@ -306,7 +333,11 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta, self._extra_services[service.service_type] = service if not self.config: - if session: + if oslo_conf: + self.config = cloud_region.from_conf( + oslo_conf, session=session, app_name=app_name, + app_version=app_version) + elif session: self.config = cloud_region.from_session( session=session, app_name=app_name, app_version=app_version, diff --git a/openstack/tests/unit/config/test_from_conf.py b/openstack/tests/unit/config/test_from_conf.py new file mode 100644 index 000000000..7308b9397 --- /dev/null +++ b/openstack/tests/unit/config/test_from_conf.py @@ -0,0 +1,165 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +from keystoneauth1 import exceptions as ks_exc +from keystoneauth1 import loading as ks_loading +from oslo_config import cfg + +from openstack.config import cloud_region +from openstack import connection +from openstack import exceptions +from openstack.tests import fakes +from openstack.tests.unit import base + + +class TestFromConf(base.TestCase): + + def setUp(self): + super(TestFromConf, self).setUp() + self.oslo_config_dict = { + # All defaults for nova + 'nova': {}, + # monasca not in the service catalog + 'monasca': {}, + # Overrides for heat + 'heat': { + 'region_name': 'SpecialRegion', + 'interface': 'internal', + 'endpoint_override': 'https://example.org:8888/heat/v2' + }, + } + + def _load_ks_cfg_opts(self): + conf = cfg.ConfigOpts() + for group, opts in self.oslo_config_dict.items(): + if opts is not None: + ks_loading.register_adapter_conf_options(conf, group) + for name, val in opts.items(): + conf.set_override(name, val, group=group) + return conf + + def _get_conn(self): + oslocfg = self._load_ks_cfg_opts() + # Throw name in here to prove **kwargs is working + config = cloud_region.from_conf( + oslocfg, session=self.cloud.session, name='from_conf.example.com') + self.assertEqual('from_conf.example.com', config.name) + + # TODO(efried): Currently region_name gets set to the last value seen + # in the config, which is nondeterministic and surely incorrect. + # Sometimes that's SpecialRegion, but some tests use the base fixtures + # which have no compute endpoint in SpecialRegion. Force override for + # now to make those tests work. + config.region_name = None + + return connection.Connection(config=config) + + def test_adapter_opts_set(self): + """Adapter opts specified in the conf.""" + conn = self._get_conn() + + discovery = { + "versions": { + "values": [ + {"status": "stable", + "updated": "2019-06-01T00:00:00Z", + "media-types": [{ + "base": "application/json", + "type": "application/vnd.openstack.heat-v2+json"}], + "id": "v2.0", + "links": [{ + "href": "https://example.org:8888/heat/v2", + "rel": "self"}] + }] + } + } + self.register_uris([ + dict(method='GET', + uri='https://example.org:8888/heat/v2', + json=discovery), + dict(method='GET', + uri='https://example.org:8888/heat/v2/foo', + json={'foo': {}}), + ]) + + adap = conn.orchestration + # TODO(efried): Fix this when region_name behaves correctly. + # self.assertEqual('SpecialRegion', adap.region_name) + self.assertEqual('orchestration', adap.service_type) + self.assertEqual('internal', adap.interface) + self.assertEqual('https://example.org:8888/heat/v2', + adap.endpoint_override) + + adap.get('/foo') + self.assert_calls() + + def test_default_adapter_opts(self): + """Adapter opts are registered, but all defaulting in conf.""" + conn = self._get_conn() + + # Nova has empty adapter config, so these default + adap = conn.compute + self.assertIsNone(adap.region_name) + self.assertEqual('compute', adap.service_type) + self.assertEqual('public', adap.interface) + self.assertIsNone(adap.endpoint_override) + + server_id = str(uuid.uuid4()) + server_name = self.getUniqueString('name') + fake_server = fakes.make_fake_server(server_id, server_name) + self.register_uris([ + self.get_nova_discovery_mock_dict(), + dict(method='GET', + uri=self.get_mock_url( + 'compute', 'public', append=['servers', 'detail']), + json={'servers': [fake_server]}), + ]) + s = next(adap.servers()) + self.assertEqual(s.id, server_id) + self.assertEqual(s.name, server_name) + self.assert_calls() + + def test_no_adapter_opts(self): + """Adapter opts for service type not registered.""" + del self.oslo_config_dict['heat'] + conn = self._get_conn() + + # TODO(efried): This works, even though adapter opts are not + # registered. Should it? + adap = conn.orchestration + self.assertIsNone(adap.region_name) + self.assertEqual('orchestration', adap.service_type) + self.assertEqual('public', adap.interface) + self.assertIsNone(adap.endpoint_override) + + self.register_uris([ + dict(method='GET', + uri=self.get_mock_url( + 'orchestration', append=['foo']), + json={'foo': {}}) + ]) + adap.get('/foo') + self.assert_calls() + + def test_no_session(self): + # TODO(efried): Currently calling without a Session is not implemented. + self.assertRaises(exceptions.ConfigException, + cloud_region.from_conf, self._load_ks_cfg_opts()) + + def test_no_endpoint(self): + """Conf contains adapter opts, but service type not in catalog.""" + conn = self._get_conn() + # Monasca is not in the service catalog + self.assertRaises(ks_exc.catalog.EndpointNotFound, + getattr, conn, 'monitoring') diff --git a/releasenotes/notes/conf-object-ctr-c0e1da0a67dad841.yaml b/releasenotes/notes/conf-object-ctr-c0e1da0a67dad841.yaml new file mode 100644 index 000000000..60bb03594 --- /dev/null +++ b/releasenotes/notes/conf-object-ctr-c0e1da0a67dad841.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added the ability to create a ``Connection`` from an ``oslo.config`` + ``CONF`` object. This is primarily intended to be used by OpenStack + services using SDK for inter-service communication. diff --git a/requirements.txt b/requirements.txt index 6181b5544..9062efb83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ requestsexceptions>=1.2.0 # Apache-2.0 jsonpatch!=1.20,>=1.16 # BSD six>=1.10.0 # MIT os-service-types>=1.2.0 # Apache-2.0 -keystoneauth1>=3.13.0 # Apache-2.0 +keystoneauth1>=3.14.0 # Apache-2.0 munch>=2.1.0 # MIT decorator>=3.4.0 # BSD diff --git a/test-requirements.txt b/test-requirements.txt index c32f63564..feb8d781c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,6 +10,7 @@ jsonschema>=2.6.0 # MIT mock>=2.0.0 # BSD prometheus-client>=0.4.2 # Apache-2.0 python-subunit>=1.0.0 # Apache-2.0/BSD +oslo.config>=6.1.0 # Apache-2.0 oslotest>=3.2.0 # Apache-2.0 requests-mock>=1.2.0 # Apache-2.0 statsd>=3.3.0