diff --git a/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml b/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml new file mode 100644 index 0000000000..64f729ac23 --- /dev/null +++ b/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml @@ -0,0 +1,12 @@ +--- +features: + - A new optional interface `TempestPlugin.get_service_clients` + is available to plugins. It allows them to declare + any service client they implement. For now this is used by + tempest only, for auto-registration of service clients + in the new class `ServiceClients`. + - A new singleton class `clients.ClientsRegistry` is + available. It holds the service clients registration data + from all plugins. It is used by `ServiceClients` for + auto-registration of the service clients implemented + in plugins. diff --git a/tempest/lib/exceptions.py b/tempest/lib/exceptions.py index 5ca78f9fbd..de2d713449 100644 --- a/tempest/lib/exceptions.py +++ b/tempest/lib/exceptions.py @@ -229,3 +229,13 @@ class SSHExecCommandFailed(TempestException): class UnknownServiceClient(TempestException): message = "Service clients named %(services)s are not known" + + +class ServiceClientRegistrationException(TempestException): + message = ("Error registering module %(name)s in path %(module_path)s, " + "with service %(service_version)s and clients " + "%(client_names)s: %(detailed_error)s") + + +class PluginRegistrationException(TempestException): + message = "Error registering plugin %(name)s: %(detailed_error)s" diff --git a/tempest/lib/services/clients.py b/tempest/lib/services/clients.py new file mode 100644 index 0000000000..8054e62695 --- /dev/null +++ b/tempest/lib/services/clients.py @@ -0,0 +1,36 @@ +# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from tempest.lib.common.utils import misc +from tempest.lib import exceptions + + +@misc.singleton +class ClientsRegistry(object): + """Registry of all service clients available from plugins""" + + def __init__(self): + self._service_clients = {} + + def register_service_client(self, plugin_name, service_client_data): + if plugin_name in self._service_clients: + detailed_error = 'Clients for plugin %s already registered' + raise exceptions.PluginRegistrationException( + name=plugin_name, + detailed_error=detailed_error % plugin_name) + self._service_clients[plugin_name] = service_client_data + + def get_service_clients(self): + return self._service_clients diff --git a/tempest/service_clients.py b/tempest/service_clients.py index 252ebf438a..d0daa2bc34 100644 --- a/tempest/service_clients.py +++ b/tempest/service_clients.py @@ -17,9 +17,13 @@ import copy import importlib import inspect +import logging from tempest.lib import auth from tempest.lib import exceptions +from tempest.lib.services import clients + +LOG = logging.getLogger(__name__) def tempest_modules(): @@ -33,9 +37,48 @@ def tempest_modules(): def available_modules(): - """List of service client modules available in Tempest and plugins""" - # TODO(andreaf) For now this returns only tempest_modules - return tempest_modules() + """List of service client modules available in Tempest and plugins + + The list of available modules can be used for automatic configuration. + + :raise PluginRegistrationException: if a plugin exposes a service_version + already defined by Tempest or another plugin. + + Examples: + + >>> from tempest import config + >>> params = {} + >>> for service_version in available_modules(): + >>> service = service_version.split('.')[0] + >>> params[service] = config.service_client_config(service) + >>> service_clients = ServiceClients(creds, identity_uri, + >>> client_parameters=params) + """ + extra_service_versions = set([]) + plugin_services = clients.ClientsRegistry().get_service_clients() + for plugin_name in plugin_services: + plug_service_versions = set([x['service_version'] for x in + plugin_services[plugin_name]]) + # If a plugin exposes a duplicate service_version raise an exception + if plug_service_versions: + if not plug_service_versions.isdisjoint(extra_service_versions): + detailed_error = ( + 'Plugin %s is trying to register a service %s already ' + 'claimed by another one' % (plugin_name, + extra_service_versions & + plug_service_versions)) + raise exceptions.PluginRegistrationException( + name=plugin_name, detailed_error=detailed_error) + if not plug_service_versions.isdisjoint(tempest_modules()): + detailed_error = ( + 'Plugin %s is trying to register a service %s already ' + 'claimed by a Tempest one' % (plugin_name, + tempest_modules() & + plug_service_versions)) + raise exceptions.PluginRegistrationException( + name=plugin_name, detailed_error=detailed_error) + extra_service_versions |= plug_service_versions + return tempest_modules() | extra_service_versions class ClientsFactory(object): @@ -49,8 +92,6 @@ class ClientsFactory(object): ClientsFactory can be used directly, or consumed via the `ServiceClients` class, which manages the authorization part. """ - # TODO(andreaf) This version includes ClientsFactory but it does not - # use it yet in ServiceClients def __init__(self, module_path, client_names, auth_provider, **kwargs): """Initialises the client factory @@ -206,6 +247,7 @@ class ServiceClients(object): >>> client_parameters['service_y'] = params_service_y """ + self._registered_services = set([]) self.credentials = credentials self.identity_uri = identity_uri if not identity_uri: @@ -247,6 +289,84 @@ class ServiceClients(object): raise exceptions.UnknownServiceClient( services=list(client_parameters.keys())) + # Register service clients from plugins + clients_registry = clients.ClientsRegistry() + plugin_service_clients = clients_registry.get_service_clients() + for plugin in plugin_service_clients: + service_clients = plugin_service_clients[plugin] + # Each plugin returns a list of service client parameters + for service_client in service_clients: + # NOTE(andreaf) If a plugin cannot register, stop the + # registration process, log some details to help + # troubleshooting, and re-raise + try: + self.register_service_client_module(**service_client) + except Exception: + LOG.exception( + 'Failed to register service client from plugin %s ' + 'with parameters %s' % (plugin, service_client)) + raise + + def register_service_client_module(self, name, service_version, + module_path, client_names, **kwargs): + """Register a service client module + + Initiates a client factory for the specified module, using this + class auth_provider, and accessible via a `name` attribute in the + service client. + + :param name: Name used to access the client + :param service_version: Name of the service complete with version. + Used to track registered services. When a plugin implements it, + it can be used by other plugins to obtain their configuration. + :param module_path: Path to module that includes all service clients. + All service client classes must be exposed by a single module. + If they are separated in different modules, defining __all__ + in the root module can help, similar to what is done by service + clients in tempest. + :param client_names: List or set of names of service client classes. + :param kwargs: Extra optional parameters to be passed to all clients. + ServiceClient provides defaults for region, dscv, ca_certs and + trace_requests. + :raise ServiceClientRegistrationException: if the provided name is + already in use or if service_version is already registered. + :raise ImportError: if module_path cannot be imported. + """ + if hasattr(self, name): + using_name = getattr(self, name) + detailed_error = 'Module name already in use: %s' % using_name + raise exceptions.ServiceClientRegistrationException( + name=name, service_version=service_version, + module_path=module_path, client_names=client_names, + detailed_error=detailed_error) + if service_version in self.registered_services: + detailed_error = 'Service %s already registered.' % service_version + raise exceptions.ServiceClientRegistrationException( + name=name, service_version=service_version, + module_path=module_path, client_names=client_names, + detailed_error=detailed_error) + params = dict(region=self.region, + disable_ssl_certificate_validation=self.dscv, + ca_certs=self.ca_certs, + trace_requests=self.trace_requests) + params.update(kwargs) + # Instantiate the client factory + _factory = ClientsFactory(module_path=module_path, + client_names=client_names, + auth_provider=self.auth_provider, + **params) + # Adds the client factory to the service_client + setattr(self, name, _factory) + # Add the name of the new service in self.SERVICES for discovery + self._registered_services.add(service_version) + + @property + def registered_services(self): + # TODO(andreaf) For now add all Tempest services. to the list of + # registered service + _default_services = tempest_modules() + return self._registered_services | _default_services + def _setup_parameters(self, parameters): """Setup default values for client parameters diff --git a/tempest/test_discover/plugins.py b/tempest/test_discover/plugins.py index d604b286b5..cfb0c7f384 100644 --- a/tempest/test_discover/plugins.py +++ b/tempest/test_discover/plugins.py @@ -19,6 +19,7 @@ import six import stevedore from tempest.lib.common.utils import misc +from tempest.lib.services import clients LOG = logging.getLogger(__name__) @@ -62,6 +63,54 @@ class TempestPlugin(object): """ return [] + def get_service_clients(self): + """Get a list of the service clients for registration + + If the plugin implements service clients for one or more APIs, it + may return their details by this method for automatic registration + in any ServiceClients object instantiated by tests. + The default implementation returns an empty list. + + :return list of dictionaries. Each element of the list represents + the service client for an API. Each dict must define all + parameters required for the invocation of + `service_clients.ServiceClients.register_service_client_module`. + :rtype: list + + Example: + + >>> # Example implementation with one service client + >>> myservice_config = config.service_client_config('myservice') + >>> params = { + >>> 'name': 'myservice', + >>> 'service_version': 'myservice', + >>> 'module_path': 'myservice_tempest_tests.services', + >>> 'client_names': ['API1Client', 'API2Client'], + >>> } + >>> params.update(myservice_config) + >>> return [params] + + >>> # Example implementation with two service clients + >>> foo1_config = config.service_client_config('foo') + >>> params_foo1 = { + >>> 'name': 'foo_v1', + >>> 'service_version': 'foo.v1', + >>> 'module_path': 'bar_tempest_tests.services.foo.v1', + >>> 'client_names': ['API1Client', 'API2Client'], + >>> } + >>> params_foo1.update(foo_config) + >>> foo2_config = config.service_client_config('foo') + >>> params_foo2 = { + >>> 'name': 'foo_v2', + >>> 'service_version': 'foo.v2', + >>> 'module_path': 'bar_tempest_tests.services.foo.v2', + >>> 'client_names': ['API1Client', 'API2Client'], + >>> } + >>> params_foo2.update(foo2_config) + >>> return [params_foo1, params_foo2] + """ + return [] + @misc.singleton class TempestTestPluginManager(object): @@ -75,6 +124,7 @@ class TempestTestPluginManager(object): 'tempest.test_plugins', invoke_on_load=True, propagate_map_exceptions=True, on_load_failure_callback=self.failure_hook) + self._register_service_clients() @staticmethod def failure_hook(_, ep, err): @@ -102,3 +152,13 @@ class TempestTestPluginManager(object): if opt_list: plugin_options.extend(opt_list) return plugin_options + + def _register_service_clients(self): + registry = clients.ClientsRegistry() + for plug in self.ext_plugins: + try: + registry.register_service_client( + plug.name, plug.obj.get_service_clients()) + except Exception: + LOG.exception('Plugin %s raised an exception trying to run ' + 'get_service_clients' % plug.name) diff --git a/tempest/tests/fake_tempest_plugin.py b/tempest/tests/fake_tempest_plugin.py index f718d0be8f..56aae1e97d 100644 --- a/tempest/tests/fake_tempest_plugin.py +++ b/tempest/tests/fake_tempest_plugin.py @@ -18,6 +18,7 @@ from tempest.test_discover import plugins class FakePlugin(plugins.TempestPlugin): expected_load_test = ["my/test/path", "/home/dir"] + expected_service_clients = [{'foo': 'bar'}] def load_tests(self): return self.expected_load_test @@ -28,6 +29,9 @@ class FakePlugin(plugins.TempestPlugin): def get_opt_lists(self): return [] + def get_service_clients(self): + return self.expected_service_clients + class FakeStevedoreObj(object): obj = FakePlugin() @@ -38,3 +42,26 @@ class FakeStevedoreObj(object): def __init__(self, name='Test1'): self._name = name + + +class FakePluginNoServiceClients(plugins.TempestPlugin): + + def load_tests(self): + return [] + + def register_opts(self, conf): + return + + def get_opt_lists(self): + return [] + + +class FakeStevedoreObjNoServiceClients(object): + obj = FakePluginNoServiceClients() + + @property + def name(self): + return self._name + + def __init__(self, name='Test2'): + self._name = name diff --git a/tempest/tests/test_service_clients.py b/tempest/tests/test_service_clients.py index 26cc93f1b0..befed68832 100644 --- a/tempest/tests/test_service_clients.py +++ b/tempest/tests/test_service_clients.py @@ -263,3 +263,82 @@ class TestServiceClients(base.TestCase): for _key in _params.keys(): self.assertEqual(expected_params[_key], _params[_key]) + + def test_register_service_client_module(self): + factory_mock = self.useFixture(fixtures.MockPatch( + 'tempest.service_clients.ClientsFactory')).mock + expected_params = {'fake_param1': 'fake_value1', + 'fake_param2': 'fake_value2'} + _manager = self._get_manager(init_region='fake_region_default') + _manager.register_service_client_module( + name='fake_module', + service_version='fake_service', + module_path='fake.path.to.module', + client_names=[], + **expected_params) + self.assertThat(_manager, has_attribute('fake_module')) + # Assert called once, without check for exact parameters + self.assertTrue(factory_mock.called) + self.assertEqual(1, factory_mock.call_count) + # Assert expected params are in with their values + actual_kwargs = factory_mock.call_args[1] + self.assertIn('region', actual_kwargs) + self.assertEqual('fake_region_default', actual_kwargs['region']) + for param in expected_params: + self.assertIn(param, actual_kwargs) + self.assertEqual(expected_params[param], actual_kwargs[param]) + # Assert the new service is registered + self.assertIn('fake_service', _manager._registered_services) + + def test_register_service_client_module_override_default(self): + factory_mock = self.useFixture(fixtures.MockPatch( + 'tempest.service_clients.ClientsFactory')).mock + new_region = 'new_region' + expected_params = {'fake_param1': 'fake_value1', + 'fake_param2': 'fake_value2', + 'region': new_region} + _manager = self._get_manager(init_region='fake_region_default') + _manager.register_service_client_module( + name='fake_module', + service_version='fake_service', + module_path='fake.path.to.module', + client_names=[], + **expected_params) + self.assertThat(_manager, has_attribute('fake_module')) + # Assert called once, without check for exact parameters + self.assertTrue(factory_mock.called) + self.assertEqual(1, factory_mock.call_count) + # Assert expected params are in with their values + actual_kwargs = factory_mock.call_args[1] + self.assertIn('region', actual_kwargs) + self.assertEqual(new_region, actual_kwargs['region']) + for param in expected_params: + self.assertIn(param, actual_kwargs) + self.assertEqual(expected_params[param], actual_kwargs[param]) + # Assert the new service is registered + self.assertIn('fake_service', _manager._registered_services) + + def test_register_service_client_module_duplicate_name(self): + self.useFixture(fixtures.MockPatch( + 'tempest.service_clients.ClientsFactory')) + _manager = self._get_manager() + name_owner = 'this_is_a_string' + setattr(_manager, 'fake_module', name_owner) + expected_error = '.*' + name_owner + with testtools.ExpectedException( + exceptions.ServiceClientRegistrationException, expected_error): + _manager.register_service_client_module( + name='fake_module', module_path='fake.path.to.module', + service_version='fake_service', client_names=[]) + + def test_register_service_client_module_duplicate_service(self): + self.useFixture(fixtures.MockPatch( + 'tempest.service_clients.ClientsFactory')) + _manager = self._get_manager() + duplicate_service = 'fake_service1' + expected_error = '.*' + duplicate_service + with testtools.ExpectedException( + exceptions.ServiceClientRegistrationException, expected_error): + _manager.register_service_client_module( + name='fake_module', module_path='fake.path.to.module', + service_version=duplicate_service, client_names=[]) diff --git a/tempest/tests/test_tempest_plugin.py b/tempest/tests/test_tempest_plugin.py index c07e98c012..dd50125b68 100644 --- a/tempest/tests/test_tempest_plugin.py +++ b/tempest/tests/test_tempest_plugin.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from tempest.lib.services import clients from tempest.test_discover import plugins from tempest.tests import base from tempest.tests import fake_tempest_plugin as fake_plugin @@ -42,3 +43,39 @@ class TestPluginDiscovery(base.TestCase): result['fake01']) self.assertEqual(fake_plugin.FakePlugin.expected_load_test, result['fake02']) + + def test__register_service_clients_with_one_plugin(self): + registry = clients.ClientsRegistry() + manager = plugins.TempestTestPluginManager() + fake_obj = fake_plugin.FakeStevedoreObj() + manager.ext_plugins = [fake_obj] + manager._register_service_clients() + expected_result = fake_plugin.FakePlugin.expected_service_clients + registered_clients = registry.get_service_clients() + self.assertIn(fake_obj.name, registered_clients) + self.assertEqual(expected_result, registered_clients[fake_obj.name]) + + def test__get_service_clients_with_two_plugins(self): + registry = clients.ClientsRegistry() + manager = plugins.TempestTestPluginManager() + obj1 = fake_plugin.FakeStevedoreObj('fake01') + obj2 = fake_plugin.FakeStevedoreObj('fake02') + manager.ext_plugins = [obj1, obj2] + manager._register_service_clients() + expected_result = fake_plugin.FakePlugin.expected_service_clients + registered_clients = registry.get_service_clients() + self.assertIn('fake01', registered_clients) + self.assertIn('fake02', registered_clients) + self.assertEqual(expected_result, registered_clients['fake01']) + self.assertEqual(expected_result, registered_clients['fake02']) + + def test__register_service_clients_one_plugin_no_service_clients(self): + registry = clients.ClientsRegistry() + manager = plugins.TempestTestPluginManager() + fake_obj = fake_plugin.FakeStevedoreObjNoServiceClients() + manager.ext_plugins = [fake_obj] + manager._register_service_clients() + expected_result = [] + registered_clients = registry.get_service_clients() + self.assertIn(fake_obj.name, registered_clients) + self.assertEqual(expected_result, registered_clients[fake_obj.name])