Service Clients registration interface for plugins

Add a new registration interface to service_clients.
Add a new optional method to the plugin interface, that exposes
the plugin service client registration details.

Tests in plugins can initialise service_clients with parmaters
common to their service clients and other ones they may need.
Parameters specific to their service clients are passed via the
registration interface, and can be overwritten at any time
by passing extra parameters at client init time.

Partially-implements: bp client-manager-refactor
Change-Id: I2d99aaa317b0d21c0968dd25b21c4ba9088136fb

releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - A new optional interface `TempestPlugin.get_service_clients`
+    is available to plugins. It allows them to declare
+    any service client they implement. For now this is used by
+    tempest only, for auto-registration of service clients
+    in the new class `ServiceClients`.
+  - A new singleton class `clients.ClientsRegistry` is
+    available. It holds the service clients registration data
+    from all plugins. It is used by `ServiceClients` for
+    auto-registration of the service clients implemented
+    in plugins.

tempest/lib/exceptions.py

@@ -229,3 +229,13 @@ class SSHExecCommandFailed(TempestException):
 
 class UnknownServiceClient(TempestException):
     message = "Service clients named %(services)s are not known"
+
+
+class ServiceClientRegistrationException(TempestException):
+    message = ("Error registering module %(name)s in path %(module_path)s, "
+               "with service %(service_version)s and clients "
+               "%(client_names)s: %(detailed_error)s")
+
+
+class PluginRegistrationException(TempestException):
+    message = "Error registering plugin %(name)s: %(detailed_error)s"

tempest/lib/services/clients.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tempest.lib.common.utils import misc
+from tempest.lib import exceptions
+
+
+@misc.singleton
+class ClientsRegistry(object):
+    """Registry of all service clients available from plugins"""
+
+    def __init__(self):
+        self._service_clients = {}
+
+    def register_service_client(self, plugin_name, service_client_data):
+        if plugin_name in self._service_clients:
+            detailed_error = 'Clients for plugin %s already registered'
+            raise exceptions.PluginRegistrationException(
+                name=plugin_name,
+                detailed_error=detailed_error % plugin_name)
+        self._service_clients[plugin_name] = service_client_data
+
+    def get_service_clients(self):
+        return self._service_clients

tempest/service_clients.py

@@ -17,9 +17,13 @@
 import copy
 import importlib
 import inspect
+import logging
 
 from tempest.lib import auth
 from tempest.lib import exceptions
+from tempest.lib.services import clients
+
+LOG = logging.getLogger(__name__)
 
 
 def tempest_modules():
@@ -33,9 +37,48 @@ def tempest_modules():
 
 
 def available_modules():
-    """List of service client modules available in Tempest and plugins"""
-    # TODO(andreaf) For now this returns only tempest_modules
-    return tempest_modules()
+    """List of service client modules available in Tempest and plugins
+
+    The list of available modules can be used for automatic configuration.
+
+    :raise PluginRegistrationException: if a plugin exposes a service_version
+        already defined by Tempest or another plugin.
+
+    Examples:
+
+        >>> from tempest import config
+        >>> params = {}
+        >>> for service_version in available_modules():
+        >>>     service = service_version.split('.')[0]
+        >>>     params[service] = config.service_client_config(service)
+        >>> service_clients = ServiceClients(creds, identity_uri,
+        >>>                                  client_parameters=params)
+    """
+    extra_service_versions = set([])
+    plugin_services = clients.ClientsRegistry().get_service_clients()
+    for plugin_name in plugin_services:
+        plug_service_versions = set([x['service_version'] for x in
+                                     plugin_services[plugin_name]])
+        # If a plugin exposes a duplicate service_version raise an exception
+        if plug_service_versions:
+            if not plug_service_versions.isdisjoint(extra_service_versions):
+                detailed_error = (
+                    'Plugin %s is trying to register a service %s already '
+                    'claimed by another one' % (plugin_name,
+                                                extra_service_versions &
+                                                plug_service_versions))
+                raise exceptions.PluginRegistrationException(
+                    name=plugin_name, detailed_error=detailed_error)
+            if not plug_service_versions.isdisjoint(tempest_modules()):
+                detailed_error = (
+                    'Plugin %s is trying to register a service %s already '
+                    'claimed by a Tempest one' % (plugin_name,
+                                                  tempest_modules() &
+                                                  plug_service_versions))
+                raise exceptions.PluginRegistrationException(
+                    name=plugin_name, detailed_error=detailed_error)
+        extra_service_versions |= plug_service_versions
+    return tempest_modules() | extra_service_versions
 
 
 class ClientsFactory(object):
@@ -49,8 +92,6 @@ class ClientsFactory(object):
     ClientsFactory can be used directly, or consumed via the `ServiceClients`
     class, which manages the authorization part.
     """
-    # TODO(andreaf) This version includes ClientsFactory but it does not
-    # use it yet in ServiceClients
 
     def __init__(self, module_path, client_names, auth_provider, **kwargs):
         """Initialises the client factory
@@ -206,6 +247,7 @@ class ServiceClients(object):
             >>> client_parameters['service_y'] = params_service_y
 
         """
+        self._registered_services = set([])
         self.credentials = credentials
         self.identity_uri = identity_uri
         if not identity_uri:
@@ -247,6 +289,84 @@ class ServiceClients(object):
             raise exceptions.UnknownServiceClient(
                 services=list(client_parameters.keys()))
 
+        # Register service clients from plugins
+        clients_registry = clients.ClientsRegistry()
+        plugin_service_clients = clients_registry.get_service_clients()
+        for plugin in plugin_service_clients:
+            service_clients = plugin_service_clients[plugin]
+            # Each plugin returns a list of service client parameters
+            for service_client in service_clients:
+                # NOTE(andreaf) If a plugin cannot register, stop the
+                # registration process, log some details to help
+                # troubleshooting, and re-raise
+                try:
+                    self.register_service_client_module(**service_client)
+                except Exception:
+                    LOG.exception(
+                        'Failed to register service client from plugin %s '
+                        'with parameters %s' % (plugin, service_client))
+                    raise
+
+    def register_service_client_module(self, name, service_version,
+                                       module_path, client_names, **kwargs):
+        """Register a service client module
+
+        Initiates a client factory for the specified module, using this
+        class auth_provider, and accessible via a `name` attribute in the
+        service client.
+
+        :param name: Name used to access the client
+        :param service_version: Name of the service complete with version.
+            Used to track registered services. When a plugin implements it,
+            it can be used by other plugins to obtain their configuration.
+        :param module_path: Path to module that includes all service clients.
+            All service client classes must be exposed by a single module.
+            If they are separated in different modules, defining __all__
+            in the root module can help, similar to what is done by service
+            clients in tempest.
+        :param client_names: List or set of names of service client classes.
+        :param kwargs: Extra optional parameters to be passed to all clients.
+            ServiceClient provides defaults for region, dscv, ca_certs and
+            trace_requests.
+        :raise ServiceClientRegistrationException: if the provided name is
+            already in use or if service_version is already registered.
+        :raise ImportError: if module_path cannot be imported.
+        """
+        if hasattr(self, name):
+            using_name = getattr(self, name)
+            detailed_error = 'Module name already in use: %s' % using_name
+            raise exceptions.ServiceClientRegistrationException(
+                name=name, service_version=service_version,
+                module_path=module_path, client_names=client_names,
+                detailed_error=detailed_error)
+        if service_version in self.registered_services:
+            detailed_error = 'Service %s already registered.' % service_version
+            raise exceptions.ServiceClientRegistrationException(
+                name=name, service_version=service_version,
+                module_path=module_path, client_names=client_names,
+                detailed_error=detailed_error)
+        params = dict(region=self.region,
+                      disable_ssl_certificate_validation=self.dscv,
+                      ca_certs=self.ca_certs,
+                      trace_requests=self.trace_requests)
+        params.update(kwargs)
+        # Instantiate the client factory
+        _factory = ClientsFactory(module_path=module_path,
+                                  client_names=client_names,
+                                  auth_provider=self.auth_provider,
+                                  **params)
+        # Adds the client factory to the service_client
+        setattr(self, name, _factory)
+        # Add the name of the new service in self.SERVICES for discovery
+        self._registered_services.add(service_version)
+
+    @property
+    def registered_services(self):
+        # TODO(andreaf) For now add all Tempest services. to the list of
+        # registered service
+        _default_services = tempest_modules()
+        return self._registered_services | _default_services
+
     def _setup_parameters(self, parameters):
         """Setup default values for client parameters
 

tempest/test_discover/plugins.py

@@ -19,6 +19,7 @@ import six
 import stevedore
 
 from tempest.lib.common.utils import misc
+from tempest.lib.services import clients
 
 LOG = logging.getLogger(__name__)
 
@@ -62,6 +63,54 @@ class TempestPlugin(object):
         """
         return []
 
+    def get_service_clients(self):
+        """Get a list of the service clients for registration
+
+        If the plugin implements service clients for one or more APIs, it
+        may return their details by this method for automatic registration
+        in any ServiceClients object instantiated by tests.
+        The default implementation returns an empty list.
+
+        :return list of dictionaries. Each element of the list represents
+            the service client for an API. Each dict must define all
+            parameters required for the invocation of
+            `service_clients.ServiceClients.register_service_client_module`.
+        :rtype: list
+
+        Example:
+
+            >>>  # Example implementation with one service client
+            >>>  myservice_config = config.service_client_config('myservice')
+            >>>  params = {
+            >>>     'name': 'myservice',
+            >>>     'service_version': 'myservice',
+            >>>     'module_path': 'myservice_tempest_tests.services',
+            >>>     'client_names': ['API1Client', 'API2Client'],
+            >>>  }
+            >>>  params.update(myservice_config)
+            >>> return [params]
+
+            >>>  # Example implementation with two service clients
+            >>>  foo1_config = config.service_client_config('foo')
+            >>>  params_foo1 = {
+            >>>     'name': 'foo_v1',
+            >>>     'service_version': 'foo.v1',
+            >>>     'module_path': 'bar_tempest_tests.services.foo.v1',
+            >>>     'client_names': ['API1Client', 'API2Client'],
+            >>>  }
+            >>>  params_foo1.update(foo_config)
+            >>>  foo2_config = config.service_client_config('foo')
+            >>>  params_foo2 = {
+            >>>     'name': 'foo_v2',
+            >>>     'service_version': 'foo.v2',
+            >>>     'module_path': 'bar_tempest_tests.services.foo.v2',
+            >>>     'client_names': ['API1Client', 'API2Client'],
+            >>>  }
+            >>>  params_foo2.update(foo2_config)
+            >>> return [params_foo1, params_foo2]
+        """
+        return []
+
 
 @misc.singleton
 class TempestTestPluginManager(object):
@@ -75,6 +124,7 @@ class TempestTestPluginManager(object):
             'tempest.test_plugins', invoke_on_load=True,
             propagate_map_exceptions=True,
             on_load_failure_callback=self.failure_hook)
+        self._register_service_clients()
 
     @staticmethod
     def failure_hook(_, ep, err):
@@ -102,3 +152,13 @@ class TempestTestPluginManager(object):
             if opt_list:
                 plugin_options.extend(opt_list)
         return plugin_options
+
+    def _register_service_clients(self):
+        registry = clients.ClientsRegistry()
+        for plug in self.ext_plugins:
+            try:
+                registry.register_service_client(
+                    plug.name, plug.obj.get_service_clients())
+            except Exception:
+                LOG.exception('Plugin %s raised an exception trying to run '
+                              'get_service_clients' % plug.name)

tempest/tests/fake_tempest_plugin.py

@@ -18,6 +18,7 @@ from tempest.test_discover import plugins
 
 class FakePlugin(plugins.TempestPlugin):
     expected_load_test = ["my/test/path", "/home/dir"]
+    expected_service_clients = [{'foo': 'bar'}]
 
     def load_tests(self):
         return self.expected_load_test
@@ -28,6 +29,9 @@ class FakePlugin(plugins.TempestPlugin):
     def get_opt_lists(self):
         return []
 
+    def get_service_clients(self):
+        return self.expected_service_clients
+
 
 class FakeStevedoreObj(object):
     obj = FakePlugin()
@@ -38,3 +42,26 @@ class FakeStevedoreObj(object):
 
     def __init__(self, name='Test1'):
         self._name = name
+
+
+class FakePluginNoServiceClients(plugins.TempestPlugin):
+
+    def load_tests(self):
+        return []
+
+    def register_opts(self, conf):
+        return
+
+    def get_opt_lists(self):
+        return []
+
+
+class FakeStevedoreObjNoServiceClients(object):
+    obj = FakePluginNoServiceClients()
+
+    @property
+    def name(self):
+        return self._name
+
+    def __init__(self, name='Test2'):
+        self._name = name

tempest/tests/test_service_clients.py

@@ -263,3 +263,82 @@ class TestServiceClients(base.TestCase):
         for _key in _params.keys():
             self.assertEqual(expected_params[_key],
                              _params[_key])
+
+    def test_register_service_client_module(self):
+        factory_mock = self.useFixture(fixtures.MockPatch(
+            'tempest.service_clients.ClientsFactory')).mock
+        expected_params = {'fake_param1': 'fake_value1',
+                           'fake_param2': 'fake_value2'}
+        _manager = self._get_manager(init_region='fake_region_default')
+        _manager.register_service_client_module(
+            name='fake_module',
+            service_version='fake_service',
+            module_path='fake.path.to.module',
+            client_names=[],
+            **expected_params)
+        self.assertThat(_manager, has_attribute('fake_module'))
+        # Assert called once, without check for exact parameters
+        self.assertTrue(factory_mock.called)
+        self.assertEqual(1, factory_mock.call_count)
+        # Assert expected params are in with their values
+        actual_kwargs = factory_mock.call_args[1]
+        self.assertIn('region', actual_kwargs)
+        self.assertEqual('fake_region_default', actual_kwargs['region'])
+        for param in expected_params:
+            self.assertIn(param, actual_kwargs)
+            self.assertEqual(expected_params[param], actual_kwargs[param])
+        # Assert the new service is registered
+        self.assertIn('fake_service', _manager._registered_services)
+
+    def test_register_service_client_module_override_default(self):
+        factory_mock = self.useFixture(fixtures.MockPatch(
+            'tempest.service_clients.ClientsFactory')).mock
+        new_region = 'new_region'
+        expected_params = {'fake_param1': 'fake_value1',
+                           'fake_param2': 'fake_value2',
+                           'region': new_region}
+        _manager = self._get_manager(init_region='fake_region_default')
+        _manager.register_service_client_module(
+            name='fake_module',
+            service_version='fake_service',
+            module_path='fake.path.to.module',
+            client_names=[],
+            **expected_params)
+        self.assertThat(_manager, has_attribute('fake_module'))
+        # Assert called once, without check for exact parameters
+        self.assertTrue(factory_mock.called)
+        self.assertEqual(1, factory_mock.call_count)
+        # Assert expected params are in with their values
+        actual_kwargs = factory_mock.call_args[1]
+        self.assertIn('region', actual_kwargs)
+        self.assertEqual(new_region, actual_kwargs['region'])
+        for param in expected_params:
+            self.assertIn(param, actual_kwargs)
+            self.assertEqual(expected_params[param], actual_kwargs[param])
+        # Assert the new service is registered
+        self.assertIn('fake_service', _manager._registered_services)
+
+    def test_register_service_client_module_duplicate_name(self):
+        self.useFixture(fixtures.MockPatch(
+            'tempest.service_clients.ClientsFactory'))
+        _manager = self._get_manager()
+        name_owner = 'this_is_a_string'
+        setattr(_manager, 'fake_module', name_owner)
+        expected_error = '.*' + name_owner
+        with testtools.ExpectedException(
+                exceptions.ServiceClientRegistrationException, expected_error):
+            _manager.register_service_client_module(
+                name='fake_module', module_path='fake.path.to.module',
+                service_version='fake_service', client_names=[])
+
+    def test_register_service_client_module_duplicate_service(self):
+        self.useFixture(fixtures.MockPatch(
+            'tempest.service_clients.ClientsFactory'))
+        _manager = self._get_manager()
+        duplicate_service = 'fake_service1'
+        expected_error = '.*' + duplicate_service
+        with testtools.ExpectedException(
+                exceptions.ServiceClientRegistrationException, expected_error):
+            _manager.register_service_client_module(
+                name='fake_module', module_path='fake.path.to.module',
+                service_version=duplicate_service, client_names=[])

tempest/tests/test_tempest_plugin.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from tempest.lib.services import clients
 from tempest.test_discover import plugins
 from tempest.tests import base
 from tempest.tests import fake_tempest_plugin as fake_plugin
@@ -42,3 +43,39 @@ class TestPluginDiscovery(base.TestCase):
                          result['fake01'])
         self.assertEqual(fake_plugin.FakePlugin.expected_load_test,
                          result['fake02'])
+
+    def test__register_service_clients_with_one_plugin(self):
+        registry = clients.ClientsRegistry()
+        manager = plugins.TempestTestPluginManager()
+        fake_obj = fake_plugin.FakeStevedoreObj()
+        manager.ext_plugins = [fake_obj]
+        manager._register_service_clients()
+        expected_result = fake_plugin.FakePlugin.expected_service_clients
+        registered_clients = registry.get_service_clients()
+        self.assertIn(fake_obj.name, registered_clients)
+        self.assertEqual(expected_result, registered_clients[fake_obj.name])
+
+    def test__get_service_clients_with_two_plugins(self):
+        registry = clients.ClientsRegistry()
+        manager = plugins.TempestTestPluginManager()
+        obj1 = fake_plugin.FakeStevedoreObj('fake01')
+        obj2 = fake_plugin.FakeStevedoreObj('fake02')
+        manager.ext_plugins = [obj1, obj2]
+        manager._register_service_clients()
+        expected_result = fake_plugin.FakePlugin.expected_service_clients
+        registered_clients = registry.get_service_clients()
+        self.assertIn('fake01', registered_clients)
+        self.assertIn('fake02', registered_clients)
+        self.assertEqual(expected_result, registered_clients['fake01'])
+        self.assertEqual(expected_result, registered_clients['fake02'])
+
+    def test__register_service_clients_one_plugin_no_service_clients(self):
+        registry = clients.ClientsRegistry()
+        manager = plugins.TempestTestPluginManager()
+        fake_obj = fake_plugin.FakeStevedoreObjNoServiceClients()
+        manager.ext_plugins = [fake_obj]
+        manager._register_service_clients()
+        expected_result = []
+        registered_clients = registry.get_service_clients()
+        self.assertIn(fake_obj.name, registered_clients)
+        self.assertEqual(expected_result, registered_clients[fake_obj.name])