Fix tempest and available modules

tempest_modules and available_modules define stable interfaces, so
they should be implemented so that they can be kept backward
compatible after the move to tempest.lib.

To achieve that, define an internal _tempest_internal_modules, which
holds the list of service modules not migrated to tempest.lib yet.
Tempest service clients module will add incremenetally to
tempest_modules (and thus available_modules), without changing the
meaning of those two sets.

Once all clients are migrated, _tempest_internal_modules will be
dropped silently.

Align the docstring to refer to "set" rather than "list".

Partially-implements: bp client-manager-refactor
Change-Id: I921ad863bce4b4680bd8ebb2887e34cb643621da

tempest/clients.py

@@ -109,7 +109,9 @@ class Manager(service_clients.ServiceClients):
         # Setup the parameters for all Tempest services.
         # NOTE(andreaf) Since client.py is an internal module of Tempest,
         # it doesn't have to consider plugin configuration.
-        for service in service_clients.tempest_modules():
+        all_tempest_modules = (set(service_clients.tempest_modules()) |
+                               service_clients._tempest_internal_modules())
+        for service in all_tempest_modules:
             try:
                 # NOTE(andreaf) Use the unversioned service name to fetch
                 # the configuration since configuration is not versioned.

tempest/service_clients.py

@@ -18,6 +18,7 @@ import copy
 import importlib
 import inspect
 import logging
+import six
 
 from tempest.lib import auth
 from tempest.lib import exceptions
@@ -28,28 +29,36 @@ from tempest.lib.services import network
 
 LOG = logging.getLogger(__name__)
 
-client_modules_by_service_name = {
-    'compute': compute,
-    'image.v1': image.v1,
-    'image.v2': image.v2,
-    'network': network
-}
-
 
 def tempest_modules():
-    """List of service client modules available in Tempest.
+    """Dict of service client modules available in Tempest.
 
-    Provides a list of service modules available Tempest.
+    Provides a dict of stable service modules available in Tempest, with
+    ``service_version`` as key, and the module object as value.
     """
-    return set(['compute', 'identity.v2', 'identity.v3', 'image.v1',
+    return {
-                'image.v2', 'network', 'object-storage', 'volume.v1',
+        'compute': compute,
+        'image.v1': image.v1,
+        'image.v2': image.v2,
+        'network': network
+    }
+
+
+def _tempest_internal_modules():
+    # Set of unstable service clients available in Tempest
+    # NOTE(andreaf) This list will exists only as long the remain clients
+    # are migrated to tempest.lib, and it will then be deleted without
+    # deprecation or advance notice
+    return set(['identity.v2', 'identity.v3', 'object-storage', 'volume.v1',
                 'volume.v2', 'volume.v3'])
 
 
 def available_modules():
-    """List of service client modules available in Tempest and plugins
+    """Set of service client modules available in Tempest and plugins
 
-    The list of available modules can be used for automatic configuration.
+    Set of stable service clients from Tempest and service clients exposed
+    by plugins. This set of available modules can be used for automatic
+    configuration.
 
     :raise PluginRegistrationException: if a plugin exposes a service_version
         already defined by Tempest or another plugin.
@@ -65,6 +74,7 @@ def available_modules():
         >>>                                  client_parameters=params)
     """
     extra_service_versions = set([])
+    _tempest_modules = set(tempest_modules())
     plugin_services = clients.ClientsRegistry().get_service_clients()
     for plugin_name in plugin_services:
         plug_service_versions = set([x['service_version'] for x in
@@ -79,16 +89,16 @@ def available_modules():
                                                 plug_service_versions))
                 raise exceptions.PluginRegistrationException(
                     name=plugin_name, detailed_error=detailed_error)
-            if not plug_service_versions.isdisjoint(tempest_modules()):
+            if not plug_service_versions.isdisjoint(_tempest_modules):
                 detailed_error = (
                     'Plugin %s is trying to register a service %s already '
                     'claimed by a Tempest one' % (plugin_name,
-                                                  tempest_modules() &
+                                                  _tempest_modules &
                                                   plug_service_versions))
                 raise exceptions.PluginRegistrationException(
                     name=plugin_name, detailed_error=detailed_error)
         extra_service_versions |= plug_service_versions
-    return tempest_modules() | extra_service_versions
+    return _tempest_modules | extra_service_versions
 
 
 class ClientsFactory(object):
@@ -310,8 +320,9 @@ class ServiceClients(object):
         client_parameters = client_parameters or {}
         self.parameters = {}
         # Parameters are provided for unversioned services
+        all_modules = available_modules() | _tempest_internal_modules()
         unversioned_services = set(
-            [x.split('.')[0] for x in available_modules()])
+            [x.split('.')[0] for x in all_modules])
         for service in unversioned_services:
             self.parameters[service] = self._setup_parameters(
                 client_parameters.pop(service, {}))
@@ -321,14 +332,12 @@ class ServiceClients(object):
                 services=list(client_parameters.keys()))
 
         # Register service clients owned by tempest
-        for service in tempest_modules():
+        for service, module in six.iteritems(tempest_modules()):
-            if service in list(client_modules_by_service_name):
+            attribute = service.replace('.', '_')
-                attribute = service.replace('.', '_')
+            configs = service.split('.')[0]
-                configs = service.split('.')[0]
+            self.register_service_client_module(
-                module = client_modules_by_service_name[service]
+                attribute, service, module.__name__,
-                self.register_service_client_module(
+                module.__all__, **self.parameters[configs])
-                    attribute, service, module.__name__,
-                    module.__all__, **self.parameters[configs])
 
         # Register service clients from plugins
         clients_registry = clients.ClientsRegistry()
@@ -403,10 +412,7 @@ class ServiceClients(object):
 
     @property
     def registered_services(self):
-        # TODO(andreaf) Temporary set needed until all services are migrated
+        return self._registered_services | _tempest_internal_modules()
-        _non_migrated_services = tempest_modules() - set(
-            client_modules_by_service_name)
-        return self._registered_services | _non_migrated_services
 
     def _setup_parameters(self, parameters):
         """Setup default values for client parameters

tempest/tests/test_service_clients.py

@@ -188,8 +188,10 @@ class TestServiceClients(base.TestCase):
     def setUp(self):
         super(TestServiceClients, self).setUp()
         self.useFixture(fixtures.MockPatch(
-            'tempest.service_clients.tempest_modules',
+            'tempest.service_clients.tempest_modules', return_value={}))
-            return_value=set(['fake_service1', 'fake_service2'])))
+        self.useFixture(fixtures.MockPatch(
+            'tempest.service_clients._tempest_internal_modules',
+            return_value=set(['fake_service1'])))
 
     def test___init___creds_v2_uri(self):
         # Verify that no API request is made, since no mock