Add ability to register non-official services

openstacksdk is in the business of being an SDK for OpenStack. While
it's tempting to support other services that people might want to run
alongside their OpenStack cloud and register in their keystone catalog,
doing so is just not feasible.

At the same time, the 95% case is that the openstacksdk will be used for
OpenStack services, so using entrpoints-based plugin loading as part of
normal usage incurs a startup cost that can be rather high (it's based
on the number of python packages installed on the system, not the number
of plugins for openstacksdk)

Add a method and a constructor parameter to Connection that allows
people to programmatically enable support for additional non-OpenStack
services. This introduce a new Service description class that maps
service_type, Proxy class and optional service_type aliases. A subclass
of Service could be provided by whoever is writing the Proxy class and
associated Resoure objects, or the base class can be instantiated with
type and proxy_class as arguments.

While doing this, rework the loading of the official OpenStack services
to use the Service descriptor objects. Add an OpenStackService subclass
which does the importlib searching of the openstack module tree for
proxy classes. This gets all of the module searching and loading into
the openstack.services module and out of Connection.

This should allow us to delete the metric service from the tree but
provide people who want to use the metric service with openstacksdk a
mechanism to do so. It also should provide a vehicle for people
developing new not-yet-official services to develop their Resource and
Proxy classes out of tree, and then move them in once they are official

Change-Id: I6d1e0c45026a2e7b3c42983df9c0565b1c501bc3

openstack/connection.py

@@ -74,18 +74,16 @@ try to find it and if that fails, you would create it::
         network = conn.network.create_network({"name": "zuul"})
 
 """
-import importlib
-
 import keystoneauth1.exceptions
 import os_service_types
+import six
 from six.moves import urllib
 
 from openstack import _log
 import openstack.config
 from openstack.config import cloud_region
 from openstack import exceptions
-from openstack import proxy
+from openstack import service_description
-from openstack import proxy2
 from openstack import task_manager
 
 _logger = _log.setup_logging('openstack')
@@ -127,6 +125,7 @@ class Connection(object):
                  # TODO(shade) Remove these once we've shifted
                  # python-openstackclient to not use the profile interface.
                  authenticator=None, profile=None,
+                 extra_services=None,
                  **kwargs):
         """Create a connection to a cloud.
 
@@ -162,12 +161,19 @@ class Connection(object):
         :param profile: DEPRECATED. Only exists for short-term backwards
                         compatibility for python-openstackclient while we
                         transition.
+        :param extra_services: List of
+            :class:`~openstack.service_description.ServiceDescription`
+            objects describing services that openstacksdk otherwise does not
+            know about.
         :param kwargs: If a config is not provided, the rest of the parameters
             provided are assumed to be arguments to be passed to the
             CloudRegion contructor.
         """
         self.config = config
-        self.service_type_manager = os_service_types.ServiceTypes()
+        self._extra_services = {}
+        if extra_services:
+            for service in extra_services:
+                self._extra_services[service.service_type] = service
 
         if not self.config:
             if profile:
@@ -193,7 +199,16 @@ class Connection(object):
 
         self.session = self.config.get_session()
 
-        self._open()
+        service_type_manager = os_service_types.ServiceTypes()
+        for service in service_type_manager.services:
+            self.add_service(
+                service_description.OpenStackServiceDescription(
+                    service, self.config))
+        # TODO(mordred) openstacksdk has support for the metric service
+        # which is not in service-types-authority. What do we do about that?
+        self.add_service(
+            service_description.OpenStackServiceDescription(
+                dict(service_type='metric'), self.config))
 
     def _get_config_from_profile(self, profile, authenticator, **kwargs):
         """Get openstack.config objects from legacy profile."""
@@ -222,31 +237,31 @@ class Connection(object):
             name=name, region_name=region_name, config=kwargs)
         config._auth = authenticator
 
-    def _open(self):
+    def add_service(self, service):
-        """Open the connection. """
+        """Add a service to the Connection.
-        for service in self.service_type_manager.services:
-            self._load(service['service_type'])
-        # TODO(mordred) openstacksdk has support for the metric service
-        # which is not in service-types-authority. What do we do about that?
-        self._load('metric')
 
-    def _load(self, service_type):
+        Attaches an instance of the :class:`~openstack.proxy2.BaseProxy`
-        service = self._get_service(service_type)
+        class contained in
+        :class:`~openstack.service_description.ServiceDescription`.
+        The :class:`~openstack.proxy2.BaseProxy` will be attached to the
+        `Connection` by its ``service_type`` and by any ``aliases`` that
+        may be specified.
 
-        if service:
+        :param openstack.service_description.ServiceDescription service:
-            module_name = service.get_module() + "._proxy"
+            Object describing the service to be attached. As a convenience,
-            module = importlib.import_module(module_name)
+            if ``service`` is a string it will be treated as a ``service_type``
-            proxy_class = getattr(module, "Proxy")
+            and a basic
-            if not (issubclass(proxy_class, proxy.BaseProxy) or
+            :class:`~openstack.service_description.ServiceDescription`
-                    issubclass(proxy_class, proxy2.BaseProxy)):
+            will be created.
-                raise TypeError("%s.Proxy must inherit from BaseProxy" %
+        """
-                                proxy_class.__module__)
+        # If we don't have a proxy, just instantiate BaseProxy so that
+        # we get an adapter.
+        if isinstance(service, six.string_types):
+            service_type = service_description
+            service = service_description.ServiceDescription(service_type)
         else:
-            # If we don't have a proxy, just instantiate BaseProxy so that
+            service_type = service.service_type
-            # we get an adapter.
+        proxy_object = service.proxy_class(
-            proxy_class = proxy2.BaseProxy
-
-        proxy_object = proxy_class(
             session=self.config.get_session(),
             task_manager=self.task_manager,
             allow_version_hack=True,
@@ -256,63 +271,10 @@ class Connection(object):
             region_name=self.config.region_name,
             version=self.config.get_api_version(service_type)
         )
-        all_types = self.service_type_manager.get_all_types(service_type)
+
         # Register the proxy class with every known alias
-        for attr_name in [name.replace('-', '_') for name in all_types]:
+        for attr_name in service.all_types:
-            setattr(self, attr_name, proxy_object)
+            setattr(self, attr_name.replace('-', '_'), proxy_object)
-
-    def _get_all_types(self, service_type):
-        # We make connection attributes for all official real type names
-        # and aliases. Three services have names they were called by in
-        # openstacksdk that are not covered by Service Types Authority aliases.
-        # Include them here - but take heed, no additional values should ever
-        # be added to this list.
-        # that were only used in openstacksdk resource naming.
-        LOCAL_ALIASES = {
-            'baremetal': 'bare_metal',
-            'block_storage': 'block_store',
-            'clustering': 'cluster',
-        }
-        all_types = self.service_type_manager.get_all_types(service_type)
-        if service_type in LOCAL_ALIASES:
-            all_types.append(LOCAL_ALIASES[service_type])
-        return all_types
-
-    def _get_service(self, official_service_type):
-        service_class = None
-        for service_type in self._get_all_types(official_service_type):
-            service_class = self._find_service_class(service_type)
-            if service_class:
-                break
-        if not service_class:
-            return None
-        # TODO(mordred) Replace this with proper discovery
-        version_string = self.config.get_api_version(official_service_type)
-        version = None
-        if version_string:
-            version = 'v{version}'.format(version=version_string[0])
-        return service_class(version=version)
-
-    def _find_service_class(self, service_type):
-        package_name = 'openstack.{service_type}'.format(
-            service_type=service_type).replace('-', '_')
-        module_name = service_type.replace('-', '_') + '_service'
-        class_name = ''.join(
-            [part.capitalize() for part in module_name.split('_')])
-        try:
-            import_name = '.'.join([package_name, module_name])
-            service_module = importlib.import_module(import_name)
-        except ImportError:
-            return None
-        service_class = getattr(service_module, class_name, None)
-        if not service_class:
-            _logger.warn(
-                'Unable to find class {class_name} in module for service'
-                ' for service {service_type}'.format(
-                    class_name=class_name,
-                    service_type=service_type))
-            return None
-        return service_class
 
     def authorize(self):
         """Authorize this Connection

openstack/service_description.py

@@ -0,0 +1,181 @@
+# Copyright 2018 Red Hat, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__all__ = [
+    'OpenStackServiceDescription',
+    'ServiceDescription',
+]
+
+import importlib
+import warnings
+
+import os_service_types
+
+from openstack import _log
+from openstack import proxy
+from openstack import proxy2
+
+_logger = _log.setup_logging('openstack')
+_service_type_manager = os_service_types.ServiceTypes()
+
+
+def _get_all_types(service_type, aliases=None):
+    # We make connection attributes for all official real type names
+    # and aliases. Three services have names they were called by in
+    # openstacksdk that are not covered by Service Types Authority aliases.
+    # Include them here - but take heed, no additional values should ever
+    # be added to this list.
+    # that were only used in openstacksdk resource naming.
+    LOCAL_ALIASES = {
+        'baremetal': 'bare_metal',
+        'block_storage': 'block_store',
+        'clustering': 'cluster',
+    }
+    all_types = set(_service_type_manager.get_all_types(service_type))
+    if aliases:
+        all_types.update(aliases)
+    if service_type in LOCAL_ALIASES:
+        all_types.add(LOCAL_ALIASES[service_type])
+    return all_types
+
+
+class ServiceDescription(object):
+
+    #: Proxy class for this service
+    proxy_class = proxy2.BaseProxy
+    #: main service_type to use to find this service in the catalog
+    service_type = None
+    #: list of aliases this service might be registered as
+    aliases = []
+    #: Internal temporary flag to control whether or not a warning is
+    #: emitted for use of old Proxy class. In-tree things should not
+    #: emit a warning - but out of tree things should only use Proxy2.
+    _warn_if_old = True
+
+    def __init__(self, service_type, proxy_class=None, aliases=None):
+        """Class describing how to interact with a REST service.
+
+        Each service in an OpenStack cloud needs to be found by looking
+        for it in the catalog. Once the endpoint is found, REST calls can
+        be made, but a Proxy class and some Resource objects are needed
+        to provide an object interface.
+
+        Instances of ServiceDescription can be passed to
+        `openstack.connection.Connection.add_service`, or a list can be
+        passed to the `openstack.connection.Connection` constructor in
+        the ``extra_services`` argument.
+
+        All three parameters can be provided at instantation time, or
+        a service-specific subclass can be used that sets the attributes
+        directly.
+
+        :param string service_type:
+            service_type to look for in the keystone catalog
+        :param proxy2.BaseProxy proxy_class:
+            subclass of :class:`~openstack.proxy2.BaseProxy` implementing
+            an interface for this service. Defaults to
+            :class:`~openstack.proxy2.BaseProxy` which provides REST operations
+            but no additional features.
+        :param list aliases:
+            Optional list of aliases, if there is more than one name that might
+            be used to register the service in the catalog.
+        """
+        self.service_type = service_type
+        self.proxy_class = proxy_class or self.proxy_class
+        self.all_types = _get_all_types(service_type, aliases)
+
+        self._validate_proxy_class()
+
+    def _validate_proxy_class(self):
+        if not issubclass(
+                self.proxy_class, (proxy.BaseProxy, proxy2.BaseProxy)):
+            raise TypeError(
+                "{module}.{proxy_class} must inherit from BaseProxy".format(
+                    module=self.proxy_class.__module__,
+                    proxy_class=self.proxy_class.__name__))
+        if issubclass(self.proxy_class, proxy.BaseProxy) and self._warn_if_old:
+            warnings.warn(
+                "Use of proxy.BaseProxy is not supported."
+                " Please update to use proxy2.BaseProxy.",
+                DeprecationWarning)
+
+
+class OpenStackServiceDescription(ServiceDescription):
+
+    #: Override _warn_if_old so we don't spam people with warnings
+    _warn_if_old = False
+
+    def __init__(self, service, config):
+        """Official OpenStack ServiceDescription.
+
+        The OpenStackServiceDescription class is a helper class for
+        services listed in Service Types Authority and that are directly
+        supported by openstacksdk.
+
+        It finds the proxy_class by looking in the openstacksdk tree for
+        appropriately named modules.
+
+        :param dict service:
+            A service dict as found in `os_service_types.ServiceTypes.services`
+        :param openstack.config.cloud_region.CloudRegion config:
+            ConfigRegion for the connection.
+        """
+        super(OpenStackServiceDescription, self).__init__(
+            service['service_type'])
+        self.config = config
+        service_filter = self._get_service_filter()
+        if service_filter:
+            module_name = service_filter.get_module() + "._proxy"
+            module = importlib.import_module(module_name)
+            self.proxy_class = getattr(module, "Proxy")
+
+    def _get_service_filter(self):
+        service_filter_class = None
+        for service_type in self.all_types:
+            service_filter_class = self._find_service_filter_class()
+            if service_filter_class:
+                break
+        if not service_filter_class:
+            return None
+        # TODO(mordred) Replace this with proper discovery
+        version_string = self.config.get_api_version(self.service_type)
+        version = None
+        if version_string:
+            version = 'v{version}'.format(version=version_string[0])
+        return service_filter_class(version=version)
+
+    def _find_service_filter_class(self):
+        package_name = 'openstack.{service_type}'.format(
+            service_type=self.service_type).replace('-', '_')
+        module_name = self.service_type.replace('-', '_') + '_service'
+        class_name = ''.join(
+            [part.capitalize() for part in module_name.split('_')])
+        try:
+            import_name = '.'.join([package_name, module_name])
+            service_filter_module = importlib.import_module(import_name)
+        except ImportError as e:
+            # ImportWarning is ignored by default. This warning is here
+            # as an opt-in for people trying to figure out why something
+            # didn't work.
+            warnings.warn(
+                "Could not import {service_type} service filter: {e}".format(
+                    service_type=self.service_type, e=str(e)),
+                ImportWarning)
+            return None
+        service_filter_class = getattr(service_filter_module, class_name, None)
+        if not service_filter_class:
+            _logger.warn(
+                'Unable to find class %s in module for service %s',
+                class_name, self.service_type)
+            return None
+        return service_filter_class

releasenotes/notes/add-service-0bcc16eb026eade3.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Added a new method `openstack.connection.Connection.add_service` which
+    allows the registration of Proxy/Resource classes defined externally.