Add a descriptor object for each service proxy

The end-goal here is to be able to do version discovery rather than
relying on config and in-code version defaults for what gets attached to
the connection as the service proxy for a service. However, since we're
currently constructing Adapter objects at instantation time we'd have to
authenticate AND do version discovery on every service when we create a
Connection to be able to do that.

That would be bad.

Add a Descriptor class that creates the Proxy object on-demand. That is,
when someone does "conn.compute", a Proxy will be created and returned.

To support doing that without a ton of duplicate copy-pasta for each
service, add a metaclass to Connection which reads os-service-types and
does the import lookup / BaseProxy fallback as part of Connection class
creation. One of the upsides to this is that we can add docstrings to
the service descriptor objects - meaning that the docs for Connection
actually list the proxy objects.

While we're in here, fix a NOTE in a connection doc string and add a
reference to BaseProxy in the docs so that it'll show up in the docs
too.

Change-Id: I3bef5de60b848146fc8563d853774769d0875c65

doc/source/user/index.rst

@@ -73,7 +73,12 @@ OpenStack services.
    connection
 
 Once you have a *Connection* instance, the following services may be exposed
-to you. The combination of your ``CloudRegion`` and the catalog of the cloud
+to you via the :class:`~openstack.proxy.BaseProxy` interface.
+
+.. autoclass:: openstack.proxy.BaseProxy
+   :members:
+
+The combination of your ``CloudRegion`` and the catalog of the cloud
 in question control which services are exposed, but listed below are the ones
 provided by the SDK.
 

openstack/_meta.py

@@ -0,0 +1,126 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import importlib
+import warnings
+
+import os_service_types
+
+from openstack import _log
+from openstack import proxy
+from openstack import service_description
+
+_logger = _log.setup_logging('openstack')
+_service_type_manager = os_service_types.ServiceTypes()
+_DOC_TEMPLATE = (
+    ":class:`{class_name}` for {service_type} aka {project}")
+_PROXY_TEMPLATE = """Proxy for {service_type} aka {project}
+
+This proxy object could be an instance of
+{class_doc_strings}
+depending on client configuration and which version of the service is
+found on remotely on the cloud.
+"""
+
+
+class ConnectionMeta(type):
+    def __new__(meta, name, bases, dct):
+        for service in _service_type_manager.services:
+            service_type = service['service_type']
+            if service_type == 'ec2-api':
+                # NOTE(mordred) It doesn't make any sense to use ec2-api
+                # from openstacksdk. The credentials API calls are all calls
+                # on identity endpoints.
+                continue
+            desc_class = service_description.ServiceDescription
+            service_filter_class = _find_service_filter_class(service_type)
+            descriptor_args = {'service_type': service_type}
+            if service_filter_class:
+                desc_class = service_description.OpenStackServiceDescription
+                descriptor_args['service_filter_class'] = service_filter_class
+                class_names = service_filter_class._get_proxy_class_names()
+                if len(class_names) == 1:
+                    doc = _DOC_TEMPLATE.format(
+                        class_name="{service_type} Proxy <{name}>".format(
+                            service_type=service_type, name=class_names[0]),
+                        **service)
+                else:
+                    class_doc_strings = "\n".join([
+                        ":class:`{class_name}`".format(class_name=class_name)
+                        for class_name in class_names])
+                    doc = _PROXY_TEMPLATE.format(
+                        class_doc_strings=class_doc_strings, **service)
+            else:
+                descriptor_args['proxy_class'] = proxy.BaseProxy
+                doc = _DOC_TEMPLATE.format(
+                    class_name='~openstack.proxy.BaseProxy', **service)
+            descriptor = desc_class(**descriptor_args)
+            descriptor.__doc__ = doc
+            dct[service_type.replace('-', '_')] = descriptor
+
+            # Register the descriptor class with every known alias. Don't
+            # add doc strings though - although they are supported, we don't
+            # want to give anybody any bad ideas. Making a second descriptor
+            # does not introduce runtime cost as the descriptors all use
+            # the same _proxies dict on the instance.
+            for alias_name in _get_aliases(service_type):
+                if alias_name[-1].isdigit():
+                    continue
+                alias_descriptor = desc_class(**descriptor_args)
+                dct[alias_name.replace('-', '_')] = alias_descriptor
+        return super(ConnectionMeta, meta).__new__(meta, name, bases, dct)
+
+
+def _get_aliases(service_type, aliases=None):
+    # We make connection attributes for all official real type names
+    # and aliases. Three services have names they were called by in
+    # openstacksdk that are not covered by Service Types Authority aliases.
+    # Include them here - but take heed, no additional values should ever
+    # be added to this list.
+    # that were only used in openstacksdk resource naming.
+    LOCAL_ALIASES = {
+        'baremetal': 'bare_metal',
+        'block_storage': 'block_store',
+        'clustering': 'cluster',
+    }
+    all_types = set(_service_type_manager.get_aliases(service_type))
+    if aliases:
+        all_types.update(aliases)
+    if service_type in LOCAL_ALIASES:
+        all_types.add(LOCAL_ALIASES[service_type])
+    return all_types
+
+
+def _find_service_filter_class(service_type):
+    package_name = 'openstack.{service_type}'.format(
+        service_type=service_type).replace('-', '_')
+    module_name = service_type.replace('-', '_') + '_service'
+    class_name = ''.join(
+        [part.capitalize() for part in module_name.split('_')])
+    try:
+        import_name = '.'.join([package_name, module_name])
+        service_filter_module = importlib.import_module(import_name)
+    except ImportError as e:
+        # ImportWarning is ignored by default. This warning is here
+        # as an opt-in for people trying to figure out why something
+        # didn't work.
+        warnings.warn(
+            "Could not import {service_type} service filter: {e}".format(
+                service_type=service_type, e=str(e)),
+            ImportWarning)
+        return None
+    # There are no cases in which we should have a module but not the class
+    # inside it.
+    service_filter_class = getattr(service_filter_module, class_name)
+    return service_filter_class

openstack/connection.py

@@ -82,11 +82,11 @@ __all__ = [
 import warnings
 
 import keystoneauth1.exceptions
-import os_service_types
 import requestsexceptions
 import six
 
 from openstack import _log
+from openstack import _meta
 from openstack import config as _config
 from openstack import exceptions
 from openstack import service_description
@@ -128,7 +128,7 @@ def from_config(cloud=None, config=None, options=None, **kwargs):
     return Connection(config=config)
 
 
-class Connection(object):
+class Connection(six.with_metaclass(_meta.ConnectionMeta)):
 
     def __init__(self, cloud=None, config=None, session=None,
                  app_name=None, app_version=None,
@@ -219,11 +219,7 @@ class Connection(object):
         # to a Resource method's session argument.
         self.session._sdk_connection = self
 
-        service_type_manager = os_service_types.ServiceTypes()
+        self._proxies = {}
-        for service in service_type_manager.services:
-            self.add_service(
-                service_description.OpenStackServiceDescription(
-                    service, self.config))
 
     def add_service(self, service):
         """Add a service to the Connection.
@@ -245,29 +241,16 @@ class Connection(object):
         # If we don't have a proxy, just instantiate BaseProxy so that
         # we get an adapter.
         if isinstance(service, six.string_types):
-            service_type = service
+            service = service_description.ServiceDescription(service)
-            service = service_description.ServiceDescription(service_type)
-        else:
-            service_type = service.service_type
-        proxy_object = service.proxy_class(
-            session=self.config.get_session(),
-            task_manager=self.task_manager,
-            allow_version_hack=True,
-            service_type=self.config.get_service_type(service_type),
-            service_name=self.config.get_service_name(service_type),
-            interface=self.config.get_interface(service_type),
-            region_name=self.config.region_name,
-            version=self.config.get_api_version(service_type),
-        )
 
         # Register the proxy class with every known alias
         for attr_name in service.all_types:
-            setattr(self, attr_name.replace('-', '_'), proxy_object)
+            setattr(self, attr_name.replace('-', '_'), service)
 
     def authorize(self):
         """Authorize this Connection
 
-        **NOTE**: This method is optional. When an application makes a call
+        .. note:: This method is optional. When an application makes a call
                   to any OpenStack service, this method allows you to request
                   a token manually before attempting to do anything else.
 

openstack/proxy.py

@@ -41,6 +41,7 @@ def _check_resource(strict=False):
 
 
 class BaseProxy(_adapter.OpenStackSDKAdapter):
+    """Represents a service."""
 
     def _get_resource(self, resource_type, value, **attrs):
         """Get a resource object to work on

openstack/service_description.py

@@ -17,7 +17,6 @@ __all__ = [
 ]
 
 import importlib
-import warnings
 
 import os_service_types
 
@@ -28,26 +27,6 @@ _logger = _log.setup_logging('openstack')
 _service_type_manager = os_service_types.ServiceTypes()
 
 
-def _get_all_types(service_type, aliases=None):
-    # We make connection attributes for all official real type names
-    # and aliases. Three services have names they were called by in
-    # openstacksdk that are not covered by Service Types Authority aliases.
-    # Include them here - but take heed, no additional values should ever
-    # be added to this list.
-    # that were only used in openstacksdk resource naming.
-    LOCAL_ALIASES = {
-        'baremetal': 'bare_metal',
-        'block_storage': 'block_store',
-        'clustering': 'cluster',
-    }
-    all_types = set(_service_type_manager.get_all_types(service_type))
-    if aliases:
-        all_types.update(aliases)
-    if service_type in LOCAL_ALIASES:
-        all_types.add(LOCAL_ALIASES[service_type])
-    return all_types
-
-
 class ServiceDescription(object):
 
     #: Proxy class for this service
@@ -85,11 +64,13 @@ class ServiceDescription(object):
             Optional list of aliases, if there is more than one name that might
             be used to register the service in the catalog.
         """
-        self.service_type = service_type
+        self.service_type = service_type or self.service_type
         self.proxy_class = proxy_class or self.proxy_class
-        self.all_types = _get_all_types(service_type, aliases)
+        if self.proxy_class:
-
+            self._validate_proxy_class()
-        self._validate_proxy_class()
+        self.aliases = aliases or self.aliases
+        self.all_types = [service_type] + self.aliases
+        self._proxy = None
 
     def _validate_proxy_class(self):
         if not issubclass(self.proxy_class, proxy.BaseProxy):
@@ -98,10 +79,36 @@ class ServiceDescription(object):
                     module=self.proxy_class.__module__,
                     proxy_class=self.proxy_class.__name__))
 
+    def get_proxy_class(self, config):
+        return self.proxy_class
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        if self.service_type not in instance._proxies:
+            config = instance.config
+            proxy_class = self.get_proxy_class(config)
+            instance._proxies[self.service_type] = proxy_class(
+                session=instance.config.get_session(),
+                task_manager=instance.task_manager,
+                allow_version_hack=True,
+                service_type=config.get_service_type(self.service_type),
+                service_name=config.get_service_name(self.service_type),
+                interface=config.get_interface(self.service_type),
+                region_name=config.region_name,
+                version=config.get_api_version(self.service_type)
+            )
+        return instance._proxies[self.service_type]
+
+    def __set__(self, instance, value):
+        raise AttributeError('Service Descriptors cannot be set')
+
+    def __delete__(self, instance):
+        raise AttributeError('Service Descriptors cannot be deleted')
+
 
 class OpenStackServiceDescription(ServiceDescription):
-
+    def __init__(self, service_filter_class, *args, **kwargs):
-    def __init__(self, service, config):
         """Official OpenStack ServiceDescription.
 
         The OpenStackServiceDescription class is a helper class for
@@ -111,57 +118,19 @@ class OpenStackServiceDescription(ServiceDescription):
         It finds the proxy_class by looking in the openstacksdk tree for
         appropriately named modules.
 
-        :param dict service:
+        :param service_filter_class:
-            A service dict as found in `os_service_types.ServiceTypes.services`
+            A subclass of :class:`~openstack.service_filter.ServiceFilter`
-        :param openstack.config.cloud_region.CloudRegion config:
-            ConfigRegion for the connection.
         """
-        super(OpenStackServiceDescription, self).__init__(
+        super(OpenStackServiceDescription, self).__init__(*args, **kwargs)
-            service['service_type'])
+        self._service_filter_class = service_filter_class
-        self.config = config
-        service_filter = self._get_service_filter()
-        if service_filter:
-            module_name = service_filter.get_module() + "._proxy"
-            module = importlib.import_module(module_name)
-            self.proxy_class = getattr(module, "Proxy")
 
-    def _get_service_filter(self):
+    def get_proxy_class(self, config):
-        service_filter_class = None
-        for service_type in self.all_types:
-            service_filter_class = self._find_service_filter_class()
-            if service_filter_class:
-                break
-        if not service_filter_class:
-            return None
         # TODO(mordred) Replace this with proper discovery
-        version_string = self.config.get_api_version(self.service_type)
+        version_string = config.get_api_version(self.service_type)
         version = None
         if version_string:
             version = 'v{version}'.format(version=version_string[0])
-        return service_filter_class(version=version)
+        service_filter = self._service_filter_class(version=version)
-
+        module_name = service_filter.get_module() + "._proxy"
-    def _find_service_filter_class(self):
+        module = importlib.import_module(module_name)
-        package_name = 'openstack.{service_type}'.format(
+        return getattr(module, "Proxy")
-            service_type=self.service_type).replace('-', '_')
-        module_name = self.service_type.replace('-', '_') + '_service'
-        class_name = ''.join(
-            [part.capitalize() for part in module_name.split('_')])
-        try:
-            import_name = '.'.join([package_name, module_name])
-            service_filter_module = importlib.import_module(import_name)
-        except ImportError as e:
-            # ImportWarning is ignored by default. This warning is here
-            # as an opt-in for people trying to figure out why something
-            # didn't work.
-            warnings.warn(
-                "Could not import {service_type} service filter: {e}".format(
-                    service_type=self.service_type, e=str(e)),
-                ImportWarning)
-            return None
-        service_filter_class = getattr(service_filter_module, class_name, None)
-        if not service_filter_class:
-            _logger.warn(
-                'Unable to find class %s in module for service %s',
-                class_name, self.service_type)
-            return None
-        return service_filter_class

openstack/service_filter.py

@@ -92,6 +92,16 @@ class ServiceFilter(dict):
         self['api_version'] = api_version
         self['requires_project_id'] = requires_project_id
 
+    @classmethod
+    def _get_proxy_class_names(cls):
+        names = []
+        module_name = ".".join(cls.__module__.split('.')[:-1])
+        for version in cls.valid_versions:
+            names.append("{module}.{version}._proxy.Proxy".format(
+                module=module_name,
+                version=version.module))
+        return names
+
     @property
     def service_type(self):
         return self['service_type']