Add returning EndpointData objects from discovery

The existing version discovery process is awesome, but in the normal flows it
ultimately returns urls, not the full endpoint data, so it's not
possible to know what version was discovered.

Make an EndpointData object that gets created and plumb that through the
stack so that it's possible to request EndpointData instead of just
endpoints. The existing discovery logic is unchanged, and the existing
methods continue to return the data they returned before.

Change-Id: Id48861e7d6d20be16f61cb375a21bca4a43a2500

keystoneauth1/access/service_catalog.py

@@ -21,6 +21,7 @@ import abc
 from positional import positional
 import six
 
+from keystoneauth1 import discover
 from keystoneauth1 import exceptions
 
 
@@ -65,11 +66,18 @@ class ServiceCatalog(object):
         """
         return interface
 
+    @abc.abstractmethod
+    def _extract_interface_url(self, endpoint, interface):
+        """Return the url for an interface from an endpoint description.
+
+        NOTE: This is a transition method and is removed in the next patch.
+        """
+
     @positional()
-    def get_endpoints(self, service_type=None, interface=None,
+    def get_endpoints_data(self, service_type=None, interface=None,
-                      region_name=None, service_name=None,
+                           region_name=None, service_name=None,
-                      service_id=None, endpoint_id=None):
+                           service_id=None, endpoint_id=None):
-        """Fetch and filter endpoints for the specified service(s).
+        """Fetch and filter endpoint data for the specified service(s).
 
         Returns endpoints for the specified service (or all) containing
         the specified type (or all) and region (or all) and service name.
@@ -77,75 +85,107 @@ class ServiceCatalog(object):
         If there is no name in the service catalog the service_name check will
         be skipped.  This allows compatibility with services that existed
         before the name was available in the catalog.
+
+        :returns: a dict, keyed by service_type, of lists of EndpointData
         """
         interface = self.normalize_interface(interface)
 
-        sc = {}
+        matching_endpoints = {}
 
         for service in (self._catalog or []):
-            try:
+            if 'type' not in service:
-                st = service['type']
-            except KeyError:
                 continue
 
-            if service_type and service_type != st:
+            found_service_type = service['type']
+            if service_type and service_type != found_service_type:
                 continue
 
             # NOTE(jamielennox): service_name is different. It is not available
             # in API < v3.3. If it is in the catalog then we enforce it, if it
             # is not then we don't because the name could be correct we just
             # don't have that information to check against.
-            if service_name:
+            found_service_name = service.get('name')
-                try:
+            if (service_name and found_service_name
-                    sn = service['name']
+                    and service_name != found_service_name):
-                except KeyError:
+                continue
-                    # assume that we're in v3.0-v3.2 and don't have the name in
-                    # the catalog. Skip the check.
-                    pass
-                else:
-                    if service_name != sn:
-                        continue
 
             # NOTE(jamielennox): there is no such thing as a service_id in v2
             # similarly to service_name we'll have to skip this check if it's
             # not available.
-            if service_id and 'id' in service and service_id != service['id']:
+            found_service_id = service.get('id')
+            if (service_id and found_service_id
+                    and service_id != found_service_id):
                 continue
 
-            endpoints = sc.setdefault(st, [])
+            matching_endpoints.setdefault(found_service_type, [])
 
             for endpoint in service.get('endpoints', []):
                 if (interface and not
                         self.is_interface_match(endpoint, interface)):
                     continue
+                found_region_name = self._get_endpoint_region(endpoint)
                 if (region_name and
-                        region_name != self._get_endpoint_region(endpoint)):
+                        region_name != found_region_name):
                     continue
-                if (endpoint_id and endpoint_id != endpoint.get('id')):
+                found_endpoint_id = endpoint.get('id')
+                if (endpoint_id and endpoint_id != found_endpoint_id):
                     continue
-                endpoints.append(endpoint)
 
-        return sc
+                # We have a matching endpoint description, grab the URL.
+                # If we're in V2 and no interface has been specified, this
+                # will be "None". That's admittedly weird - but the things
+                # that expect to be able to not request interface then later
+                # grab a publicURL out of a dict won't be using the .url
+                # attribute anyway. A better approach would be to normalize
+                # the catalog into the v3 format at the outset, then have
+                # a v2 and v3 specific versions of get_endpoints() that return
+                # the raw endpoint dicts.
+                url = self._extract_interface_url(endpoint, interface)
 
-    def _get_service_endpoints(self, service_type=None, **kwargs):
+                matching_endpoints[found_service_type].append(
-        sc_endpoints = self.get_endpoints(service_type=service_type, **kwargs)
+                    discover.EndpointData(
+                        catalog_url=url,
+                        service_type=found_service_type,
+                        service_name=found_service_name,
+                        service_id=found_service_id,
+                        # EndpointData expects interface values in v3 format
+                        interface=ServiceCatalogV3.normalize_interface(
+                            interface),
+                        region_name=found_region_name,
+                        endpoint_id=found_service_id,
+                        raw_endpoint=endpoint))
 
-        if service_type:
+        return matching_endpoints
-            endpoints = sc_endpoints.get(service_type, [])
-        else:
-            # flatten list of lists
-            endpoints = [x
-                         for endpoint in six.itervalues(sc_endpoints)
-                         for x in endpoint]
 
+    @positional()
+    def get_endpoints(self, service_type=None, interface=None,
+                      region_name=None, service_name=None,
+                      service_id=None, endpoint_id=None):
+        """Fetch and filter endpoint data for the specified service(s).
+
+        Returns endpoints for the specified service (or all) containing
+        the specified type (or all) and region (or all) and service name.
+
+        If there is no name in the service catalog the service_name check will
+        be skipped.  This allows compatibility with services that existed
+        before the name was available in the catalog.
+
+        Returns a dict keyed by service_type with a list of endpoint dicts
+        """
+        endpoints_data = self.get_endpoints_data(
+            service_type=service_type, interface=interface,
+            region_name=region_name, service_name=service_name,
+            service_id=service_id, endpoint_id=endpoint_id)
+        endpoints = {}
+        for service_type, data in endpoints_data.items():
+            endpoints[service_type] = [d.raw_endpoint for d in data]
         return endpoints
 
-    @abc.abstractmethod
     @positional()
-    def get_urls(self, service_type=None, interface='public',
+    def get_endpoint_data_list(self, service_type=None, interface='public',
-                 region_name=None, service_name=None,
+                               region_name=None, service_name=None,
-                 service_id=None, endpoint_id=None):
+                               service_id=None, endpoint_id=None):
-        """Fetch endpoint urls from the service catalog.
+        """Fetch a flat list of matching EndpointData objects.
 
         Fetch the endpoints from the service catalog for a particular
         endpoint attribute. If no attribute is given, return the first
@@ -161,9 +201,46 @@ class ServiceCatalog(object):
         :param string service_id: The identifier of a service.
         :param string endpoint_id: The identifier of an endpoint.
 
-        :returns: tuple of urls or None (if no match found)
+        :returns: a list of matching EndpointData objects
+        :rtype: list(`keystoneauth1.discover.EndpointData`)
         """
-        raise NotImplementedError()
+        endpoints = self.get_endpoints_data(service_type=service_type,
+                                            interface=interface,
+                                            region_name=region_name,
+                                            service_name=service_name,
+                                            service_id=service_id,
+                                            endpoint_id=endpoint_id)
+        return [endpoint for data in endpoints.values() for endpoint in data]
+
+    @positional()
+    def get_urls(self, service_type=None, interface='public',
+                 region_name=None, service_name=None,
+                 service_id=None, endpoint_id=None):
+        """Fetch endpoint urls from the service catalog.
+
+        Fetch the urls of endpoints from the service catalog for a particular
+        endpoint attribute. If no attribute is given, return the url of the
+        first endpoint of the specified type.
+
+        :param string service_type: Service type of the endpoint.
+        :param string interface: Type of endpoint.
+                                     Possible values: public or publicURL,
+                                     internal or internalURL, admin or
+                                     adminURL
+        :param string region_name: Region of the endpoint.
+        :param string service_name: The assigned name of the service.
+        :param string service_id: The identifier of a service.
+        :param string endpoint_id: The identifier of an endpoint.
+
+        :returns: tuple of urls
+        """
+        endpoints = self.get_endpoint_data_list(service_type=service_type,
+                                                interface=interface,
+                                                region_name=region_name,
+                                                service_name=service_name,
+                                                service_id=service_id,
+                                                endpoint_id=endpoint_id)
+        return tuple([endpoint.url for endpoint in endpoints])
 
     @positional()
     def url_for(self, service_type=None, interface='public',
@@ -175,6 +252,34 @@ class ServiceCatalog(object):
         a particular endpoint attribute. If no attribute is given, return
         the first endpoint of the specified type.
 
+        Valid interface types: `public` or `publicURL`,
+                               `internal` or `internalURL`,
+                               `admin` or 'adminURL`
+
+        :param string service_type: Service type of the endpoint.
+        :param string interface: Type of endpoint.
+        :param string region_name: Region of the endpoint.
+        :param string service_name: The assigned name of the service.
+        :param string service_id: The identifier of a service.
+        :param string endpoint_id: The identifier of an endpoint.
+        """
+        return self.endpoint_data_for(service_type=service_type,
+                                      interface=interface,
+                                      region_name=region_name,
+                                      service_name=service_name,
+                                      service_id=service_id,
+                                      endpoint_id=endpoint_id).url
+
+    @positional()
+    def endpoint_data_for(self, service_type=None, interface='public',
+                          region_name=None, service_name=None,
+                          service_id=None, endpoint_id=None):
+        """Fetch endpoint data from the service catalog.
+
+        Fetch the specified endpoint data from the service catalog for
+        a particular endpoint attribute. If no attribute is given, return
+        the first endpoint of the specified type.
+
         Valid interface types: `public` or `publicURL`,
                                `internal` or `internalURL`,
                                `admin` or 'adminURL`
@@ -189,17 +294,16 @@ class ServiceCatalog(object):
         if not self._catalog:
             raise exceptions.EmptyCatalog('The service catalog is empty.')
 
-        urls = self.get_urls(service_type=service_type,
+        endpoint_data_list = self.get_endpoint_data_list(
-                             interface=interface,
+            service_type=service_type,
-                             region_name=region_name,
+            interface=interface,
-                             service_name=service_name,
+            region_name=region_name,
-                             service_id=service_id,
+            service_name=service_name,
-                             endpoint_id=endpoint_id)
+            service_id=service_id,
+            endpoint_id=endpoint_id)
 
-        try:
+        if endpoint_data_list:
-            return urls[0]
+            return endpoint_data_list[0]
-        except Exception:
-            pass
 
         if service_name and region_name:
             msg = ('%(interface)s endpoint for %(service_type)s service '
@@ -251,20 +355,10 @@ class ServiceCatalogV2(ServiceCatalog):
     def is_interface_match(self, endpoint, interface):
         return interface in endpoint
 
-    @positional()
+    def _extract_interface_url(self, endpoint, interface):
-    def get_urls(self, service_type=None, interface='publicURL',
+        if not interface:
-                 region_name=None, service_name=None,
+            return None
-                 service_id=None, endpoint_id=None):
+        return endpoint[self.normalize_interface(interface)]
-        interface = self.normalize_interface(interface)
-
-        endpoints = self._get_service_endpoints(service_type=service_type,
-                                                interface=interface,
-                                                region_name=region_name,
-                                                service_name=service_name,
-                                                service_id=service_id,
-                                                endpoint_id=endpoint_id)
-
-        return tuple([endpoint[interface] for endpoint in endpoints])
 
 
 class ServiceCatalogV3(ServiceCatalog):
@@ -293,15 +387,5 @@ class ServiceCatalogV3(ServiceCatalog):
         except KeyError:
             return False
 
-    @positional()
+    def _extract_interface_url(self, endpoint, interface):
-    def get_urls(self, service_type=None, interface='publicURL',
+        return endpoint['url']
-                 region_name=None, service_name=None,
-                 service_id=None, endpoint_id=None):
-        endpoints = self._get_service_endpoints(service_type=service_type,
-                                                interface=interface,
-                                                region_name=region_name,
-                                                service_name=service_name,
-                                                service_id=service_id,
-                                                endpoint_id=endpoint_id)
-
-        return tuple([endpoint['url'] for endpoint in endpoints])

keystoneauth1/discover.py

@@ -272,6 +272,51 @@ class Discover(object):
         return data['url'] if data else None
 
 
+class EndpointData(object):
+    """Normalized information about a discovered endpoint.
+
+    Contains url, version, microversion, interface and region information.
+    This is essentially the data contained in the catalog and the version
+    discovery documents about an endpoint that is used to select the endpoint
+    desired by the user. It is returned so that a user can know which qualities
+    a discovered endpoint had, in case their request allowed for a range of
+    possibilities.
+    """
+
+    @positional()
+    def __init__(self,
+                 catalog_url=None,
+                 service_url=None,
+                 service_type=None,
+                 service_name=None,
+                 service_id=None,
+                 region_name=None,
+                 interface=None,
+                 endpoint_id=None,
+                 raw_endpoint=None,
+                 api_version=None,
+                 major_version=None,
+                 min_microversion=None,
+                 max_microversion=None):
+        self.catalog_url = catalog_url
+        self.service_url = service_url
+        self.service_type = service_type
+        self.service_name = service_name
+        self.service_id = service_id
+        self.interface = interface
+        self.region_name = region_name
+        self.endpoint_id = endpoint_id
+        self.raw_endpoint = raw_endpoint
+        self.api_version = api_version
+        self.major_version = major_version
+        self.min_microversion = min_microversion
+        self.max_microversion = max_microversion
+
+    @property
+    def url(self):
+        return self.service_url or self.catalog_url
+
+
 class _VersionHacks(object):
     """A container to abstract the list of version hacks.
 
@@ -313,17 +358,21 @@ _VERSION_HACKS = _VersionHacks()
 _VERSION_HACKS.add_discover_hack('identity', re.compile('/v2.0/?$'), '/')
 
 
-def _get_catalog_discover_hack(service_type, url):
+def _get_catalog_discover_hack(endpoint_data, allow_version_hack=True):
     """Apply the catalog hacks and figure out an unversioned endpoint.
 
     This function is internal to keystoneauth1.
 
-    :param str service_type: the service_type to look up.
+    :param str endpoint_data: the endpoint_data in question
-    :param str url: The original url that came from a service_catalog.
+    :param bool allow_version_hacks: Whether or not to allow version hacks
+                                     to be applied. (defaults to True)
 
     :returns: Either the unversioned url or the one from the catalog to try.
     """
-    return _VERSION_HACKS.get_discover_hack(service_type, url)
+    if allow_version_hack:
+        return _VERSION_HACKS.get_discover_hack(endpoint_data.service_type,
+                                                endpoint_data.url)
+    return endpoint_data.url
 
 
 def add_catalog_discover_hack(service_type, old, new):

keystoneauth1/identity/base.py

@@ -157,6 +157,129 @@ class BaseIdentityPlugin(plugin.BaseAuthPlugin):
 
         return False
 
+    def get_endpoint_data(self, session, service_type=None, interface=None,
+                          region_name=None, service_name=None, version=None,
+                          allow={}, allow_version_hack=True, **kwargs):
+        """Return a valid endpoint data for a service.
+
+        If a valid token is not present then a new one will be fetched using
+        the session and kwargs.
+
+        :param session: A session object that can be used for communication.
+        :type session: keystoneauth1.session.Session
+        :param string service_type: The type of service to lookup the endpoint
+                                    for. This plugin will return None (failure)
+                                    if service_type is not provided.
+        :param string interface: The exposure of the endpoint. Should be
+                                 `public`, `internal`, `admin`, or `auth`.
+                                 `auth` is special here to use the `auth_url`
+                                 rather than a URL extracted from the service
+                                 catalog. Defaults to `public`.
+        :param string region_name: The region the endpoint should exist in.
+                                   (optional)
+        :param string service_name: The name of the service in the catalog.
+                                   (optional)
+        :param tuple version: The minimum version number required for this
+                              endpoint. (optional)
+        :param dict allow: Extra filters to pass when discovering API
+                           versions. (optional)
+        :param bool allow_version_hack: Allow keystoneauth to hack up catalog
+                                        URLS to support older schemes.
+                                        (optional, default True)
+
+        :raises keystoneauth1.exceptions.http.HttpError: An error from an
+                                                         invalid HTTP response.
+
+        :return: Valid EndpointData or None if not available.
+        :rtype: `keystoneauth1.discover.EndpointData` or None
+        """
+        # NOTE(jamielennox): if you specifically ask for requests to be sent to
+        # the auth url then we can ignore many of the checks. Typically if you
+        # are asking for the auth endpoint it means that there is no catalog to
+        # query however we still need to support asking for a specific version
+        # of the auth_url for generic plugins.
+        if interface is plugin.AUTH_INTERFACE:
+            endpoint_data = discover.EndpointData(
+                service_url=self.auth_url,
+                service_type=service_type or 'identity')
+        else:
+            if not service_type:
+                LOG.warning('Plugin cannot return an endpoint without '
+                            'knowing the service type that is required. Add '
+                            'service_type to endpoint filtering data.')
+                return None
+
+            # It's possible for things higher in the stack, because of
+            # defaults, to explicitly pass None.
+            if not interface:
+                interface = 'public'
+
+            service_catalog = self.get_access(session).service_catalog
+            endpoint_data = service_catalog.endpoint_data_for(
+                service_type=service_type,
+                interface=interface,
+                region_name=region_name,
+                service_name=service_name)
+            if not endpoint_data:
+                return None
+
+        if not version:
+            # NOTE(jamielennox): This may not be the best thing to default to
+            # but is here for backwards compatibility. It may be worth
+            # defaulting to the most recent version.
+            return endpoint_data
+
+        # NOTE(jamielennox): For backwards compatibility people might have a
+        # versioned endpoint in their catalog even though they want to use
+        # other endpoint versions. So we support a list of client defined
+        # situations where we can strip the version component from a URL before
+        # doing discovery.
+        vers_url = discover._get_catalog_discover_hack(
+            endpoint_data, allow_version_hack=allow_version_hack)
+
+        try:
+            disc = self.get_discovery(session, vers_url, authenticated=False)
+        except (exceptions.DiscoveryFailure,
+                exceptions.HttpError,
+                exceptions.ConnectionError):
+            # NOTE(jamielennox): The logic here is required for backwards
+            # compatibility. By itself it is not ideal.
+
+            if allow_version_hack:
+                # NOTE(jamielennox): Again if we can't contact the server we
+                # fall back to just returning the URL from the catalog.  This
+                # is backwards compatible behaviour and used when there is no
+                # other choice. Realistically if you have provided a version
+                # you should be able to rely on that version being returned or
+                # the request failing.
+                LOG.warning('Failed to contact the endpoint at %s for '
+                            'discovery. Fallback to using that endpoint as '
+                            'the base url.', endpoint_data.url)
+
+            else:
+                # NOTE(jamielennox): If you've said no to allow_version_hack
+                # and you can't determine the actual URL this is a failure
+                # because we are specifying that the deployment must be up to
+                # date enough to properly specify a version and keystoneauth
+                # can't deliver.
+                return None
+
+        else:
+            # NOTE(jamielennox): urljoin allows the url to be relative or even
+            # protocol-less. The additional trailing '/' make urljoin respect
+            # the current path as canonical even if the url doesn't include it.
+            # for example a "v2" path from http://host/admin should resolve as
+            # http://host/admin/v2 where it would otherwise be host/v2.
+            # This has no effect on absolute urls returned from url_for.
+            url = disc.url_for(version, **allow)
+            if not url:
+                return None
+
+            url = urllib.parse.urljoin(vers_url.rstrip('/') + '/', url)
+            endpoint_data.service_url = url
+
+        return endpoint_data
+
     def get_endpoint(self, session, service_type=None, interface=None,
                      region_name=None, service_name=None, version=None,
                      allow={}, allow_version_hack=True, **kwargs):
@@ -193,87 +316,12 @@ class BaseIdentityPlugin(plugin.BaseAuthPlugin):
         :return: A valid endpoint URL or None if not available.
         :rtype: string or None
         """
-        # NOTE(jamielennox): if you specifically ask for requests to be sent to
+        endpoint_data = self.get_endpoint_data(
-        # the auth url then we can ignore many of the checks. Typically if you
+            session, service_type=service_type, interface=interface,
-        # are asking for the auth endpoint it means that there is no catalog to
+            region_name=region_name, service_name=service_name,
-        # query however we still need to support asking for a specific version
+            version=version, allow=allow,
-        # of the auth_url for generic plugins.
+            allow_version_hack=allow_version_hack, **kwargs)
-        if interface is plugin.AUTH_INTERFACE:
+        return endpoint_data.url if endpoint_data else None
-            url = self.auth_url
-            service_type = service_type or 'identity'
-
-        else:
-            if not service_type:
-                LOG.warning('Plugin cannot return an endpoint without '
-                            'knowing the service type that is required. Add '
-                            'service_type to endpoint filtering data.')
-                return None
-
-            if not interface:
-                interface = 'public'
-
-            service_catalog = self.get_access(session).service_catalog
-            url = service_catalog.url_for(service_type=service_type,
-                                          interface=interface,
-                                          region_name=region_name,
-                                          service_name=service_name)
-
-        if not version:
-            # NOTE(jamielennox): This may not be the best thing to default to
-            # but is here for backwards compatibility. It may be worth
-            # defaulting to the most recent version.
-            return url
-
-        # NOTE(jamielennox): For backwards compatibility people might have a
-        # versioned endpoint in their catalog even though they want to use
-        # other endpoint versions. So we support a list of client defined
-        # situations where we can strip the version component from a URL before
-        # doing discovery.
-        if allow_version_hack:
-            vers_url = discover._get_catalog_discover_hack(service_type, url)
-        else:
-            vers_url = url
-
-        try:
-            disc = self.get_discovery(session, vers_url, authenticated=False)
-        except (exceptions.DiscoveryFailure,
-                exceptions.HttpError,
-                exceptions.ConnectionError):
-            # NOTE(jamielennox): The logic here is required for backwards
-            # compatibility. By itself it is not ideal.
-
-            if allow_version_hack:
-                # NOTE(jamielennox): Again if we can't contact the server we
-                # fall back to just returning the URL from the catalog.  This
-                # is backwards compatible behaviour and used when there is no
-                # other choice. Realistically if you have provided a version
-                # you should be able to rely on that version being returned or
-                # the request failing.
-                LOG.warning('Failed to contact the endpoint at %s for '
-                            'discovery. Fallback to using that endpoint as '
-                            'the base url.', url)
-
-            else:
-                # NOTE(jamielennox): If you've said no to allow_version_hack
-                # and you can't determine the actual URL this is a failure
-                # because we are specifying that the deployment must be up to
-                # date enough to properly specify a version and keystoneauth
-                # can't deliver.
-                return None
-
-        else:
-            # NOTE(jamielennox): urljoin allows the url to be relative or even
-            # protocol-less. The additional trailing '/' make urljoin respect
-            # the current path as canonical even if the url doesn't include it.
-            # for example a "v2" path from http://host/admin should resolve as
-            # http://host/admin/v2 where it would otherwise be host/v2.
-            # This has no effect on absolute urls returned from url_for.
-            url = disc.url_for(version, **allow)
-
-            if url:
-                url = urllib.parse.urljoin(vers_url.rstrip('/') + '/', url)
-
-        return url
 
     def get_user_id(self, session, **kwargs):
         return self.get_access(session).user_id