openstack
/
keystoneauth
Code Issues Proposed changes

Add returning EndpointData objects from discovery 

The existing version discovery process is awesome, but in the normal flows it
ultimately returns urls, not the full endpoint data, so it's not
possible to know what version was discovered.

Make an EndpointData object that gets created and plumb that through the
stack so that it's possible to request EndpointData instead of just
endpoints. The existing discovery logic is unchanged, and the existing
methods continue to return the data they returned before.

Change-Id: Id48861e7d6d20be16f61cb375a21bca4a43a2500
This commit is contained in:
Monty Taylor 2017-05-29 12:47:11 -05:00
parent 0906fb0424
commit 337e5af637
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
3 changed files with 343 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
keystoneauth1/access/service_catalog.py
View File

@ -21,6 +21,7 @@ import abc
from positional import positional
import six
from keystoneauth1 import discover
from keystoneauth1 import exceptions
@ -65,11 +66,18 @@ class ServiceCatalog(object):
"""
return interface
@abc.abstractmethod
def _extract_interface_url(self, endpoint, interface):
"""Return the url for an interface from an endpoint description.
NOTE: This is a transition method and is removed in the next patch.
"""
@positional()
def get_endpoints(self, service_type=None, interface=None,
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch and filter endpoints for the specified service(s).
def get_endpoints_data(self, service_type=None, interface=None,
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch and filter endpoint data for the specified service(s).
Returns endpoints for the specified service (or all) containing
the specified type (or all) and region (or all) and service name.
@ -77,75 +85,107 @@ class ServiceCatalog(object):
If there is no name in the service catalog the service_name check will
be skipped. This allows compatibility with services that existed
before the name was available in the catalog.
:returns: a dict, keyed by service_type, of lists of EndpointData
"""
interface = self.normalize_interface(interface)
sc = {}
matching_endpoints = {}
for service in (self._catalog or []):
try:
st = service['type']
except KeyError:
if 'type' not in service:
continue
if service_type and service_type != st:
found_service_type = service['type']
if service_type and service_type != found_service_type:
continue
# NOTE(jamielennox): service_name is different. It is not available
# in API < v3.3. If it is in the catalog then we enforce it, if it
# is not then we don't because the name could be correct we just
# don't have that information to check against.
if service_name:
try:
sn = service['name']
except KeyError:
# assume that we're in v3.0-v3.2 and don't have the name in
# the catalog. Skip the check.
pass
else:
if service_name != sn:
continue
found_service_name = service.get('name')
if (service_name and found_service_name
and service_name != found_service_name):
continue
# NOTE(jamielennox): there is no such thing as a service_id in v2
# similarly to service_name we'll have to skip this check if it's
# not available.
if service_id and 'id' in service and service_id != service['id']:
found_service_id = service.get('id')
if (service_id and found_service_id
and service_id != found_service_id):
continue
endpoints = sc.setdefault(st, [])
matching_endpoints.setdefault(found_service_type, [])
for endpoint in service.get('endpoints', []):
if (interface and not
self.is_interface_match(endpoint, interface)):
continue
found_region_name = self._get_endpoint_region(endpoint)
if (region_name and
region_name != self._get_endpoint_region(endpoint)):
region_name != found_region_name):
continue
if (endpoint_id and endpoint_id != endpoint.get('id')):
found_endpoint_id = endpoint.get('id')
if (endpoint_id and endpoint_id != found_endpoint_id):
continue
endpoints.append(endpoint)
return sc
# We have a matching endpoint description, grab the URL.
# If we're in V2 and no interface has been specified, this
# will be "None". That's admittedly weird - but the things
# that expect to be able to not request interface then later
# grab a publicURL out of a dict won't be using the .url
# attribute anyway. A better approach would be to normalize
# the catalog into the v3 format at the outset, then have
# a v2 and v3 specific versions of get_endpoints() that return
# the raw endpoint dicts.
url = self._extract_interface_url(endpoint, interface)
def _get_service_endpoints(self, service_type=None, **kwargs):
sc_endpoints = self.get_endpoints(service_type=service_type, **kwargs)
matching_endpoints[found_service_type].append(
discover.EndpointData(
catalog_url=url,
service_type=found_service_type,
service_name=found_service_name,
service_id=found_service_id,
# EndpointData expects interface values in v3 format
interface=ServiceCatalogV3.normalize_interface(
interface),
region_name=found_region_name,
endpoint_id=found_service_id,
raw_endpoint=endpoint))
if service_type:
endpoints = sc_endpoints.get(service_type, [])
else:
# flatten list of lists
endpoints = [x
for endpoint in six.itervalues(sc_endpoints)
for x in endpoint]
return matching_endpoints
@positional()
def get_endpoints(self, service_type=None, interface=None,
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch and filter endpoint data for the specified service(s).
Returns endpoints for the specified service (or all) containing
the specified type (or all) and region (or all) and service name.
If there is no name in the service catalog the service_name check will
be skipped. This allows compatibility with services that existed
before the name was available in the catalog.
Returns a dict keyed by service_type with a list of endpoint dicts
"""
endpoints_data = self.get_endpoints_data(
service_type=service_type, interface=interface,
region_name=region_name, service_name=service_name,
service_id=service_id, endpoint_id=endpoint_id)
endpoints = {}
for service_type, data in endpoints_data.items():
endpoints[service_type] = [d.raw_endpoint for d in data]
return endpoints
@abc.abstractmethod
@positional()
def get_urls(self, service_type=None, interface='public',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch endpoint urls from the service catalog.
def get_endpoint_data_list(self, service_type=None, interface='public',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch a flat list of matching EndpointData objects.
Fetch the endpoints from the service catalog for a particular
endpoint attribute. If no attribute is given, return the first
@ -161,9 +201,46 @@ class ServiceCatalog(object):
:param string service_id: The identifier of a service.
:param string endpoint_id: The identifier of an endpoint.
:returns: tuple of urls or None (if no match found)
:returns: a list of matching EndpointData objects
:rtype: list(`keystoneauth1.discover.EndpointData`)
"""
raise NotImplementedError()
endpoints = self.get_endpoints_data(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
return [endpoint for data in endpoints.values() for endpoint in data]
@positional()
def get_urls(self, service_type=None, interface='public',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch endpoint urls from the service catalog.
Fetch the urls of endpoints from the service catalog for a particular
endpoint attribute. If no attribute is given, return the url of the
first endpoint of the specified type.
:param string service_type: Service type of the endpoint.
:param string interface: Type of endpoint.
Possible values: public or publicURL,
internal or internalURL, admin or
adminURL
:param string region_name: Region of the endpoint.
:param string service_name: The assigned name of the service.
:param string service_id: The identifier of a service.
:param string endpoint_id: The identifier of an endpoint.
:returns: tuple of urls
"""
endpoints = self.get_endpoint_data_list(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
return tuple([endpoint.url for endpoint in endpoints])
@positional()
def url_for(self, service_type=None, interface='public',
@ -175,6 +252,34 @@ class ServiceCatalog(object):
a particular endpoint attribute. If no attribute is given, return
the first endpoint of the specified type.
Valid interface types: `public` or `publicURL`,
`internal` or `internalURL`,
`admin` or 'adminURL`
:param string service_type: Service type of the endpoint.
:param string interface: Type of endpoint.
:param string region_name: Region of the endpoint.
:param string service_name: The assigned name of the service.
:param string service_id: The identifier of a service.
:param string endpoint_id: The identifier of an endpoint.
"""
return self.endpoint_data_for(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id).url
@positional()
def endpoint_data_for(self, service_type=None, interface='public',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
"""Fetch endpoint data from the service catalog.
Fetch the specified endpoint data from the service catalog for
a particular endpoint attribute. If no attribute is given, return
the first endpoint of the specified type.
Valid interface types: `public` or `publicURL`,
`internal` or `internalURL`,
`admin` or 'adminURL`
@ -189,17 +294,16 @@ class ServiceCatalog(object):
if not self._catalog:
raise exceptions.EmptyCatalog('The service catalog is empty.')
urls = self.get_urls(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
endpoint_data_list = self.get_endpoint_data_list(
service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
try:
return urls[0]
except Exception:
pass
if endpoint_data_list:
return endpoint_data_list[0]
if service_name and region_name:
msg = ('%(interface)s endpoint for %(service_type)s service '
@ -251,20 +355,10 @@ class ServiceCatalogV2(ServiceCatalog):
def is_interface_match(self, endpoint, interface):
return interface in endpoint
@positional()
def get_urls(self, service_type=None, interface='publicURL',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
interface = self.normalize_interface(interface)
endpoints = self._get_service_endpoints(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
return tuple([endpoint[interface] for endpoint in endpoints])
def _extract_interface_url(self, endpoint, interface):
if not interface:
return None
return endpoint[self.normalize_interface(interface)]
class ServiceCatalogV3(ServiceCatalog):
@ -293,15 +387,5 @@ class ServiceCatalogV3(ServiceCatalog):
except KeyError:
return False
@positional()
def get_urls(self, service_type=None, interface='publicURL',
region_name=None, service_name=None,
service_id=None, endpoint_id=None):
endpoints = self._get_service_endpoints(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name,
service_id=service_id,
endpoint_id=endpoint_id)
return tuple([endpoint['url'] for endpoint in endpoints])
def _extract_interface_url(self, endpoint, interface):
return endpoint['url']

57
keystoneauth1/discover.py
View File

@ -272,6 +272,51 @@ class Discover(object):
return data['url'] if data else None
class EndpointData(object):
"""Normalized information about a discovered endpoint.
Contains url, version, microversion, interface and region information.
This is essentially the data contained in the catalog and the version
discovery documents about an endpoint that is used to select the endpoint
desired by the user. It is returned so that a user can know which qualities
a discovered endpoint had, in case their request allowed for a range of
possibilities.
"""
@positional()
def __init__(self,
catalog_url=None,
service_url=None,
service_type=None,
service_name=None,
service_id=None,
region_name=None,
interface=None,
endpoint_id=None,
raw_endpoint=None,
api_version=None,
major_version=None,
min_microversion=None,
max_microversion=None):
self.catalog_url = catalog_url
self.service_url = service_url
self.service_type = service_type
self.service_name = service_name
self.service_id = service_id
self.interface = interface
self.region_name = region_name
self.endpoint_id = endpoint_id
self.raw_endpoint = raw_endpoint
self.api_version = api_version
self.major_version = major_version
self.min_microversion = min_microversion
self.max_microversion = max_microversion
@property
def url(self):
return self.service_url or self.catalog_url
class _VersionHacks(object):
"""A container to abstract the list of version hacks.
@ -313,17 +358,21 @@ _VERSION_HACKS = _VersionHacks()
_VERSION_HACKS.add_discover_hack('identity', re.compile('/v2.0/?$'), '/')
def _get_catalog_discover_hack(service_type, url):
def _get_catalog_discover_hack(endpoint_data, allow_version_hack=True):
"""Apply the catalog hacks and figure out an unversioned endpoint.
This function is internal to keystoneauth1.
:param str service_type: the service_type to look up.
:param str url: The original url that came from a service_catalog.
:param str endpoint_data: the endpoint_data in question
:param bool allow_version_hacks: Whether or not to allow version hacks
to be applied. (defaults to True)
:returns: Either the unversioned url or the one from the catalog to try.
"""
return _VERSION_HACKS.get_discover_hack(service_type, url)
if allow_version_hack:
return _VERSION_HACKS.get_discover_hack(endpoint_data.service_type,
endpoint_data.url)
return endpoint_data.url
def add_catalog_discover_hack(service_type, old, new):

210
keystoneauth1/identity/base.py
View File

@ -157,6 +157,129 @@ class BaseIdentityPlugin(plugin.BaseAuthPlugin):
return False
def get_endpoint_data(self, session, service_type=None, interface=None,
region_name=None, service_name=None, version=None,
allow={}, allow_version_hack=True, **kwargs):
"""Return a valid endpoint data for a service.
If a valid token is not present then a new one will be fetched using
the session and kwargs.
:param session: A session object that can be used for communication.
:type session: keystoneauth1.session.Session
:param string service_type: The type of service to lookup the endpoint
for. This plugin will return None (failure)
if service_type is not provided.
:param string interface: The exposure of the endpoint. Should be
`public`, `internal`, `admin`, or `auth`.
`auth` is special here to use the `auth_url`
rather than a URL extracted from the service
catalog. Defaults to `public`.
:param string region_name: The region the endpoint should exist in.
(optional)
:param string service_name: The name of the service in the catalog.
(optional)
:param tuple version: The minimum version number required for this
endpoint. (optional)
:param dict allow: Extra filters to pass when discovering API
versions. (optional)
:param bool allow_version_hack: Allow keystoneauth to hack up catalog
URLS to support older schemes.
(optional, default True)
:raises keystoneauth1.exceptions.http.HttpError: An error from an
invalid HTTP response.
:return: Valid EndpointData or None if not available.
:rtype: `keystoneauth1.discover.EndpointData` or None
"""
# NOTE(jamielennox): if you specifically ask for requests to be sent to
# the auth url then we can ignore many of the checks. Typically if you
# are asking for the auth endpoint it means that there is no catalog to
# query however we still need to support asking for a specific version
# of the auth_url for generic plugins.
if interface is plugin.AUTH_INTERFACE:
endpoint_data = discover.EndpointData(
service_url=self.auth_url,
service_type=service_type or 'identity')
else:
if not service_type:
LOG.warning('Plugin cannot return an endpoint without '
'knowing the service type that is required. Add '
'service_type to endpoint filtering data.')
return None
# It's possible for things higher in the stack, because of
# defaults, to explicitly pass None.
if not interface:
interface = 'public'
service_catalog = self.get_access(session).service_catalog
endpoint_data = service_catalog.endpoint_data_for(
service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name)
if not endpoint_data:
return None
if not version:
# NOTE(jamielennox): This may not be the best thing to default to
# but is here for backwards compatibility. It may be worth
# defaulting to the most recent version.
return endpoint_data
# NOTE(jamielennox): For backwards compatibility people might have a
# versioned endpoint in their catalog even though they want to use
# other endpoint versions. So we support a list of client defined
# situations where we can strip the version component from a URL before
# doing discovery.
vers_url = discover._get_catalog_discover_hack(
endpoint_data, allow_version_hack=allow_version_hack)
try:
disc = self.get_discovery(session, vers_url, authenticated=False)
except (exceptions.DiscoveryFailure,
exceptions.HttpError,
exceptions.ConnectionError):
# NOTE(jamielennox): The logic here is required for backwards
# compatibility. By itself it is not ideal.
if allow_version_hack:
# NOTE(jamielennox): Again if we can't contact the server we
# fall back to just returning the URL from the catalog. This
# is backwards compatible behaviour and used when there is no
# other choice. Realistically if you have provided a version
# you should be able to rely on that version being returned or
# the request failing.
LOG.warning('Failed to contact the endpoint at %s for '
'discovery. Fallback to using that endpoint as '
'the base url.', endpoint_data.url)
else:
# NOTE(jamielennox): If you've said no to allow_version_hack
# and you can't determine the actual URL this is a failure
# because we are specifying that the deployment must be up to
# date enough to properly specify a version and keystoneauth
# can't deliver.
return None
else:
# NOTE(jamielennox): urljoin allows the url to be relative or even
# protocol-less. The additional trailing '/' make urljoin respect
# the current path as canonical even if the url doesn't include it.
# for example a "v2" path from http://host/admin should resolve as
# http://host/admin/v2 where it would otherwise be host/v2.
# This has no effect on absolute urls returned from url_for.
url = disc.url_for(version, **allow)
if not url:
return None
url = urllib.parse.urljoin(vers_url.rstrip('/') + '/', url)
endpoint_data.service_url = url
return endpoint_data
def get_endpoint(self, session, service_type=None, interface=None,
region_name=None, service_name=None, version=None,
allow={}, allow_version_hack=True, **kwargs):
@ -193,87 +316,12 @@ class BaseIdentityPlugin(plugin.BaseAuthPlugin):
:return: A valid endpoint URL or None if not available.
:rtype: string or None
"""
# NOTE(jamielennox): if you specifically ask for requests to be sent to
# the auth url then we can ignore many of the checks. Typically if you
# are asking for the auth endpoint it means that there is no catalog to
# query however we still need to support asking for a specific version
# of the auth_url for generic plugins.
if interface is plugin.AUTH_INTERFACE:
url = self.auth_url
service_type = service_type or 'identity'
else:
if not service_type:
LOG.warning('Plugin cannot return an endpoint without '
'knowing the service type that is required. Add '
'service_type to endpoint filtering data.')
return None
if not interface:
interface = 'public'
service_catalog = self.get_access(session).service_catalog
url = service_catalog.url_for(service_type=service_type,
interface=interface,
region_name=region_name,
service_name=service_name)
if not version:
# NOTE(jamielennox): This may not be the best thing to default to
# but is here for backwards compatibility. It may be worth
# defaulting to the most recent version.
return url
# NOTE(jamielennox): For backwards compatibility people might have a
# versioned endpoint in their catalog even though they want to use
# other endpoint versions. So we support a list of client defined
# situations where we can strip the version component from a URL before
# doing discovery.
if allow_version_hack:
vers_url = discover._get_catalog_discover_hack(service_type, url)
else:
vers_url = url
try:
disc = self.get_discovery(session, vers_url, authenticated=False)
except (exceptions.DiscoveryFailure,
exceptions.HttpError,
exceptions.ConnectionError):
# NOTE(jamielennox): The logic here is required for backwards
# compatibility. By itself it is not ideal.
if allow_version_hack:
# NOTE(jamielennox): Again if we can't contact the server we
# fall back to just returning the URL from the catalog. This
# is backwards compatible behaviour and used when there is no
# other choice. Realistically if you have provided a version
# you should be able to rely on that version being returned or
# the request failing.
LOG.warning('Failed to contact the endpoint at %s for '
'discovery. Fallback to using that endpoint as '
'the base url.', url)
else:
# NOTE(jamielennox): If you've said no to allow_version_hack
# and you can't determine the actual URL this is a failure
# because we are specifying that the deployment must be up to
# date enough to properly specify a version and keystoneauth
# can't deliver.
return None
else:
# NOTE(jamielennox): urljoin allows the url to be relative or even
# protocol-less. The additional trailing '/' make urljoin respect
# the current path as canonical even if the url doesn't include it.
# for example a "v2" path from http://host/admin should resolve as
# http://host/admin/v2 where it would otherwise be host/v2.
# This has no effect on absolute urls returned from url_for.
url = disc.url_for(version, **allow)
if url:
url = urllib.parse.urljoin(vers_url.rstrip('/') + '/', url)
return url
endpoint_data = self.get_endpoint_data(
session, service_type=service_type, interface=interface,
region_name=region_name, service_name=service_name,
version=version, allow=allow,
allow_version_hack=allow_version_hack, **kwargs)
return endpoint_data.url if endpoint_data else None
def get_user_id(self, session, **kwargs):
return self.get_access(session).user_id