Get endpoints directly from services

While investigating an issue with us improperly determining the Identity
endpoints between v2 and v3, it was mentioned several times that we
should really be getting endpoints directly from the services, not from
the service catalog. Part of this was a forward looking mention, as
eventually services will be unversioned in the catalog. It's also a
current issue as Identity themselves do not provide a v3 endpoint in the
service catalog--only v2.

This change introduces an implementation of Session.get_endpoint that
uses the service catalog to get started, taking only the root of a
service's endpoint, and then making a GET call to that root to find the
versions and endpoints it is configured for. Once we've received the
supported versions, we use somewhat of a fuzzy match to determine the
exact endpoint.

When a profile specifies v2 for a service, we'll look for the latest v2
offered, which could be v2.2 while it also offers v2.0 and v2.1. When a
user asks specifically for v2.1, they will get exactly that. The one
place we don't fuzzily match is across major versions, so if a user asks
for a v3 but the service is only providing v2 at the moment, an
exception will be raised.

This adds an additional field to the
ServiceFilter, requires_project_id, because there are some services that
require a project id in their URI and some don't. We were apparently
getting that straight from the service catalog before, but now we need
services to tell us when they need it.

Now for the special cases:
* object_store doesn't help us with any of this. It still comes straight
  out of the service catalog.
* some services only provide version fragments, not full URIs, so we need
  to reconstruct them to prepend the root.
* identity nests the versions within another dictionary of "values"

For any given combination of service_type and interface, a request to
determine the versions is only made once. The endpoints are cached in a
dictionary that is per-Connection instance.

Change-Id: I0958ce5d7477da106890dc7770ac013d5b9098f6

openstack/block_store/block_store_service.py

@@ -21,4 +21,5 @@ class BlockStoreService(service_filter.ServiceFilter):
     def __init__(self, version=None):
         """Create a block store service."""
         super(BlockStoreService, self).__init__(service_type='volume',
-                                                version=version)
+                                                version=version,
+                                                requires_project_id=True)

openstack/exceptions.py

@@ -27,6 +27,12 @@ class SDKException(Exception):
         super(SDKException, self).__init__(self.message)
 
 
+class EndpointNotFound(SDKException):
+    """A mismatch occurred between what the client and server expect."""
+    def __init__(self, message=None):
+        super(EndpointNotFound, self).__init__(message)
+
+
 class InvalidResponse(SDKException):
     """The response from the server is not valid for this request."""
 

openstack/profile.py

@@ -89,18 +89,20 @@ class Profile(object):
         'compute', etc.
         """
         self._services = {}
-        self._add_service(cluster_service.ClusterService())
+        self._add_service(cluster_service.ClusterService(version="v1"))
-        self._add_service(compute_service.ComputeService())
+        self._add_service(compute_service.ComputeService(version="v2"))
-        self._add_service(database_service.DatabaseService())
+        self._add_service(database_service.DatabaseService(version="v1"))
-        self._add_service(identity_service.IdentityService())
+        self._add_service(identity_service.IdentityService(version="v3"))
-        self._add_service(image_service.ImageService())
+        self._add_service(image_service.ImageService(version="v2"))
-        self._add_service(network_service.NetworkService())
+        self._add_service(network_service.NetworkService(version="v2"))
-        self._add_service(object_store_service.ObjectStoreService())
+        self._add_service(
-        self._add_service(orchestration_service.OrchestrationService())
+            object_store_service.ObjectStoreService(version="v1"))
-        self._add_service(key_manager_service.KeyManagerService())
+        self._add_service(
-        self._add_service(telemetry_service.TelemetryService())
+            orchestration_service.OrchestrationService(version="v1"))
-        self._add_service(block_store_service.BlockStoreService())
+        self._add_service(key_manager_service.KeyManagerService(version="v1"))
-        self._add_service(message_service.MessageService())
+        self._add_service(telemetry_service.TelemetryService(version="v1"))
+        self._add_service(block_store_service.BlockStoreService(version="v2"))
+        self._add_service(message_service.MessageService(version="v1"))
 
         # NOTE: The Metric service is not added here as it currently
         # only retrieves the /capabilities API.

openstack/service_filter.py

@@ -72,7 +72,8 @@ class ServiceFilter(dict):
     valid_versions = []
 
     def __init__(self, service_type, interface=PUBLIC, region=None,
-                 service_name=None, version=None, api_version=None):
+                 service_name=None, version=None, api_version=None,
+                 requires_project_id=False):
         """Create a service identifier.
 
         :param string service_type: The desired type of service.
@@ -82,6 +83,8 @@ class ServiceFilter(dict):
         :param string service_name: Name of the service
         :param string version: Version of service to use.
         :param string api_version: Microversion of service supported.
+        :param bool requires_project_id: True if this service's endpoint
+                                         expects project id to be included.
         """
         self['service_type'] = service_type.lower()
         self['interface'] = interface
@@ -89,6 +92,7 @@ class ServiceFilter(dict):
         self['service_name'] = service_name
         self['version'] = version
         self['api_version'] = api_version
+        self['requires_project_id'] = requires_project_id
 
     @property
     def service_type(self):
@@ -134,6 +138,14 @@ class ServiceFilter(dict):
     def api_version(self, value):
         self['api_version'] = value
 
+    @property
+    def requires_project_id(self):
+        return self['requires_project_id']
+
+    @requires_project_id.setter
+    def requires_project_id(self, value):
+        self['requires_project_id'] = value
+
     @property
     def path(self):
         return self['path']

openstack/session.py

@@ -16,34 +16,21 @@ The :class:`~openstack.session.Session` overrides
 mapping KSA exceptions to SDK exceptions.
 
 """
-import re
+from collections import namedtuple
 
 from keystoneauth1 import exceptions as _exceptions
 from keystoneauth1 import session as _session
 
 from openstack import exceptions
+from openstack import utils
 from openstack import version as openstack_version
 
 from six.moves.urllib import parse
 
 DEFAULT_USER_AGENT = "openstacksdk/%s" % openstack_version.__version__
-VERSION_PATTERN = re.compile('/v\d[\d.]*')
 API_REQUEST_HEADER = "openstack-api-version"
 
-
+Version = namedtuple("Version", ["major", "minor"])
-def parse_url(filt, url):
-    result = parse.urlparse(url)
-    path = result.path
-    vstr = VERSION_PATTERN.search(path)
-    if not vstr:
-        return (result.scheme + "://" + result.netloc + path.rstrip('/') +
-                '/' + filt.get_path())
-    start, end = vstr.span()
-    prefix = path[:start]
-    version = '/' + filt.get_path(path[start + 1:end])
-    postfix = path[end:].rstrip('/') if path[end:] else ''
-    url = result.scheme + "://" + result.netloc + prefix + version + postfix
-    return url
 
 
 def map_exceptions(func):
@@ -93,6 +80,8 @@ class Session(_session.Session):
 
         self.profile = profile
         api_version_header = self._get_api_requests()
+        self.endpoint_cache = {}
+
         super(Session, self).__init__(user_agent=self.user_agent,
                                       additional_headers=api_version_header,
                                       **kwargs)
@@ -118,15 +107,137 @@ class Session(_session.Session):
 
         return None
 
-    def get_endpoint(self, auth=None, interface=None, **kwargs):
+    def _get_endpoint_versions(self, service_type, endpoint):
-        """Override get endpoint to automate endpoint filtering"""
+        """Get available endpoints from the remote service
+
+        Take the endpoint that the Service Catalog gives us, then split off
+        anything and just take the root. We need to make a request there
+        to get the versions the API exposes.
+        """
+        parts = parse.urlparse(endpoint)
+        root_endpoint = "://".join([parts.scheme, parts.netloc])
+        response = self.get(root_endpoint)
+
+        # Normalize the version response. Identity nests the versions
+        # a level deeper than others, inside of a "values" dictionary.
+        response_body = response.json()
+        if "versions" in response_body:
+            versions = response_body["versions"]
+            if "values" in versions:
+                versions = versions["values"]
+            return root_endpoint, versions
+
+        raise exceptions.EndpointNotFound(
+            "Unable to parse endpoints for %s" % service_type)
+
+    def _parse_version(self, version):
+        """Parse the version and return major and minor components
+
+        If the version was given with a leading "v", e.g., "v3", strip
+        that off to just numerals.
+        """
+        version_num = version[version.find("v") + 1:]
+        components = version_num.split(".")
+        if len(components) == 1:
+            # The minor version of a v2 ends up being -1 so that we can
+            # loop through versions taking the highest available match
+            # while also working around a direct match for 2.0.
+            rv = Version(int(components[0]), -1)
+        elif len(components) == 2:
+            rv = Version(*[int(component) for component in components])
+        else:
+            raise ValueError("Unable to parse version string %s" % version)
+
+        return rv
+
+    def _get_version_match(self, versions, profile_version, service_type,
+                           root_endpoint, requires_project_id):
+        """Return the best matching version
+
+        Look through each version trying to find the best match for
+        the version specified in this profile.
+         * The best match will only ever be found within the same
+           major version, meaning a v2 profile will never match if
+           only v3 is available on the server.
+         * The search for the best match is fuzzy if needed.
+           * If the profile specifies v2 and the server has
+             v2.0, v2.1, and v2.2, the match will be v2.2.
+           * When an exact major/minor is specified, e.g., v2.0,
+             it will only match v2.0.
+        """
+        match = None
+        for version in versions:
+            api_version = self._parse_version(version["id"])
+            if profile_version.major != api_version.major:
+                continue
+
+            if profile_version.minor <= api_version.minor:
+                for link in version["links"]:
+                    if link["rel"] == "self":
+                        match = link["href"]
+
+            # Only break out of the loop on an exact match,
+            # otherwise keep trying.
+            if profile_version.minor == api_version.minor:
+                break
+
+        if match is None:
+            raise exceptions.EndpointNotFound(
+                "Unable to determine endpoint for %s" % service_type)
+
+        # Some services return only the path fragment of a URI.
+        # If we split and see that we're not given the scheme and netloc,
+        # construct the match with the root from the service catalog.
+        match_split = parse.urlsplit(match)
+        if not all([match_split.scheme, match_split.netloc]):
+            match = root_endpoint + match
+
+        # For services that require the project id in the request URI,
+        # add them in here.
+        if requires_project_id:
+            match = utils.urljoin(match, self.get_project_id())
+
+        return match
+
+    def get_endpoint(self, auth=None, interface=None, service_type=None,
+                     **kwargs):
+        """Override get endpoint to automate endpoint filtering
+
+        This method uses the service catalog to find the root URI of
+        each service and then gets all available versions directly
+        from the service, not from the service catalog.
+
+        Endpoints are cached per service type and interface combination
+        so that they're only requested from the remote service once
+        per instance of this class.
+        """
+        key = (service_type, interface)
+        if key in self.endpoint_cache:
+            return self.endpoint_cache[key]
 
-        service_type = kwargs.get('service_type')
         filt = self.profile.get_filter(service_type)
         if filt.interface is None:
             filt.interface = interface
-        url = super(Session, self).get_endpoint(auth, **filt.get_filter())
+        sc_endpoint = super(Session, self).get_endpoint(auth,
-        return parse_url(filt, url)
+                                                        **filt.get_filter())
+
+        # Object Storage is, of course, different. Just use what we get
+        # back from the service catalog as not only does it not offer
+        # a list of supported versions, it appends an "AUTH_" prefix to
+        # the project id so we'd have to special case that as well.
+        if service_type == "object-store":
+            self.endpoint_cache[key] = sc_endpoint
+            return sc_endpoint
+
+        root_endpoint, versions = self._get_endpoint_versions(service_type,
+                                                              sc_endpoint)
+        profile_version = self._parse_version(filt.version)
+        match = self._get_version_match(versions, profile_version,
+                                        service_type, root_endpoint,
+                                        filt.requires_project_id)
+
+        self.endpoint_cache[key] = match
+        return match
 
     @map_exceptions
     def request(self, *args, **kwargs):

openstack/tests/unit/test_profile.py

@@ -34,6 +34,21 @@ class TestProfile(base.TestCase):
         ]
         self.assertEqual(expected, prof.service_keys)
 
+    def test_default_versions(self):
+        prof = profile.Profile()
+        self.assertEqual('v1', prof.get_filter('clustering').version)
+        self.assertEqual('v2', prof.get_filter('compute').version)
+        self.assertEqual('v1', prof.get_filter('database').version)
+        self.assertEqual('v3', prof.get_filter('identity').version)
+        self.assertEqual('v2', prof.get_filter('image').version)
+        self.assertEqual('v2', prof.get_filter('network').version)
+        self.assertEqual('v1', prof.get_filter('object-store').version)
+        self.assertEqual('v1', prof.get_filter('orchestration').version)
+        self.assertEqual('v1', prof.get_filter('key-manager').version)
+        self.assertEqual('v1', prof.get_filter('metering').version)
+        self.assertEqual('v2', prof.get_filter('volume').version)
+        self.assertEqual('v1', prof.get_filter('messaging').version)
+
     def test_set(self):
         prof = profile.Profile()
         prof.set_version('compute', 'v2')

openstack/tests/unit/test_service_filter.py

@@ -27,12 +27,13 @@ class TestServiceFilter(testtools.TestCase):
     def test_init(self):
         sot = service_filter.ServiceFilter(
             'ServiceType', region='REGION1', service_name='ServiceName',
-            version='1', api_version='1.23')
+            version='1', api_version='1.23', requires_project_id=True)
         self.assertEqual('servicetype', sot.service_type)
         self.assertEqual('REGION1', sot.region)
         self.assertEqual('ServiceName', sot.service_name)
         self.assertEqual('1', sot.version)
         self.assertEqual('1.23', sot.api_version)
+        self.assertTrue(sot.requires_project_id)
 
     def test_get_module(self):
         sot = identity_service.IdentityService()

openstack/tests/unit/test_session.py

@@ -16,32 +16,12 @@ import testtools
 from keystoneauth1 import exceptions as _exceptions
 
 from openstack import exceptions
-from openstack.image import image_service
 from openstack import profile
 from openstack import session
 
 
 class TestSession(testtools.TestCase):
 
-    def test_parse_url(self):
-        filt = image_service.ImageService()
-        self.assertEqual(
-            "http://127.0.0.1:9292/v2",
-            session.parse_url(filt, "http://127.0.0.1:9292"))
-        self.assertEqual(
-            "http://127.0.0.1:9292/foo/v2",
-            session.parse_url(filt, "http://127.0.0.1:9292/foo"))
-        self.assertEqual(
-            "http://127.0.0.1:9292/v2",
-            session.parse_url(filt, "http://127.0.0.1:9292/v2.0"))
-        filt.version = 'v1'
-        self.assertEqual(
-            "http://127.0.0.1:9292/v1/mytenant",
-            session.parse_url(filt, "http://127.0.0.1:9292/v2.0/mytenant/"))
-        self.assertEqual(
-            "http://127.0.0.1:9292/wot/v1/mytenant",
-            session.parse_url(filt, "http://127.0.0.1:9292/wot/v2.0/mytenant"))
-
     def test_init_user_agent_none(self):
         sot = session.Session(None)
         self.assertTrue(sot.user_agent.startswith("openstacksdk"))
@@ -118,3 +98,118 @@ class TestSession(testtools.TestCase):
             exceptions.SDKException, session.map_exceptions(func))
         self.assertIsInstance(os_exc, exceptions.SDKException)
         self.assertEqual(ksa_exc, os_exc.cause)
+
+    def _test__get_endpoint_versions(self, body, versions):
+        sot = session.Session(None)
+
+        fake_response = mock.Mock()
+        fake_response.json = mock.Mock(return_value=body)
+        sot.get = mock.Mock(return_value=fake_response)
+
+        scheme = "https"
+        netloc = "devstack"
+        root = scheme + "://" + netloc
+
+        rv = sot._get_endpoint_versions(
+            "compute", "%s://%s/v2.1/projectidblahblah" % (scheme, netloc))
+
+        sot.get.assert_called_with(root)
+
+        self.assertEqual(rv[0], root)
+        self.assertEqual(rv[1], versions)
+
+    def test__get_endpoint_versions_nested(self):
+        versions = [{"id": "v2.0"}, {"id": "v2.1"}]
+        body = {"versions": {"values": versions}}
+        self._test__get_endpoint_versions(body, versions)
+
+    def test__get_endpoint_versions(self):
+        versions = [{"id": "v2.0"}, {"id": "v2.1"}]
+        body = {"versions": versions}
+        self._test__get_endpoint_versions(body, versions)
+
+    def test__get_endpoint_versions_exception(self):
+        sot = session.Session(None)
+
+        fake_response = mock.Mock()
+        fake_response.json = mock.Mock(return_value={})
+        sot.get = mock.Mock(return_value=fake_response)
+
+        self.assertRaises(exceptions.EndpointNotFound,
+                          sot._get_endpoint_versions, "service", "endpoint")
+
+    def test__parse_version(self):
+        sot = session.Session(None)
+
+        self.assertEqual(sot._parse_version("2"), (2, -1))
+        self.assertEqual(sot._parse_version("v2"), (2, -1))
+        self.assertEqual(sot._parse_version("v2.1"), (2, 1))
+        self.assertRaises(ValueError, sot._parse_version, "lol")
+
+    def test__get_version_match_none(self):
+        sot = session.Session(None)
+
+        self.assertRaises(
+            exceptions.EndpointNotFound,
+            sot._get_version_match, [], None, "service", "root", False)
+
+    def test__get_version_match_fuzzy(self):
+        match = "http://devstack/v2.1/"
+        versions = [{"id": "v2.0",
+                     "links": [{"href": "http://devstack/v2/",
+                                "rel": "self"}]},
+                    {"id": "v2.1",
+                     "links": [{"href": match,
+                                "rel": "self"}]}]
+
+        sot = session.Session(None)
+        # Look for a v2 match, which we internally denote as a minor
+        # version of -1 so we can find the highest matching minor.
+        rv = sot._get_version_match(versions, session.Version(2, -1),
+                                    "service", "root", False)
+        self.assertEqual(rv, match)
+
+    def test__get_version_match_exact(self):
+        match = "http://devstack/v2/"
+        versions = [{"id": "v2.0",
+                     "links": [{"href": match,
+                                "rel": "self"}]},
+                    {"id": "v2.1",
+                     "links": [{"href": "http://devstack/v2.1/",
+                                "rel": "self"}]}]
+
+        sot = session.Session(None)
+        rv = sot._get_version_match(versions, session.Version(2, 0),
+                                    "service", "root", False)
+        self.assertEqual(rv, match)
+
+    def test__get_version_match_fragment(self):
+        root = "http://cloud.net"
+        match = "/v2/"
+        versions = [{"id": "v2.0", "links": [{"href": match, "rel": "self"}]}]
+
+        sot = session.Session(None)
+        rv = sot._get_version_match(versions, session.Version(2, 0),
+                                    "service", root, False)
+        self.assertEqual(rv, root+match)
+
+    def test__get_version_match_project_id(self):
+        match = "http://devstack/v2/"
+        project_id = "asdf123"
+        versions = [{"id": "v2.0", "links": [{"href": match, "rel": "self"}]}]
+
+        sot = session.Session(None)
+        sot.get_project_id = mock.Mock(return_value=project_id)
+        rv = sot._get_version_match(versions, session.Version(2, 0),
+                                    "service", "root", True)
+        self.assertEqual(rv, match + project_id)
+
+    def test_get_endpoint_cached(self):
+        sot = session.Session(None)
+        service_type = "compute"
+        interface = "public"
+        endpoint = "the world wide web"
+
+        sot.endpoint_cache[(service_type, interface)] = endpoint
+        rv = sot.get_endpoint(service_type=service_type, interface=interface)
+        self.assertEqual(rv, endpoint)