From fab3e136c892acc9879060f8da8e4e8a0cf933d5 Mon Sep 17 00:00:00 2001 From: Brian Curtin Date: Mon, 12 Sep 2016 17:47:48 -0400 Subject: [PATCH] Generalize endpoint determination Since moving away from the service catalog for per-service endpoint determination, we've covered most of the cases we're familiar with, but of course a cloud can configure endpoints in other ways, including some we're not handling properly. Rather than just use the root of the service catalog's entry, or using the entire entry when it doesn't include a port, we need to automate this further to cover a number of cases at once. We can't make a smart determination on the root endpoint by just looking at the URI anymore--we have to make some requests and see what happens. This change now splits the URI and looks at the parts of the path and reconstructs a number of sub-URIs. It first pops off any tenant/project from the given URL, and then constructs successively longer paths eminating from the root. We then make requests to each of them until we find a URL which gives us the response we're aiming to parse. Given a URL like http://mycloud.com/nova/v2/blahblahtenantid, we would make the following requests while looking for a JSON body with a list of versions: 1. http://mycloud.com/ 2. http://mycloud.com/nova/ (this is the one that would work, and we would exit) 3. http://mycloud.com/nova/v2 In some cases, services are configured either at a particular port or under a subdomain. This was the original case we were solving for. Given a URL like http://nova.mycloud.com/v2/blahblahtenantid, we would make the following requests: 1. http://nova.mycloud.com/ (this is the one that would work, and we would exit) 2. http://nova.mycloud.com/v2/ So, this does unfortunately add one or more extra GET requests to some cases (one time on startup only), but there is no other automated way to properly determine a root endpoint. The only other way we could reliably know what root endpoint to go to would involve end-users submitting some kind of parameter to a profile or other configuration in order to specify what their environment looks like. They really should not have to know that, and that complicates our user experience if we need to enforce something like that. Change-Id: I333638aee889202d8a17bf9e3df5c797c23a03e4 --- openstack/session.py | 164 +++++++++++++++++++----- openstack/tests/unit/test_session.py | 179 +++++++++++++++++++-------- 2 files changed, 263 insertions(+), 80 deletions(-) diff --git a/openstack/session.py b/openstack/session.py index 5e5b459da..5b087fcf3 100644 --- a/openstack/session.py +++ b/openstack/session.py @@ -17,6 +17,29 @@ mapping KSA exceptions to SDK exceptions. """ from collections import namedtuple +import logging + +try: + from itertools import accumulate +except ImportError: + # itertools.accumulate was added to Python 3.2, and since we have to + # support Python 2 for some reason, we include this equivalent from + # the 3.x docs. While it's stated that it's a rough equivalent, it's + # good enough for the purposes we're using it for. + # https://docs.python.org/dev/library/itertools.html#itertools.accumulate + def accumulate(iterable, func=None): + """Return running totals""" + # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 + # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 + it = iter(iterable) + try: + total = next(it) + except StopIteration: + return + yield total + for element in it: + total = func(total, element) + yield total from keystoneauth1 import exceptions as _exceptions from keystoneauth1 import session as _session @@ -32,6 +55,8 @@ API_REQUEST_HEADER = "openstack-api-version" Version = namedtuple("Version", ["major", "minor"]) +_logger = logging.getLogger(__name__) + def map_exceptions(func): def map_exceptions_wrapper(*args, **kwargs): @@ -107,29 +132,111 @@ class Session(_session.Session): return None + class _Endpoint(object): + + def __init__(self, uri, versions, + needs_project_id=False, project_id=None): + self.uri = uri + self.versions = versions + self.needs_project_id = needs_project_id + self.project_id = project_id + + def __eq__(self, other): + return all([self.uri == other.uri, + self.versions == other.versions, + self.needs_project_id == other.needs_project_id, + self.project_id == other.project_id]) + + def _parse_versions_response(self, uri): + """Look for a "versions" JSON response at `uri` + + Return versions if we get them, otherwise return None. + """ + _logger.debug("Looking for versions at %s", uri) + + try: + response = self.get(uri) + except exceptions.HttpException: + return None + + try: + response_body = response.json() + except Exception: + # This could raise a number of things, all of which are bad. + # ValueError, JSONDecodeError, etc. Rather than pick and choose + # a bunch of things that might happen, catch 'em all. + return None + + if "versions" in response_body: + versions = response_body["versions"] + # Normalize the version response. Identity nests the versions + # a level deeper than others, inside of a "values" dictionary. + if "values" in versions: + versions = versions["values"] + return self._Endpoint(uri, versions) + + return None + def _get_endpoint_versions(self, service_type, endpoint): """Get available endpoints from the remote service - Take the endpoint that the Service Catalog gives us, then split off - anything and just take the root. We need to make a request there - to get the versions the API exposes. + Take the endpoint that the Service Catalog gives us as a base + and then work from there. In most cases, the path-less 'root' + of the URI is the base of the service which contains the versions. + In other cases, we need to discover it by trying the paths that + eminate from that root. Generally this is achieved in one roundtrip + request/response, but depending on how the service is installed, + it may require multiple requests. """ parts = parse.urlparse(endpoint) - if ':' in parts.netloc: - root_endpoint = "://".join([parts.scheme, parts.netloc]) + + just_root = "://".join([parts.scheme, parts.netloc]) + + # If we need to try using a portion of the parts, + # the project id won't be one worth asking for so remove it. + # However, we do need to know that the project id was + # previously there, so keep it. + project_id = self.get_project_id() + project_id_location = parts.path.find(project_id) + if project_id_location > -1: + usable_path = parts.path[slice(0, project_id_location)] + needs_project_id = True else: - root_endpoint = endpoint + usable_path = parts.path + needs_project_id = False - response = self.get(root_endpoint) + # Generate a series of paths that might contain our version + # information. This will build successively longer paths from + # the split, so /nova/v2 would return "", "/nova", + # "/nova/v2" out of it. Based on what we've normally seen, + # the match will be found early on within those. + paths = accumulate(usable_path.split("/"), + func=lambda *fragments: "/".join(fragments)) - # Normalize the version response. Identity nests the versions - # a level deeper than others, inside of a "values" dictionary. - response_body = response.json() - if "versions" in response_body: - versions = response_body["versions"] - if "values" in versions: - versions = versions["values"] - return root_endpoint, versions + result = None + + # If we have paths, try them from the root outwards. + # NOTE: Both the body of the for loop and the else clause + # cover the request for `just_root`. The else clause is explicit + # in only testing it because there are no path parts. In the for + # loop, it gets requested in the first iteration. + for path in paths: + response = self._parse_versions_response(just_root + path) + if response is not None: + result = response + break + else: + # If we didn't have paths, root is all we can do anyway. + response = self._parse_versions_response(just_root) + if response is not None: + result = response + + if result is not None: + if needs_project_id: + result.needs_project_id = True + result.project_id = project_id + + return result raise exceptions.EndpointNotFound( "Unable to parse endpoints for %s" % service_type) @@ -154,8 +261,7 @@ class Session(_session.Session): return rv - def _get_version_match(self, versions, profile_version, service_type, - root_endpoint, requires_project_id): + def _get_version_match(self, endpoint, profile_version, service_type): """Return the best matching version Look through each version trying to find the best match for @@ -172,7 +278,7 @@ class Session(_session.Session): match_version = None - for version in versions: + for version in endpoint.versions: api_version = self._parse_version(version["id"]) if profile_version.major != api_version.major: continue @@ -192,15 +298,15 @@ class Session(_session.Session): raise exceptions.EndpointNotFound( "Unable to determine endpoint for %s" % service_type) - # Make sure "root_endpoint" has no overlap with match_version - root_parts = parse.urlsplit(root_endpoint) + # Make sure the root endpoint has no overlap with match_version + root_parts = parse.urlsplit(endpoint.uri) match_version = match_version.replace(root_parts.path, "", 1) - match = utils.urljoin(root_endpoint, match_version) + match = utils.urljoin(endpoint.uri, match_version) # For services that require the project id in the request URI, # add them in here. - if requires_project_id: - match = utils.urljoin(match, self.get_project_id()) + if endpoint.needs_project_id: + match = utils.urljoin(match, endpoint.project_id) return match @@ -234,12 +340,14 @@ class Session(_session.Session): self.endpoint_cache[key] = sc_endpoint return sc_endpoint - root_endpoint, versions = self._get_endpoint_versions(service_type, - sc_endpoint) + endpoint = self._get_endpoint_versions(service_type, sc_endpoint) + profile_version = self._parse_version(filt.version) - match = self._get_version_match(versions, profile_version, - service_type, root_endpoint, - filt.requires_project_id) + match = self._get_version_match(endpoint, profile_version, + service_type) + + _logger.debug("Using %s as %s %s endpoint", + match, interface, service_type) self.endpoint_cache[key] = match return match diff --git a/openstack/tests/unit/test_session.py b/openstack/tests/unit/test_session.py index 77c9d8c35..a1f8ebc85 100644 --- a/openstack/tests/unit/test_session.py +++ b/openstack/tests/unit/test_session.py @@ -100,56 +100,124 @@ class TestSession(testtools.TestCase): self.assertIsInstance(os_exc, exceptions.SDKException) self.assertEqual(ksa_exc, os_exc.cause) - def _test__get_endpoint_versions(self, body, versions, endpoint=None): + def test__parse_versions_response_exception(self): + uri = "http://www.openstack.org" + level = "DEBUG" sot = session.Session(None) + sot.get = mock.Mock(side_effect=exceptions.NotFoundException) - fake_response = mock.Mock() - fake_response.json = mock.Mock(return_value=body) - sot.get = mock.Mock(return_value=fake_response) + with self.assertLogs(logger=session.__name__, level=level) as log: + self.assertIsNone(sot._parse_versions_response(uri)) - if endpoint is None: - # default case with port numbers, we strip the path to get base - endpoint = 'https://hostname:1234/v2.1/project_id' - root = 'https://hostname:1234' - else: - # otherwise we preserve the whole URI - root = endpoint + self.assertEqual(len(log.output), 1, + "Too many warnings were logged") + self.assertEqual( + log.output[0], + "%s:%s:Looking for versions at %s" % (level, session.__name__, + uri)) - rv = sot._get_endpoint_versions("compute", endpoint) - - sot.get.assert_called_with(root) - - self.assertEqual(root, rv[0]) - self.assertEqual(versions, rv[1]) - - def test__get_endpoint_versions_nested(self): - versions = [{"id": "v2.0"}, {"id": "v2.1"}] - body = {"versions": {"values": versions}} - self._test__get_endpoint_versions(body, versions) - - def test__get_endpoint_versions(self): - versions = [{"id": "v2.0"}, {"id": "v2.1"}] - body = {"versions": versions} - self._test__get_endpoint_versions(body, versions) - - def test__get_endpoint_versions_apache_case(self): - # test for service running in apache, i.e. those that doesn't have a - # port number in its endpoint URI but the path in its URI should be - # preserved. - versions = [{"id": "v2.0"}, {"id": "v2.1"}] - body = {"versions": versions} - self._test__get_endpoint_versions( - body, versions, endpoint='http://hostname/service_path') - - def test__get_endpoint_versions_exception(self): + def test__parse_versions_response_no_json(self): sot = session.Session(None) + retval = mock.Mock() + retval.json = mock.Mock(side_effect=ValueError) + sot.get = mock.Mock(return_value=retval) - fake_response = mock.Mock() - fake_response.json = mock.Mock(return_value={}) - sot.get = mock.Mock(return_value=fake_response) + self.assertIsNone(sot._parse_versions_response("test")) - self.assertRaises(exceptions.EndpointNotFound, - sot._get_endpoint_versions, "service", "endpoint") + def test__parse_versions_response_no_versions(self): + sot = session.Session(None) + retval = mock.Mock() + retval.json = mock.Mock(return_value={"no_versions_here": "blarga"}) + sot.get = mock.Mock(return_value=retval) + + self.assertIsNone(sot._parse_versions_response("test")) + + def test__parse_versions_response_with_versions(self): + uri = "http://openstack.org" + versions = [1, 2, 3] + + sot = session.Session(None) + retval = mock.Mock() + retval.json = mock.Mock(return_value={"versions": versions}) + sot.get = mock.Mock(return_value=retval) + + expected = session.Session._Endpoint(uri, versions) + self.assertEqual(expected, sot._parse_versions_response(uri)) + + def test__parse_versions_response_with_nested_versions(self): + uri = "http://openstack.org" + versions = [1, 2, 3] + + sot = session.Session(None) + retval = mock.Mock() + retval.json = mock.Mock(return_value={"versions": + {"values": versions}}) + sot.get = mock.Mock(return_value=retval) + + expected = session.Session._Endpoint(uri, versions) + self.assertEqual(expected, sot._parse_versions_response(uri)) + + def test__get_endpoint_versions_at_subdomain(self): + # This test covers a common case of services deployed under + # subdomains. Additionally, it covers the case of a service + # deployed at the root, which will be the first request made + # for versions. + sc_uri = "https://service.cloud.com/v1/" + versions_uri = "https://service.cloud.com" + + sot = session.Session(None) + sot.get_project_id = mock.Mock(return_value="project_id") + + responses = [session.Session._Endpoint(versions_uri, "versions")] + sot._parse_versions_response = mock.Mock(side_effect=responses) + + result = sot._get_endpoint_versions("type", sc_uri) + + sot._parse_versions_response.assert_called_once_with(versions_uri) + self.assertEqual(result, responses[0]) + self.assertFalse(result.needs_project_id) + + def test__get_endpoint_versions_at_path(self): + # This test covers a common case of services deployed under + # a path. Additionally, it covers the case of a service + # deployed at a path deeper than the root, which will mean + # more than one request will need to be made. + sc_uri = "https://cloud.com/api/service/v2/project_id" + versions_uri = "https://cloud.com/api/service" + + sot = session.Session(None) + sot.get_project_id = mock.Mock(return_value="project_id") + + responses = [None, None, + session.Session._Endpoint(versions_uri, "versions")] + sot._parse_versions_response = mock.Mock(side_effect=responses) + + result = sot._get_endpoint_versions("type", sc_uri) + + sot._parse_versions_response.assert_has_calls( + [mock.call("https://cloud.com"), + mock.call("https://cloud.com/api"), + mock.call(versions_uri)]) + self.assertEqual(result, responses[2]) + self.assertTrue(result.needs_project_id) + + def test__get_endpoint_versions_at_port(self): + # This test covers a common case of services deployed under + # a port. + sc_uri = "https://cloud.com:1234/v3" + versions_uri = "https://cloud.com:1234" + + sot = session.Session(None) + sot.get_project_id = mock.Mock(return_value="project_id") + + responses = [session.Session._Endpoint(versions_uri, "versions")] + sot._parse_versions_response = mock.Mock(side_effect=responses) + + result = sot._get_endpoint_versions("type", sc_uri) + + sot._parse_versions_response.assert_called_once_with(versions_uri) + self.assertEqual(result, responses[0]) + self.assertFalse(result.needs_project_id) def test__parse_version(self): sot = session.Session(None) @@ -162,9 +230,10 @@ class TestSession(testtools.TestCase): def test__get_version_match_none(self): sot = session.Session(None) + endpoint = session.Session._Endpoint("root", []) self.assertRaises( exceptions.EndpointNotFound, - sot._get_version_match, [], None, "service", "root", False) + sot._get_version_match, endpoint, None, "service") def test__get_version_match_fuzzy(self): match = "http://devstack/v2.1" @@ -177,10 +246,12 @@ class TestSession(testtools.TestCase): "rel": "self"}]}] sot = session.Session(None) + + endpoint = session.Session._Endpoint(root_endpoint, versions) # Look for a v2 match, which we internally denote as a minor # version of -1 so we can find the highest matching minor. - rv = sot._get_version_match(versions, session.Version(2, -1), - "service", root_endpoint, False) + rv = sot._get_version_match(endpoint, session.Version(2, -1), + "service") self.assertEqual(rv, match) def test__get_version_match_exact(self): @@ -194,8 +265,9 @@ class TestSession(testtools.TestCase): "rel": "self"}]}] sot = session.Session(None) - rv = sot._get_version_match(versions, session.Version(2, 0), - "service", root_endpoint, False) + endpoint = session.Session._Endpoint(root_endpoint, versions) + rv = sot._get_version_match(endpoint, session.Version(2, 0), + "service") self.assertEqual(rv, match) def test__get_version_match_fragment(self): @@ -204,8 +276,8 @@ class TestSession(testtools.TestCase): versions = [{"id": "v2.0", "links": [{"href": match, "rel": "self"}]}] sot = session.Session(None) - rv = sot._get_version_match(versions, session.Version(2, 0), - "service", root, False) + endpoint = session.Session._Endpoint(root, versions) + rv = sot._get_version_match(endpoint, session.Version(2, 0), "service") self.assertEqual(rv, root+match) def test__get_version_match_project_id(self): @@ -216,8 +288,11 @@ class TestSession(testtools.TestCase): sot = session.Session(None) sot.get_project_id = mock.Mock(return_value=project_id) - rv = sot._get_version_match(versions, session.Version(2, 0), - "service", root_endpoint, True) + endpoint = session.Session._Endpoint(root_endpoint, versions, + project_id=project_id, + needs_project_id=True) + rv = sot._get_version_match(endpoint, session.Version(2, 0), + "service") match_endpoint = utils.urljoin(match, project_id) self.assertEqual(rv, match_endpoint)