diff --git a/openstack/block_store/block_store_service.py b/openstack/block_store/block_store_service.py index 586c3d1c..fa133a6a 100644 --- a/openstack/block_store/block_store_service.py +++ b/openstack/block_store/block_store_service.py @@ -21,4 +21,5 @@ class BlockStoreService(service_filter.ServiceFilter): def __init__(self, version=None): """Create a block store service.""" super(BlockStoreService, self).__init__(service_type='volume', - version=version) + version=version, + requires_project_id=True) diff --git a/openstack/exceptions.py b/openstack/exceptions.py index ad2c81ac..ff3176fd 100644 --- a/openstack/exceptions.py +++ b/openstack/exceptions.py @@ -27,6 +27,12 @@ class SDKException(Exception): super(SDKException, self).__init__(self.message) +class EndpointNotFound(SDKException): + """A mismatch occurred between what the client and server expect.""" + def __init__(self, message=None): + super(EndpointNotFound, self).__init__(message) + + class InvalidResponse(SDKException): """The response from the server is not valid for this request.""" diff --git a/openstack/profile.py b/openstack/profile.py index 8e4dd260..534b03d2 100644 --- a/openstack/profile.py +++ b/openstack/profile.py @@ -89,18 +89,20 @@ class Profile(object): 'compute', etc. """ self._services = {} - self._add_service(cluster_service.ClusterService()) - self._add_service(compute_service.ComputeService()) - self._add_service(database_service.DatabaseService()) - self._add_service(identity_service.IdentityService()) - self._add_service(image_service.ImageService()) - self._add_service(network_service.NetworkService()) - self._add_service(object_store_service.ObjectStoreService()) - self._add_service(orchestration_service.OrchestrationService()) - self._add_service(key_manager_service.KeyManagerService()) - self._add_service(telemetry_service.TelemetryService()) - self._add_service(block_store_service.BlockStoreService()) - self._add_service(message_service.MessageService()) + self._add_service(cluster_service.ClusterService(version="v1")) + self._add_service(compute_service.ComputeService(version="v2")) + self._add_service(database_service.DatabaseService(version="v1")) + self._add_service(identity_service.IdentityService(version="v3")) + self._add_service(image_service.ImageService(version="v2")) + self._add_service(network_service.NetworkService(version="v2")) + self._add_service( + object_store_service.ObjectStoreService(version="v1")) + self._add_service( + orchestration_service.OrchestrationService(version="v1")) + self._add_service(key_manager_service.KeyManagerService(version="v1")) + self._add_service(telemetry_service.TelemetryService(version="v1")) + self._add_service(block_store_service.BlockStoreService(version="v2")) + self._add_service(message_service.MessageService(version="v1")) # NOTE: The Metric service is not added here as it currently # only retrieves the /capabilities API. diff --git a/openstack/service_filter.py b/openstack/service_filter.py index b3d28d4e..fdb54ba1 100644 --- a/openstack/service_filter.py +++ b/openstack/service_filter.py @@ -72,7 +72,8 @@ class ServiceFilter(dict): valid_versions = [] def __init__(self, service_type, interface=PUBLIC, region=None, - service_name=None, version=None, api_version=None): + service_name=None, version=None, api_version=None, + requires_project_id=False): """Create a service identifier. :param string service_type: The desired type of service. @@ -82,6 +83,8 @@ class ServiceFilter(dict): :param string service_name: Name of the service :param string version: Version of service to use. :param string api_version: Microversion of service supported. + :param bool requires_project_id: True if this service's endpoint + expects project id to be included. """ self['service_type'] = service_type.lower() self['interface'] = interface @@ -89,6 +92,7 @@ class ServiceFilter(dict): self['service_name'] = service_name self['version'] = version self['api_version'] = api_version + self['requires_project_id'] = requires_project_id @property def service_type(self): @@ -134,6 +138,14 @@ class ServiceFilter(dict): def api_version(self, value): self['api_version'] = value + @property + def requires_project_id(self): + return self['requires_project_id'] + + @requires_project_id.setter + def requires_project_id(self, value): + self['requires_project_id'] = value + @property def path(self): return self['path'] diff --git a/openstack/session.py b/openstack/session.py index b91d34a8..4818100e 100644 --- a/openstack/session.py +++ b/openstack/session.py @@ -16,34 +16,21 @@ The :class:`~openstack.session.Session` overrides mapping KSA exceptions to SDK exceptions. """ -import re +from collections import namedtuple from keystoneauth1 import exceptions as _exceptions from keystoneauth1 import session as _session from openstack import exceptions +from openstack import utils from openstack import version as openstack_version from six.moves.urllib import parse DEFAULT_USER_AGENT = "openstacksdk/%s" % openstack_version.__version__ -VERSION_PATTERN = re.compile('/v\d[\d.]*') API_REQUEST_HEADER = "openstack-api-version" - -def parse_url(filt, url): - result = parse.urlparse(url) - path = result.path - vstr = VERSION_PATTERN.search(path) - if not vstr: - return (result.scheme + "://" + result.netloc + path.rstrip('/') + - '/' + filt.get_path()) - start, end = vstr.span() - prefix = path[:start] - version = '/' + filt.get_path(path[start + 1:end]) - postfix = path[end:].rstrip('/') if path[end:] else '' - url = result.scheme + "://" + result.netloc + prefix + version + postfix - return url +Version = namedtuple("Version", ["major", "minor"]) def map_exceptions(func): @@ -93,6 +80,8 @@ class Session(_session.Session): self.profile = profile api_version_header = self._get_api_requests() + self.endpoint_cache = {} + super(Session, self).__init__(user_agent=self.user_agent, additional_headers=api_version_header, **kwargs) @@ -118,15 +107,137 @@ class Session(_session.Session): return None - def get_endpoint(self, auth=None, interface=None, **kwargs): - """Override get endpoint to automate endpoint filtering""" + def _get_endpoint_versions(self, service_type, endpoint): + """Get available endpoints from the remote service + + Take the endpoint that the Service Catalog gives us, then split off + anything and just take the root. We need to make a request there + to get the versions the API exposes. + """ + parts = parse.urlparse(endpoint) + root_endpoint = "://".join([parts.scheme, parts.netloc]) + response = self.get(root_endpoint) + + # Normalize the version response. Identity nests the versions + # a level deeper than others, inside of a "values" dictionary. + response_body = response.json() + if "versions" in response_body: + versions = response_body["versions"] + if "values" in versions: + versions = versions["values"] + return root_endpoint, versions + + raise exceptions.EndpointNotFound( + "Unable to parse endpoints for %s" % service_type) + + def _parse_version(self, version): + """Parse the version and return major and minor components + + If the version was given with a leading "v", e.g., "v3", strip + that off to just numerals. + """ + version_num = version[version.find("v") + 1:] + components = version_num.split(".") + if len(components) == 1: + # The minor version of a v2 ends up being -1 so that we can + # loop through versions taking the highest available match + # while also working around a direct match for 2.0. + rv = Version(int(components[0]), -1) + elif len(components) == 2: + rv = Version(*[int(component) for component in components]) + else: + raise ValueError("Unable to parse version string %s" % version) + + return rv + + def _get_version_match(self, versions, profile_version, service_type, + root_endpoint, requires_project_id): + """Return the best matching version + + Look through each version trying to find the best match for + the version specified in this profile. + * The best match will only ever be found within the same + major version, meaning a v2 profile will never match if + only v3 is available on the server. + * The search for the best match is fuzzy if needed. + * If the profile specifies v2 and the server has + v2.0, v2.1, and v2.2, the match will be v2.2. + * When an exact major/minor is specified, e.g., v2.0, + it will only match v2.0. + """ + match = None + for version in versions: + api_version = self._parse_version(version["id"]) + if profile_version.major != api_version.major: + continue + + if profile_version.minor <= api_version.minor: + for link in version["links"]: + if link["rel"] == "self": + match = link["href"] + + # Only break out of the loop on an exact match, + # otherwise keep trying. + if profile_version.minor == api_version.minor: + break + + if match is None: + raise exceptions.EndpointNotFound( + "Unable to determine endpoint for %s" % service_type) + + # Some services return only the path fragment of a URI. + # If we split and see that we're not given the scheme and netloc, + # construct the match with the root from the service catalog. + match_split = parse.urlsplit(match) + if not all([match_split.scheme, match_split.netloc]): + match = root_endpoint + match + + # For services that require the project id in the request URI, + # add them in here. + if requires_project_id: + match = utils.urljoin(match, self.get_project_id()) + + return match + + def get_endpoint(self, auth=None, interface=None, service_type=None, + **kwargs): + """Override get endpoint to automate endpoint filtering + + This method uses the service catalog to find the root URI of + each service and then gets all available versions directly + from the service, not from the service catalog. + + Endpoints are cached per service type and interface combination + so that they're only requested from the remote service once + per instance of this class. + """ + key = (service_type, interface) + if key in self.endpoint_cache: + return self.endpoint_cache[key] - service_type = kwargs.get('service_type') filt = self.profile.get_filter(service_type) if filt.interface is None: filt.interface = interface - url = super(Session, self).get_endpoint(auth, **filt.get_filter()) - return parse_url(filt, url) + sc_endpoint = super(Session, self).get_endpoint(auth, + **filt.get_filter()) + + # Object Storage is, of course, different. Just use what we get + # back from the service catalog as not only does it not offer + # a list of supported versions, it appends an "AUTH_" prefix to + # the project id so we'd have to special case that as well. + if service_type == "object-store": + self.endpoint_cache[key] = sc_endpoint + return sc_endpoint + + root_endpoint, versions = self._get_endpoint_versions(service_type, + sc_endpoint) + profile_version = self._parse_version(filt.version) + match = self._get_version_match(versions, profile_version, + service_type, root_endpoint, + filt.requires_project_id) + + self.endpoint_cache[key] = match + return match @map_exceptions def request(self, *args, **kwargs): diff --git a/openstack/tests/unit/test_profile.py b/openstack/tests/unit/test_profile.py index ac6b77b4..97b7716a 100644 --- a/openstack/tests/unit/test_profile.py +++ b/openstack/tests/unit/test_profile.py @@ -34,6 +34,21 @@ class TestProfile(base.TestCase): ] self.assertEqual(expected, prof.service_keys) + def test_default_versions(self): + prof = profile.Profile() + self.assertEqual('v1', prof.get_filter('clustering').version) + self.assertEqual('v2', prof.get_filter('compute').version) + self.assertEqual('v1', prof.get_filter('database').version) + self.assertEqual('v3', prof.get_filter('identity').version) + self.assertEqual('v2', prof.get_filter('image').version) + self.assertEqual('v2', prof.get_filter('network').version) + self.assertEqual('v1', prof.get_filter('object-store').version) + self.assertEqual('v1', prof.get_filter('orchestration').version) + self.assertEqual('v1', prof.get_filter('key-manager').version) + self.assertEqual('v1', prof.get_filter('metering').version) + self.assertEqual('v2', prof.get_filter('volume').version) + self.assertEqual('v1', prof.get_filter('messaging').version) + def test_set(self): prof = profile.Profile() prof.set_version('compute', 'v2') diff --git a/openstack/tests/unit/test_service_filter.py b/openstack/tests/unit/test_service_filter.py index 6f30aa2a..7d012991 100644 --- a/openstack/tests/unit/test_service_filter.py +++ b/openstack/tests/unit/test_service_filter.py @@ -27,12 +27,13 @@ class TestServiceFilter(testtools.TestCase): def test_init(self): sot = service_filter.ServiceFilter( 'ServiceType', region='REGION1', service_name='ServiceName', - version='1', api_version='1.23') + version='1', api_version='1.23', requires_project_id=True) self.assertEqual('servicetype', sot.service_type) self.assertEqual('REGION1', sot.region) self.assertEqual('ServiceName', sot.service_name) self.assertEqual('1', sot.version) self.assertEqual('1.23', sot.api_version) + self.assertTrue(sot.requires_project_id) def test_get_module(self): sot = identity_service.IdentityService() diff --git a/openstack/tests/unit/test_session.py b/openstack/tests/unit/test_session.py index 4c8b7d38..5b2af152 100644 --- a/openstack/tests/unit/test_session.py +++ b/openstack/tests/unit/test_session.py @@ -16,32 +16,12 @@ import testtools from keystoneauth1 import exceptions as _exceptions from openstack import exceptions -from openstack.image import image_service from openstack import profile from openstack import session class TestSession(testtools.TestCase): - def test_parse_url(self): - filt = image_service.ImageService() - self.assertEqual( - "http://127.0.0.1:9292/v2", - session.parse_url(filt, "http://127.0.0.1:9292")) - self.assertEqual( - "http://127.0.0.1:9292/foo/v2", - session.parse_url(filt, "http://127.0.0.1:9292/foo")) - self.assertEqual( - "http://127.0.0.1:9292/v2", - session.parse_url(filt, "http://127.0.0.1:9292/v2.0")) - filt.version = 'v1' - self.assertEqual( - "http://127.0.0.1:9292/v1/mytenant", - session.parse_url(filt, "http://127.0.0.1:9292/v2.0/mytenant/")) - self.assertEqual( - "http://127.0.0.1:9292/wot/v1/mytenant", - session.parse_url(filt, "http://127.0.0.1:9292/wot/v2.0/mytenant")) - def test_init_user_agent_none(self): sot = session.Session(None) self.assertTrue(sot.user_agent.startswith("openstacksdk")) @@ -118,3 +98,118 @@ class TestSession(testtools.TestCase): exceptions.SDKException, session.map_exceptions(func)) self.assertIsInstance(os_exc, exceptions.SDKException) self.assertEqual(ksa_exc, os_exc.cause) + + def _test__get_endpoint_versions(self, body, versions): + sot = session.Session(None) + + fake_response = mock.Mock() + fake_response.json = mock.Mock(return_value=body) + sot.get = mock.Mock(return_value=fake_response) + + scheme = "https" + netloc = "devstack" + root = scheme + "://" + netloc + + rv = sot._get_endpoint_versions( + "compute", "%s://%s/v2.1/projectidblahblah" % (scheme, netloc)) + + sot.get.assert_called_with(root) + + self.assertEqual(rv[0], root) + self.assertEqual(rv[1], versions) + + def test__get_endpoint_versions_nested(self): + versions = [{"id": "v2.0"}, {"id": "v2.1"}] + body = {"versions": {"values": versions}} + self._test__get_endpoint_versions(body, versions) + + def test__get_endpoint_versions(self): + versions = [{"id": "v2.0"}, {"id": "v2.1"}] + body = {"versions": versions} + self._test__get_endpoint_versions(body, versions) + + def test__get_endpoint_versions_exception(self): + sot = session.Session(None) + + fake_response = mock.Mock() + fake_response.json = mock.Mock(return_value={}) + sot.get = mock.Mock(return_value=fake_response) + + self.assertRaises(exceptions.EndpointNotFound, + sot._get_endpoint_versions, "service", "endpoint") + + def test__parse_version(self): + sot = session.Session(None) + + self.assertEqual(sot._parse_version("2"), (2, -1)) + self.assertEqual(sot._parse_version("v2"), (2, -1)) + self.assertEqual(sot._parse_version("v2.1"), (2, 1)) + self.assertRaises(ValueError, sot._parse_version, "lol") + + def test__get_version_match_none(self): + sot = session.Session(None) + + self.assertRaises( + exceptions.EndpointNotFound, + sot._get_version_match, [], None, "service", "root", False) + + def test__get_version_match_fuzzy(self): + match = "http://devstack/v2.1/" + versions = [{"id": "v2.0", + "links": [{"href": "http://devstack/v2/", + "rel": "self"}]}, + {"id": "v2.1", + "links": [{"href": match, + "rel": "self"}]}] + + sot = session.Session(None) + # Look for a v2 match, which we internally denote as a minor + # version of -1 so we can find the highest matching minor. + rv = sot._get_version_match(versions, session.Version(2, -1), + "service", "root", False) + self.assertEqual(rv, match) + + def test__get_version_match_exact(self): + match = "http://devstack/v2/" + versions = [{"id": "v2.0", + "links": [{"href": match, + "rel": "self"}]}, + {"id": "v2.1", + "links": [{"href": "http://devstack/v2.1/", + "rel": "self"}]}] + + sot = session.Session(None) + rv = sot._get_version_match(versions, session.Version(2, 0), + "service", "root", False) + self.assertEqual(rv, match) + + def test__get_version_match_fragment(self): + root = "http://cloud.net" + match = "/v2/" + versions = [{"id": "v2.0", "links": [{"href": match, "rel": "self"}]}] + + sot = session.Session(None) + rv = sot._get_version_match(versions, session.Version(2, 0), + "service", root, False) + self.assertEqual(rv, root+match) + + def test__get_version_match_project_id(self): + match = "http://devstack/v2/" + project_id = "asdf123" + versions = [{"id": "v2.0", "links": [{"href": match, "rel": "self"}]}] + + sot = session.Session(None) + sot.get_project_id = mock.Mock(return_value=project_id) + rv = sot._get_version_match(versions, session.Version(2, 0), + "service", "root", True) + self.assertEqual(rv, match + project_id) + + def test_get_endpoint_cached(self): + sot = session.Session(None) + service_type = "compute" + interface = "public" + endpoint = "the world wide web" + + sot.endpoint_cache[(service_type, interface)] = endpoint + rv = sot.get_endpoint(service_type=service_type, interface=interface) + self.assertEqual(rv, endpoint)