Add v2 API to list endpoints middleware

The new API adds better support for storage policies and changes the
response from a list of backend urls to a dictionary with a key
"endpoints" that's a list of of the backend urls and a new key headers
that's a dictionary of headers to send along with the backend request.
In the v2 response format for object requests, there the headers key
includes "X-Backend-Storage-Policy-Index" which indicates the storage
policy index for the endpoints returned in the response.

Change-Id: I706a5b5be8002c633fe97b2429256735800a902e
This commit is contained in:
Clay Gerrard 2014-06-20 15:43:40 -07:00 committed by anc
parent 9fdd86bf44
commit 216aaab638
2 changed files with 272 additions and 18 deletions
swift/common/middleware
test/unit/common/middleware

@ -64,6 +64,8 @@ from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
from swift.common.storage_policy import POLICIES
from swift.proxy.controllers.base import get_container_info
RESPONSE_VERSIONS = (1.0, 2.0)
class ListEndpointsMiddleware(object):
"""
@ -87,6 +89,11 @@ class ListEndpointsMiddleware(object):
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
if not self.endpoints_path.endswith('/'):
self.endpoints_path += '/'
self.default_response_version = 1.0
self.response_map = {
1.0: self.v1_format_response,
2.0: self.v2_format_response,
}
def get_object_ring(self, policy_idx):
"""
@ -97,6 +104,71 @@ class ListEndpointsMiddleware(object):
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def _parse_version(self, raw_version):
err_msg = 'Unsupported version %r' % raw_version
try:
version = float(raw_version.lstrip('v'))
except ValueError:
raise ValueError(err_msg)
if not any(version == v for v in RESPONSE_VERSIONS):
raise ValueError(err_msg)
return version
def _parse_path(self, request):
"""
Parse path parts of request into a tuple of version, account,
container, obj. Unspecified path parts are filled in as None,
except version which is always returned as a float using the
configured default response version if not specified in the
request.
:param request: the swob request
:returns: parsed path parts as a tuple with version filled in as
configured default response version if not specified.
:raises: ValueError if path is invalid, message will say why.
"""
clean_path = request.path[len(self.endpoints_path) - 1:]
# try to peel off version
try:
raw_version, rest = split_path(clean_path, 1, 2, True)
except ValueError:
raise ValueError('No account specified')
try:
version = self._parse_version(raw_version)
except ValueError:
if raw_version.startswith('v') and '_' not in raw_version:
# looks more like a invalid version than an account
raise
# probably no version specified, but if the client really
# said /endpoints/v_3/account they'll probably be sorta
# confused by the useless response and lack of error.
version = self.default_response_version
rest = clean_path
else:
rest = '/' + rest if rest else '/'
try:
account, container, obj = split_path(rest, 1, 3, True)
except ValueError:
raise ValueError('No account specified')
return version, account, container, obj
def v1_format_response(self, req, endpoints, **kwargs):
return Response(json.dumps(endpoints),
content_type='application/json')
def v2_format_response(self, req, endpoints, storage_policy_index,
**kwargs):
resp = {
'endpoints': endpoints,
'headers': {},
}
if storage_policy_index is not None:
resp['headers'][
'X-Backend-Storage-Policy-Index'] = str(storage_policy_index)
return Response(json.dumps(resp),
content_type='application/json')
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
@ -107,11 +179,9 @@ class ListEndpointsMiddleware(object):
req=request, headers={"Allow": "GET"})(env, start_response)
try:
clean_path = request.path[len(self.endpoints_path) - 1:]
account, container, obj = \
split_path(clean_path, 1, 3, True)
except ValueError:
return HTTPBadRequest('No account specified')(env, start_response)
version, account, container, obj = self._parse_path(request)
except ValueError as err:
return HTTPBadRequest(str(err))(env, start_response)
if account is not None:
account = unquote(account)
@ -120,16 +190,13 @@ class ListEndpointsMiddleware(object):
if obj is not None:
obj = unquote(obj)
storage_policy_index = None
if obj is not None:
# remove 'endpoints' from call to get_container_info
stripped = request.environ
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
self.endpoints_path:
stripped['PATH_INFO'] = "/v1/" + \
stripped['PATH_INFO'][len(self.endpoints_path):]
container_info = get_container_info(
stripped, self.app, swift_source='LE')
obj_ring = self.get_object_ring(container_info['storage_policy'])
{'PATH_INFO': '/v1/%s/%s' % (account, container)},
self.app, swift_source='LE')
storage_policy_index = container_info['storage_policy']
obj_ring = self.get_object_ring(storage_policy_index)
partition, nodes = obj_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
@ -157,8 +224,10 @@ class ListEndpointsMiddleware(object):
obj=quote(obj or ''))
endpoints.append(endpoint)
return Response(json.dumps(endpoints),
content_type='application/json')(env, start_response)
resp = self.response_map[version](
request, endpoints=endpoints,
storage_policy_index=storage_policy_index)
return resp(env, start_response)
def filter_factory(global_conf, **local_conf):

@ -110,10 +110,87 @@ class TestListEndpoints(unittest.TestCase):
info['storage_policy'] = self.policy_to_test
(version, account, container, unused) = \
split_path(env['PATH_INFO'], 3, 4, True)
self.assertEquals((version, account, container, unused),
self.expected_path)
self.assertEquals((version, account, container),
self.expected_path[:3])
return info
def test_parse_response_version(self):
expectations = {
'': 1.0, # legacy compat
'/1': 1.0,
'/v1': 1.0,
'/1.0': 1.0,
'/v1.0': 1.0,
'/2': 2.0,
'/v2': 2.0,
'/2.0': 2.0,
'/v2.0': 2.0,
}
accounts = (
'AUTH_test',
'test',
'verybadreseller_prefix'
'verybadaccount'
)
for expected_account in accounts:
for version, expected in expectations.items():
path = '/endpoints%s/%s/c/o' % (version, expected_account)
req = Request.blank(path)
version, account, container, obj = \
self.list_endpoints._parse_path(req)
try:
self.assertEqual(version, expected)
self.assertEqual(account, expected_account)
except AssertionError:
self.fail('Unexpected result from parse path %r: %r != %r'
% (path, (version, account),
(expected, expected_account)))
def test_parse_version_that_looks_like_account(self):
"""
Demonstrate the failure mode for versions that look like accounts,
if you can make _parse_path better and this is the *only* test that
fails you can delete it ;)
"""
bad_versions = (
'v_3',
'verybadreseller_prefix',
)
for bad_version in bad_versions:
req = Request.blank('/endpoints/%s/a/c/o' % bad_version)
version, account, container, obj = \
self.list_endpoints._parse_path(req)
self.assertEqual(version, 1.0)
self.assertEqual(account, bad_version)
self.assertEqual(container, 'a')
self.assertEqual(obj, 'c/o')
def test_parse_account_that_looks_like_version(self):
"""
Demonstrate the failure mode for accounts that looks like versions,
if you can make _parse_path better and this is the *only* test that
fails you can delete it ;)
"""
bad_accounts = (
'v3.0', 'verybaddaccountwithnoprefix',
)
for bad_account in bad_accounts:
req = Request.blank('/endpoints/%s/c/o' % bad_account)
self.assertRaises(ValueError,
self.list_endpoints._parse_path, req)
even_worse_accounts = {
'v1': 1.0,
'v2.0': 2.0,
}
for bad_account, guessed_version in even_worse_accounts.items():
req = Request.blank('/endpoints/%s/c/o' % bad_account)
version, account, container, obj = \
self.list_endpoints._parse_path(req)
self.assertEqual(version, guessed_version)
self.assertEqual(account, 'c')
self.assertEqual(container, 'o')
self.assertEqual(obj, None)
def test_get_object_ring(self):
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0),
ring.Ring), True)
@ -121,6 +198,38 @@ class TestListEndpoints(unittest.TestCase):
ring.Ring), True)
self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99)
def test_parse_path_no_version_specified(self):
req = Request.blank('/endpoints/a/c/o1')
version, account, container, obj = \
self.list_endpoints._parse_path(req)
self.assertEqual(account, 'a')
self.assertEqual(container, 'c')
self.assertEqual(obj, 'o1')
def test_parse_path_with_valid_version(self):
req = Request.blank('/endpoints/v2/a/c/o1')
version, account, container, obj = \
self.list_endpoints._parse_path(req)
self.assertEqual(version, 2.0)
self.assertEqual(account, 'a')
self.assertEqual(container, 'c')
self.assertEqual(obj, 'o1')
def test_parse_path_with_invalid_version(self):
req = Request.blank('/endpoints/v3/a/c/o1')
self.assertRaises(ValueError, self.list_endpoints._parse_path,
req)
def test_parse_path_with_no_account(self):
bad_paths = ('v1', 'v2', '')
for path in bad_paths:
req = Request.blank('/endpoints/%s' % path)
try:
self.list_endpoints._parse_path(req)
self.fail('Expected ValueError to be raised')
except ValueError as err:
self.assertEqual(str(err), 'No account specified')
def test_get_endpoint(self):
# Expected results for objects taken from test_ring
# Expected results for others computed by manually invoking
@ -134,7 +243,7 @@ class TestListEndpoints(unittest.TestCase):
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# test policies with default endpoint name
# test policies with no version endpoint name
expected = [[
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"], [
@ -245,6 +354,82 @@ class TestListEndpoints(unittest.TestCase):
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
def test_v1_response(self):
req = Request.blank('/endpoints/v1/a/c/o1')
resp = req.get_response(self.list_endpoints)
expected = ["http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"]
self.assertEqual(resp.body, json.dumps(expected))
def test_v2_obj_response(self):
req = Request.blank('/endpoints/v2/a/c/o1')
resp = req.get_response(self.list_endpoints)
expected = {
'endpoints': ["http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"],
'headers': {'X-Backend-Storage-Policy-Index': "0"},
}
self.assertEqual(resp.body, json.dumps(expected))
for policy in POLICIES:
patch_path = 'swift.common.middleware.list_endpoints' \
'.get_container_info'
mock_get_container_info = lambda *args, **kwargs: \
{'storage_policy': int(policy)}
with mock.patch(patch_path, mock_get_container_info):
resp = req.get_response(self.list_endpoints)
part, nodes = policy.object_ring.get_nodes('a', 'c', 'o1')
[node.update({'part': part}) for node in nodes]
path = 'http://%(ip)s:%(port)s/%(device)s/%(part)s/a/c/o1'
expected = {
'headers': {
'X-Backend-Storage-Policy-Index': str(int(policy))},
'endpoints': [path % node for node in nodes],
}
self.assertEqual(resp.body, json.dumps(expected))
def test_v2_non_obj_response(self):
# account
req = Request.blank('/endpoints/v2/a')
resp = req.get_response(self.list_endpoints)
expected = {
'endpoints': ["http://10.1.2.1:6000/sdc1/0/a",
"http://10.1.1.1:6000/sda1/0/a",
"http://10.1.1.1:6000/sdb1/0/a"],
'headers': {},
}
# container
self.assertEqual(resp.body, json.dumps(expected))
req = Request.blank('/endpoints/v2/a/c')
resp = req.get_response(self.list_endpoints)
expected = {
'endpoints': ["http://10.1.2.2:6000/sdd1/0/a/c",
"http://10.1.1.1:6000/sda1/0/a/c",
"http://10.1.2.1:6000/sdc1/0/a/c"],
'headers': {},
}
self.assertEqual(resp.body, json.dumps(expected))
def test_version_account_response(self):
req = Request.blank('/endpoints/a')
resp = req.get_response(self.list_endpoints)
expected = ["http://10.1.2.1:6000/sdc1/0/a",
"http://10.1.1.1:6000/sda1/0/a",
"http://10.1.1.1:6000/sdb1/0/a"]
self.assertEqual(resp.body, json.dumps(expected))
req = Request.blank('/endpoints/v1.0/a')
resp = req.get_response(self.list_endpoints)
self.assertEqual(resp.body, json.dumps(expected))
req = Request.blank('/endpoints/v2/a')
resp = req.get_response(self.list_endpoints)
expected = {
'endpoints': ["http://10.1.2.1:6000/sdc1/0/a",
"http://10.1.1.1:6000/sda1/0/a",
"http://10.1.1.1:6000/sdb1/0/a"],
'headers': {},
}
self.assertEqual(resp.body, json.dumps(expected))
if __name__ == '__main__':
unittest.main()