Add v2 API to list endpoints middleware
The new API adds better support for storage policies and changes the response from a list of backend urls to a dictionary with a key "endpoints" that's a list of of the backend urls and a new key headers that's a dictionary of headers to send along with the backend request. In the v2 response format for object requests, there the headers key includes "X-Backend-Storage-Policy-Index" which indicates the storage policy index for the endpoints returned in the response. Change-Id: I706a5b5be8002c633fe97b2429256735800a902e
This commit is contained in:
parent
9fdd86bf44
commit
216aaab638
@ -64,6 +64,8 @@ from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.proxy.controllers.base import get_container_info
|
||||
|
||||
RESPONSE_VERSIONS = (1.0, 2.0)
|
||||
|
||||
|
||||
class ListEndpointsMiddleware(object):
|
||||
"""
|
||||
@ -87,6 +89,11 @@ class ListEndpointsMiddleware(object):
|
||||
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
|
||||
if not self.endpoints_path.endswith('/'):
|
||||
self.endpoints_path += '/'
|
||||
self.default_response_version = 1.0
|
||||
self.response_map = {
|
||||
1.0: self.v1_format_response,
|
||||
2.0: self.v2_format_response,
|
||||
}
|
||||
|
||||
def get_object_ring(self, policy_idx):
|
||||
"""
|
||||
@ -97,6 +104,71 @@ class ListEndpointsMiddleware(object):
|
||||
"""
|
||||
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
|
||||
|
||||
def _parse_version(self, raw_version):
|
||||
err_msg = 'Unsupported version %r' % raw_version
|
||||
try:
|
||||
version = float(raw_version.lstrip('v'))
|
||||
except ValueError:
|
||||
raise ValueError(err_msg)
|
||||
if not any(version == v for v in RESPONSE_VERSIONS):
|
||||
raise ValueError(err_msg)
|
||||
return version
|
||||
|
||||
def _parse_path(self, request):
|
||||
"""
|
||||
Parse path parts of request into a tuple of version, account,
|
||||
container, obj. Unspecified path parts are filled in as None,
|
||||
except version which is always returned as a float using the
|
||||
configured default response version if not specified in the
|
||||
request.
|
||||
|
||||
:param request: the swob request
|
||||
|
||||
:returns: parsed path parts as a tuple with version filled in as
|
||||
configured default response version if not specified.
|
||||
:raises: ValueError if path is invalid, message will say why.
|
||||
"""
|
||||
clean_path = request.path[len(self.endpoints_path) - 1:]
|
||||
# try to peel off version
|
||||
try:
|
||||
raw_version, rest = split_path(clean_path, 1, 2, True)
|
||||
except ValueError:
|
||||
raise ValueError('No account specified')
|
||||
try:
|
||||
version = self._parse_version(raw_version)
|
||||
except ValueError:
|
||||
if raw_version.startswith('v') and '_' not in raw_version:
|
||||
# looks more like a invalid version than an account
|
||||
raise
|
||||
# probably no version specified, but if the client really
|
||||
# said /endpoints/v_3/account they'll probably be sorta
|
||||
# confused by the useless response and lack of error.
|
||||
version = self.default_response_version
|
||||
rest = clean_path
|
||||
else:
|
||||
rest = '/' + rest if rest else '/'
|
||||
try:
|
||||
account, container, obj = split_path(rest, 1, 3, True)
|
||||
except ValueError:
|
||||
raise ValueError('No account specified')
|
||||
return version, account, container, obj
|
||||
|
||||
def v1_format_response(self, req, endpoints, **kwargs):
|
||||
return Response(json.dumps(endpoints),
|
||||
content_type='application/json')
|
||||
|
||||
def v2_format_response(self, req, endpoints, storage_policy_index,
|
||||
**kwargs):
|
||||
resp = {
|
||||
'endpoints': endpoints,
|
||||
'headers': {},
|
||||
}
|
||||
if storage_policy_index is not None:
|
||||
resp['headers'][
|
||||
'X-Backend-Storage-Policy-Index'] = str(storage_policy_index)
|
||||
return Response(json.dumps(resp),
|
||||
content_type='application/json')
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
request = Request(env)
|
||||
if not request.path.startswith(self.endpoints_path):
|
||||
@ -107,11 +179,9 @@ class ListEndpointsMiddleware(object):
|
||||
req=request, headers={"Allow": "GET"})(env, start_response)
|
||||
|
||||
try:
|
||||
clean_path = request.path[len(self.endpoints_path) - 1:]
|
||||
account, container, obj = \
|
||||
split_path(clean_path, 1, 3, True)
|
||||
except ValueError:
|
||||
return HTTPBadRequest('No account specified')(env, start_response)
|
||||
version, account, container, obj = self._parse_path(request)
|
||||
except ValueError as err:
|
||||
return HTTPBadRequest(str(err))(env, start_response)
|
||||
|
||||
if account is not None:
|
||||
account = unquote(account)
|
||||
@ -120,16 +190,13 @@ class ListEndpointsMiddleware(object):
|
||||
if obj is not None:
|
||||
obj = unquote(obj)
|
||||
|
||||
storage_policy_index = None
|
||||
if obj is not None:
|
||||
# remove 'endpoints' from call to get_container_info
|
||||
stripped = request.environ
|
||||
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
|
||||
self.endpoints_path:
|
||||
stripped['PATH_INFO'] = "/v1/" + \
|
||||
stripped['PATH_INFO'][len(self.endpoints_path):]
|
||||
container_info = get_container_info(
|
||||
stripped, self.app, swift_source='LE')
|
||||
obj_ring = self.get_object_ring(container_info['storage_policy'])
|
||||
{'PATH_INFO': '/v1/%s/%s' % (account, container)},
|
||||
self.app, swift_source='LE')
|
||||
storage_policy_index = container_info['storage_policy']
|
||||
obj_ring = self.get_object_ring(storage_policy_index)
|
||||
partition, nodes = obj_ring.get_nodes(
|
||||
account, container, obj)
|
||||
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
|
||||
@ -157,8 +224,10 @@ class ListEndpointsMiddleware(object):
|
||||
obj=quote(obj or ''))
|
||||
endpoints.append(endpoint)
|
||||
|
||||
return Response(json.dumps(endpoints),
|
||||
content_type='application/json')(env, start_response)
|
||||
resp = self.response_map[version](
|
||||
request, endpoints=endpoints,
|
||||
storage_policy_index=storage_policy_index)
|
||||
return resp(env, start_response)
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
|
@ -110,10 +110,87 @@ class TestListEndpoints(unittest.TestCase):
|
||||
info['storage_policy'] = self.policy_to_test
|
||||
(version, account, container, unused) = \
|
||||
split_path(env['PATH_INFO'], 3, 4, True)
|
||||
self.assertEquals((version, account, container, unused),
|
||||
self.expected_path)
|
||||
self.assertEquals((version, account, container),
|
||||
self.expected_path[:3])
|
||||
return info
|
||||
|
||||
def test_parse_response_version(self):
|
||||
expectations = {
|
||||
'': 1.0, # legacy compat
|
||||
'/1': 1.0,
|
||||
'/v1': 1.0,
|
||||
'/1.0': 1.0,
|
||||
'/v1.0': 1.0,
|
||||
'/2': 2.0,
|
||||
'/v2': 2.0,
|
||||
'/2.0': 2.0,
|
||||
'/v2.0': 2.0,
|
||||
}
|
||||
accounts = (
|
||||
'AUTH_test',
|
||||
'test',
|
||||
'verybadreseller_prefix'
|
||||
'verybadaccount'
|
||||
)
|
||||
for expected_account in accounts:
|
||||
for version, expected in expectations.items():
|
||||
path = '/endpoints%s/%s/c/o' % (version, expected_account)
|
||||
req = Request.blank(path)
|
||||
version, account, container, obj = \
|
||||
self.list_endpoints._parse_path(req)
|
||||
try:
|
||||
self.assertEqual(version, expected)
|
||||
self.assertEqual(account, expected_account)
|
||||
except AssertionError:
|
||||
self.fail('Unexpected result from parse path %r: %r != %r'
|
||||
% (path, (version, account),
|
||||
(expected, expected_account)))
|
||||
|
||||
def test_parse_version_that_looks_like_account(self):
|
||||
"""
|
||||
Demonstrate the failure mode for versions that look like accounts,
|
||||
if you can make _parse_path better and this is the *only* test that
|
||||
fails you can delete it ;)
|
||||
"""
|
||||
bad_versions = (
|
||||
'v_3',
|
||||
'verybadreseller_prefix',
|
||||
)
|
||||
for bad_version in bad_versions:
|
||||
req = Request.blank('/endpoints/%s/a/c/o' % bad_version)
|
||||
version, account, container, obj = \
|
||||
self.list_endpoints._parse_path(req)
|
||||
self.assertEqual(version, 1.0)
|
||||
self.assertEqual(account, bad_version)
|
||||
self.assertEqual(container, 'a')
|
||||
self.assertEqual(obj, 'c/o')
|
||||
|
||||
def test_parse_account_that_looks_like_version(self):
|
||||
"""
|
||||
Demonstrate the failure mode for accounts that looks like versions,
|
||||
if you can make _parse_path better and this is the *only* test that
|
||||
fails you can delete it ;)
|
||||
"""
|
||||
bad_accounts = (
|
||||
'v3.0', 'verybaddaccountwithnoprefix',
|
||||
)
|
||||
for bad_account in bad_accounts:
|
||||
req = Request.blank('/endpoints/%s/c/o' % bad_account)
|
||||
self.assertRaises(ValueError,
|
||||
self.list_endpoints._parse_path, req)
|
||||
even_worse_accounts = {
|
||||
'v1': 1.0,
|
||||
'v2.0': 2.0,
|
||||
}
|
||||
for bad_account, guessed_version in even_worse_accounts.items():
|
||||
req = Request.blank('/endpoints/%s/c/o' % bad_account)
|
||||
version, account, container, obj = \
|
||||
self.list_endpoints._parse_path(req)
|
||||
self.assertEqual(version, guessed_version)
|
||||
self.assertEqual(account, 'c')
|
||||
self.assertEqual(container, 'o')
|
||||
self.assertEqual(obj, None)
|
||||
|
||||
def test_get_object_ring(self):
|
||||
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0),
|
||||
ring.Ring), True)
|
||||
@ -121,6 +198,38 @@ class TestListEndpoints(unittest.TestCase):
|
||||
ring.Ring), True)
|
||||
self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99)
|
||||
|
||||
def test_parse_path_no_version_specified(self):
|
||||
req = Request.blank('/endpoints/a/c/o1')
|
||||
version, account, container, obj = \
|
||||
self.list_endpoints._parse_path(req)
|
||||
self.assertEqual(account, 'a')
|
||||
self.assertEqual(container, 'c')
|
||||
self.assertEqual(obj, 'o1')
|
||||
|
||||
def test_parse_path_with_valid_version(self):
|
||||
req = Request.blank('/endpoints/v2/a/c/o1')
|
||||
version, account, container, obj = \
|
||||
self.list_endpoints._parse_path(req)
|
||||
self.assertEqual(version, 2.0)
|
||||
self.assertEqual(account, 'a')
|
||||
self.assertEqual(container, 'c')
|
||||
self.assertEqual(obj, 'o1')
|
||||
|
||||
def test_parse_path_with_invalid_version(self):
|
||||
req = Request.blank('/endpoints/v3/a/c/o1')
|
||||
self.assertRaises(ValueError, self.list_endpoints._parse_path,
|
||||
req)
|
||||
|
||||
def test_parse_path_with_no_account(self):
|
||||
bad_paths = ('v1', 'v2', '')
|
||||
for path in bad_paths:
|
||||
req = Request.blank('/endpoints/%s' % path)
|
||||
try:
|
||||
self.list_endpoints._parse_path(req)
|
||||
self.fail('Expected ValueError to be raised')
|
||||
except ValueError as err:
|
||||
self.assertEqual(str(err), 'No account specified')
|
||||
|
||||
def test_get_endpoint(self):
|
||||
# Expected results for objects taken from test_ring
|
||||
# Expected results for others computed by manually invoking
|
||||
@ -134,7 +243,7 @@ class TestListEndpoints(unittest.TestCase):
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
|
||||
])
|
||||
|
||||
# test policies with default endpoint name
|
||||
# test policies with no version endpoint name
|
||||
expected = [[
|
||||
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"], [
|
||||
@ -245,6 +354,82 @@ class TestListEndpoints(unittest.TestCase):
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), expected[pol.idx])
|
||||
|
||||
def test_v1_response(self):
|
||||
req = Request.blank('/endpoints/v1/a/c/o1')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = ["http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"]
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
|
||||
def test_v2_obj_response(self):
|
||||
req = Request.blank('/endpoints/v2/a/c/o1')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = {
|
||||
'endpoints': ["http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"],
|
||||
'headers': {'X-Backend-Storage-Policy-Index': "0"},
|
||||
}
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
for policy in POLICIES:
|
||||
patch_path = 'swift.common.middleware.list_endpoints' \
|
||||
'.get_container_info'
|
||||
mock_get_container_info = lambda *args, **kwargs: \
|
||||
{'storage_policy': int(policy)}
|
||||
with mock.patch(patch_path, mock_get_container_info):
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
part, nodes = policy.object_ring.get_nodes('a', 'c', 'o1')
|
||||
[node.update({'part': part}) for node in nodes]
|
||||
path = 'http://%(ip)s:%(port)s/%(device)s/%(part)s/a/c/o1'
|
||||
expected = {
|
||||
'headers': {
|
||||
'X-Backend-Storage-Policy-Index': str(int(policy))},
|
||||
'endpoints': [path % node for node in nodes],
|
||||
}
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
|
||||
def test_v2_non_obj_response(self):
|
||||
# account
|
||||
req = Request.blank('/endpoints/v2/a')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = {
|
||||
'endpoints': ["http://10.1.2.1:6000/sdc1/0/a",
|
||||
"http://10.1.1.1:6000/sda1/0/a",
|
||||
"http://10.1.1.1:6000/sdb1/0/a"],
|
||||
'headers': {},
|
||||
}
|
||||
# container
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
req = Request.blank('/endpoints/v2/a/c')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = {
|
||||
'endpoints': ["http://10.1.2.2:6000/sdd1/0/a/c",
|
||||
"http://10.1.1.1:6000/sda1/0/a/c",
|
||||
"http://10.1.2.1:6000/sdc1/0/a/c"],
|
||||
'headers': {},
|
||||
}
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
|
||||
def test_version_account_response(self):
|
||||
req = Request.blank('/endpoints/a')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = ["http://10.1.2.1:6000/sdc1/0/a",
|
||||
"http://10.1.1.1:6000/sda1/0/a",
|
||||
"http://10.1.1.1:6000/sdb1/0/a"]
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
req = Request.blank('/endpoints/v1.0/a')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
|
||||
req = Request.blank('/endpoints/v2/a')
|
||||
resp = req.get_response(self.list_endpoints)
|
||||
expected = {
|
||||
'endpoints': ["http://10.1.2.1:6000/sdc1/0/a",
|
||||
"http://10.1.1.1:6000/sda1/0/a",
|
||||
"http://10.1.1.1:6000/sdb1/0/a"],
|
||||
'headers': {},
|
||||
}
|
||||
self.assertEqual(resp.body, json.dumps(expected))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
Loading…
x
Reference in New Issue
Block a user