openstack
/
swift
Code Issues Proposed changes

Merge "update SLO delete error handling"

This commit is contained in:
Jenkins 2013-09-25 01:04:20 +00:00 committed by Gerrit Code Review
parent bb3f9657f2 3e6f9293b8
commit d65cbecb01
4 changed files with 249 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
swift/common/middleware/bulk.py
View File

@ -190,6 +190,8 @@ class Bulk(object):
conf.get('max_containers_per_extraction', 10000))
self.max_failed_extractions = int(
conf.get('max_failed_extractions', 1000))
self.max_failed_deletes = int(
conf.get('max_failed_deletes', 1000))
self.max_deletes_per_request = int(
conf.get('max_deletes_per_request', 10000))
self.yield_frequency = int(conf.get('yield_frequency', 60))
@ -239,7 +241,8 @@ class Bulk(object):
while data_remaining:
if '\n' in line:
obj_to_delete, line = line.split('\n', 1)
objs_to_delete.append(unquote(obj_to_delete))
objs_to_delete.append(
{'name': unquote(obj_to_delete)})
else:
data = req.body_file.read(MAX_PATH_LENGTH)
if data:
@ -247,7 +250,8 @@ class Bulk(object):
else:
data_remaining = False
if line.strip():
objs_to_delete.append(unquote(line))
objs_to_delete.append(
{'name': unquote(line)})
if len(objs_to_delete) > self.max_deletes_per_request:
raise HTTPRequestEntityTooLarge(
'Maximum Bulk Deletes: %d per request' %
@ -304,13 +308,22 @@ class Bulk(object):
separator = '\r\n\r\n'
last_yield = time()
yield ' '
obj_to_delete = obj_to_delete.strip()
if not obj_to_delete:
obj_name = obj_to_delete['name'].strip()
if not obj_name:
continue
if len(failed_files) >= self.max_failed_deletes:
raise HTTPBadRequest('Max delete failures exceeded')
if obj_to_delete.get('error'):
if obj_to_delete['error']['code'] == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
else:
failed_files.append([quote(obj_name),
obj_to_delete['error']['message']])
continue
delete_path = '/'.join(['', vrs, account,
obj_to_delete.lstrip('/')])
obj_name.lstrip('/')])
if not check_utf8(delete_path):
failed_files.append([quote(obj_to_delete),
failed_files.append([quote(obj_name),
HTTPPreconditionFailed().status])
continue
new_env = req.environ.copy()
@ -327,13 +340,12 @@ class Bulk(object):
elif resp.status_int == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
elif resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append([quote(obj_to_delete),
failed_files.append([quote(obj_name),
HTTPUnauthorized().status])
raise HTTPUnauthorized(request=req)
else:
if resp.status_int // 100 == 5:
failed_file_response_type = HTTPBadGateway
failed_files.append([quote(obj_to_delete), resp.status])
failed_files.append([quote(obj_name), resp.status])
if failed_files:
resp_dict['Response Status'] = \

117
swift/common/middleware/slo.py
View File

@ -141,10 +141,11 @@ import mimetypes
from hashlib import md5
from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \
HTTPOk, HTTPPreconditionFailed, HTTPException
HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \
HTTPUnauthorized
from swift.common.utils import json, get_logger, config_true_value
from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS
from swift.common.http import HTTP_NOT_FOUND
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED
from swift.common.wsgi import WSGIContext
from swift.common.middleware.bulk import get_response_body, \
ACCEPTABLE_FORMATS, Bulk
@ -216,8 +217,7 @@ class StaticLargeObject(object):
1024 * 1024 * 2))
self.min_segment_size = int(self.conf.get('min_segment_size',
1024 * 1024))
self.bulk_deleter = Bulk(
app, {'max_deletes_per_request': self.max_manifest_segments})
self.bulk_deleter = Bulk(app, {})
def handle_multipart_put(self, req, start_response):
"""
@ -333,66 +333,91 @@ class StaticLargeObject(object):
A generator function to be used to delete all the segments and
sub-segments referenced in a manifest.
:raises HTTPBadRequest: on sub manifest not manifest anymore or
on too many buffered sub segments
:raises HTTPServerError: on unable to load manifest
:params req: a swob.Request with an SLO manifest in path
:raises HTTPPreconditionFailed: on invalid UTF8 in request path
:raises HTTPBadRequest: on too many buffered sub segments and
on invalid SLO manifest path
"""
if not check_utf8(req.path_info):
raise HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
try:
vrs, account, container, obj = req.split_path(4, 4, True)
except ValueError:
raise HTTPBadRequest('Not a SLO manifest')
sub_segments = [{
raise HTTPBadRequest('Invalid SLO manifiest path')
segments = [{
'sub_slo': True,
'name': ('/%s/%s' % (container, obj)).decode('utf-8')}]
while sub_segments:
if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS:
while segments:
if len(segments) > MAX_BUFFERED_SLO_SEGMENTS:
raise HTTPBadRequest(
'Too many buffered slo segments to delete.')
seg_data = sub_segments.pop(0)
seg_data = segments.pop(0)
if seg_data.get('sub_slo'):
new_env = req.environ.copy()
new_env['REQUEST_METHOD'] = 'GET'
del(new_env['wsgi.input'])
new_env['QUERY_STRING'] = 'multipart-manifest=get'
new_env['CONTENT_LENGTH'] = 0
new_env['HTTP_USER_AGENT'] = \
'%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')
new_env['swift.source'] = 'SLO'
new_env['PATH_INFO'] = (
'/%s/%s/%s' % (
vrs, account,
seg_data['name'].lstrip('/'))).encode('utf-8')
sub_resp = Request.blank('', new_env).get_response(self.app)
if sub_resp.is_success:
try:
# if its still a SLO, load its segments
if config_true_value(
sub_resp.headers.get('X-Static-Large-Object')):
sub_segments.extend(json.loads(sub_resp.body))
except ValueError:
raise HTTPServerError('Unable to load SLO manifest')
# add sub-manifest back to be deleted after sub segments
# (even if obj is not a SLO)
seg_data['sub_slo'] = False
sub_segments.append(seg_data)
elif sub_resp.status_int != HTTP_NOT_FOUND:
# on deletes treat not found as success
raise HTTPServerError('Sub SLO unable to load.')
try:
segments.extend(
self.get_slo_segments(seg_data['name'], req))
except HTTPException as err:
# allow bulk delete response to report errors
seg_data['error'] = {'code': err.status_int,
'message': err.body}
# add manifest back to be deleted after segments
seg_data['sub_slo'] = False
segments.append(seg_data)
else:
yield seg_data['name'].encode('utf-8')
seg_data['name'] = seg_data['name'].encode('utf-8')
yield seg_data
def get_slo_segments(self, obj_name, req):
"""
Performs a swob.Request and returns the SLO manifest's segments.
:raises HTTPServerError: on unable to load obj_name or
on unable to load the SLO manifest data.
:raises HTTPBadRequest: on not an SLO manifest
:raises HTTPNotFound: on SLO manifest not found
:returns: SLO manifest's segments
"""
vrs, account, _junk = req.split_path(2, 3, True)
new_env = req.environ.copy()
new_env['REQUEST_METHOD'] = 'GET'
del(new_env['wsgi.input'])
new_env['QUERY_STRING'] = 'multipart-manifest=get'
new_env['CONTENT_LENGTH'] = 0
new_env['HTTP_USER_AGENT'] = \
'%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')
new_env['swift.source'] = 'SLO'
new_env['PATH_INFO'] = (
'/%s/%s/%s' % (
vrs, account,
obj_name.lstrip('/'))).encode('utf-8')
resp = Request.blank('', new_env).get_response(self.app)
if resp.is_success:
if config_true_value(resp.headers.get('X-Static-Large-Object')):
try:
return json.loads(resp.body)
except ValueError:
raise HTTPServerError('Unable to load SLO manifest')
else:
raise HTTPBadRequest('Not an SLO manifest')
elif resp.status_int == HTTP_NOT_FOUND:
raise HTTPNotFound('SLO manifest not found')
elif resp.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized('401 Unauthorized')
else:
raise HTTPServerError('Unable to load SLO manifest or segment.')
def handle_multipart_delete(self, req):
"""
Will delete all the segments in the SLO manifest and then, if
successful, will delete the manifest file.
:params req: a swob.Request with an obj in path
:raises HTTPServerError: on invalid manifest
:returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
"""
if not check_utf8(req.path_info):
raise HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
resp = HTTPOk(request=req)
out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
if out_content_type:

54
test/unit/common/middleware/test_bulk.py
View File

@ -23,6 +23,7 @@ from StringIO import StringIO
from mock import patch
from swift.common.middleware import bulk
from swift.common.swob import Request, Response, HTTPException
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED
from swift.common.utils import json
@ -35,6 +36,8 @@ class FakeApp(object):
def __call__(self, env, start_response):
self.calls += 1
if env['PATH_INFO'].startswith('/unauth/'):
if env['PATH_INFO'].endswith('/c/f_ok'):
return Response(status='204 No Content')(env, start_response)
return Response(status=401)(env, start_response)
if env['PATH_INFO'].startswith('/create_cont/'):
if env['REQUEST_METHOD'] == 'HEAD':
@ -493,6 +496,29 @@ class TestDelete(unittest.TestCase):
req, out_content_type=out_content_type))
return resp_body
def test_bulk_delete_uses_predefined_object_errors(self):
req = Request.blank('/delete_works/AUTH_Acc')
objs_to_delete = [
{'name': '/c/file_a'},
{'name': '/c/file_b', 'error': {'code': HTTP_NOT_FOUND,
'message': 'not found'}},
{'name': '/c/file_c', 'error': {'code': HTTP_UNAUTHORIZED,
'message': 'unauthorized'}},
{'name': '/c/file_d'}]
resp_body = ''.join(self.bulk.handle_delete_iter(
req, objs_to_delete=objs_to_delete,
out_content_type='application/json'))
self.assertEquals(
self.app.delete_paths, ['/delete_works/AUTH_Acc/c/file_a',
'/delete_works/AUTH_Acc/c/file_d'])
self.assertEquals(self.app.calls, 2)
resp_data = json.loads(resp_body)
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Number Deleted'], 2)
self.assertEquals(resp_data['Number Not Found'], 1)
self.assertEquals(resp_data['Errors'],
[['/c/file_c', 'unauthorized']])
def test_bulk_delete_works(self):
req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n/c/f404',
headers={'Accept': 'application/json'})
@ -538,13 +564,14 @@ class TestDelete(unittest.TestCase):
req.method = 'DELETE'
with patch.object(self.bulk, 'max_deletes_per_request', 2):
results = self.bulk.get_objs_to_delete(req)
self.assertEquals(results, ['1\r', '2\r'])
self.assertEquals(results, [{'name': '1\r'}, {'name': '2\r'}])
with patch.object(bulk, 'MAX_PATH_LENGTH', 2):
results = []
req.environ['wsgi.input'] = StringIO('1\n2\n3')
results = self.bulk.get_objs_to_delete(req)
self.assertEquals(results, ['1', '2', '3'])
self.assertEquals(results,
[{'name': '1'}, {'name': '2'}, {'name': '3'}])
with patch.object(self.bulk, 'max_deletes_per_request', 9):
with patch.object(bulk, 'MAX_PATH_LENGTH', 1):
@ -611,14 +638,15 @@ class TestDelete(unittest.TestCase):
self.assertTrue('400 Bad Request' in resp_body)
def test_bulk_delete_unauth(self):
req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f2\n',
req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f_ok\n',
headers={'Accept': 'application/json'})
req.method = 'DELETE'
resp_body = self.handle_delete_and_iter(req)
self.assertEquals(self.app.calls, 1)
self.assertEquals(self.app.calls, 2)
resp_data = json.loads(resp_body)
self.assertEquals(resp_data['Errors'], [['/c/f', '401 Unauthorized']])
self.assertEquals(resp_data['Response Status'], '401 Unauthorized')
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Number Deleted'], 1)
def test_bulk_delete_500_resp(self):
req = Request.blank('/broke/AUTH_acc/', body='/c/f\nc/f2\n',
@ -667,5 +695,21 @@ class TestDelete(unittest.TestCase):
resp_body = self.handle_delete_and_iter(req)
self.assertTrue('400 Bad Request' in resp_body)
def test_bulk_delete_max_failures(self):
req = Request.blank('/unauth/AUTH_Acc', body='/c/f1\n/c/f2\n/c/f3',
headers={'Accept': 'application/json'})
req.method = 'DELETE'
with patch.object(self.bulk, 'max_failed_deletes', 2):
resp_body = self.handle_delete_and_iter(req)
self.assertEquals(self.app.calls, 2)
resp_data = json.loads(resp_body)
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Response Body'],
'Max delete failures exceeded')
self.assertEquals(resp_data['Errors'],
[['/c/f1', '401 Unauthorized'],
['/c/f2', '401 Unauthorized']])
if __name__ == '__main__':
unittest.main()

129
test/unit/common/middleware/test_slo.py
View File

@ -81,9 +81,18 @@ class FakeApp(object):
return Response(status=200, body='lalala')(env, start_response)
if env['PATH_INFO'].startswith('/test_delete_404/'):
good_data = json.dumps(
[{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'},
{'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}])
self.req_method_paths.append((env['REQUEST_METHOD'],
env['PATH_INFO']))
return Response(status=404)(env, start_response)
if env['PATH_INFO'].endswith('/c/man_404'):
return Response(status=404)(env, start_response)
if env['PATH_INFO'].endswith('/c/a_1'):
return Response(status=404)(env, start_response)
return Response(status=200,
headers={'X-Static-Large-Object': 'True'},
body=good_data)(env, start_response)
if env['PATH_INFO'].startswith('/test_delete/'):
good_data = json.dumps(
@ -115,6 +124,21 @@ class FakeApp(object):
headers={'X-Static-Large-Object': 'True'},
body=good_data)(env, start_response)
if env['PATH_INFO'].startswith('/test_delete_nested_404/'):
good_data = json.dumps(
[{'name': '/a/a_1', 'hash': 'a', 'bytes': '1'},
{'name': '/a/sub_nest', 'hash': 'a', 'bytes': '2',
'sub_slo': True},
{'name': '/d/d_3', 'hash': 'b', 'bytes': '3'}])
self.req_method_paths.append((env['REQUEST_METHOD'],
env['PATH_INFO']))
if 'sub_nest' in env['PATH_INFO']:
return Response(status=404)(env, start_response)
else:
return Response(status=200,
headers={'X-Static-Large-Object': 'True'},
body=good_data)(env, start_response)
if env['PATH_INFO'].startswith('/test_delete_bad_json/'):
self.req_method_paths.append((env['REQUEST_METHOD'],
env['PATH_INFO']))
@ -127,13 +151,13 @@ class FakeApp(object):
env['PATH_INFO']))
return Response(status=200, body='')(env, start_response)
if env['PATH_INFO'].startswith('/test_delete_bad/'):
if env['PATH_INFO'].startswith('/test_delete_401/'):
good_data = json.dumps(
[{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'},
{'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}])
self.req_method_paths.append((env['REQUEST_METHOD'],
env['PATH_INFO']))
if env['PATH_INFO'].endswith('/c/a_1'):
if env['PATH_INFO'].endswith('/d/b_2'):
return Response(status=401)(env, start_response)
return Response(status=200,
headers={'X-Static-Large-Object': 'True'},
@ -368,13 +392,38 @@ class TestStaticLargeObject(unittest.TestCase):
def test_handle_multipart_delete_whole_404(self):
req = Request.blank(
'/test_delete_404/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE'})
'/test_delete_404/A/c/man_404?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
'HTTP_ACCEPT': 'application/json'})
app_iter = self.slo(req.environ, fake_start_response)
list(app_iter) # iterate through whole response
app_iter = list(app_iter) # iterate through whole response
resp_data = json.loads(app_iter[0])
self.assertEquals(self.app.calls, 1)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_404/A/c/man')])
[('GET', '/test_delete_404/A/c/man_404')])
self.assertEquals(resp_data['Response Status'], '200 OK')
self.assertEquals(resp_data['Response Body'], '')
self.assertEquals(resp_data['Number Deleted'], 0)
self.assertEquals(resp_data['Number Not Found'], 1)
self.assertEquals(resp_data['Errors'], [])
def test_handle_multipart_delete_segment_404(self):
req = Request.blank(
'/test_delete_404/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
'HTTP_ACCEPT': 'application/json'})
app_iter = self.slo(req.environ, fake_start_response)
app_iter = list(app_iter) # iterate through whole response
resp_data = json.loads(app_iter[0])
self.assertEquals(self.app.calls, 4)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_404/A/c/man'),
('DELETE', '/test_delete_404/A/c/a_1'),
('DELETE', '/test_delete_404/A/d/b_2'),
('DELETE', '/test_delete_404/A/c/man')])
self.assertEquals(resp_data['Response Status'], '200 OK')
self.assertEquals(resp_data['Number Deleted'], 2)
self.assertEquals(resp_data['Number Not Found'], 1)
def test_handle_multipart_delete_whole(self):
req = Request.blank(
@ -407,9 +456,28 @@ class TestStaticLargeObject(unittest.TestCase):
('DELETE', '/test_delete_nested/A/d/d_3'),
('DELETE', '/test_delete_nested/A/c/man')]))
def test_handle_multipart_delete_nested_404(self):
req = Request.blank(
'/test_delete_nested_404/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
'HTTP_ACCEPT': 'application/json'})
app_iter = self.slo(req.environ, fake_start_response)
app_iter = list(app_iter) # iterate through whole response
resp_data = json.loads(app_iter[0])
self.assertEquals(self.app.calls, 5)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_nested_404/A/c/man'),
('DELETE', '/test_delete_nested_404/A/a/a_1'),
('GET', '/test_delete_nested_404/A/a/sub_nest'),
('DELETE', '/test_delete_nested_404/A/d/d_3'),
('DELETE', '/test_delete_nested_404/A/c/man')])
self.assertEquals(resp_data['Response Status'], '200 OK')
self.assertEquals(resp_data['Response Body'], '')
self.assertEquals(resp_data['Number Deleted'], 3)
self.assertEquals(resp_data['Number Not Found'], 1)
self.assertEquals(resp_data['Errors'], [])
def test_handle_multipart_delete_not_a_manifest(self):
# when trying to delete a SLO and its not an SLO, just go ahead
# and delete it
req = Request.blank(
'/test_delete_bad_man/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
@ -417,11 +485,15 @@ class TestStaticLargeObject(unittest.TestCase):
app_iter = self.slo(req.environ, fake_start_response)
app_iter = list(app_iter) # iterate through whole response
resp_data = json.loads(app_iter[0])
self.assertEquals(self.app.calls, 2)
self.assertEquals(self.app.calls, 1)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_bad_man/A/c/man'),
('DELETE', '/test_delete_bad_man/A/c/man')])
self.assertEquals(resp_data['Response Status'], '200 OK')
[('GET', '/test_delete_bad_man/A/c/man')])
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Response Body'], '')
self.assertEquals(resp_data['Number Deleted'], 0)
self.assertEquals(resp_data['Number Not Found'], 0)
self.assertEquals(resp_data['Errors'],
[['/c/man', 'Not an SLO manifest']])
def test_handle_multipart_delete_bad_json(self):
req = Request.blank(
@ -434,18 +506,33 @@ class TestStaticLargeObject(unittest.TestCase):
self.assertEquals(self.app.calls, 1)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_bad_json/A/c/man')])
self.assertEquals(resp_data["Response Status"], "500 Internal Error")
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Response Body'], '')
self.assertEquals(resp_data['Number Deleted'], 0)
self.assertEquals(resp_data['Number Not Found'], 0)
self.assertEquals(resp_data['Errors'],
[['/c/man', 'Unable to load SLO manifest']])
def test_handle_multipart_delete_whole_bad(self):
def test_handle_multipart_delete_401(self):
req = Request.blank(
'/test_delete_bad/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE'})
'/test_delete_401/A/c/man?multipart-manifest=delete',
environ={'REQUEST_METHOD': 'DELETE',
'HTTP_ACCEPT': 'application/json'})
app_iter = self.slo(req.environ, fake_start_response)
list(app_iter) # iterate through whole response
self.assertEquals(self.app.calls, 2)
app_iter = list(app_iter) # iterate through whole response
resp_data = json.loads(app_iter[0])
self.assertEquals(self.app.calls, 4)
self.assertEquals(self.app.req_method_paths,
[('GET', '/test_delete_bad/A/c/man'),
('DELETE', '/test_delete_bad/A/c/a_1')])
[('GET', '/test_delete_401/A/c/man'),
('DELETE', '/test_delete_401/A/c/a_1'),
('DELETE', '/test_delete_401/A/d/b_2'),
('DELETE', '/test_delete_401/A/c/man')])
self.assertEquals(resp_data['Response Status'], '400 Bad Request')
self.assertEquals(resp_data['Response Body'], '')
self.assertEquals(resp_data['Number Deleted'], 2)
self.assertEquals(resp_data['Number Not Found'], 0)
self.assertEquals(resp_data['Errors'],
[['/d/b_2', '401 Unauthorized']])
if __name__ == '__main__':
unittest.main()