diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index f404c70d59..f63b122361 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -133,6 +133,9 @@ use = egg:swift#proxy # be set to false if slo is not used in pipeline. # allow_static_large_object = true # +# The maximum time (seconds) that a large object connection is allowed to last. +# max_large_object_get_time = 86400 +# # Set to the number of nodes to contact for a normal request. You can use # '* replicas' at the end to have it use the number given times the number of # replicas for the ring being used for the request. diff --git a/swift/common/constraints.py b/swift/common/constraints.py index 41a6ee4cbf..fc5e0f0662 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -56,6 +56,8 @@ MAX_ACCOUNT_NAME_LENGTH = constraints_conf_int('max_account_name_length', 256) #: Max container name length MAX_CONTAINER_NAME_LENGTH = constraints_conf_int('max_container_name_length', 256) +# Maximum slo segments in buffer +MAX_BUFFERED_SLO_SEGMENTS = 10000 #: Query string format= values to their corresponding content-type values diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index ea7be528c9..dc089ae6b2 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -112,5 +112,5 @@ class ListingIterNotAuthorized(ListingIterError): self.aresp = aresp -class SloSegmentError(SwiftException): +class SegmentError(SwiftException): pass diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index a44fcb1c7e..b0f0131680 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -61,9 +61,11 @@ appended to the existing Content-Type, where total_size is the sum of all the included segments' size_bytes. This extra parameter will be hidden from the user. -Manifest files can reference objects in separate containers, which -will improve concurrent upload speed. Objects can be referenced by -multiple manifests. +Manifest files can reference objects in separate containers, which will improve +concurrent upload speed. Objects can be referenced by multiple manifests. The +segments of a SLO manifest can even be other SLO manifests. Treat them as any +other object i.e., use the Etag and Content-Length given on the PUT of the +sub-SLO in the manifest to the parent SLO. ------------------------- Retrieving a Large Object @@ -107,9 +109,8 @@ A DELETE with a query parameter:: ?multipart-manifest=delete -will delete all the segments referenced in the manifest and then, if -successful, the manifest itself. The failure response will be similar to -the bulk delete middleware. +will delete all the segments referenced in the manifest and then the manifest +itself. The failure response will be similar to the bulk delete middleware. ------------------------ Modifying a Large Object @@ -141,7 +142,8 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ HTTPOk, HTTPPreconditionFailed, wsgify from swift.common.utils import json, get_logger, config_true_value -from swift.common.constraints import check_utf8 +from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS +from swift.common.http import HTTP_NOT_FOUND from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk @@ -254,16 +256,12 @@ class StaticLargeObject(object): '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self.app) - if head_seg_resp.status_int // 100 == 2: + if head_seg_resp.is_success: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_path), 'Size Mismatch']) if seg_dict['etag'] != head_seg_resp.etag: problem_segments.append([quote(obj_path), 'Etag Mismatch']) - if 'X-Static-Large-Object' in head_seg_resp.headers or \ - 'X-Object-Manifest' in head_seg_resp.headers: - problem_segments.append( - [quote(obj_path), 'Segments cannot be Large Objects']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: @@ -272,12 +270,15 @@ class StaticLargeObject(object): last_modified_formatted = \ last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f') - data_for_storage.append( - {'name': '/' + seg_dict['path'].lstrip('/'), - 'bytes': seg_size, - 'hash': seg_dict['etag'], - 'content_type': head_seg_resp.content_type, - 'last_modified': last_modified_formatted}) + seg_data = {'name': '/' + seg_dict['path'].lstrip('/'), + 'bytes': seg_size, + 'hash': seg_dict['etag'], + 'content_type': head_seg_resp.content_type, + 'last_modified': last_modified_formatted} + if config_true_value( + head_seg_resp.headers.get('X-Static-Large-Object')): + seg_data['sub_slo'] = True + data_for_storage.append(seg_data) else: problem_segments.append([quote(obj_path), @@ -299,6 +300,60 @@ class StaticLargeObject(object): env['wsgi.input'] = StringIO(json_data) return self.app + def get_segments_to_delete_iter(self, req): + """ + A generator function to be used to delete all the segments and + sub-segments referenced in a manifest. + + :raises HTTPBadRequest: on sub manifest not manifest anymore or + on too many buffered sub segments + :raises HTTPServerError: on unable to load manifest + """ + try: + vrs, account, container, obj = req.split_path(4, 4, True) + except ValueError: + raise HTTPBadRequest('Not a SLO manifest') + sub_segments = [{ + 'sub_slo': True, + 'name': ('/%s/%s' % (container, obj)).decode('utf-8')}] + while sub_segments: + if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS: + raise HTTPBadRequest( + 'Too many buffered slo segments to delete.') + if sub_segments: + seg_data = sub_segments.pop(0) + if seg_data.get('sub_slo'): + new_env = req.environ.copy() + new_env['REQUEST_METHOD'] = 'GET' + del(new_env['wsgi.input']) + new_env['QUERY_STRING'] = 'multipart-manifest=get' + new_env['CONTENT_LENGTH'] = 0 + new_env['HTTP_USER_AGENT'] = \ + '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') + new_env['swift.source'] = 'SLO' + new_env['PATH_INFO'] = ( + '/%s/%s/%s' % ( + vrs, account, + seg_data['name'].lstrip('/'))).encode('utf-8') + sub_resp = Request.blank('', new_env).get_response(self.app) + if sub_resp.is_success: + try: + # if its still a SLO, load its segments + if config_true_value( + sub_resp.headers.get('X-Static-Large-Object')): + sub_segments.extend(json.loads(sub_resp.body)) + except ValueError: + raise HTTPServerError('Unable to load SLO manifest') + # add sub-manifest back to be deleted after sub segments + # (even if obj is not a SLO) + seg_data['sub_slo'] = False + sub_segments.append(seg_data) + elif sub_resp.status_int != HTTP_NOT_FOUND: + # on deletes treat not found as success + raise HTTPServerError('Sub SLO unable to load.') + else: + yield seg_data['name'].encode('utf-8') + def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if @@ -310,38 +365,16 @@ class StaticLargeObject(object): if not check_utf8(req.path_info): raise HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') - try: - vrs, account, container, obj = req.split_path(4, 4, True) - except ValueError: - raise HTTPBadRequest('Not an SLO manifest') - new_env = req.environ.copy() - new_env['REQUEST_METHOD'] = 'GET' - del(new_env['wsgi.input']) - new_env['QUERY_STRING'] = 'multipart-manifest=get' - new_env['CONTENT_LENGTH'] = 0 - new_env['HTTP_USER_AGENT'] = \ - '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT') - new_env['swift.source'] = 'SLO' - get_man_resp = \ - Request.blank('', new_env).get_response(self.app) - if get_man_resp.status_int // 100 == 2: - if not config_true_value( - get_man_resp.headers.get('X-Static-Large-Object')): - raise HTTPBadRequest('Not an SLO manifest') - try: - manifest = json.loads(get_man_resp.body) - # append the manifest file for deletion at the end - manifest.append( - {'name': '/'.join(['', container, obj]).decode('utf-8')}) - except ValueError: - raise HTTPServerError('Invalid manifest file') - resp = HTTPOk(request=req) - resp.app_iter = self.bulk_deleter.handle_delete_iter( - req, - objs_to_delete=[o['name'].encode('utf-8') for o in manifest], - user_agent='MultipartDELETE', swift_source='SLO') - return resp - return get_man_resp + + resp = HTTPOk(request=req) + out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) + if out_content_type: + resp.content_type = out_content_type + resp.app_iter = self.bulk_deleter.handle_delete_iter( + req, objs_to_delete=self.get_segments_to_delete_iter(req), + user_agent='MultipartDELETE', swift_source='SLO', + out_content_type=out_content_type) + return resp @wsgify def __call__(self, req): diff --git a/swift/common/swob.py b/swift/common/swob.py index 5ad202638a..8e63456c5f 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -1103,6 +1103,10 @@ class Response(object): return self.location return self.host_url + self.location + @property + def is_success(self): + return self.status_int // 100 == 2 + def __call__(self, env, start_response): if not self.request: self.request = Request(env) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index c236580177..458d466bcb 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -40,10 +40,10 @@ from swift.common.utils import ContextPool, normalize_timestamp, \ config_true_value, public, json, csv_append, GreenthreadSafeIterator from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ - CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE + CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE, MAX_BUFFERED_SLO_SEGMENTS from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ListingIterNotFound, \ - ListingIterNotAuthorized, ListingIterError, SloSegmentError + ListingIterNotAuthorized, ListingIterError, SegmentError from swift.common.http import is_success, is_client_error, HTTP_CONTINUE, \ HTTP_CREATED, HTTP_MULTIPLE_CHOICES, HTTP_NOT_FOUND, HTTP_CONFLICT, \ HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, \ @@ -56,13 +56,31 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPClientDisconnect, HTTPNotImplemented -def segment_listing_iter(listing): - listing = iter(listing) - while True: - seg_dict = listing.next() - if isinstance(seg_dict['name'], unicode): - seg_dict['name'] = seg_dict['name'].encode('utf-8') - yield seg_dict +class SegmentListing(object): + + def __init__(self, listing): + self.listing = iter(listing) + self._prepended_segments = [] + + def prepend_segments(self, new_segs): + """ + Will prepend given segments to listing when iterating. + :raises SegmentError: when # segments > MAX_BUFFERED_SLO_SEGMENTS + """ + new_segs.extend(self._prepended_segments) + if len(new_segs) > MAX_BUFFERED_SLO_SEGMENTS: + raise SegmentError('Too many unread slo segments in buffer') + self._prepended_segments = new_segs + + def listing_iter(self): + while True: + if self._prepended_segments: + seg_dict = self._prepended_segments.pop(0) + else: + seg_dict = self.listing.next() + if isinstance(seg_dict['name'], unicode): + seg_dict['name'] = seg_dict['name'].encode('utf-8') + yield seg_dict def copy_headers_into(from_r, to_r): @@ -104,14 +122,20 @@ class SegmentedIterable(object): 'bytes' keys. :param response: The swob.Response this iterable is associated with, if any (default: None) + :param is_slo: A boolean, defaults to False, as to whether this references + a SLO object. + :param max_lo_time: Defaults to 86400. The connection for the + SegmentedIterable will drop after that many seconds. """ def __init__(self, controller, container, listing, response=None, - is_slo=False): + is_slo=False, max_lo_time=86400): self.controller = controller self.container = container - self.listing = segment_listing_iter(listing) + self.segment_listing = SegmentListing(listing) + self.listing = self.segment_listing.listing_iter() self.is_slo = is_slo + self.max_lo_time = max_lo_time self.ratelimit_index = 0 self.segment_dict = None self.segment_peek = None @@ -121,10 +145,12 @@ class SegmentedIterable(object): # See NOTE: swift_conn at top of file about this. self.segment_iter_swift_conn = None self.position = 0 + self.have_yielded_data = False self.response = response if not self.response: self.response = Response() self.next_get_time = 0 + self.start_time = time.time() def _load_next_segment(self): """ @@ -135,6 +161,9 @@ class SegmentedIterable(object): """ try: self.ratelimit_index += 1 + if time.time() - self.start_time > self.max_lo_time: + raise SegmentError( + _('Max LO GET time of %s exceeded.') % self.max_lo_time) self.segment_dict = self.segment_peek or self.listing.next() self.segment_peek = None if self.container is None: @@ -167,7 +196,7 @@ class SegmentedIterable(object): req, _('Object'), self.controller.app.object_ring, partition, path) if self.is_slo and resp.status_int == HTTP_NOT_FOUND: - raise SloSegmentError(_( + raise SegmentError(_( 'Could not load object segment %(path)s:' ' %(status)s') % {'path': path, 'status': resp.status_int}) if not is_success(resp.status_int): @@ -175,21 +204,41 @@ class SegmentedIterable(object): 'Could not load object segment %(path)s:' ' %(status)s') % {'path': path, 'status': resp.status_int}) if self.is_slo: - if resp.etag != self.segment_dict['hash']: - raise SloSegmentError(_( - 'Object segment no longer valid: ' - '%(path)s etag: %(r_etag)s != %(s_etag)s.' % - {'path': path, 'r_etag': resp.etag, - 's_etag': self.segment_dict['hash']})) if 'X-Static-Large-Object' in resp.headers: - raise SloSegmentError(_( - 'SLO can not be made of other SLOs: %s' % path)) + # this segment is a nested slo object. read in the body + # and add its segments into this slo. + try: + sub_manifest = json.loads(resp.body) + self.segment_listing.prepend_segments(sub_manifest) + sub_etag = md5(''.join( + o['hash'] for o in sub_manifest)).hexdigest() + if sub_etag != self.segment_dict['hash']: + raise SegmentError(_( + 'Object segment does not match sub-slo: ' + '%(path)s etag: %(r_etag)s != %(s_etag)s.' % + {'path': path, 'r_etag': sub_etag, + 's_etag': self.segment_dict['hash']})) + return self._load_next_segment() + except ValueError: + raise SegmentError(_( + 'Sub SLO has invalid manifest: %s' % path)) + + elif resp.etag != self.segment_dict['hash'] or \ + resp.content_length != self.segment_dict['bytes']: + raise SegmentError(_( + 'Object segment no longer valid: ' + '%(path)s etag: %(r_etag)s != %(s_etag)s or ' + '%(r_size)s != %(s_size)s.' % + {'path': path, 'r_etag': resp.etag, + 'r_size': resp.content_length, + 's_etag': self.segment_dict['hash'], + 's_size': self.segment_dict['bytes']})) self.segment_iter = resp.app_iter # See NOTE: swift_conn at top of file about this. self.segment_iter_swift_conn = getattr(resp, 'swift_conn', None) except StopIteration: raise - except SloSegmentError, err: + except SegmentError, err: if not getattr(err, 'swift_logged', False): self.controller.app.logger.error(_( 'ERROR: While processing manifest ' @@ -232,9 +281,21 @@ class SegmentedIterable(object): else: return self.position += len(chunk) + self.have_yielded_data = True yield chunk except StopIteration: raise + except SegmentError: + if not self.have_yielded_data: + # Normally, exceptions before any data has been yielded will + # cause Eventlet to send a 5xx response. In this particular + # case of SegmentError we don't want that and we'd rather + # just send the normal 2xx response and then hang up early + # since 5xx codes are often used to judge Service Level + # Agreements and this SegmentError indicates the user has + # created an invalid condition. + yield ' ' + raise except (Exception, Timeout), err: if not getattr(err, 'swift_logged', False): self.controller.app.logger.exception(_( @@ -532,7 +593,8 @@ class ObjectController(Controller): else: resp.app_iter = SegmentedIterable( self, lcontainer, listing, resp, - is_slo=(large_object == 'SLO')) + is_slo=(large_object == 'SLO'), + max_lo_time=self.app.max_large_object_get_time) else: # For objects with a reasonable number of segments, we'll serve @@ -559,7 +621,8 @@ class ObjectController(Controller): conditional_response=True) resp.app_iter = SegmentedIterable( self, lcontainer, listing, resp, - is_slo=(large_object == 'SLO')) + is_slo=(large_object == 'SLO'), + max_lo_time=self.app.max_large_object_get_time) resp.content_length = content_length resp.last_modified = last_modified resp.etag = etag diff --git a/swift/proxy/server.py b/swift/proxy/server.py index cc7c8bb5dd..ba920378d0 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -116,6 +116,8 @@ class Application(object): self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.allow_static_large_object = config_true_value( conf.get('allow_static_large_object', 'true')) + self.max_large_object_get_time = float( + conf.get('max_large_object_get_time', '86400')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 8a39a79a5d..206ce9d6b9 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -94,6 +94,26 @@ class FakeApp(object): headers={'X-Static-Large-Object': 'True'}, body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_nested/'): + nested_data = json.dumps( + [{'name': '/b/b_2', 'hash': 'a', 'bytes': '1'}, + {'name': '/c/c_3', 'hash': 'b', 'bytes': '2'}]) + good_data = json.dumps( + [{'name': '/a/a_1', 'hash': 'a', 'bytes': '1'}, + {'name': '/a/sub_nest', 'hash': 'a', 'sub_slo': True, + 'bytes': len(nested_data)}, + {'name': '/d/d_3', 'hash': 'b', 'bytes': '2'}]) + self.req_method_paths.append((env['REQUEST_METHOD'], + env['PATH_INFO'])) + if 'sub_nest' in env['PATH_INFO']: + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=nested_data)(env, start_response) + else: + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_bad_json/'): self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) @@ -309,7 +329,7 @@ class TestStaticLargeObject(unittest.TestCase): [{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'}, {'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'}, {'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}, - {'path': '/d/slob', 'etag': 'b', 'size_bytes': '2'}]) + {'path': '/d/slob', 'etag': 'a', 'size_bytes': '2'}]) req = Request.blank( '/test_good/A/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, @@ -327,8 +347,7 @@ class TestStaticLargeObject(unittest.TestCase): self.assertEquals(errors[4][0], '/test_good/A/d/b_2') self.assertEquals(errors[4][1], 'Etag Mismatch') self.assertEquals(errors[-1][0], '/test_good/A/d/slob') - self.assertEquals(errors[-1][1], - 'Segments cannot be Large Objects') + self.assertEquals(errors[-1][1], 'Etag Mismatch') else: self.assert_(False) @@ -342,7 +361,8 @@ class TestStaticLargeObject(unittest.TestCase): req = Request.blank( '/test_delete_404/A/c/man?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE'}) - self.slo(req.environ, fake_start_response) + app_iter = self.slo(req.environ, fake_start_response) + list(app_iter) # iterate through whole response self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, [('GET', '/test_delete_404/A/c/man')]) @@ -360,25 +380,52 @@ class TestStaticLargeObject(unittest.TestCase): ('DELETE', '/test_delete/A/d/b_2'), ('DELETE', '/test_delete/A/c/man')]) - def test_handle_multipart_delete_bad_manifest(self): + def test_handle_multipart_delete_nested(self): + req = Request.blank( + '/test_delete_nested/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE'}) + app_iter = self.slo(req.environ, fake_start_response) + list(app_iter) # iterate through whole response + self.assertEquals(self.app.calls, 8) + self.assertEquals( + set(self.app.req_method_paths), + set([('GET', '/test_delete_nested/A/c/man'), + ('GET', '/test_delete_nested/A/a/sub_nest'), + ('DELETE', '/test_delete_nested/A/a/a_1'), + ('DELETE', '/test_delete_nested/A/b/b_2'), + ('DELETE', '/test_delete_nested/A/c/c_3'), + ('DELETE', '/test_delete_nested/A/a/sub_nest'), + ('DELETE', '/test_delete_nested/A/d/d_3'), + ('DELETE', '/test_delete_nested/A/c/man')])) + + def test_handle_multipart_delete_not_a_manifest(self): + # when trying to delete a SLO and its not an SLO, just go ahead + # and delete it req = Request.blank( '/test_delete_bad_man/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) - resp = self.slo(req.environ, fake_start_response) - self.assertEquals(self.app.calls, 1) + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 2) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_bad_man/A/c/man')]) - self.assertEquals(resp, ['Not an SLO manifest']) + [('GET', '/test_delete_bad_man/A/c/man'), + ('DELETE', '/test_delete_bad_man/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '200 OK') def test_handle_multipart_delete_bad_json(self): req = Request.blank( '/test_delete_bad_json/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) - resp = self.slo(req.environ, fake_start_response) + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, [('GET', '/test_delete_bad_json/A/c/man')]) - self.assertEquals(resp, ['Invalid manifest file']) + self.assertEquals(resp_data["Response Status"], "500 Internal Error") def test_handle_multipart_delete_whole_bad(self): req = Request.blank( diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 7f56de9a6b..962cece739 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -41,7 +41,7 @@ from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server from swift.common import ring -from swift.common.exceptions import ChunkReadTimeout, SloSegmentError +from swift.common.exceptions import ChunkReadTimeout, SegmentError from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH @@ -1370,7 +1370,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(resp.status_int, 200) self.assertEqual(resp.content_length, 4) # content incomplete self.assertEqual(resp.content_type, 'text/html') - self.assertRaises(SloSegmentError, lambda: resp.body) + self.assertRaises(SegmentError, lambda: resp.body) # dropped connection, exception is caught by eventlet as it is # iterating over response @@ -1388,18 +1388,37 @@ class TestObjectController(unittest.TestCase): "bytes": 2, "name": "/d1/seg01", "content_type": "application/octet-stream"}, - {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", + {"hash": "8681fb3ada2715c8754706ee5f23d4f8", "last_modified": "2012-11-08T04:05:37.846710", + "bytes": 4, + "name": "/d2/sub_manifest", + "content_type": "application/octet-stream"}, + {"hash": "419af6d362a14b7a789ba1c7e772bbae", + "last_modified": "2012-11-08T04:05:37.866820", "bytes": 2, - "name": "/d2/seg02", + "name": "/d1/seg04", "content_type": "application/octet-stream"}] + sub_listing = [{"hash": "d526f1c8ef6c1e4e980e2b8471352d23", + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg02", + "content_type": "application/octet-stream"}, + {"hash": "e4c8f1de1c0855c7c2be33196d3c3537", + "last_modified": "2012-11-08T04:05:37.846710", + "bytes": 2, + "name": "/d2/seg03", + "content_type": "application/octet-stream"}] + response_bodies = ( '', # HEAD /a '', # HEAD /a/c simplejson.dumps(listing), # GET manifest 'Aa', # GET seg01 - 'Bb') # GET seg02 + simplejson.dumps(sub_listing), # GET sub_manifest + 'Bb', # GET seg02 + 'Cc', # GET seg03 + 'Dd') # GET seg04 with save_globals(): controller = proxy_server.ObjectController( self.app, 'a', 'c', 'manifest') @@ -1419,26 +1438,37 @@ class TestObjectController(unittest.TestCase): 200, # HEAD /a/c 200, # GET listing1 200, # GET seg01 + 200, # GET sub listing1 200, # GET seg02 - headers=[{}, {}, slob_headers, {}, slob_headers], + 200, # GET seg03 + 200, # GET seg04 + headers=[{}, {}, slob_headers, {}, slob_headers, {}, {}, {}], body_iter=response_bodies, give_connect=capture_requested_paths) req = Request.blank('/a/c/manifest') resp = controller.GET(req) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.content_length, 4) # content incomplete + self.assertEqual(resp.content_length, 8) self.assertEqual(resp.content_type, 'text/html') - self.assertRaises(SloSegmentError, lambda: resp.body) - # dropped connection, exception is caught by eventlet as it is - # iterating over response + self.assertEqual( + requested, + [['HEAD', '/a', {}], + ['HEAD', '/a/c', {}], + ['GET', '/a/c/manifest', {}]]) + # iterating over body will retrieve manifest and sub manifest's + # objects + self.assertEqual(resp.body, 'AaBbCcDd') self.assertEqual( requested, [['HEAD', '/a', {}], ['HEAD', '/a/c', {}], ['GET', '/a/c/manifest', {}], ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/seg02', {}]]) + ['GET', '/a/d2/sub_manifest', {}], + ['GET', '/a/d1/seg02', {}], + ['GET', '/a/d2/seg03', {}], + ['GET', '/a/d1/seg04', {}]]) def test_GET_bad_404_manifest_slo(self): listing = [{"hash": "98568d540134639be4655198a36614a4", @@ -1490,7 +1520,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual(resp.status_int, 200) self.assertEqual(resp.content_length, 6) # content incomplete self.assertEqual(resp.content_type, 'text/html') - self.assertRaises(SloSegmentError, lambda: resp.body) + self.assertRaises(SegmentError, lambda: resp.body) # dropped connection, exception is caught by eventlet as it is # iterating over response