openstack/swift
Code Issues Proposed changes

Merge "slobjects can not be made of other slobjects"

Browse Source
This commit is contained in:
Jenkins
2013-04-17 19:06:34 +00:00
committed by Gerrit Code Review
parent dfcfb767d3 3f21e14398
commit 47eef53a35
5 changed files with 92 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/middleware/slo.py
View File

@@ -258,6 +258,10 @@ class StaticLargeObject(object):
problem_segments.append([quote(obj_path), 'Size Mismatch'])
if seg_dict['etag'] != head_seg_resp.etag:
problem_segments.append([quote(obj_path), 'Etag Mismatch'])
if 'X-Static-Large-Object' in head_seg_resp.headers or \
'X-Object-Manifest' in head_seg_resp.headers:
problem_segments.append(
[quote(obj_path), 'Segments cannot be Large Objects'])
if head_seg_resp.last_modified:
last_modified = head_seg_resp.last_modified
else:

3
swift/proxy/controllers/obj.py
View File

@@ -170,6 +170,9 @@ class SegmentedIterable(object):
'%(path)s etag: %(r_etag)s != %(s_etag)s.' %
{'path': path, 'r_etag': resp.etag,
's_etag': self.segment_dict['hash']}))
if 'X-Static-Large-Object' in resp.headers:
raise SloSegmentError(_(
'SLO can not be made of other SLOs: %s' % path))
self.segment_iter = resp.app_iter
# See NOTE: swift_conn at top of file about this.
self.segment_iter_swift_conn = getattr(resp, 'swift_conn', None)

14
test/unit/__init__.py
View File

@@ -273,7 +273,7 @@ def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
def __init__(self, status, etag=None, body='', timestamp='1',
expect_status=None):
expect_status=None, headers=None):
self.status = status
if expect_status is None:
self.expect_status = self.status
@@ -286,6 +286,7 @@ def fake_http_connect(*code_iter, **kwargs):
self.received = 0
self.etag = etag
self.body = body
self.headers = headers or {}
self.timestamp = timestamp
def getresponse(self):
@@ -329,8 +330,7 @@ def fake_http_connect(*code_iter, **kwargs):
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
if 'headers' in kwargs:
headers.update(kwargs['headers'])
headers.update(self.headers)
return headers.items()
def read(self, amt=None):
@@ -354,6 +354,11 @@ def fake_http_connect(*code_iter, **kwargs):
timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
if isinstance(kwargs.get('headers'), list):
headers_iter = iter(kwargs['headers'])
else:
headers_iter = iter([kwargs.get('headers', {})] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
x = [x] * len(code_iter)
@@ -378,6 +383,7 @@ def fake_http_connect(*code_iter, **kwargs):
else:
expect_status = status
etag = etag_iter.next()
headers = headers_iter.next()
timestamp = timestamps_iter.next()
if status <= 0:
@@ -387,6 +393,6 @@ def fake_http_connect(*code_iter, **kwargs):
else:
body = body_iter.next()
return FakeConn(status, etag, body=body, timestamp=timestamp,
expect_status=expect_status)
expect_status=expect_status, headers=headers)
return connect

21
test/unit/common/middleware/test_slo.py
View File

@@ -36,10 +36,11 @@ class FakeApp(object):
cont_len = 100
if obj == 'small_object':
cont_len = 10
return Response(
status=200,
headers={'etag': 'etagoftheobjectsegment',
'Content-Length': cont_len})(env, start_response)
headers = {'etag': 'etagoftheobjectsegment',
'Content-Length': cont_len}
if obj == 'slob':
headers['X-Static-Large-Object'] = 'true'
return Response(status=200, headers=headers)(env, start_response)
if env['PATH_INFO'].startswith('/test_good_check/'):
j, v, a, cont, obj = env['PATH_INFO'].split('/')
etag, size = obj.split('_')
@@ -307,7 +308,8 @@ class TestStaticLargeObject(unittest.TestCase):
bad_data = json.dumps(
[{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'},
{'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'},
{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}])
{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'},
{'path': '/d/slob', 'etag': 'b', 'size_bytes': '2'}])
req = Request.blank(
'/test_good/A/c/man?multipart-manifest=put',
environ={'REQUEST_METHOD': 'PUT'},
@@ -316,14 +318,17 @@ class TestStaticLargeObject(unittest.TestCase):
try:
self.slo.handle_multipart_put(req)
except HTTPException, e:
self.assertEquals(self.app.calls, 3)
self.assertEquals(self.app.calls, 4)
data = json.loads(e.body)
errors = data['Errors']
self.assertEquals(errors[0][0], '/test_good/A/c/a_1')
self.assertEquals(errors[0][1], 'Size Mismatch')
self.assertEquals(errors[2][1], '400 Bad Request')
self.assertEquals(errors[-1][0], '/test_good/A/d/b_2')
self.assertEquals(errors[-1][1], 'Etag Mismatch')
self.assertEquals(errors[4][0], '/test_good/A/d/b_2')
self.assertEquals(errors[4][1], 'Etag Mismatch')
self.assertEquals(errors[-1][0], '/test_good/A/d/slob')
self.assertEquals(errors[-1][1],
'Segments cannot be Large Objects')
else:
self.assert_(False)

68
test/unit/proxy/test_server.py
View File

@@ -1040,8 +1040,8 @@ class TestObjectController(unittest.TestCase):
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)
@@ -1195,8 +1195,64 @@ class TestObjectController(unittest.TestCase):
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')
resp = controller.GET(req)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.body, 'Aa') # dropped connection
self.assertEqual(resp.content_length, 4) # content incomplete
self.assertEqual(resp.content_type, 'text/html')
self.assertEqual(
requested,
[['HEAD', '/a', {}],
['HEAD', '/a/c', {}],
['GET', '/a/c/manifest', {}],
['GET', '/a/d1/seg01', {}],
['GET', '/a/d2/seg02', {}]])
def test_GET_nested_slo(self):
listing = [{"hash": "98568d540134639be4655198a36614a4",
"last_modified": "2012-11-08T04:05:37.866820",
"bytes": 2,
"name": "/d1/seg01",
"content_type": "application/octet-stream"},
{"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
"last_modified": "2012-11-08T04:05:37.846710",
"bytes": 2,
"name": "/d2/seg02",
"content_type": "application/octet-stream"}]
response_bodies = (
'', # HEAD /a
'', # HEAD /a/c
simplejson.dumps(listing), # GET manifest
'Aa', # GET seg01
'Bb') # GET seg02
with save_globals():
controller = proxy_server.ObjectController(
self.app, 'a', 'c', 'manifest')
requested = []
def capture_requested_paths(ipaddr, port, device, partition,
method, path, headers=None,
query_string=None):
qs_dict = dict(urlparse.parse_qsl(query_string or ''))
requested.append([method, path, qs_dict])
slob_headers = {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}
set_http_connect(
200, # HEAD /a
200, # HEAD /a/c
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers=[{}, {}, slob_headers, {}, slob_headers],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')
@@ -1255,8 +1311,8 @@ class TestObjectController(unittest.TestCase):
200, # GET listing1
200, # GET seg01
404, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')