Check object segment MD5s
Now if a client is downloading a large object and one of the segment fetches gets bad data, we'll abort the request right away instead of letting the client carry on. We do this by checking each segment's body against the etag in the GET response's headers, like we tell clients to do. To make sure that the etag is the MD5, we add ?multipart-manifest=get to all segment GET requests. That way, a segment won't be another large object, because a large object's etag isn't its MD5. This does mean that clients can't make, say, a DLO manifest that points to a bunch of SLO manifests. However, that capability was only introduced in Swift 1.13.0 (post-Havana); prior to that, clients couldn't mix large-object types. I'm more than happy to trade that capability away in exchange for data-integrity checks. Change-Id: I98c78a291ee1a863a36d54a22867bd01c126644e
This commit is contained in:
Samuel Merritt
parent
1e22e42235
commit
cd149ae19f
|
|
@ -20,6 +20,7 @@ Why not swift.common.utils, you ask? Because this way we can import things
|
|||
from swob in here without creating circular imports.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
|
@ -281,8 +282,11 @@ class SegmentedIterable(object):
|
|||
'ERROR: While processing manifest %s, '
|
||||
'max LO GET time of %ds exceeded' %
|
||||
(self.name, self.max_get_time))
|
||||
# Make sure that the segment is a plain old object, not some
|
||||
# flavor of large object, so that we can check its MD5.
|
||||
path = seg_path + '?multipart-manifest=get'
|
||||
seg_req = make_subrequest(
|
||||
self.req.environ, path=seg_path, method='GET',
|
||||
self.req.environ, path=path, method='GET',
|
||||
headers={'x-auth-token': self.req.headers.get(
|
||||
'x-auth-token')},
|
||||
agent=('%(orig)s ' + self.ua_suffix),
|
||||
|
|
@ -322,7 +326,9 @@ class SegmentedIterable(object):
|
|||
's_etag': seg_etag,
|
||||
's_size': seg_size})
|
||||
|
||||
seg_hash = hashlib.md5()
|
||||
for chunk in seg_resp.app_iter:
|
||||
seg_hash.update(chunk)
|
||||
have_yielded_data = True
|
||||
if bytes_left is None:
|
||||
yield chunk
|
||||
|
|
@ -340,6 +346,14 @@ class SegmentedIterable(object):
|
|||
'left': bytes_left})
|
||||
close_if_possible(seg_resp.app_iter)
|
||||
|
||||
if seg_resp.etag and seg_hash.hexdigest() != seg_resp.etag \
|
||||
and first_byte is None and last_byte is None:
|
||||
raise SegmentError(
|
||||
"Bad MD5 checksum in %(name)s for %(seg)s: headers had"
|
||||
" %(etag)s, but object MD5 was actually %(actual)s" %
|
||||
{'seg': seg_req.path, 'etag': seg_resp.etag,
|
||||
'name': self.name, 'actual': seg_hash.hexdigest()})
|
||||
|
||||
if bytes_left:
|
||||
raise SegmentError(
|
||||
'Not enough bytes for %s; closing connection' %
|
||||
|
|
|
|||
|
|
@ -29,6 +29,10 @@ from textwrap import dedent
|
|||
LIMIT = 'swift.common.middleware.dlo.CONTAINER_LISTING_LIMIT'
|
||||
|
||||
|
||||
def md5hex(s):
|
||||
return hashlib.md5(s).hexdigest()
|
||||
|
||||
|
||||
class DloTestCase(unittest.TestCase):
|
||||
def call_dlo(self, req, app=None, expect_exception=False):
|
||||
if app is None:
|
||||
|
|
@ -69,29 +73,30 @@ class DloTestCase(unittest.TestCase):
|
|||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_01',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
||||
'aaaaa')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_02',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||
'bbbbb')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
||||
'ccccc')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_04',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
||||
'ddddd')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_05',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
||||
'eeeee')
|
||||
|
||||
# an unrelated object (not seg*) to test the prefix matching
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
||||
swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '9',
|
||||
'Etag': md5hex("meow meow meow meow")},
|
||||
'meow meow meow meow')
|
||||
|
||||
self.app.register(
|
||||
|
|
@ -102,16 +107,16 @@ class DloTestCase(unittest.TestCase):
|
|||
|
||||
lm = '2013-11-22T02:42:13.781760'
|
||||
ct = 'application/octet-stream'
|
||||
segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
|
||||
"last_modified": lm, "content_type": ct}]
|
||||
segs = [{"hash": md5hex("aaaaa"), "bytes": 5,
|
||||
"name": "seg_01", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("bbbbb"), "bytes": 5,
|
||||
"name": "seg_02", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("ccccc"), "bytes": 5,
|
||||
"name": "seg_03", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("ddddd"), "bytes": 5,
|
||||
"name": "seg_04", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("eeeee"), "bytes": 5,
|
||||
"name": "seg_05", "last_modified": lm, "content_type": ct}]
|
||||
|
||||
full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
|
||||
"name": "catpicture.jpg",
|
||||
|
|
@ -232,9 +237,9 @@ class TestDloPutManifest(DloTestCase):
|
|||
|
||||
class TestDloHeadManifest(DloTestCase):
|
||||
def test_head_large_object(self):
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'HEAD'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
|
|
@ -258,8 +263,7 @@ class TestDloHeadManifest(DloTestCase):
|
|||
environ={'REQUEST_METHOD': 'HEAD'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
self.assertEqual(headers["Etag"],
|
||||
'"' + hashlib.md5("").hexdigest() + '"')
|
||||
self.assertEqual(headers["Etag"], '"%s"' % md5hex(""))
|
||||
self.assertEqual(headers["Content-Length"], "0")
|
||||
|
||||
# one request to HEAD the manifest
|
||||
|
|
@ -273,9 +277,9 @@ class TestDloHeadManifest(DloTestCase):
|
|||
|
||||
class TestDloGetManifest(DloTestCase):
|
||||
def test_get_manifest(self):
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
|
|
@ -346,9 +350,9 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.assertEqual(status, "206 Partial Content")
|
||||
self.assertEqual(headers["Content-Length"], "10")
|
||||
self.assertEqual(body, "bbcccccddd")
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
self.assertEqual(headers.get("Etag"), expected_etag)
|
||||
|
||||
def test_get_range_on_segment_boundaries(self):
|
||||
|
|
@ -426,9 +430,9 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/mancon/manifest-many-segments'),
|
||||
('GET', '/v1/AUTH_test/c?format=json&prefix=seg_'),
|
||||
('GET', '/v1/AUTH_test/c/seg_01'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03')])
|
||||
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
|
||||
|
||||
def test_get_range_many_segments_satisfiability_unknown(self):
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
||||
|
|
@ -480,9 +484,9 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
||||
|
||||
def test_if_match_matches(self):
|
||||
manifest_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
manifest_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag})
|
||||
|
|
@ -507,9 +511,9 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_none_match_matches(self):
|
||||
manifest_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
manifest_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': manifest_etag})
|
||||
|
|
@ -603,6 +607,21 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.assertEqual(status, "200 OK")
|
||||
self.assertEqual(body, "aaaaabbbbbccccc")
|
||||
|
||||
def test_mismatched_etag_fetching_second_segment(self):
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_02',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||
'bbWRONGbb')
|
||||
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
self.assertTrue(isinstance(exc, exceptions.SegmentError))
|
||||
self.assertEqual(status, "200 OK")
|
||||
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
|
||||
|
||||
def test_etag_comparison_ignores_quotes(self):
|
||||
# a little future-proofing here in case we ever fix this
|
||||
self.app.register(
|
||||
|
|
@ -632,8 +651,8 @@ class TestDloGetManifest(DloTestCase):
|
|||
swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah',
|
||||
'X-Object-Manifest': u'c/é'.encode('utf-8')}, None)
|
||||
|
||||
segs = [{"hash": "etag1", "bytes": 5, "name": u"é1"},
|
||||
{"hash": "etag2", "bytes": 5, "name": u"é2"}]
|
||||
segs = [{"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é1"},
|
||||
{"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é2"}]
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c?format=json&prefix=%C3%A9',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json'},
|
||||
|
|
@ -641,11 +660,11 @@ class TestDloGetManifest(DloTestCase):
|
|||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/\xC3\xa91',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'etag1'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
||||
"AAAAA")
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/\xC3\xA92',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'etag2'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
||||
"BBBBB")
|
||||
|
||||
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
||||
|
|
@ -709,9 +728,9 @@ class TestDloGetManifest(DloTestCase):
|
|||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
||||
('GET', '/v1/AUTH_test/c?format=json&prefix=seg'),
|
||||
('GET', '/v1/AUTH_test/c/seg_01'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03')])
|
||||
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
|
||||
|
||||
def test_get_undersize_segment(self):
|
||||
# If we send a Content-Length header to the client, it's based on the
|
||||
|
|
@ -725,7 +744,7 @@ class TestDloGetManifest(DloTestCase):
|
|||
# Shrink it by a single byte
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
|
||||
'cccc')
|
||||
|
||||
req = swob.Request.blank(
|
||||
|
|
@ -743,7 +762,7 @@ class TestDloGetManifest(DloTestCase):
|
|||
# Shrink it by a single byte
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
|
||||
'cccc')
|
||||
|
||||
req = swob.Request.blank(
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
import unittest
|
||||
from contextlib import nested
|
||||
|
|
@ -45,6 +46,10 @@ def fake_start_response(*args, **kwargs):
|
|||
pass
|
||||
|
||||
|
||||
def md5hex(s):
|
||||
return hashlib.md5(s).hexdigest()
|
||||
|
||||
|
||||
class SloTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = FakeSwift()
|
||||
|
|
@ -741,31 +746,31 @@ class TestSloGetManifest(SloTestCase):
|
|||
super(TestSloGetManifest, self).setUp()
|
||||
|
||||
_bc_manifest_json = json.dumps(
|
||||
[{'name': '/gettest/b_10', 'hash': 'b', 'bytes': '10',
|
||||
[{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10',
|
||||
'content_type': 'text/plain'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c', 'bytes': '15',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15',
|
||||
'content_type': 'text/plain'}])
|
||||
|
||||
# some plain old objects
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/a_5',
|
||||
swob.HTTPOk, {'Content-Length': '5',
|
||||
'Etag': 'a'},
|
||||
'Etag': md5hex('a' * 5)},
|
||||
'a' * 5)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/b_10',
|
||||
swob.HTTPOk, {'Content-Length': '10',
|
||||
'Etag': 'b'},
|
||||
'Etag': md5hex('b' * 10)},
|
||||
'b' * 10)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/c_15',
|
||||
swob.HTTPOk, {'Content-Length': '15',
|
||||
'Etag': 'c'},
|
||||
'Etag': md5hex('c' * 15)},
|
||||
'c' * 15)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/d_20',
|
||||
swob.HTTPOk, {'Content-Length': '20',
|
||||
'Etag': 'd'},
|
||||
'Etag': md5hex('d' * 20)},
|
||||
'd' * 20)
|
||||
|
||||
self.app.register(
|
||||
|
|
@ -773,17 +778,17 @@ class TestSloGetManifest(SloTestCase):
|
|||
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
|
||||
'X-Static-Large-Object': 'true',
|
||||
'X-Object-Meta-Plant': 'Ficus',
|
||||
'Etag': md5(_bc_manifest_json).hexdigest()},
|
||||
'Etag': md5hex(_bc_manifest_json)},
|
||||
_bc_manifest_json)
|
||||
|
||||
_abcd_manifest_json = json.dumps(
|
||||
[{'name': '/gettest/a_5', 'hash': 'a',
|
||||
[{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/manifest-bc', 'sub_slo': True,
|
||||
'content_type': 'application/json;swift_bytes=25',
|
||||
'hash': md5("bc").hexdigest(),
|
||||
'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
|
||||
'bytes': len(_bc_manifest_json)},
|
||||
{'name': '/gettest/d_20', 'hash': 'd',
|
||||
{'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
|
||||
'content_type': 'text/plain', 'bytes': '20'}])
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-abcd',
|
||||
|
|
@ -792,6 +797,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
'Etag': md5(_abcd_manifest_json).hexdigest()},
|
||||
_abcd_manifest_json)
|
||||
|
||||
self.manifest_abcd_etag = md5hex(
|
||||
md5hex("a" * 5) + md5hex(md5hex("b" * 10) + md5hex("c" * 15)) +
|
||||
md5hex("d" * 20))
|
||||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-badjson',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
|
|
@ -817,9 +826,9 @@ class TestSloGetManifest(SloTestCase):
|
|||
|
||||
self.assertEqual(
|
||||
resp_data,
|
||||
[{'hash': 'b', 'bytes': '10', 'name': '/gettest/b_10',
|
||||
[{'hash': md5hex('b' * 10), 'bytes': '10', 'name': '/gettest/b_10',
|
||||
'content_type': 'text/plain'},
|
||||
{'hash': 'c', 'bytes': '15', 'name': '/gettest/c_15',
|
||||
{'hash': md5hex('c' * 15), 'bytes': '15', 'name': '/gettest/c_15',
|
||||
'content_type': 'text/plain'}],
|
||||
body)
|
||||
|
||||
|
|
@ -839,7 +848,7 @@ class TestSloGetManifest(SloTestCase):
|
|||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("bc").hexdigest()
|
||||
manifest_etag = md5hex(md5hex("b" * 10) + md5hex("c" * 15))
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '25')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
|
|
@ -856,12 +865,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
"SLO MultipartGET" in first_ua)
|
||||
|
||||
def test_if_none_match_matches(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': manifest_etag})
|
||||
headers={'If-None-Match': self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
|
|
@ -870,12 +877,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_none_match_does_not_match(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': "not-%s" % manifest_etag})
|
||||
headers={'If-None-Match': "not-%s" % self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
|
|
@ -884,12 +889,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
def test_if_match_matches(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag})
|
||||
headers={'If-Match': self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
|
|
@ -898,12 +901,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
def test_if_match_does_not_match(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': "not-%s" % manifest_etag})
|
||||
headers={'If-Match': "not-%s" % self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
|
|
@ -912,12 +913,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_match_matches_and_range(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag,
|
||||
headers={'If-Match': self.manifest_abcd_etag,
|
||||
'Range': 'bytes=3-6'})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
|
@ -933,10 +932,9 @@ class TestSloGetManifest(SloTestCase):
|
|||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '50')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
|
||||
self.assertEqual(
|
||||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
|
|
@ -957,10 +955,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
headers = [c[2] for c in self.app.calls_with_headers]
|
||||
self.assertEqual(headers[0].get('Range'), 'bytes=3-17')
|
||||
|
|
@ -992,15 +990,15 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual(
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_beyond_manifest(self):
|
||||
big = 'e' * 1024 * 1024
|
||||
big_etag = md5(big).hexdigest()
|
||||
big_etag = md5hex(big)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/big_seg',
|
||||
swob.HTTPOk, {'Content-Type': 'application/foo',
|
||||
|
|
@ -1031,7 +1029,8 @@ class TestSloGetManifest(SloTestCase):
|
|||
('GET', '/v1/AUTH_test/gettest/big_manifest'),
|
||||
# retry the first one
|
||||
('GET', '/v1/AUTH_test/gettest/big_manifest'),
|
||||
('GET', '/v1/AUTH_test/gettest/big_seg')])
|
||||
('GET',
|
||||
'/v1/AUTH_test/gettest/big_seg?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_bogus_content_range(self):
|
||||
# Just a little paranoia; Swift currently sends back valid
|
||||
|
|
@ -1064,11 +1063,11 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_manifest_on_segment_boundaries(self):
|
||||
req = Request.blank(
|
||||
|
|
@ -1088,8 +1087,8 @@ class TestSloGetManifest(SloTestCase):
|
|||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
headers = [c[2] for c in self.app.calls_with_headers]
|
||||
self.assertEqual(headers[0].get('Range'), 'bytes=5-29')
|
||||
|
|
@ -1116,7 +1115,7 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5')])
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_manifest_overlapping_end(self):
|
||||
req = Request.blank(
|
||||
|
|
@ -1158,11 +1157,11 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.app.register(
|
||||
'GET', u'/v1/AUTH_test/ünicode/öbject-segment'.encode('utf-8'),
|
||||
swob.HTTPOk, {'Content-Length': str(len(segment_body)),
|
||||
'Etag': "moose"},
|
||||
'Etag': md5hex(segment_body)},
|
||||
segment_body)
|
||||
|
||||
manifest_json = json.dumps([{'name': u'/ünicode/öbject-segment',
|
||||
'hash': 'moose',
|
||||
'hash': md5hex(segment_body),
|
||||
'content_type': 'text/plain',
|
||||
'bytes': len(segment_body)}])
|
||||
self.app.register(
|
||||
|
|
@ -1199,10 +1198,9 @@ class TestSloGetManifest(SloTestCase):
|
|||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '50')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
|
||||
self.assertEqual(body, '')
|
||||
# Note the lack of recursive descent into manifest-bc. We know the
|
||||
# content-length from the outer manifest, so there's no need for any
|
||||
|
|
@ -1217,11 +1215,11 @@ class TestSloGetManifest(SloTestCase):
|
|||
for i in xrange(20):
|
||||
self.app.register('GET', '/v1/AUTH_test/gettest/obj%d' % i,
|
||||
swob.HTTPOk, {'Content-Type': 'text/plain',
|
||||
'Etag': 'hash%d' % i},
|
||||
'Etag': md5hex('body%02d' % i)},
|
||||
'body%02d' % i)
|
||||
|
||||
manifest_json = json.dumps([{'name': '/gettest/obj20',
|
||||
'hash': 'hash20',
|
||||
'hash': md5hex('body20'),
|
||||
'content_type': 'text/plain',
|
||||
'bytes': '6'}])
|
||||
self.app.register(
|
||||
|
|
@ -1234,7 +1232,7 @@ class TestSloGetManifest(SloTestCase):
|
|||
for i in xrange(19, 0, -1):
|
||||
manifest_data = [
|
||||
{'name': '/gettest/obj%d' % i,
|
||||
'hash': 'hash%d' % i,
|
||||
'hash': md5hex('body%02d' % i),
|
||||
'bytes': '6',
|
||||
'content_type': 'text/plain'},
|
||||
{'name': '/gettest/man%d' % (i + 1),
|
||||
|
|
@ -1296,11 +1294,11 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
# This one has the error, and so is the last one we fetch.
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
def test_error_fetching_submanifest(self):
|
||||
self.app.register('GET', '/v1/AUTH_test/gettest/manifest-bc',
|
||||
|
|
@ -1315,7 +1313,7 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual("aaaaa", body)
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
# This one has the error, and so is the last one we fetch.
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc')])
|
||||
|
||||
|
|
@ -1365,11 +1363,11 @@ class TestSloGetManifest(SloTestCase):
|
|||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': 'a',
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/b_10', 'hash': 'wrong!',
|
||||
'content_type': 'text/plain', 'bytes': '10'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15),
|
||||
'content_type': 'text/plain', 'bytes': '15'}]))
|
||||
|
||||
req = Request.blank(
|
||||
|
|
@ -1383,18 +1381,18 @@ class TestSloGetManifest(SloTestCase):
|
|||
|
||||
def test_mismatched_size(self):
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badsize-c',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': 'a',
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/b_10', 'hash': 'b',
|
||||
{'name': '/gettest/b_10', 'hash': md5hex('b' * 10),
|
||||
'content_type': 'text/plain', 'bytes': '999999'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15),
|
||||
'content_type': 'text/plain', 'bytes': '15'}]))
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
'/v1/AUTH_test/gettest/manifest-a-b-badsize-c',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body, exc = self.call_slo(req, expect_exception=True)
|
||||
|
||||
|
|
@ -1430,10 +1428,10 @@ class TestSloGetManifest(SloTestCase):
|
|||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
|
||||
class TestSloBulkLogger(unittest.TestCase):
|
||||
|
|
@ -1448,12 +1446,13 @@ class TestSloCopyHook(SloTestCase):
|
|||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/o', swob.HTTPOk,
|
||||
{'Content-Length': '3', 'Etag': 'obj-etag'}, "obj")
|
||||
{'Content-Length': '3', 'Etag': md5hex("obj")}, "obj")
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/man',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/c/o', 'hash': 'obj-etag', 'bytes': '3'}]))
|
||||
json.dumps([{'name': '/c/o', 'hash': md5hex("obj"),
|
||||
'bytes': '3'}]))
|
||||
|
||||
copy_hook = [None]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue