Merge "Check object segment MD5s"
This commit is contained in:
commit
89905636f0
@ -20,6 +20,7 @@ Why not swift.common.utils, you ask? Because this way we can import things
|
||||
from swob in here without creating circular imports.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
@ -281,8 +282,11 @@ class SegmentedIterable(object):
|
||||
'ERROR: While processing manifest %s, '
|
||||
'max LO GET time of %ds exceeded' %
|
||||
(self.name, self.max_get_time))
|
||||
# Make sure that the segment is a plain old object, not some
|
||||
# flavor of large object, so that we can check its MD5.
|
||||
path = seg_path + '?multipart-manifest=get'
|
||||
seg_req = make_subrequest(
|
||||
self.req.environ, path=seg_path, method='GET',
|
||||
self.req.environ, path=path, method='GET',
|
||||
headers={'x-auth-token': self.req.headers.get(
|
||||
'x-auth-token')},
|
||||
agent=('%(orig)s ' + self.ua_suffix),
|
||||
@ -322,7 +326,9 @@ class SegmentedIterable(object):
|
||||
's_etag': seg_etag,
|
||||
's_size': seg_size})
|
||||
|
||||
seg_hash = hashlib.md5()
|
||||
for chunk in seg_resp.app_iter:
|
||||
seg_hash.update(chunk)
|
||||
have_yielded_data = True
|
||||
if bytes_left is None:
|
||||
yield chunk
|
||||
@ -340,6 +346,14 @@ class SegmentedIterable(object):
|
||||
'left': bytes_left})
|
||||
close_if_possible(seg_resp.app_iter)
|
||||
|
||||
if seg_resp.etag and seg_hash.hexdigest() != seg_resp.etag \
|
||||
and first_byte is None and last_byte is None:
|
||||
raise SegmentError(
|
||||
"Bad MD5 checksum in %(name)s for %(seg)s: headers had"
|
||||
" %(etag)s, but object MD5 was actually %(actual)s" %
|
||||
{'seg': seg_req.path, 'etag': seg_resp.etag,
|
||||
'name': self.name, 'actual': seg_hash.hexdigest()})
|
||||
|
||||
if bytes_left:
|
||||
raise SegmentError(
|
||||
'Not enough bytes for %s; closing connection' %
|
||||
|
@ -29,6 +29,10 @@ from textwrap import dedent
|
||||
LIMIT = 'swift.common.middleware.dlo.CONTAINER_LISTING_LIMIT'
|
||||
|
||||
|
||||
def md5hex(s):
|
||||
return hashlib.md5(s).hexdigest()
|
||||
|
||||
|
||||
class DloTestCase(unittest.TestCase):
|
||||
def call_dlo(self, req, app=None, expect_exception=False):
|
||||
if app is None:
|
||||
@ -69,29 +73,30 @@ class DloTestCase(unittest.TestCase):
|
||||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_01',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
||||
'aaaaa')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_02',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||
'bbbbb')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
||||
'ccccc')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_04',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
||||
'ddddd')
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_05',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
||||
'eeeee')
|
||||
|
||||
# an unrelated object (not seg*) to test the prefix matching
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
||||
swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '9',
|
||||
'Etag': md5hex("meow meow meow meow")},
|
||||
'meow meow meow meow')
|
||||
|
||||
self.app.register(
|
||||
@ -102,16 +107,16 @@ class DloTestCase(unittest.TestCase):
|
||||
|
||||
lm = '2013-11-22T02:42:13.781760'
|
||||
ct = 'application/octet-stream'
|
||||
segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
|
||||
"last_modified": lm, "content_type": ct},
|
||||
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
|
||||
"last_modified": lm, "content_type": ct}]
|
||||
segs = [{"hash": md5hex("aaaaa"), "bytes": 5,
|
||||
"name": "seg_01", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("bbbbb"), "bytes": 5,
|
||||
"name": "seg_02", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("ccccc"), "bytes": 5,
|
||||
"name": "seg_03", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("ddddd"), "bytes": 5,
|
||||
"name": "seg_04", "last_modified": lm, "content_type": ct},
|
||||
{"hash": md5hex("eeeee"), "bytes": 5,
|
||||
"name": "seg_05", "last_modified": lm, "content_type": ct}]
|
||||
|
||||
full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
|
||||
"name": "catpicture.jpg",
|
||||
@ -232,9 +237,9 @@ class TestDloPutManifest(DloTestCase):
|
||||
|
||||
class TestDloHeadManifest(DloTestCase):
|
||||
def test_head_large_object(self):
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'HEAD'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
@ -258,8 +263,7 @@ class TestDloHeadManifest(DloTestCase):
|
||||
environ={'REQUEST_METHOD': 'HEAD'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
self.assertEqual(headers["Etag"],
|
||||
'"' + hashlib.md5("").hexdigest() + '"')
|
||||
self.assertEqual(headers["Etag"], '"%s"' % md5hex(""))
|
||||
self.assertEqual(headers["Content-Length"], "0")
|
||||
|
||||
# one request to HEAD the manifest
|
||||
@ -273,9 +277,9 @@ class TestDloHeadManifest(DloTestCase):
|
||||
|
||||
class TestDloGetManifest(DloTestCase):
|
||||
def test_get_manifest(self):
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body = self.call_dlo(req)
|
||||
@ -346,9 +350,9 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.assertEqual(status, "206 Partial Content")
|
||||
self.assertEqual(headers["Content-Length"], "10")
|
||||
self.assertEqual(body, "bbcccccddd")
|
||||
expected_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
expected_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
self.assertEqual(headers.get("Etag"), expected_etag)
|
||||
|
||||
def test_get_range_on_segment_boundaries(self):
|
||||
@ -426,9 +430,9 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/mancon/manifest-many-segments'),
|
||||
('GET', '/v1/AUTH_test/c?format=json&prefix=seg_'),
|
||||
('GET', '/v1/AUTH_test/c/seg_01'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03')])
|
||||
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
|
||||
|
||||
def test_get_range_many_segments_satisfiability_unknown(self):
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
||||
@ -480,9 +484,9 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
||||
|
||||
def test_if_match_matches(self):
|
||||
manifest_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
manifest_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag})
|
||||
@ -507,9 +511,9 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_none_match_matches(self):
|
||||
manifest_etag = '"%s"' % hashlib.md5(
|
||||
"seg01-etag" + "seg02-etag" + "seg03-etag" +
|
||||
"seg04-etag" + "seg05-etag").hexdigest()
|
||||
manifest_etag = '"%s"' % md5hex(
|
||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||
md5hex("ddddd") + md5hex("eeeee"))
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': manifest_etag})
|
||||
@ -603,6 +607,21 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.assertEqual(status, "200 OK")
|
||||
self.assertEqual(body, "aaaaabbbbbccccc")
|
||||
|
||||
def test_mismatched_etag_fetching_second_segment(self):
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_02',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||
'bbWRONGbb')
|
||||
|
||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
self.assertTrue(isinstance(exc, exceptions.SegmentError))
|
||||
self.assertEqual(status, "200 OK")
|
||||
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
|
||||
|
||||
def test_etag_comparison_ignores_quotes(self):
|
||||
# a little future-proofing here in case we ever fix this
|
||||
self.app.register(
|
||||
@ -632,8 +651,8 @@ class TestDloGetManifest(DloTestCase):
|
||||
swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah',
|
||||
'X-Object-Manifest': u'c/é'.encode('utf-8')}, None)
|
||||
|
||||
segs = [{"hash": "etag1", "bytes": 5, "name": u"é1"},
|
||||
{"hash": "etag2", "bytes": 5, "name": u"é2"}]
|
||||
segs = [{"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é1"},
|
||||
{"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é2"}]
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c?format=json&prefix=%C3%A9',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json'},
|
||||
@ -641,11 +660,11 @@ class TestDloGetManifest(DloTestCase):
|
||||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/\xC3\xa91',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'etag1'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
||||
"AAAAA")
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/\xC3\xA92',
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'etag2'},
|
||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
||||
"BBBBB")
|
||||
|
||||
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
||||
@ -709,9 +728,9 @@ class TestDloGetManifest(DloTestCase):
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
||||
('GET', '/v1/AUTH_test/c?format=json&prefix=seg'),
|
||||
('GET', '/v1/AUTH_test/c/seg_01'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03')])
|
||||
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
|
||||
|
||||
def test_get_undersize_segment(self):
|
||||
# If we send a Content-Length header to the client, it's based on the
|
||||
@ -725,7 +744,7 @@ class TestDloGetManifest(DloTestCase):
|
||||
# Shrink it by a single byte
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
|
||||
'cccc')
|
||||
|
||||
req = swob.Request.blank(
|
||||
@ -743,7 +762,7 @@ class TestDloGetManifest(DloTestCase):
|
||||
# Shrink it by a single byte
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/seg_03',
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': 'seg03-etag'},
|
||||
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
|
||||
'cccc')
|
||||
|
||||
req = swob.Request.blank(
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
import unittest
|
||||
from contextlib import nested
|
||||
@ -45,6 +46,10 @@ def fake_start_response(*args, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
def md5hex(s):
|
||||
return hashlib.md5(s).hexdigest()
|
||||
|
||||
|
||||
class SloTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = FakeSwift()
|
||||
@ -741,31 +746,31 @@ class TestSloGetManifest(SloTestCase):
|
||||
super(TestSloGetManifest, self).setUp()
|
||||
|
||||
_bc_manifest_json = json.dumps(
|
||||
[{'name': '/gettest/b_10', 'hash': 'b', 'bytes': '10',
|
||||
[{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10',
|
||||
'content_type': 'text/plain'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c', 'bytes': '15',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15',
|
||||
'content_type': 'text/plain'}])
|
||||
|
||||
# some plain old objects
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/a_5',
|
||||
swob.HTTPOk, {'Content-Length': '5',
|
||||
'Etag': 'a'},
|
||||
'Etag': md5hex('a' * 5)},
|
||||
'a' * 5)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/b_10',
|
||||
swob.HTTPOk, {'Content-Length': '10',
|
||||
'Etag': 'b'},
|
||||
'Etag': md5hex('b' * 10)},
|
||||
'b' * 10)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/c_15',
|
||||
swob.HTTPOk, {'Content-Length': '15',
|
||||
'Etag': 'c'},
|
||||
'Etag': md5hex('c' * 15)},
|
||||
'c' * 15)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/d_20',
|
||||
swob.HTTPOk, {'Content-Length': '20',
|
||||
'Etag': 'd'},
|
||||
'Etag': md5hex('d' * 20)},
|
||||
'd' * 20)
|
||||
|
||||
self.app.register(
|
||||
@ -773,17 +778,17 @@ class TestSloGetManifest(SloTestCase):
|
||||
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
|
||||
'X-Static-Large-Object': 'true',
|
||||
'X-Object-Meta-Plant': 'Ficus',
|
||||
'Etag': md5(_bc_manifest_json).hexdigest()},
|
||||
'Etag': md5hex(_bc_manifest_json)},
|
||||
_bc_manifest_json)
|
||||
|
||||
_abcd_manifest_json = json.dumps(
|
||||
[{'name': '/gettest/a_5', 'hash': 'a',
|
||||
[{'name': '/gettest/a_5', 'hash': md5hex("a" * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/manifest-bc', 'sub_slo': True,
|
||||
'content_type': 'application/json;swift_bytes=25',
|
||||
'hash': md5("bc").hexdigest(),
|
||||
'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)),
|
||||
'bytes': len(_bc_manifest_json)},
|
||||
{'name': '/gettest/d_20', 'hash': 'd',
|
||||
{'name': '/gettest/d_20', 'hash': md5hex("d" * 20),
|
||||
'content_type': 'text/plain', 'bytes': '20'}])
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-abcd',
|
||||
@ -792,6 +797,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
'Etag': md5(_abcd_manifest_json).hexdigest()},
|
||||
_abcd_manifest_json)
|
||||
|
||||
self.manifest_abcd_etag = md5hex(
|
||||
md5hex("a" * 5) + md5hex(md5hex("b" * 10) + md5hex("c" * 15)) +
|
||||
md5hex("d" * 20))
|
||||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-badjson',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
@ -817,9 +826,9 @@ class TestSloGetManifest(SloTestCase):
|
||||
|
||||
self.assertEqual(
|
||||
resp_data,
|
||||
[{'hash': 'b', 'bytes': '10', 'name': '/gettest/b_10',
|
||||
[{'hash': md5hex('b' * 10), 'bytes': '10', 'name': '/gettest/b_10',
|
||||
'content_type': 'text/plain'},
|
||||
{'hash': 'c', 'bytes': '15', 'name': '/gettest/c_15',
|
||||
{'hash': md5hex('c' * 15), 'bytes': '15', 'name': '/gettest/c_15',
|
||||
'content_type': 'text/plain'}],
|
||||
body)
|
||||
|
||||
@ -839,7 +848,7 @@ class TestSloGetManifest(SloTestCase):
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("bc").hexdigest()
|
||||
manifest_etag = md5hex(md5hex("b" * 10) + md5hex("c" * 15))
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '25')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
@ -856,12 +865,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
"SLO MultipartGET" in first_ua)
|
||||
|
||||
def test_if_none_match_matches(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': manifest_etag})
|
||||
headers={'If-None-Match': self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
@ -870,12 +877,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_none_match_does_not_match(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-None-Match': "not-%s" % manifest_etag})
|
||||
headers={'If-None-Match': "not-%s" % self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
@ -884,12 +889,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
def test_if_match_matches(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag})
|
||||
headers={'If-Match': self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
@ -898,12 +901,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
def test_if_match_does_not_match(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': "not-%s" % manifest_etag})
|
||||
headers={'If-Match': "not-%s" % self.manifest_abcd_etag})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
@ -912,12 +913,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual(body, '')
|
||||
|
||||
def test_if_match_matches_and_range(self):
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-abcd',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'If-Match': manifest_etag,
|
||||
headers={'If-Match': self.manifest_abcd_etag,
|
||||
'Range': 'bytes=3-6'})
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
@ -933,10 +932,9 @@ class TestSloGetManifest(SloTestCase):
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '50')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
|
||||
self.assertEqual(
|
||||
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
|
||||
|
||||
@ -957,10 +955,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
headers = [c[2] for c in self.app.calls_with_headers]
|
||||
self.assertEqual(headers[0].get('Range'), 'bytes=3-17')
|
||||
@ -992,15 +990,15 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual(
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_beyond_manifest(self):
|
||||
big = 'e' * 1024 * 1024
|
||||
big_etag = md5(big).hexdigest()
|
||||
big_etag = md5hex(big)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/big_seg',
|
||||
swob.HTTPOk, {'Content-Type': 'application/foo',
|
||||
@ -1031,7 +1029,8 @@ class TestSloGetManifest(SloTestCase):
|
||||
('GET', '/v1/AUTH_test/gettest/big_manifest'),
|
||||
# retry the first one
|
||||
('GET', '/v1/AUTH_test/gettest/big_manifest'),
|
||||
('GET', '/v1/AUTH_test/gettest/big_seg')])
|
||||
('GET',
|
||||
'/v1/AUTH_test/gettest/big_seg?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_bogus_content_range(self):
|
||||
# Just a little paranoia; Swift currently sends back valid
|
||||
@ -1064,11 +1063,11 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_manifest_on_segment_boundaries(self):
|
||||
req = Request.blank(
|
||||
@ -1088,8 +1087,8 @@ class TestSloGetManifest(SloTestCase):
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
headers = [c[2] for c in self.app.calls_with_headers]
|
||||
self.assertEqual(headers[0].get('Range'), 'bytes=5-29')
|
||||
@ -1116,7 +1115,7 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.app.calls,
|
||||
[('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5')])
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')])
|
||||
|
||||
def test_range_get_manifest_overlapping_end(self):
|
||||
req = Request.blank(
|
||||
@ -1158,11 +1157,11 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.app.register(
|
||||
'GET', u'/v1/AUTH_test/ünicode/öbject-segment'.encode('utf-8'),
|
||||
swob.HTTPOk, {'Content-Length': str(len(segment_body)),
|
||||
'Etag': "moose"},
|
||||
'Etag': md5hex(segment_body)},
|
||||
segment_body)
|
||||
|
||||
manifest_json = json.dumps([{'name': u'/ünicode/öbject-segment',
|
||||
'hash': 'moose',
|
||||
'hash': md5hex(segment_body),
|
||||
'content_type': 'text/plain',
|
||||
'bytes': len(segment_body)}])
|
||||
self.app.register(
|
||||
@ -1199,10 +1198,9 @@ class TestSloGetManifest(SloTestCase):
|
||||
status, headers, body = self.call_slo(req)
|
||||
headers = swob.HeaderKeyDict(headers)
|
||||
|
||||
manifest_etag = md5("a" + md5("bc").hexdigest() + "d").hexdigest()
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(headers['Content-Length'], '50')
|
||||
self.assertEqual(headers['Etag'], '"%s"' % manifest_etag)
|
||||
self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag)
|
||||
self.assertEqual(body, '')
|
||||
# Note the lack of recursive descent into manifest-bc. We know the
|
||||
# content-length from the outer manifest, so there's no need for any
|
||||
@ -1217,11 +1215,11 @@ class TestSloGetManifest(SloTestCase):
|
||||
for i in xrange(20):
|
||||
self.app.register('GET', '/v1/AUTH_test/gettest/obj%d' % i,
|
||||
swob.HTTPOk, {'Content-Type': 'text/plain',
|
||||
'Etag': 'hash%d' % i},
|
||||
'Etag': md5hex('body%02d' % i)},
|
||||
'body%02d' % i)
|
||||
|
||||
manifest_json = json.dumps([{'name': '/gettest/obj20',
|
||||
'hash': 'hash20',
|
||||
'hash': md5hex('body20'),
|
||||
'content_type': 'text/plain',
|
||||
'bytes': '6'}])
|
||||
self.app.register(
|
||||
@ -1234,7 +1232,7 @@ class TestSloGetManifest(SloTestCase):
|
||||
for i in xrange(19, 0, -1):
|
||||
manifest_data = [
|
||||
{'name': '/gettest/obj%d' % i,
|
||||
'hash': 'hash%d' % i,
|
||||
'hash': md5hex('body%02d' % i),
|
||||
'bytes': '6',
|
||||
'content_type': 'text/plain'},
|
||||
{'name': '/gettest/man%d' % (i + 1),
|
||||
@ -1296,11 +1294,11 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
# This one has the error, and so is the last one we fetch.
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
def test_error_fetching_submanifest(self):
|
||||
self.app.register('GET', '/v1/AUTH_test/gettest/manifest-bc',
|
||||
@ -1315,7 +1313,7 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual("aaaaa", body)
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
# This one has the error, and so is the last one we fetch.
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc')])
|
||||
|
||||
@ -1365,11 +1363,11 @@ class TestSloGetManifest(SloTestCase):
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': 'a',
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/b_10', 'hash': 'wrong!',
|
||||
'content_type': 'text/plain', 'bytes': '10'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15),
|
||||
'content_type': 'text/plain', 'bytes': '15'}]))
|
||||
|
||||
req = Request.blank(
|
||||
@ -1383,18 +1381,18 @@ class TestSloGetManifest(SloTestCase):
|
||||
|
||||
def test_mismatched_size(self):
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-a-b-badsize-c',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': 'a',
|
||||
json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5),
|
||||
'content_type': 'text/plain', 'bytes': '5'},
|
||||
{'name': '/gettest/b_10', 'hash': 'b',
|
||||
{'name': '/gettest/b_10', 'hash': md5hex('b' * 10),
|
||||
'content_type': 'text/plain', 'bytes': '999999'},
|
||||
{'name': '/gettest/c_15', 'hash': 'c',
|
||||
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15),
|
||||
'content_type': 'text/plain', 'bytes': '15'}]))
|
||||
|
||||
req = Request.blank(
|
||||
'/v1/AUTH_test/gettest/manifest-a-b-badetag-c',
|
||||
'/v1/AUTH_test/gettest/manifest-a-b-badsize-c',
|
||||
environ={'REQUEST_METHOD': 'GET'})
|
||||
status, headers, body, exc = self.call_slo(req, expect_exception=True)
|
||||
|
||||
@ -1430,10 +1428,10 @@ class TestSloGetManifest(SloTestCase):
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertEqual(self.app.calls, [
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-abcd'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5'),
|
||||
('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/manifest-bc'),
|
||||
('GET', '/v1/AUTH_test/gettest/b_10'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15')])
|
||||
('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'),
|
||||
('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')])
|
||||
|
||||
|
||||
class TestSloBulkLogger(unittest.TestCase):
|
||||
@ -1448,12 +1446,13 @@ class TestSloCopyHook(SloTestCase):
|
||||
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/o', swob.HTTPOk,
|
||||
{'Content-Length': '3', 'Etag': 'obj-etag'}, "obj")
|
||||
{'Content-Length': '3', 'Etag': md5hex("obj")}, "obj")
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/c/man',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json',
|
||||
'X-Static-Large-Object': 'true'},
|
||||
json.dumps([{'name': '/c/o', 'hash': 'obj-etag', 'bytes': '3'}]))
|
||||
json.dumps([{'name': '/c/o', 'hash': md5hex("obj"),
|
||||
'bytes': '3'}]))
|
||||
|
||||
copy_hook = [None]
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user