py3: port dlo
Change-Id: I7236ddea0acde93d0789ad8affa76df0097a86aa
This commit is contained in:
parent
5090a15f52
commit
893acffbc0
@ -121,14 +121,14 @@ Here's an example using ``curl`` with tiny 1-byte segments::
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
import six
|
import six
|
||||||
from six.moves.urllib.parse import unquote
|
|
||||||
|
|
||||||
from hashlib import md5
|
from hashlib import md5
|
||||||
from swift.common import constraints
|
from swift.common import constraints
|
||||||
from swift.common.exceptions import ListingIterError, SegmentError
|
from swift.common.exceptions import ListingIterError, SegmentError
|
||||||
from swift.common.http import is_success
|
from swift.common.http import is_success
|
||||||
from swift.common.swob import Request, Response, \
|
from swift.common.swob import Request, Response, \
|
||||||
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict
|
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
|
||||||
|
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
|
||||||
from swift.common.utils import get_logger, \
|
from swift.common.utils import get_logger, \
|
||||||
RateLimitedIterator, quote, close_if_possible, closing_if_possible
|
RateLimitedIterator, quote, close_if_possible, closing_if_possible
|
||||||
from swift.common.request_helpers import SegmentedIterable
|
from swift.common.request_helpers import SegmentedIterable
|
||||||
@ -143,9 +143,18 @@ class GetContext(WSGIContext):
|
|||||||
|
|
||||||
def _get_container_listing(self, req, version, account, container,
|
def _get_container_listing(self, req, version, account, container,
|
||||||
prefix, marker=''):
|
prefix, marker=''):
|
||||||
|
'''
|
||||||
|
:param version: whatever
|
||||||
|
:param account: native
|
||||||
|
:param container: native
|
||||||
|
:param prefix: native
|
||||||
|
:param marker: native
|
||||||
|
'''
|
||||||
con_req = make_subrequest(
|
con_req = make_subrequest(
|
||||||
req.environ,
|
req.environ,
|
||||||
path=quote('/'.join(['', version, account, container])),
|
path=wsgi_quote('/'.join([
|
||||||
|
'', str_to_wsgi(version),
|
||||||
|
str_to_wsgi(account), str_to_wsgi(container)])),
|
||||||
method='GET',
|
method='GET',
|
||||||
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
||||||
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
||||||
@ -156,14 +165,24 @@ class GetContext(WSGIContext):
|
|||||||
con_resp = con_req.get_response(self.dlo.app)
|
con_resp = con_req.get_response(self.dlo.app)
|
||||||
if not is_success(con_resp.status_int):
|
if not is_success(con_resp.status_int):
|
||||||
if req.method == 'HEAD':
|
if req.method == 'HEAD':
|
||||||
con_resp.body = ''
|
con_resp.body = b''
|
||||||
return con_resp, None
|
return con_resp, None
|
||||||
with closing_if_possible(con_resp.app_iter):
|
with closing_if_possible(con_resp.app_iter):
|
||||||
return None, json.loads(''.join(con_resp.app_iter))
|
return None, json.loads(b''.join(con_resp.app_iter))
|
||||||
|
|
||||||
def _segment_listing_iterator(self, req, version, account, container,
|
def _segment_listing_iterator(self, req, version, account, container,
|
||||||
prefix, segments, first_byte=None,
|
prefix, segments, first_byte=None,
|
||||||
last_byte=None):
|
last_byte=None):
|
||||||
|
'''
|
||||||
|
:param req: upstream request
|
||||||
|
:param version: native
|
||||||
|
:param account: native
|
||||||
|
:param container: native
|
||||||
|
:param prefix: native
|
||||||
|
:param segments: array of dicts, with native strings
|
||||||
|
:param first_byte: number
|
||||||
|
:param last_byte: number
|
||||||
|
'''
|
||||||
# It's sort of hokey that this thing takes in the first page of
|
# It's sort of hokey that this thing takes in the first page of
|
||||||
# segments as an argument, but we need to compute the etag and content
|
# segments as an argument, but we need to compute the etag and content
|
||||||
# length from the first page, and it's better to have a hokey
|
# length from the first page, and it's better to have a hokey
|
||||||
@ -173,7 +192,6 @@ class GetContext(WSGIContext):
|
|||||||
if last_byte is None:
|
if last_byte is None:
|
||||||
last_byte = float("inf")
|
last_byte = float("inf")
|
||||||
|
|
||||||
marker = ''
|
|
||||||
while True:
|
while True:
|
||||||
for segment in segments:
|
for segment in segments:
|
||||||
seg_length = int(segment['bytes'])
|
seg_length = int(segment['bytes'])
|
||||||
@ -188,7 +206,7 @@ class GetContext(WSGIContext):
|
|||||||
break
|
break
|
||||||
|
|
||||||
seg_name = segment['name']
|
seg_name = segment['name']
|
||||||
if isinstance(seg_name, six.text_type):
|
if six.PY2:
|
||||||
seg_name = seg_name.encode("utf-8")
|
seg_name = seg_name.encode("utf-8")
|
||||||
|
|
||||||
# We deliberately omit the etag and size here;
|
# We deliberately omit the etag and size here;
|
||||||
@ -227,16 +245,18 @@ class GetContext(WSGIContext):
|
|||||||
"Got status %d listing container /%s/%s" %
|
"Got status %d listing container /%s/%s" %
|
||||||
(error_response.status_int, account, container))
|
(error_response.status_int, account, container))
|
||||||
|
|
||||||
def get_or_head_response(self, req, x_object_manifest,
|
def get_or_head_response(self, req, x_object_manifest):
|
||||||
response_headers=None):
|
'''
|
||||||
if response_headers is None:
|
:param req: user's request
|
||||||
response_headers = self._response_headers
|
:param x_object_manifest: as unquoted, native string
|
||||||
|
'''
|
||||||
|
response_headers = self._response_headers
|
||||||
|
|
||||||
container, obj_prefix = x_object_manifest.split('/', 1)
|
container, obj_prefix = x_object_manifest.split('/', 1)
|
||||||
container = unquote(container)
|
|
||||||
obj_prefix = unquote(obj_prefix)
|
|
||||||
|
|
||||||
version, account, _junk = req.split_path(2, 3, True)
|
version, account, _junk = req.split_path(2, 3, True)
|
||||||
|
version = wsgi_to_str(version)
|
||||||
|
account = wsgi_to_str(account)
|
||||||
error_response, segments = self._get_container_listing(
|
error_response, segments = self._get_container_listing(
|
||||||
req, version, account, container, obj_prefix)
|
req, version, account, container, obj_prefix)
|
||||||
if error_response:
|
if error_response:
|
||||||
@ -311,7 +331,7 @@ class GetContext(WSGIContext):
|
|||||||
if h.lower() != "etag"]
|
if h.lower() != "etag"]
|
||||||
etag = md5()
|
etag = md5()
|
||||||
for seg_dict in segments:
|
for seg_dict in segments:
|
||||||
etag.update(seg_dict['hash'].strip('"'))
|
etag.update(seg_dict['hash'].strip('"').encode('utf8'))
|
||||||
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
|
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
|
||||||
|
|
||||||
app_iter = None
|
app_iter = None
|
||||||
@ -353,7 +373,8 @@ class GetContext(WSGIContext):
|
|||||||
for header, value in self._response_headers:
|
for header, value in self._response_headers:
|
||||||
if (header.lower() == 'x-object-manifest'):
|
if (header.lower() == 'x-object-manifest'):
|
||||||
close_if_possible(resp_iter)
|
close_if_possible(resp_iter)
|
||||||
response = self.get_or_head_response(req, value)
|
response = self.get_or_head_response(
|
||||||
|
req, wsgi_to_str(wsgi_unquote(value)))
|
||||||
return response(req.environ, start_response)
|
return response(req.environ, start_response)
|
||||||
# Not a dynamic large object manifest; just pass it through.
|
# Not a dynamic large object manifest; just pass it through.
|
||||||
start_response(self._response_status,
|
start_response(self._response_status,
|
||||||
|
@ -23,6 +23,8 @@ from textwrap import dedent
|
|||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
from swift.common import swob
|
from swift.common import swob
|
||||||
from swift.common.header_key_dict import HeaderKeyDict
|
from swift.common.header_key_dict import HeaderKeyDict
|
||||||
from swift.common.middleware import dlo
|
from swift.common.middleware import dlo
|
||||||
@ -34,6 +36,8 @@ LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
|
|||||||
|
|
||||||
|
|
||||||
def md5hex(s):
|
def md5hex(s):
|
||||||
|
if isinstance(s, six.text_type):
|
||||||
|
s = s.encode('utf-8')
|
||||||
return hashlib.md5(s).hexdigest()
|
return hashlib.md5(s).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
@ -52,7 +56,7 @@ class DloTestCase(unittest.TestCase):
|
|||||||
headers[0] = h
|
headers[0] = h
|
||||||
|
|
||||||
body_iter = app(req.environ, start_response)
|
body_iter = app(req.environ, start_response)
|
||||||
body = ''
|
body = b''
|
||||||
# appease the close-checker
|
# appease the close-checker
|
||||||
with closing_if_possible(body_iter):
|
with closing_if_possible(body_iter):
|
||||||
for chunk in body_iter:
|
for chunk in body_iter:
|
||||||
@ -69,36 +73,36 @@ class DloTestCase(unittest.TestCase):
|
|||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_01',
|
'GET', '/v1/AUTH_test/c/seg_01',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
|
||||||
'aaaaa')
|
b'aaaaa')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_02',
|
'GET', '/v1/AUTH_test/c/seg_02',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
|
||||||
'bbbbb')
|
b'bbbbb')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_03',
|
'GET', '/v1/AUTH_test/c/seg_03',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
|
||||||
'ccccc')
|
b'ccccc')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_04',
|
'GET', '/v1/AUTH_test/c/seg_04',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
|
||||||
'ddddd')
|
b'ddddd')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/seg_05',
|
'GET', '/v1/AUTH_test/c/seg_05',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
|
||||||
'eeeee')
|
b'eeeee')
|
||||||
|
|
||||||
# an unrelated object (not seg*) to test the prefix matching
|
# an unrelated object (not seg*) to test the prefix matching
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
'GET', '/v1/AUTH_test/c/catpicture.jpg',
|
||||||
swob.HTTPOk, {'Content-Length': '9',
|
swob.HTTPOk, {'Content-Length': '9',
|
||||||
'Etag': md5hex("meow meow meow meow")},
|
'Etag': md5hex("meow meow meow meow")},
|
||||||
'meow meow meow meow')
|
b'meow meow meow meow')
|
||||||
|
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/mancon/manifest',
|
'GET', '/v1/AUTH_test/mancon/manifest',
|
||||||
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
|
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
|
||||||
'X-Object-Manifest': 'c/seg'},
|
'X-Object-Manifest': 'c/seg'},
|
||||||
'manifest-contents')
|
b'manifest-contents')
|
||||||
|
|
||||||
lm = '2013-11-22T02:42:13.781760'
|
lm = '2013-11-22T02:42:13.781760'
|
||||||
ct = 'application/octet-stream'
|
ct = 'application/octet-stream'
|
||||||
@ -120,11 +124,11 @@ class DloTestCase(unittest.TestCase):
|
|||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c',
|
'GET', '/v1/AUTH_test/c',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(full_container_listing))
|
json.dumps(full_container_listing).encode('ascii'))
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg',
|
'GET', '/v1/AUTH_test/c?prefix=seg',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs))
|
json.dumps(segs).encode('ascii'))
|
||||||
|
|
||||||
# This is to let us test multi-page container listings; we use the
|
# This is to let us test multi-page container listings; we use the
|
||||||
# trailing underscore to send small (pagesize=3) listings.
|
# trailing underscore to send small (pagesize=3) listings.
|
||||||
@ -135,26 +139,26 @@ class DloTestCase(unittest.TestCase):
|
|||||||
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
|
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
|
||||||
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
|
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
|
||||||
'X-Object-Manifest': 'c/seg_'},
|
'X-Object-Manifest': 'c/seg_'},
|
||||||
'manyseg')
|
b'manyseg')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg_',
|
'GET', '/v1/AUTH_test/c?prefix=seg_',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs[:3]))
|
json.dumps(segs[:3]).encode('ascii'))
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
|
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps(segs[3:]))
|
json.dumps(segs[3:]).encode('ascii'))
|
||||||
|
|
||||||
# Here's a manifest with 0 segments
|
# Here's a manifest with 0 segments
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
|
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
|
||||||
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
|
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
|
||||||
'X-Object-Manifest': 'c/noseg_'},
|
'X-Object-Manifest': 'c/noseg_'},
|
||||||
'noseg')
|
b'noseg')
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=noseg_',
|
'GET', '/v1/AUTH_test/c?prefix=noseg_',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
|
||||||
json.dumps([]))
|
json.dumps([]).encode('ascii'))
|
||||||
|
|
||||||
|
|
||||||
class TestDloPutManifest(DloTestCase):
|
class TestDloPutManifest(DloTestCase):
|
||||||
@ -284,7 +288,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"], expected_etag)
|
self.assertEqual(headers["Etag"], expected_etag)
|
||||||
self.assertEqual(headers["Content-Length"], "25")
|
self.assertEqual(headers["Content-Length"], "25")
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
for _, _, hdrs in self.app.calls_with_headers[1:]:
|
for _, _, hdrs in self.app.calls_with_headers[1:]:
|
||||||
ua = hdrs.get("User-Agent", "")
|
ua = hdrs.get("User-Agent", "")
|
||||||
@ -302,7 +306,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
|
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
|
||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(body, "meow meow meow meow")
|
self.assertEqual(body, b"meow meow meow meow")
|
||||||
|
|
||||||
def test_get_non_object_passthrough(self):
|
def test_get_non_object_passthrough(self):
|
||||||
self.app.register('GET', '/info', swob.HTTPOk,
|
self.app.register('GET', '/info', swob.HTTPOk,
|
||||||
@ -311,7 +315,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(body, 'useful stuff here')
|
self.assertEqual(body, b'useful stuff here')
|
||||||
self.assertEqual(self.app.call_count, 1)
|
self.assertEqual(self.app.call_count, 1)
|
||||||
|
|
||||||
def test_get_manifest_passthrough(self):
|
def test_get_manifest_passthrough(self):
|
||||||
@ -328,7 +332,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"], "manifest-etag")
|
self.assertEqual(headers["Etag"], "manifest-etag")
|
||||||
self.assertEqual(body, "manifest-contents")
|
self.assertEqual(body, b'manifest-contents')
|
||||||
|
|
||||||
def test_error_passthrough(self):
|
def test_error_passthrough(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
@ -347,7 +351,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "10")
|
self.assertEqual(headers["Content-Length"], "10")
|
||||||
self.assertEqual(body, "bbcccccddd")
|
self.assertEqual(body, b'bbcccccddd')
|
||||||
expected_etag = '"%s"' % md5hex(
|
expected_etag = '"%s"' % md5hex(
|
||||||
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
|
||||||
md5hex("ddddd") + md5hex("eeeee"))
|
md5hex("ddddd") + md5hex("eeeee"))
|
||||||
@ -361,7 +365,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "10")
|
self.assertEqual(headers["Content-Length"], "10")
|
||||||
self.assertEqual(body, "cccccddddd")
|
self.assertEqual(body, b'cccccddddd')
|
||||||
|
|
||||||
def test_get_range_first_byte(self):
|
def test_get_range_first_byte(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -371,7 +375,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "1")
|
self.assertEqual(headers["Content-Length"], "1")
|
||||||
self.assertEqual(body, "a")
|
self.assertEqual(body, b'a')
|
||||||
|
|
||||||
def test_get_range_last_byte(self):
|
def test_get_range_last_byte(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -381,7 +385,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "1")
|
self.assertEqual(headers["Content-Length"], "1")
|
||||||
self.assertEqual(body, "e")
|
self.assertEqual(body, b'e')
|
||||||
|
|
||||||
def test_get_range_overlapping_end(self):
|
def test_get_range_overlapping_end(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -392,7 +396,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "7")
|
self.assertEqual(headers["Content-Length"], "7")
|
||||||
self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
|
self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
|
||||||
self.assertEqual(body, "ddeeeee")
|
self.assertEqual(body, b'ddeeeee')
|
||||||
|
|
||||||
def test_get_range_unsatisfiable(self):
|
def test_get_range_unsatisfiable(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -428,7 +432,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
#
|
#
|
||||||
# Since the truth is forbidden, we lie.
|
# Since the truth is forbidden, we lie.
|
||||||
self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
|
self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
|
||||||
self.assertEqual(body, "aabbbbbccc")
|
self.assertEqual(body, b"aabbbbbccc")
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.app.calls,
|
self.app.calls,
|
||||||
@ -449,7 +453,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
# this requires multiple pages of container listing, so we can't send
|
# this requires multiple pages of container listing, so we can't send
|
||||||
# a Content-Length header
|
# a Content-Length header
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_suffix_range(self):
|
def test_get_suffix_range(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -459,7 +463,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(status, "206 Partial Content")
|
self.assertEqual(status, "206 Partial Content")
|
||||||
self.assertEqual(headers["Content-Length"], "25")
|
self.assertEqual(headers["Content-Length"], "25")
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_suffix_range_many_segments(self):
|
def test_get_suffix_range_many_segments(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
|
||||||
@ -471,7 +475,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertIsNone(headers.get("Content-Range"))
|
self.assertIsNone(headers.get("Content-Range"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
|
||||||
|
|
||||||
def test_get_multi_range(self):
|
def test_get_multi_range(self):
|
||||||
# DLO doesn't support multi-range GETs. The way that you express that
|
# DLO doesn't support multi-range GETs. The way that you express that
|
||||||
@ -485,7 +489,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertIsNone(headers.get("Content-Length"))
|
self.assertIsNone(headers.get("Content-Length"))
|
||||||
self.assertIsNone(headers.get("Content-Range"))
|
self.assertIsNone(headers.get("Content-Range"))
|
||||||
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_if_match_matches(self):
|
def test_if_match_matches(self):
|
||||||
manifest_etag = '"%s"' % md5hex(
|
manifest_etag = '"%s"' % md5hex(
|
||||||
@ -500,7 +504,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(headers['Content-Length'], '25')
|
self.assertEqual(headers['Content-Length'], '25')
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_if_match_does_not_match(self):
|
def test_if_match_does_not_match(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -512,7 +516,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '412 Precondition Failed')
|
self.assertEqual(status, '412 Precondition Failed')
|
||||||
self.assertEqual(headers['Content-Length'], '0')
|
self.assertEqual(headers['Content-Length'], '0')
|
||||||
self.assertEqual(body, '')
|
self.assertEqual(body, b'')
|
||||||
|
|
||||||
def test_if_none_match_matches(self):
|
def test_if_none_match_matches(self):
|
||||||
manifest_etag = '"%s"' % md5hex(
|
manifest_etag = '"%s"' % md5hex(
|
||||||
@ -527,7 +531,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '304 Not Modified')
|
self.assertEqual(status, '304 Not Modified')
|
||||||
self.assertEqual(headers['Content-Length'], '0')
|
self.assertEqual(headers['Content-Length'], '0')
|
||||||
self.assertEqual(body, '')
|
self.assertEqual(body, b'')
|
||||||
|
|
||||||
def test_if_none_match_does_not_match(self):
|
def test_if_none_match_does_not_match(self):
|
||||||
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
|
||||||
@ -539,7 +543,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(headers['Content-Length'], '25')
|
self.assertEqual(headers['Content-Length'], '25')
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
|
||||||
|
|
||||||
def test_get_with_if_modified_since(self):
|
def test_get_with_if_modified_since(self):
|
||||||
# It's important not to pass the If-[Un]Modified-Since header to the
|
# It's important not to pass the If-[Un]Modified-Since header to the
|
||||||
@ -581,7 +585,11 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
|
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(''.join(body), "aaaaa") # first segment made it out
|
# first segment made it out
|
||||||
|
if six.PY2:
|
||||||
|
self.assertEqual(''.join(body), "aaaaa")
|
||||||
|
else:
|
||||||
|
self.assertEqual(body, b'aaaaa')
|
||||||
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
|
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
|
||||||
'While processing manifest /v1/AUTH_test/mancon/manifest, '
|
'While processing manifest /v1/AUTH_test/mancon/manifest, '
|
||||||
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
|
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
|
||||||
@ -610,7 +618,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
with mock.patch(LIMIT, 3):
|
with mock.patch(LIMIT, 3):
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(body, "aaaaabbbbbccccc")
|
self.assertEqual(body, b'aaaaabbbbbccccc')
|
||||||
|
|
||||||
def test_error_listing_container_HEAD(self):
|
def test_error_listing_container_HEAD(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
@ -639,7 +647,11 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
|
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
|
if six.PY2:
|
||||||
|
# stop after error
|
||||||
|
self.assertEqual(''.join(body), "aaaaabbWRONGbb")
|
||||||
|
else:
|
||||||
|
self.assertEqual(body, b"aaaaabbWRONGbb")
|
||||||
|
|
||||||
def test_etag_comparison_ignores_quotes(self):
|
def test_etag_comparison_ignores_quotes(self):
|
||||||
# a little future-proofing here in case we ever fix this in swob
|
# a little future-proofing here in case we ever fix this in swob
|
||||||
@ -662,7 +674,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
headers = HeaderKeyDict(headers)
|
headers = HeaderKeyDict(headers)
|
||||||
self.assertEqual(headers["Etag"],
|
self.assertEqual(headers["Etag"],
|
||||||
'"' + hashlib.md5("abcdef").hexdigest() + '"')
|
'"' + hashlib.md5(b"abcdef").hexdigest() + '"')
|
||||||
|
|
||||||
def test_object_prefix_quoting(self):
|
def test_object_prefix_quoting(self):
|
||||||
self.app.register(
|
self.app.register(
|
||||||
@ -675,22 +687,26 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
|
'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
|
||||||
swob.HTTPOk, {'Content-Type': 'application/json'},
|
swob.HTTPOk, {'Content-Type': 'application/json'},
|
||||||
json.dumps(segs))
|
json.dumps(segs).encode('ascii'))
|
||||||
|
|
||||||
|
if six.PY2:
|
||||||
|
path = b'/v1/AUTH_test/c/\xC3\xa9'
|
||||||
|
else:
|
||||||
|
path = u'/v1/AUTH_test/c/\xc3\xa9'
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/\xC3\xa91',
|
'GET', path + '1',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
|
||||||
"AAAAA")
|
b"AAAAA")
|
||||||
self.app.register(
|
self.app.register(
|
||||||
'GET', '/v1/AUTH_test/c/\xC3\xA92',
|
'GET', path + '2',
|
||||||
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
|
||||||
"BBBBB")
|
b"BBBBB")
|
||||||
|
|
||||||
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
req = swob.Request.blank('/v1/AUTH_test/man/accent',
|
||||||
environ={'REQUEST_METHOD': 'GET'})
|
environ={'REQUEST_METHOD': 'GET'})
|
||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
self.assertEqual(status, "200 OK")
|
self.assertEqual(status, "200 OK")
|
||||||
self.assertEqual(body, "AAAAABBBBB")
|
self.assertEqual(body, b'AAAAABBBBB')
|
||||||
|
|
||||||
def test_get_taking_too_long(self):
|
def test_get_taking_too_long(self):
|
||||||
the_time = [time.time()]
|
the_time = [time.time()]
|
||||||
@ -715,7 +731,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
status, headers, body = self.call_dlo(req)
|
status, headers, body = self.call_dlo(req)
|
||||||
|
|
||||||
self.assertEqual(status, '200 OK')
|
self.assertEqual(status, '200 OK')
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccc')
|
self.assertEqual(body, b'aaaaabbbbbccccc')
|
||||||
|
|
||||||
def test_get_oversize_segment(self):
|
def test_get_oversize_segment(self):
|
||||||
# If we send a Content-Length header to the client, it's based on the
|
# If we send a Content-Length header to the client, it's based on the
|
||||||
@ -738,7 +754,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '200 OK') # sanity check
|
self.assertEqual(status, '200 OK') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccccccccccccc')
|
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.app.calls,
|
self.app.calls,
|
||||||
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
[('GET', '/v1/AUTH_test/mancon/manifest'),
|
||||||
@ -770,7 +786,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '200 OK') # sanity check
|
self.assertEqual(status, '200 OK') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
|
self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
|
||||||
|
|
||||||
def test_get_undersize_segment_range(self):
|
def test_get_undersize_segment_range(self):
|
||||||
# Shrink it by a single byte
|
# Shrink it by a single byte
|
||||||
@ -788,7 +804,7 @@ class TestDloGetManifest(DloTestCase):
|
|||||||
|
|
||||||
self.assertEqual(status, '206 Partial Content') # sanity check
|
self.assertEqual(status, '206 Partial Content') # sanity check
|
||||||
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
|
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
|
||||||
self.assertEqual(body, 'aaaaabbbbbcccc')
|
self.assertEqual(body, b'aaaaabbbbbcccc')
|
||||||
|
|
||||||
def test_get_with_auth_overridden(self):
|
def test_get_with_auth_overridden(self):
|
||||||
auth_got_called = [0]
|
auth_got_called = [0]
|
||||||
@ -840,7 +856,7 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
max_get_time = 2900
|
max_get_time = 2900
|
||||||
""")
|
""")
|
||||||
|
|
||||||
conffile = tempfile.NamedTemporaryFile()
|
conffile = tempfile.NamedTemporaryFile(mode='w')
|
||||||
conffile.write(proxy_conf)
|
conffile.write(proxy_conf)
|
||||||
conffile.flush()
|
conffile.flush()
|
||||||
|
|
||||||
@ -853,6 +869,8 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
self.assertEqual(10, mware.rate_limit_after_segment)
|
self.assertEqual(10, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(3600, mware.max_get_time)
|
self.assertEqual(3600, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile.close()
|
||||||
|
|
||||||
def test_finding_defaults_from_file(self):
|
def test_finding_defaults_from_file(self):
|
||||||
# If DLO has no config vars, go pull them from the proxy server's
|
# If DLO has no config vars, go pull them from the proxy server's
|
||||||
# config section
|
# config section
|
||||||
@ -875,7 +893,7 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
set max_get_time = 2900
|
set max_get_time = 2900
|
||||||
""")
|
""")
|
||||||
|
|
||||||
conffile = tempfile.NamedTemporaryFile()
|
conffile = tempfile.NamedTemporaryFile(mode='w')
|
||||||
conffile.write(proxy_conf)
|
conffile.write(proxy_conf)
|
||||||
conffile.flush()
|
conffile.flush()
|
||||||
|
|
||||||
@ -887,6 +905,8 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
self.assertEqual(13, mware.rate_limit_after_segment)
|
self.assertEqual(13, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(2900, mware.max_get_time)
|
self.assertEqual(2900, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile.close()
|
||||||
|
|
||||||
def test_finding_defaults_from_dir(self):
|
def test_finding_defaults_from_dir(self):
|
||||||
# If DLO has no config vars, go pull them from the proxy server's
|
# If DLO has no config vars, go pull them from the proxy server's
|
||||||
# config section
|
# config section
|
||||||
@ -913,11 +933,13 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
|
|
||||||
conf_dir = self.tmpdir
|
conf_dir = self.tmpdir
|
||||||
|
|
||||||
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
|
conffile1 = tempfile.NamedTemporaryFile(mode='w',
|
||||||
|
dir=conf_dir, suffix='.conf')
|
||||||
conffile1.write(proxy_conf1)
|
conffile1.write(proxy_conf1)
|
||||||
conffile1.flush()
|
conffile1.flush()
|
||||||
|
|
||||||
conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
|
conffile2 = tempfile.NamedTemporaryFile(mode='w',
|
||||||
|
dir=conf_dir, suffix='.conf')
|
||||||
conffile2.write(proxy_conf2)
|
conffile2.write(proxy_conf2)
|
||||||
conffile2.flush()
|
conffile2.flush()
|
||||||
|
|
||||||
@ -929,6 +951,9 @@ class TestDloConfiguration(unittest.TestCase):
|
|||||||
self.assertEqual(13, mware.rate_limit_after_segment)
|
self.assertEqual(13, mware.rate_limit_after_segment)
|
||||||
self.assertEqual(2900, mware.max_get_time)
|
self.assertEqual(2900, mware.max_get_time)
|
||||||
|
|
||||||
|
conffile1.close()
|
||||||
|
conffile2.close()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
1
tox.ini
1
tox.ini
@ -48,6 +48,7 @@ commands =
|
|||||||
test/unit/common/middleware/test_container_sync.py \
|
test/unit/common/middleware/test_container_sync.py \
|
||||||
test/unit/common/middleware/test_copy.py \
|
test/unit/common/middleware/test_copy.py \
|
||||||
test/unit/common/middleware/test_crossdomain.py \
|
test/unit/common/middleware/test_crossdomain.py \
|
||||||
|
test/unit/common/middleware/test_dlo.py \
|
||||||
test/unit/common/middleware/test_domain_remap.py \
|
test/unit/common/middleware/test_domain_remap.py \
|
||||||
test/unit/common/middleware/test_formpost.py \
|
test/unit/common/middleware/test_formpost.py \
|
||||||
test/unit/common/middleware/test_gatekeeper.py \
|
test/unit/common/middleware/test_gatekeeper.py \
|
||||||
|
Loading…
Reference in New Issue
Block a user