py3: port dlo

Change-Id: I7236ddea0acde93d0789ad8affa76df0097a86aa
This commit is contained in:
Pete Zaitcev 2019-02-20 17:09:08 -06:00
parent 5090a15f52
commit 893acffbc0
3 changed files with 113 additions and 66 deletions

View File

@ -121,14 +121,14 @@ Here's an example using ``curl`` with tiny 1-byte segments::
import json import json
import six import six
from six.moves.urllib.parse import unquote
from hashlib import md5 from hashlib import md5
from swift.common import constraints from swift.common import constraints
from swift.common.exceptions import ListingIterError, SegmentError from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success from swift.common.http import is_success
from swift.common.swob import Request, Response, \ from swift.common.swob import Request, Response, \
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
from swift.common.utils import get_logger, \ from swift.common.utils import get_logger, \
RateLimitedIterator, quote, close_if_possible, closing_if_possible RateLimitedIterator, quote, close_if_possible, closing_if_possible
from swift.common.request_helpers import SegmentedIterable from swift.common.request_helpers import SegmentedIterable
@ -143,9 +143,18 @@ class GetContext(WSGIContext):
def _get_container_listing(self, req, version, account, container, def _get_container_listing(self, req, version, account, container,
prefix, marker=''): prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest( con_req = make_subrequest(
req.environ, req.environ,
path=quote('/'.join(['', version, account, container])), path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET', method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')}, headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
@ -156,14 +165,24 @@ class GetContext(WSGIContext):
con_resp = con_req.get_response(self.dlo.app) con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int): if not is_success(con_resp.status_int):
if req.method == 'HEAD': if req.method == 'HEAD':
con_resp.body = '' con_resp.body = b''
return con_resp, None return con_resp, None
with closing_if_possible(con_resp.app_iter): with closing_if_possible(con_resp.app_iter):
return None, json.loads(''.join(con_resp.app_iter)) return None, json.loads(b''.join(con_resp.app_iter))
def _segment_listing_iterator(self, req, version, account, container, def _segment_listing_iterator(self, req, version, account, container,
prefix, segments, first_byte=None, prefix, segments, first_byte=None,
last_byte=None): last_byte=None):
'''
:param req: upstream request
:param version: native
:param account: native
:param container: native
:param prefix: native
:param segments: array of dicts, with native strings
:param first_byte: number
:param last_byte: number
'''
# It's sort of hokey that this thing takes in the first page of # It's sort of hokey that this thing takes in the first page of
# segments as an argument, but we need to compute the etag and content # segments as an argument, but we need to compute the etag and content
# length from the first page, and it's better to have a hokey # length from the first page, and it's better to have a hokey
@ -173,7 +192,6 @@ class GetContext(WSGIContext):
if last_byte is None: if last_byte is None:
last_byte = float("inf") last_byte = float("inf")
marker = ''
while True: while True:
for segment in segments: for segment in segments:
seg_length = int(segment['bytes']) seg_length = int(segment['bytes'])
@ -188,7 +206,7 @@ class GetContext(WSGIContext):
break break
seg_name = segment['name'] seg_name = segment['name']
if isinstance(seg_name, six.text_type): if six.PY2:
seg_name = seg_name.encode("utf-8") seg_name = seg_name.encode("utf-8")
# We deliberately omit the etag and size here; # We deliberately omit the etag and size here;
@ -227,16 +245,18 @@ class GetContext(WSGIContext):
"Got status %d listing container /%s/%s" % "Got status %d listing container /%s/%s" %
(error_response.status_int, account, container)) (error_response.status_int, account, container))
def get_or_head_response(self, req, x_object_manifest, def get_or_head_response(self, req, x_object_manifest):
response_headers=None): '''
if response_headers is None: :param req: user's request
response_headers = self._response_headers :param x_object_manifest: as unquoted, native string
'''
response_headers = self._response_headers
container, obj_prefix = x_object_manifest.split('/', 1) container, obj_prefix = x_object_manifest.split('/', 1)
container = unquote(container)
obj_prefix = unquote(obj_prefix)
version, account, _junk = req.split_path(2, 3, True) version, account, _junk = req.split_path(2, 3, True)
version = wsgi_to_str(version)
account = wsgi_to_str(account)
error_response, segments = self._get_container_listing( error_response, segments = self._get_container_listing(
req, version, account, container, obj_prefix) req, version, account, container, obj_prefix)
if error_response: if error_response:
@ -311,7 +331,7 @@ class GetContext(WSGIContext):
if h.lower() != "etag"] if h.lower() != "etag"]
etag = md5() etag = md5()
for seg_dict in segments: for seg_dict in segments:
etag.update(seg_dict['hash'].strip('"')) etag.update(seg_dict['hash'].strip('"').encode('utf8'))
response_headers.append(('Etag', '"%s"' % etag.hexdigest())) response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
app_iter = None app_iter = None
@ -353,7 +373,8 @@ class GetContext(WSGIContext):
for header, value in self._response_headers: for header, value in self._response_headers:
if (header.lower() == 'x-object-manifest'): if (header.lower() == 'x-object-manifest'):
close_if_possible(resp_iter) close_if_possible(resp_iter)
response = self.get_or_head_response(req, value) response = self.get_or_head_response(
req, wsgi_to_str(wsgi_unquote(value)))
return response(req.environ, start_response) return response(req.environ, start_response)
# Not a dynamic large object manifest; just pass it through. # Not a dynamic large object manifest; just pass it through.
start_response(self._response_status, start_response(self._response_status,

View File

@ -23,6 +23,8 @@ from textwrap import dedent
import time import time
import unittest import unittest
import six
from swift.common import swob from swift.common import swob
from swift.common.header_key_dict import HeaderKeyDict from swift.common.header_key_dict import HeaderKeyDict
from swift.common.middleware import dlo from swift.common.middleware import dlo
@ -34,6 +36,8 @@ LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
def md5hex(s): def md5hex(s):
if isinstance(s, six.text_type):
s = s.encode('utf-8')
return hashlib.md5(s).hexdigest() return hashlib.md5(s).hexdigest()
@ -52,7 +56,7 @@ class DloTestCase(unittest.TestCase):
headers[0] = h headers[0] = h
body_iter = app(req.environ, start_response) body_iter = app(req.environ, start_response)
body = '' body = b''
# appease the close-checker # appease the close-checker
with closing_if_possible(body_iter): with closing_if_possible(body_iter):
for chunk in body_iter: for chunk in body_iter:
@ -69,36 +73,36 @@ class DloTestCase(unittest.TestCase):
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/seg_01', 'GET', '/v1/AUTH_test/c/seg_01',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
'aaaaa') b'aaaaa')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/seg_02', 'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
'bbbbb') b'bbbbb')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/seg_03', 'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
'ccccc') b'ccccc')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/seg_04', 'GET', '/v1/AUTH_test/c/seg_04',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
'ddddd') b'ddddd')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/seg_05', 'GET', '/v1/AUTH_test/c/seg_05',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
'eeeee') b'eeeee')
# an unrelated object (not seg*) to test the prefix matching # an unrelated object (not seg*) to test the prefix matching
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/catpicture.jpg', 'GET', '/v1/AUTH_test/c/catpicture.jpg',
swob.HTTPOk, {'Content-Length': '9', swob.HTTPOk, {'Content-Length': '9',
'Etag': md5hex("meow meow meow meow")}, 'Etag': md5hex("meow meow meow meow")},
'meow meow meow meow') b'meow meow meow meow')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest', 'GET', '/v1/AUTH_test/mancon/manifest',
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'}, 'X-Object-Manifest': 'c/seg'},
'manifest-contents') b'manifest-contents')
lm = '2013-11-22T02:42:13.781760' lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream' ct = 'application/octet-stream'
@ -120,11 +124,11 @@ class DloTestCase(unittest.TestCase):
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c', 'GET', '/v1/AUTH_test/c',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing)) json.dumps(full_container_listing).encode('ascii'))
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg', 'GET', '/v1/AUTH_test/c?prefix=seg',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs)) json.dumps(segs).encode('ascii'))
# This is to let us test multi-page container listings; we use the # This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings. # trailing underscore to send small (pagesize=3) listings.
@ -135,26 +139,26 @@ class DloTestCase(unittest.TestCase):
'GET', '/v1/AUTH_test/mancon/manifest-many-segments', 'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'}, 'X-Object-Manifest': 'c/seg_'},
'manyseg') b'manyseg')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg_', 'GET', '/v1/AUTH_test/c?prefix=seg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3])) json.dumps(segs[:3]).encode('ascii'))
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03', 'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:])) json.dumps(segs[3:]).encode('ascii'))
# Here's a manifest with 0 segments # Here's a manifest with 0 segments
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments', 'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'}, 'X-Object-Manifest': 'c/noseg_'},
'noseg') b'noseg')
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c?prefix=noseg_', 'GET', '/v1/AUTH_test/c?prefix=noseg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps([])) json.dumps([]).encode('ascii'))
class TestDloPutManifest(DloTestCase): class TestDloPutManifest(DloTestCase):
@ -284,7 +288,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Etag"], expected_etag)
self.assertEqual(headers["Content-Length"], "25") self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
for _, _, hdrs in self.app.calls_with_headers[1:]: for _, _, hdrs in self.app.calls_with_headers[1:]:
ua = hdrs.get("User-Agent", "") ua = hdrs.get("User-Agent", "")
@ -302,7 +306,7 @@ class TestDloGetManifest(DloTestCase):
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg', req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
environ={'REQUEST_METHOD': 'GET'}) environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
self.assertEqual(body, "meow meow meow meow") self.assertEqual(body, b"meow meow meow meow")
def test_get_non_object_passthrough(self): def test_get_non_object_passthrough(self):
self.app.register('GET', '/info', swob.HTTPOk, self.app.register('GET', '/info', swob.HTTPOk,
@ -311,7 +315,7 @@ class TestDloGetManifest(DloTestCase):
environ={'REQUEST_METHOD': 'GET'}) environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
self.assertEqual(status, '200 OK') self.assertEqual(status, '200 OK')
self.assertEqual(body, 'useful stuff here') self.assertEqual(body, b'useful stuff here')
self.assertEqual(self.app.call_count, 1) self.assertEqual(self.app.call_count, 1)
def test_get_manifest_passthrough(self): def test_get_manifest_passthrough(self):
@ -328,7 +332,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], "manifest-etag") self.assertEqual(headers["Etag"], "manifest-etag")
self.assertEqual(body, "manifest-contents") self.assertEqual(body, b'manifest-contents')
def test_error_passthrough(self): def test_error_passthrough(self):
self.app.register( self.app.register(
@ -347,7 +351,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10") self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, "bbcccccddd") self.assertEqual(body, b'bbcccccddd')
expected_etag = '"%s"' % md5hex( expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee")) md5hex("ddddd") + md5hex("eeeee"))
@ -361,7 +365,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10") self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, "cccccddddd") self.assertEqual(body, b'cccccddddd')
def test_get_range_first_byte(self): def test_get_range_first_byte(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -371,7 +375,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "1") self.assertEqual(headers["Content-Length"], "1")
self.assertEqual(body, "a") self.assertEqual(body, b'a')
def test_get_range_last_byte(self): def test_get_range_last_byte(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -381,7 +385,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "1") self.assertEqual(headers["Content-Length"], "1")
self.assertEqual(body, "e") self.assertEqual(body, b'e')
def test_get_range_overlapping_end(self): def test_get_range_overlapping_end(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -392,7 +396,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "7") self.assertEqual(headers["Content-Length"], "7")
self.assertEqual(headers["Content-Range"], "bytes 18-24/25") self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
self.assertEqual(body, "ddeeeee") self.assertEqual(body, b'ddeeeee')
def test_get_range_unsatisfiable(self): def test_get_range_unsatisfiable(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -428,7 +432,7 @@ class TestDloGetManifest(DloTestCase):
# #
# Since the truth is forbidden, we lie. # Since the truth is forbidden, we lie.
self.assertEqual(headers["Content-Range"], "bytes 3-12/15") self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
self.assertEqual(body, "aabbbbbccc") self.assertEqual(body, b"aabbbbbccc")
self.assertEqual( self.assertEqual(
self.app.calls, self.app.calls,
@ -449,7 +453,7 @@ class TestDloGetManifest(DloTestCase):
# this requires multiple pages of container listing, so we can't send # this requires multiple pages of container listing, so we can't send
# a Content-Length header # a Content-Length header
self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_suffix_range(self): def test_get_suffix_range(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -459,7 +463,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content") self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "25") self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_suffix_range_many_segments(self): def test_get_suffix_range_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
@ -471,7 +475,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range")) self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_multi_range(self): def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that # DLO doesn't support multi-range GETs. The way that you express that
@ -485,7 +489,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range")) self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_if_match_matches(self): def test_if_match_matches(self):
manifest_etag = '"%s"' % md5hex( manifest_etag = '"%s"' % md5hex(
@ -500,7 +504,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '25') self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_if_match_does_not_match(self): def test_if_match_does_not_match(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -512,7 +516,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '412 Precondition Failed') self.assertEqual(status, '412 Precondition Failed')
self.assertEqual(headers['Content-Length'], '0') self.assertEqual(headers['Content-Length'], '0')
self.assertEqual(body, '') self.assertEqual(body, b'')
def test_if_none_match_matches(self): def test_if_none_match_matches(self):
manifest_etag = '"%s"' % md5hex( manifest_etag = '"%s"' % md5hex(
@ -527,7 +531,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '304 Not Modified') self.assertEqual(status, '304 Not Modified')
self.assertEqual(headers['Content-Length'], '0') self.assertEqual(headers['Content-Length'], '0')
self.assertEqual(body, '') self.assertEqual(body, b'')
def test_if_none_match_does_not_match(self): def test_if_none_match_does_not_match(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -539,7 +543,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '25') self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_get_with_if_modified_since(self): def test_get_with_if_modified_since(self):
# It's important not to pass the If-[Un]Modified-Since header to the # It's important not to pass the If-[Un]Modified-Since header to the
@ -581,7 +585,11 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaa") # first segment made it out # first segment made it out
if six.PY2:
self.assertEqual(''.join(body), "aaaaa")
else:
self.assertEqual(body, b'aaaaa')
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [ self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
'While processing manifest /v1/AUTH_test/mancon/manifest, ' 'While processing manifest /v1/AUTH_test/mancon/manifest, '
'got 403 while retrieving /v1/AUTH_test/c/seg_02', 'got 403 while retrieving /v1/AUTH_test/c/seg_02',
@ -610,7 +618,7 @@ class TestDloGetManifest(DloTestCase):
with mock.patch(LIMIT, 3): with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertEqual(body, "aaaaabbbbbccccc") self.assertEqual(body, b'aaaaabbbbbccccc')
def test_error_listing_container_HEAD(self): def test_error_listing_container_HEAD(self):
self.app.register( self.app.register(
@ -639,7 +647,11 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error if six.PY2:
# stop after error
self.assertEqual(''.join(body), "aaaaabbWRONGbb")
else:
self.assertEqual(body, b"aaaaabbWRONGbb")
def test_etag_comparison_ignores_quotes(self): def test_etag_comparison_ignores_quotes(self):
# a little future-proofing here in case we ever fix this in swob # a little future-proofing here in case we ever fix this in swob
@ -662,7 +674,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers) headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], self.assertEqual(headers["Etag"],
'"' + hashlib.md5("abcdef").hexdigest() + '"') '"' + hashlib.md5(b"abcdef").hexdigest() + '"')
def test_object_prefix_quoting(self): def test_object_prefix_quoting(self):
self.app.register( self.app.register(
@ -675,22 +687,26 @@ class TestDloGetManifest(DloTestCase):
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c?prefix=%C3%A9', 'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
swob.HTTPOk, {'Content-Type': 'application/json'}, swob.HTTPOk, {'Content-Type': 'application/json'},
json.dumps(segs)) json.dumps(segs).encode('ascii'))
if six.PY2:
path = b'/v1/AUTH_test/c/\xC3\xa9'
else:
path = u'/v1/AUTH_test/c/\xc3\xa9'
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/\xC3\xa91', 'GET', path + '1',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
"AAAAA") b"AAAAA")
self.app.register( self.app.register(
'GET', '/v1/AUTH_test/c/\xC3\xA92', 'GET', path + '2',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")}, swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
"BBBBB") b"BBBBB")
req = swob.Request.blank('/v1/AUTH_test/man/accent', req = swob.Request.blank('/v1/AUTH_test/man/accent',
environ={'REQUEST_METHOD': 'GET'}) environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
self.assertEqual(status, "200 OK") self.assertEqual(status, "200 OK")
self.assertEqual(body, "AAAAABBBBB") self.assertEqual(body, b'AAAAABBBBB')
def test_get_taking_too_long(self): def test_get_taking_too_long(self):
the_time = [time.time()] the_time = [time.time()]
@ -715,7 +731,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req) status, headers, body = self.call_dlo(req)
self.assertEqual(status, '200 OK') self.assertEqual(status, '200 OK')
self.assertEqual(body, 'aaaaabbbbbccccc') self.assertEqual(body, b'aaaaabbbbbccccc')
def test_get_oversize_segment(self): def test_get_oversize_segment(self):
# If we send a Content-Length header to the client, it's based on the # If we send a Content-Length header to the client, it's based on the
@ -738,7 +754,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') # sanity check self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccccccccccccc') self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
self.assertEqual( self.assertEqual(
self.app.calls, self.app.calls,
[('GET', '/v1/AUTH_test/mancon/manifest'), [('GET', '/v1/AUTH_test/mancon/manifest'),
@ -770,7 +786,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') # sanity check self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee') self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
def test_get_undersize_segment_range(self): def test_get_undersize_segment_range(self):
# Shrink it by a single byte # Shrink it by a single byte
@ -788,7 +804,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '206 Partial Content') # sanity check self.assertEqual(status, '206 Partial Content') # sanity check
self.assertEqual(headers.get('Content-Length'), '15') # sanity check self.assertEqual(headers.get('Content-Length'), '15') # sanity check
self.assertEqual(body, 'aaaaabbbbbcccc') self.assertEqual(body, b'aaaaabbbbbcccc')
def test_get_with_auth_overridden(self): def test_get_with_auth_overridden(self):
auth_got_called = [0] auth_got_called = [0]
@ -840,7 +856,7 @@ class TestDloConfiguration(unittest.TestCase):
max_get_time = 2900 max_get_time = 2900
""") """)
conffile = tempfile.NamedTemporaryFile() conffile = tempfile.NamedTemporaryFile(mode='w')
conffile.write(proxy_conf) conffile.write(proxy_conf)
conffile.flush() conffile.flush()
@ -853,6 +869,8 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(10, mware.rate_limit_after_segment) self.assertEqual(10, mware.rate_limit_after_segment)
self.assertEqual(3600, mware.max_get_time) self.assertEqual(3600, mware.max_get_time)
conffile.close()
def test_finding_defaults_from_file(self): def test_finding_defaults_from_file(self):
# If DLO has no config vars, go pull them from the proxy server's # If DLO has no config vars, go pull them from the proxy server's
# config section # config section
@ -875,7 +893,7 @@ class TestDloConfiguration(unittest.TestCase):
set max_get_time = 2900 set max_get_time = 2900
""") """)
conffile = tempfile.NamedTemporaryFile() conffile = tempfile.NamedTemporaryFile(mode='w')
conffile.write(proxy_conf) conffile.write(proxy_conf)
conffile.flush() conffile.flush()
@ -887,6 +905,8 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time) self.assertEqual(2900, mware.max_get_time)
conffile.close()
def test_finding_defaults_from_dir(self): def test_finding_defaults_from_dir(self):
# If DLO has no config vars, go pull them from the proxy server's # If DLO has no config vars, go pull them from the proxy server's
# config section # config section
@ -913,11 +933,13 @@ class TestDloConfiguration(unittest.TestCase):
conf_dir = self.tmpdir conf_dir = self.tmpdir
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile1 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile1.write(proxy_conf1) conffile1.write(proxy_conf1)
conffile1.flush() conffile1.flush()
conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile2 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile2.write(proxy_conf2) conffile2.write(proxy_conf2)
conffile2.flush() conffile2.flush()
@ -929,6 +951,9 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time) self.assertEqual(2900, mware.max_get_time)
conffile1.close()
conffile2.close()
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -48,6 +48,7 @@ commands =
test/unit/common/middleware/test_container_sync.py \ test/unit/common/middleware/test_container_sync.py \
test/unit/common/middleware/test_copy.py \ test/unit/common/middleware/test_copy.py \
test/unit/common/middleware/test_crossdomain.py \ test/unit/common/middleware/test_crossdomain.py \
test/unit/common/middleware/test_dlo.py \
test/unit/common/middleware/test_domain_remap.py \ test/unit/common/middleware/test_domain_remap.py \
test/unit/common/middleware/test_formpost.py \ test/unit/common/middleware/test_formpost.py \
test/unit/common/middleware/test_gatekeeper.py \ test/unit/common/middleware/test_gatekeeper.py \