blueprint Multi-range support implementation
This change adds multi range retrieval to OpenStack Swift. For non- segmented data object, a client can use HTTP Range header to specify multiple ranges to retrieve sections of the data object. This implementation currently does not support segmented data object multi range retrieval. When a client sends a multi range request against a segmented data object, Swift will return HTTP status code 200. Support for segmented data multi range retrieval will be added in near future. This implementation is to bring Swift closer to CDMI multi range data retrieval standard. Once support for segemented data multi range is added, Swift will be compliant with CDMI standard in this area. DocImpact Change-Id: I4ed1fb0a0a93c037ddb2f551ea62afe447945107
This commit is contained in:
litong01
parent
c7948ec5d9
commit
ce274b3532
|
|
@ -30,6 +30,7 @@ from email.utils import parsedate
|
|||
import urlparse
|
||||
import urllib2
|
||||
import re
|
||||
import random
|
||||
|
||||
from swift.common.utils import reiterate
|
||||
|
||||
|
|
@ -440,6 +441,22 @@ class Range(object):
|
|||
invalid byte-range-spec values MUST ignore the header field that includes
|
||||
that byte-range-set."
|
||||
|
||||
According to the RFC 2616 specification, the following cases will be all
|
||||
considered as syntactically invalid, thus, a ValueError is thrown so that
|
||||
the range header will be ignored. If the range value contains at least
|
||||
one of the following cases, the entire range is considered invalid,
|
||||
ValueError will be thrown so that the header will be ignored.
|
||||
|
||||
1. value not starts with bytes=
|
||||
2. range value start is greater than the end, eg. bytes=5-3
|
||||
3. range does not have start or end, eg. bytes=-
|
||||
4. range does not have hyphen, eg. bytes=45
|
||||
5. range value is non numeric
|
||||
6. any combination of the above
|
||||
|
||||
Every syntactically valid range will be added into the ranges list
|
||||
even when some of the ranges may not be satisfied by underlying content.
|
||||
|
||||
:param headerval: value of the header as a str
|
||||
"""
|
||||
def __init__(self, headerval):
|
||||
|
|
@ -448,22 +465,26 @@ class Range(object):
|
|||
raise ValueError('Invalid Range header: %s' % headerval)
|
||||
self.ranges = []
|
||||
for rng in headerval[6:].split(','):
|
||||
# Check if the range has required hyphen.
|
||||
if rng.find('-') == -1:
|
||||
raise ValueError('Invalid Range header: %s' % headerval)
|
||||
start, end = rng.split('-', 1)
|
||||
if start:
|
||||
# when start contains non numeric value, this also causes
|
||||
# ValueError
|
||||
start = int(start)
|
||||
else:
|
||||
start = None
|
||||
if end:
|
||||
# when end contains non numeric value, this also causes
|
||||
# ValueError
|
||||
end = int(end)
|
||||
if start is not None and not end >= start:
|
||||
# If the last-byte-pos value is present, it MUST be greater
|
||||
# than or equal to the first-byte-pos in that
|
||||
# byte-range-spec, or the byte- range-spec is syntactically
|
||||
# invalid. [which "MUST" be ignored]
|
||||
self.ranges = []
|
||||
break
|
||||
if start is not None and end < start:
|
||||
raise ValueError('Invalid Range header: %s' % headerval)
|
||||
else:
|
||||
end = None
|
||||
if start is None:
|
||||
raise ValueError('Invalid Range header: %s' % headerval)
|
||||
self.ranges.append((start, end))
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -477,38 +498,68 @@ class Range(object):
|
|||
string += ','
|
||||
return string.rstrip(',')
|
||||
|
||||
def range_for_length(self, length):
|
||||
def ranges_for_length(self, length):
|
||||
"""
|
||||
range_for_length is used to determine the correct range of bytes to
|
||||
serve from a body, given body length argument and the Range's ranges.
|
||||
This method is used to return multiple ranges for a given length
|
||||
which should represent the length of the underlying content.
|
||||
The constructor method __init__ made sure that any range in ranges
|
||||
list is syntactically valid. So if length is None or size of the
|
||||
ranges is zero, then the Range header should be ignored which will
|
||||
eventually make the response to be 200.
|
||||
|
||||
A limitation of this method is that it can't handle multiple ranges,
|
||||
for compatibility with webob. This should be fairly easy to extend.
|
||||
If an empty list is returned by this method, it indicates that there
|
||||
are unsatisfiable ranges found in the Range header, 416 will be
|
||||
returned.
|
||||
|
||||
:param length: length of the response body
|
||||
if a returned list has at least one element, the list indicates that
|
||||
there is at least one range valid and the server should serve the
|
||||
request with a 206 status code.
|
||||
|
||||
The start value of each range represents the starting position in
|
||||
the content, the end value represents the ending position. This
|
||||
method purposely adds 1 to the end number because the spec defines
|
||||
the Range to be inclusive.
|
||||
|
||||
The Range spec can be found at the following link:
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.1
|
||||
|
||||
:param length: length of the underlying content
|
||||
"""
|
||||
if length is None or not self.ranges or len(self.ranges) != 1:
|
||||
# not syntactically valid ranges, must ignore
|
||||
if length is None or not self.ranges or self.ranges == []:
|
||||
return None
|
||||
begin, end = self.ranges[0]
|
||||
if begin is None:
|
||||
if end == 0:
|
||||
return None
|
||||
if end > length:
|
||||
return (0, length)
|
||||
return (length - end, length)
|
||||
if end is None:
|
||||
if begin < length:
|
||||
# If a syntactically valid byte-range-set includes at least one
|
||||
# byte-range-spec whose first-byte-pos is LESS THAN THE CURRENT
|
||||
# LENGTH OF THE ENTITY-BODY..., then the byte-range-set is
|
||||
# satisfiable.
|
||||
return (begin, length)
|
||||
else:
|
||||
# Otherwise, the byte-range-set is unsatisfiable.
|
||||
return None
|
||||
if begin > length:
|
||||
return None
|
||||
return (begin, min(end + 1, length))
|
||||
all_ranges = []
|
||||
for single_range in self.ranges:
|
||||
begin, end = single_range
|
||||
# The possible values for begin and end are
|
||||
# None, 0, or a positive numeric number
|
||||
if begin is None:
|
||||
if end == 0:
|
||||
# this is the bytes=-0 case
|
||||
continue
|
||||
elif end > length:
|
||||
# This is the case where the end is greater than the
|
||||
# content length, as the RFC 2616 stated, the entire
|
||||
# content should be returned.
|
||||
all_ranges.append((0, length))
|
||||
else:
|
||||
all_ranges.append((length - end, length))
|
||||
continue
|
||||
# begin can only be 0 and numeric value from this point on
|
||||
if end is None:
|
||||
if begin < length:
|
||||
all_ranges.append((begin, length))
|
||||
else:
|
||||
# the begin position is greater than or equal to the
|
||||
# content length; skip and move on to the next range
|
||||
continue
|
||||
# end can only be 0 or numeric value
|
||||
elif begin < length:
|
||||
# the begin position is valid, take the min of end + 1 or
|
||||
# the total length of the content
|
||||
all_ranges.append((begin, min(end + 1, length)))
|
||||
|
||||
return all_ranges
|
||||
|
||||
|
||||
class Match(object):
|
||||
|
|
@ -759,6 +810,25 @@ class Request(object):
|
|||
app_iter=app_iter, request=self)
|
||||
|
||||
|
||||
def content_range_header(start, stop, size, value_only=True):
|
||||
if value_only:
|
||||
range_str = 'bytes %s-%s/%s'
|
||||
else:
|
||||
range_str = 'Content-Range: bytes %s-%s/%s'
|
||||
return range_str % (start, (stop - 1), size)
|
||||
|
||||
|
||||
def multi_range_iterator(ranges, content_type, boundary, size, sub_iter_gen):
|
||||
for start, stop in ranges:
|
||||
yield ''.join(['\r\n--', boundary, '\r\n',
|
||||
'Content-Type: ', content_type, '\r\n'])
|
||||
yield content_range_header(start, stop, size, False) + '\r\n\r\n'
|
||||
sub_iter = sub_iter_gen(start, stop)
|
||||
for chunk in sub_iter:
|
||||
yield chunk
|
||||
yield '\r\n--' + boundary + '--\r\n'
|
||||
|
||||
|
||||
class Response(object):
|
||||
"""
|
||||
WSGI Response object.
|
||||
|
|
@ -783,6 +853,7 @@ class Response(object):
|
|||
self.body = body
|
||||
self.app_iter = app_iter
|
||||
self.status = status
|
||||
self.boundary = "%x" % random.randint(0, 256 ** 16)
|
||||
if request:
|
||||
self.environ = request.environ
|
||||
if request.range and self.status == 200:
|
||||
|
|
@ -793,6 +864,35 @@ class Response(object):
|
|||
for key, value in kw.iteritems():
|
||||
setattr(self, key, value)
|
||||
|
||||
def _prepare_for_ranges(self, ranges):
|
||||
"""
|
||||
Prepare the Response for multiple ranges.
|
||||
"""
|
||||
|
||||
content_size = self.content_length
|
||||
content_type = self.content_type
|
||||
self.content_type = ''.join(['multipart/byteranges;',
|
||||
'boundary=', self.boundary])
|
||||
|
||||
# This section calculate the total size of the targeted response
|
||||
# The value 12 is the length of total bytes of hyphen, new line
|
||||
# form feed for each section header. The value 8 is the length of
|
||||
# total bytes of hyphen, new line, form feed characters for the
|
||||
# closing boundary which appears only once
|
||||
section_header_fixed_len = 12 + (len(self.boundary) +
|
||||
len('Content-Type: ') +
|
||||
len(content_type) +
|
||||
len('Content-Range: bytes '))
|
||||
body_size = 0
|
||||
for start, end in ranges:
|
||||
body_size += section_header_fixed_len
|
||||
body_size += len(str(start) + '-' + str(end - 1) + '/' +
|
||||
str(content_size)) + (end - start)
|
||||
body_size += 8 + len(self.boundary)
|
||||
self.content_length = body_size
|
||||
self.content_range = None
|
||||
return content_size, content_type
|
||||
|
||||
def _response_iter(self, app_iter, body):
|
||||
if self.request and self.request.method == 'HEAD':
|
||||
# We explicitly do NOT want to set self.content_length to 0 here
|
||||
|
|
@ -800,23 +900,54 @@ class Response(object):
|
|||
if self.conditional_response and self.request and \
|
||||
self.request.range and self.request.range.ranges and \
|
||||
not self.content_range:
|
||||
args = self.request.range.range_for_length(self.content_length)
|
||||
if not args:
|
||||
ranges = self.request.range.ranges_for_length(self.content_length)
|
||||
if ranges == []:
|
||||
self.status = 416
|
||||
self.content_length = 0
|
||||
return ['']
|
||||
else:
|
||||
start, end = args
|
||||
self.status = 206
|
||||
self.content_range = self.request.range
|
||||
self.content_length = (end - start)
|
||||
if app_iter and hasattr(app_iter, 'app_iter_range'):
|
||||
return app_iter.app_iter_range(start, end)
|
||||
elif app_iter:
|
||||
# this could be improved, but we don't actually use it
|
||||
return [''.join(app_iter)[start:end]]
|
||||
elif body:
|
||||
return [body[start:end]]
|
||||
elif ranges:
|
||||
range_size = len(ranges)
|
||||
if range_size > 0:
|
||||
# There is at least one valid range in the request, so try
|
||||
# to satisfy the request
|
||||
if range_size == 1:
|
||||
start, end = ranges[0]
|
||||
if app_iter and hasattr(app_iter, 'app_iter_range'):
|
||||
self.status = 206
|
||||
self.content_range = \
|
||||
content_range_header(start, end,
|
||||
self.content_length,
|
||||
True)
|
||||
self.content_length = (end - start)
|
||||
return app_iter.app_iter_range(start, end)
|
||||
elif body:
|
||||
self.status = 206
|
||||
self.content_range = \
|
||||
content_range_header(start, end,
|
||||
self.content_length,
|
||||
True)
|
||||
self.content_length = (end - start)
|
||||
return [body[start:end]]
|
||||
elif range_size > 1:
|
||||
if app_iter and hasattr(app_iter, 'app_iter_ranges'):
|
||||
self.status = 206
|
||||
content_size, content_type = \
|
||||
self._prepare_for_ranges(ranges)
|
||||
return app_iter.app_iter_ranges(ranges,
|
||||
content_type,
|
||||
self.boundary,
|
||||
content_size)
|
||||
elif body:
|
||||
self.status = 206
|
||||
content_size, content_type, = \
|
||||
self._prepare_for_ranges(ranges)
|
||||
|
||||
def _body_slicer(start, stop):
|
||||
yield body[start:stop]
|
||||
return multi_range_iterator(ranges, content_type,
|
||||
self.boundary,
|
||||
content_size,
|
||||
_body_slicer)
|
||||
if app_iter:
|
||||
return app_iter
|
||||
if body:
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
|
|||
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
|
||||
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
|
||||
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \
|
||||
HTTPInsufficientStorage
|
||||
HTTPInsufficientStorage, multi_range_iterator
|
||||
|
||||
|
||||
DATADIR = 'objects'
|
||||
|
|
@ -128,6 +128,7 @@ class DiskFile(object):
|
|||
self.read_to_eof = False
|
||||
self.quarantined_dir = None
|
||||
self.keep_cache = False
|
||||
self.suppress_file_closing = False
|
||||
if not os.path.exists(self.datadir):
|
||||
return
|
||||
files = sorted(os.listdir(self.datadir), reverse=True)
|
||||
|
|
@ -183,11 +184,12 @@ class DiskFile(object):
|
|||
read - dropped_cache)
|
||||
break
|
||||
finally:
|
||||
self.close()
|
||||
if not self.suppress_file_closing:
|
||||
self.close()
|
||||
|
||||
def app_iter_range(self, start, stop):
|
||||
"""Returns an iterator over the data file for range (start, stop)"""
|
||||
if start:
|
||||
if start or start == 0:
|
||||
self.fp.seek(start)
|
||||
if stop is not None:
|
||||
length = stop - start
|
||||
|
|
@ -202,6 +204,21 @@ class DiskFile(object):
|
|||
break
|
||||
yield chunk
|
||||
|
||||
def app_iter_ranges(self, ranges, content_type, boundary, size):
|
||||
"""Returns an iterator over the data file for a set of ranges"""
|
||||
if (not ranges or len(ranges) == 0):
|
||||
yield ''
|
||||
else:
|
||||
try:
|
||||
self.suppress_file_closing = True
|
||||
for chunk in multi_range_iterator(
|
||||
ranges, content_type, boundary, size,
|
||||
self.app_iter_range):
|
||||
yield chunk
|
||||
finally:
|
||||
self.suppress_file_closing = False
|
||||
self.close()
|
||||
|
||||
def _handle_close_quarantine(self):
|
||||
"""Check if file needs to be quarantined"""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
import unittest
|
||||
import datetime
|
||||
import re
|
||||
from StringIO import StringIO
|
||||
|
||||
import swift.common.swob
|
||||
|
|
@ -108,7 +109,7 @@ class TestRange(unittest.TestCase):
|
|||
|
||||
def test_upsidedown_range(self):
|
||||
range = swift.common.swob.Range('bytes=5-10')
|
||||
self.assertEquals(range.range_for_length(2), None)
|
||||
self.assertEquals(range.ranges_for_length(2), [])
|
||||
|
||||
def test_str(self):
|
||||
for range_str in ('bytes=1-7', 'bytes=1-', 'bytes=-1',
|
||||
|
|
@ -116,31 +117,94 @@ class TestRange(unittest.TestCase):
|
|||
range = swift.common.swob.Range(range_str)
|
||||
self.assertEquals(str(range), range_str)
|
||||
|
||||
def test_range_for_length(self):
|
||||
def test_ranges_for_length(self):
|
||||
range = swift.common.swob.Range('bytes=1-7')
|
||||
self.assertEquals(range.range_for_length(10), (1, 8))
|
||||
self.assertEquals(range.range_for_length(5), (1, 5))
|
||||
self.assertEquals(range.range_for_length(None), None)
|
||||
self.assertEquals(range.ranges_for_length(10), [(1, 8)])
|
||||
self.assertEquals(range.ranges_for_length(5), [(1, 5)])
|
||||
self.assertEquals(range.ranges_for_length(None), None)
|
||||
|
||||
def test_range_for_length_no_end(self):
|
||||
def test_ranges_for_large_length(self):
|
||||
range = swift.common.swob.Range('bytes=-1000000000000000000000000000')
|
||||
self.assertEquals(range.ranges_for_length(100), [(0, 100)])
|
||||
|
||||
def test_ranges_for_length_no_end(self):
|
||||
range = swift.common.swob.Range('bytes=1-')
|
||||
self.assertEquals(range.range_for_length(10), (1, 10))
|
||||
self.assertEquals(range.range_for_length(5), (1, 5))
|
||||
self.assertEquals(range.range_for_length(None), None)
|
||||
self.assertEquals(range.ranges_for_length(10), [(1, 10)])
|
||||
self.assertEquals(range.ranges_for_length(5), [(1, 5)])
|
||||
self.assertEquals(range.ranges_for_length(None), None)
|
||||
# This used to freak out:
|
||||
range = swift.common.swob.Range('bytes=100-')
|
||||
self.assertEquals(range.range_for_length(5), None)
|
||||
self.assertEquals(range.range_for_length(None), None)
|
||||
self.assertEquals(range.ranges_for_length(5), [])
|
||||
self.assertEquals(range.ranges_for_length(None), None)
|
||||
|
||||
def test_range_for_length_no_start(self):
|
||||
range = swift.common.swob.Range('bytes=4-6,100-')
|
||||
self.assertEquals(range.ranges_for_length(5), [(4, 5)])
|
||||
|
||||
def test_ranges_for_length_no_start(self):
|
||||
range = swift.common.swob.Range('bytes=-7')
|
||||
self.assertEquals(range.range_for_length(10), (3, 10))
|
||||
self.assertEquals(range.range_for_length(5), (0, 5))
|
||||
self.assertEquals(range.range_for_length(None), None)
|
||||
self.assertEquals(range.ranges_for_length(10), [(3, 10)])
|
||||
self.assertEquals(range.ranges_for_length(5), [(0, 5)])
|
||||
self.assertEquals(range.ranges_for_length(None), None)
|
||||
|
||||
range = swift.common.swob.Range('bytes=4-6,-100')
|
||||
self.assertEquals(range.ranges_for_length(5), [(4, 5), (0, 5)])
|
||||
|
||||
def test_ranges_for_length_multi(self):
|
||||
range = swift.common.swob.Range('bytes=-20,4-,30-150,-10')
|
||||
# the length of the ranges should be 4
|
||||
self.assertEquals(len(range.ranges_for_length(200)), 4)
|
||||
|
||||
# the actual length less than any of the range
|
||||
self.assertEquals(range.ranges_for_length(90),
|
||||
[(70, 90), (4, 90), (30, 90), (80, 90)])
|
||||
|
||||
# the actual length greater than any of the range
|
||||
self.assertEquals(range.ranges_for_length(200),
|
||||
[(180, 200), (4, 200), (30, 151), (190, 200)])
|
||||
|
||||
self.assertEquals(range.ranges_for_length(None), None)
|
||||
|
||||
def test_ranges_for_length_edges(self):
|
||||
range = swift.common.swob.Range('bytes=0-1, -7')
|
||||
self.assertEquals(range.ranges_for_length(10),
|
||||
[(0, 2), (3, 10)])
|
||||
|
||||
range = swift.common.swob.Range('bytes=-7, 0-1')
|
||||
self.assertEquals(range.ranges_for_length(10),
|
||||
[(3, 10), (0, 2)])
|
||||
|
||||
range = swift.common.swob.Range('bytes=-7, 0-1')
|
||||
self.assertEquals(range.ranges_for_length(5),
|
||||
[(0, 5), (0, 2)])
|
||||
|
||||
def test_range_invalid_syntax(self):
|
||||
range = swift.common.swob.Range('bytes=10-2')
|
||||
self.assertEquals(range.ranges, [])
|
||||
|
||||
def _check_invalid_range(range_value):
|
||||
try:
|
||||
swift.common.swob.Range(range_value)
|
||||
return False
|
||||
except ValueError:
|
||||
return True
|
||||
|
||||
"""
|
||||
All the following cases should result ValueError exception
|
||||
1. value not starts with bytes=
|
||||
2. range value start is greater than the end, eg. bytes=5-3
|
||||
3. range does not have start or end, eg. bytes=-
|
||||
4. range does not have hyphen, eg. bytes=45
|
||||
5. range value is non numeric
|
||||
6. any combination of the above
|
||||
"""
|
||||
|
||||
self.assert_(_check_invalid_range('nonbytes=foobar,10-2'))
|
||||
self.assert_(_check_invalid_range('bytes=5-3'))
|
||||
self.assert_(_check_invalid_range('bytes=-'))
|
||||
self.assert_(_check_invalid_range('bytes=45'))
|
||||
self.assert_(_check_invalid_range('bytes=foo-bar,3-5'))
|
||||
self.assert_(_check_invalid_range('bytes=4-10,45'))
|
||||
self.assert_(_check_invalid_range('bytes=foobar,3-5'))
|
||||
self.assert_(_check_invalid_range('bytes=nonumber-5'))
|
||||
self.assert_(_check_invalid_range('bytes=nonumber'))
|
||||
|
||||
|
||||
class TestMatch(unittest.TestCase):
|
||||
|
|
@ -387,6 +451,106 @@ class TestResponse(unittest.TestCase):
|
|||
body = ''.join(resp({}, start_response))
|
||||
self.assertEquals(body, 'abc')
|
||||
|
||||
def test_multi_ranges_wo_iter_ranges(self):
|
||||
def test_app(environ, start_response):
|
||||
start_response('200 OK', [('Content-Length', '10')])
|
||||
return ['1234567890']
|
||||
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=0-9,10-19,20-29'})
|
||||
|
||||
resp = req.get_response(test_app)
|
||||
resp.conditional_response = True
|
||||
resp.content_length = 10
|
||||
|
||||
content = ''.join(resp._response_iter(resp.app_iter, ''))
|
||||
|
||||
self.assertEquals(resp.status, '200 OK')
|
||||
self.assertEqual(10, resp.content_length)
|
||||
|
||||
def test_single_range_wo_iter_range(self):
|
||||
def test_app(environ, start_response):
|
||||
start_response('200 OK', [('Content-Length', '10')])
|
||||
return ['1234567890']
|
||||
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=0-9'})
|
||||
|
||||
resp = req.get_response(test_app)
|
||||
resp.conditional_response = True
|
||||
resp.content_length = 10
|
||||
|
||||
content = ''.join(resp._response_iter(resp.app_iter, ''))
|
||||
|
||||
self.assertEquals(resp.status, '200 OK')
|
||||
self.assertEqual(10, resp.content_length)
|
||||
|
||||
def test_multi_range_body(self):
|
||||
def test_app(environ, start_response):
|
||||
start_response('200 OK', [('Content-Length', '4')])
|
||||
return ['abcd']
|
||||
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=0-9,10-19,20-29'})
|
||||
|
||||
resp = req.get_response(test_app)
|
||||
resp.conditional_response = True
|
||||
resp.content_length = 100
|
||||
|
||||
resp.content_type = 'text/plain'
|
||||
content = ''.join(resp._response_iter(None,
|
||||
('0123456789112345678'
|
||||
'92123456789')))
|
||||
|
||||
self.assert_(re.match(('\r\n'
|
||||
'--[a-f0-9]{32}\r\n'
|
||||
'Content-Type: text/plain\r\n'
|
||||
'Content-Range: bytes '
|
||||
'0-9/100\r\n\r\n0123456789\r\n'
|
||||
'--[a-f0-9]{32}\r\n'
|
||||
'Content-Type: text/plain\r\n'
|
||||
'Content-Range: bytes '
|
||||
'10-19/100\r\n\r\n1123456789\r\n'
|
||||
'--[a-f0-9]{32}\r\n'
|
||||
'Content-Type: text/plain\r\n'
|
||||
'Content-Range: bytes '
|
||||
'20-29/100\r\n\r\n2123456789\r\n'
|
||||
'--[a-f0-9]{32}--\r\n'), content))
|
||||
|
||||
def test_multi_response_iter(self):
|
||||
def test_app(environ, start_response):
|
||||
start_response('200 OK', [('Content-Length', '10'),
|
||||
('Content-Type', 'application/xml')])
|
||||
return ['0123456789']
|
||||
|
||||
app_iter_ranges_args = []
|
||||
|
||||
class App_iter(object):
|
||||
def app_iter_ranges(self, ranges, content_type, boundary, size):
|
||||
app_iter_ranges_args.append((ranges, content_type, boundary,
|
||||
size))
|
||||
for i in xrange(3):
|
||||
yield str(i) + 'fun'
|
||||
yield boundary
|
||||
|
||||
def __iter__(self):
|
||||
for i in xrange(3):
|
||||
yield str(i) + 'fun'
|
||||
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=1-5,8-11'})
|
||||
|
||||
resp = req.get_response(test_app)
|
||||
resp.conditional_response = True
|
||||
resp.content_length = 12
|
||||
|
||||
content = ''.join(resp._response_iter(App_iter(), ''))
|
||||
boundary = content[-32:]
|
||||
self.assertEqual(content[:-32], '0fun1fun2fun')
|
||||
self.assertEqual(app_iter_ranges_args,
|
||||
[([(1, 6), (8, 12)], 'application/xml',
|
||||
boundary, 12)])
|
||||
|
||||
def test_range_body(self):
|
||||
|
||||
def test_app(environ, start_response):
|
||||
|
|
@ -398,20 +562,17 @@ class TestResponse(unittest.TestCase):
|
|||
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=1-3'})
|
||||
resp = req.get_response(test_app)
|
||||
resp.conditional_response = True
|
||||
body = ''.join(resp([], start_response))
|
||||
self.assertEquals(body, '234')
|
||||
self.assertEquals(resp.status, '206 Partial Content')
|
||||
|
||||
resp = swift.common.swob.Response(
|
||||
body='1234567890', request=req,
|
||||
conditional_response=True)
|
||||
body = ''.join(resp([], start_response))
|
||||
self.assertEquals(body, '234')
|
||||
self.assertEquals(resp.content_range, 'bytes 1-3/10')
|
||||
self.assertEquals(resp.status, '206 Partial Content')
|
||||
|
||||
# No body for 416
|
||||
# syntactically valid, but does not make sense, so returning 416
|
||||
# in next couple of cases.
|
||||
req = swift.common.swob.Request.blank(
|
||||
'/', headers={'Range': 'bytes=-0'})
|
||||
resp = req.get_response(test_app)
|
||||
|
|
@ -426,6 +587,7 @@ class TestResponse(unittest.TestCase):
|
|||
conditional_response=True)
|
||||
body = ''.join(resp([], start_response))
|
||||
self.assertEquals(body, '')
|
||||
self.assertEquals(resp.content_length, 0)
|
||||
self.assertEquals(resp.status, '416 Requested Range Not Satisfiable')
|
||||
|
||||
# Syntactically-invalid Range headers "MUST" be ignored
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
import cPickle as pickle
|
||||
import os
|
||||
import unittest
|
||||
import email
|
||||
from shutil import rmtree
|
||||
from StringIO import StringIO
|
||||
from time import gmtime, sleep, strftime, time
|
||||
|
|
@ -55,31 +56,89 @@ class TestDiskFile(unittest.TestCase):
|
|||
""" Tear down for testing swift.object_server.ObjectController """
|
||||
rmtree(os.path.dirname(self.testdir))
|
||||
|
||||
def test_disk_file_app_iter_corners(self):
|
||||
def _create_test_file(self, data, keep_data_fp=True):
|
||||
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
|
||||
FakeLogger())
|
||||
mkdirs(df.datadir)
|
||||
f = open(os.path.join(df.datadir,
|
||||
normalize_timestamp(time()) + '.data'), 'wb')
|
||||
f.write('1234567890')
|
||||
f.write(data)
|
||||
setxattr(f.fileno(), object_server.METADATA_KEY,
|
||||
pickle.dumps({}, object_server.PICKLE_PROTOCOL))
|
||||
f.close()
|
||||
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
|
||||
FakeLogger(), keep_data_fp=True)
|
||||
it = df.app_iter_range(0, None)
|
||||
sio = StringIO()
|
||||
for chunk in it:
|
||||
sio.write(chunk)
|
||||
self.assertEquals(sio.getvalue(), '1234567890')
|
||||
FakeLogger(), keep_data_fp=keep_data_fp)
|
||||
return df
|
||||
|
||||
def test_disk_file_app_iter_corners(self):
|
||||
df = self._create_test_file('1234567890')
|
||||
self.assertEquals(''.join(df.app_iter_range(0, None)), '1234567890')
|
||||
|
||||
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
|
||||
FakeLogger(), keep_data_fp=True)
|
||||
it = df.app_iter_range(5, None)
|
||||
sio = StringIO()
|
||||
for chunk in it:
|
||||
sio.write(chunk)
|
||||
self.assertEquals(sio.getvalue(), '67890')
|
||||
self.assertEqual(''.join(df.app_iter_range(5, None)), '67890')
|
||||
|
||||
def test_disk_file_app_iter_ranges(self):
|
||||
df = self._create_test_file('012345678911234567892123456789')
|
||||
it = df.app_iter_ranges([(0, 10), (10, 20), (20, 30)], 'plain/text',
|
||||
'\r\n--someheader\r\n', 30)
|
||||
value = ''.join(it)
|
||||
self.assert_('0123456789' in value)
|
||||
self.assert_('1123456789' in value)
|
||||
self.assert_('2123456789' in value)
|
||||
|
||||
def test_disk_file_app_iter_ranges_edges(self):
|
||||
df = self._create_test_file('012345678911234567892123456789')
|
||||
it = df.app_iter_ranges([(3, 10), (0, 2)], 'application/whatever',
|
||||
'\r\n--someheader\r\n', 30)
|
||||
value = ''.join(it)
|
||||
self.assert_('3456789' in value)
|
||||
self.assert_('01' in value)
|
||||
|
||||
def test_disk_file_large_app_iter_ranges(self):
|
||||
"""
|
||||
This test case is to make sure that the disk file app_iter_ranges
|
||||
method all the paths being tested.
|
||||
"""
|
||||
long_str = '01234567890' * 65536
|
||||
target_strs = ['3456789', long_str[0:65590]]
|
||||
df = self._create_test_file(long_str)
|
||||
|
||||
it = df.app_iter_ranges([(3, 10), (0, 65590)], 'plain/text',
|
||||
'5e816ff8b8b8e9a5d355497e5d9e0301', 655360)
|
||||
|
||||
"""
|
||||
the produced string actually missing the MIME headers
|
||||
need to add these headers to make it as real MIME message.
|
||||
The body of the message is produced by method app_iter_ranges
|
||||
off of DiskFile object.
|
||||
"""
|
||||
header = ''.join(['Content-Type: multipart/byteranges;',
|
||||
'boundary=',
|
||||
'5e816ff8b8b8e9a5d355497e5d9e0301\r\n'])
|
||||
|
||||
value = header + ''.join(it)
|
||||
|
||||
parts = map(lambda p: p.get_payload(decode=True),
|
||||
email.message_from_string(value).walk())[1:3]
|
||||
self.assertEqual(parts, target_strs)
|
||||
|
||||
def test_disk_file_app_iter_ranges_empty(self):
|
||||
"""
|
||||
This test case tests when empty value passed into app_iter_ranges
|
||||
When ranges passed into the method is either empty array or None,
|
||||
this method will yield empty string
|
||||
"""
|
||||
df = self._create_test_file('012345678911234567892123456789')
|
||||
it = df.app_iter_ranges([], 'application/whatever',
|
||||
'\r\n--someheader\r\n', 100)
|
||||
self.assertEqual(''.join(it), '')
|
||||
|
||||
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
|
||||
FakeLogger(), keep_data_fp=True)
|
||||
it = df.app_iter_ranges(None, 'app/something',
|
||||
'\r\n--someheader\r\n', 150)
|
||||
self.assertEqual(''.join(it), '')
|
||||
|
||||
def test_disk_file_mkstemp_creates_dir(self):
|
||||
tmpdir = os.path.join(self.testdir, 'sda1', 'tmp')
|
||||
|
|
|
|||
|
|
@ -4450,9 +4450,9 @@ class FakeObjectController(object):
|
|||
path = args[4]
|
||||
body = data = path[-1] * int(path[-1])
|
||||
if req.range:
|
||||
r = req.range.range_for_length(len(data))
|
||||
r = req.range.ranges_for_length(len(data))
|
||||
if r:
|
||||
(start, stop) = r
|
||||
(start, stop) = r[0]
|
||||
body = data[start:stop]
|
||||
resp = Response(app_iter=iter(body))
|
||||
return resp
|
||||
|
|
|
|||
Loading…
Reference in New Issue