Merge "tests: refactor SLO size/etag sysmeta tests"
This commit is contained in:
commit
ed15681349
@ -460,15 +460,17 @@ class SegmentedIterable(object):
|
|||||||
:param app: WSGI application from which segments will come
|
:param app: WSGI application from which segments will come
|
||||||
|
|
||||||
:param listing_iter: iterable yielding the object segments to fetch,
|
:param listing_iter: iterable yielding the object segments to fetch,
|
||||||
along with the byte subranges to fetch, in the form of a 5-tuple
|
along with the byte sub-ranges to fetch. Each yielded item should be a
|
||||||
(object-path, object-etag, object-size, first-byte, last-byte).
|
dict with the following keys: ``path`` or ``raw_data``,
|
||||||
|
``first-byte``, ``last-byte``, ``hash`` (optional), ``bytes``
|
||||||
|
(optional).
|
||||||
|
|
||||||
If object-etag is None, no MD5 verification will be done.
|
If ``hash`` is None, no MD5 verification will be done.
|
||||||
|
|
||||||
If object-size is None, no length verification will be done.
|
If ``bytes`` is None, no length verification will be done.
|
||||||
|
|
||||||
If first-byte and last-byte are None, then the entire object will be
|
If ``first-byte`` and ``last-byte`` are None, then the entire object
|
||||||
fetched.
|
will be fetched.
|
||||||
|
|
||||||
:param max_get_time: maximum permitted duration of a GET request (seconds)
|
:param max_get_time: maximum permitted duration of a GET request (seconds)
|
||||||
:param logger: logger object
|
:param logger: logger object
|
||||||
|
@ -63,6 +63,7 @@ def normalize_query_string(qs):
|
|||||||
if not qs:
|
if not qs:
|
||||||
return ''
|
return ''
|
||||||
else:
|
else:
|
||||||
|
# N.B. sort params so app.call asserts can hard code qs
|
||||||
return '?%s' % parse.urlencode(sorted(parse.parse_qsl(qs)))
|
return '?%s' % parse.urlencode(sorted(parse.parse_qsl(qs)))
|
||||||
|
|
||||||
|
|
||||||
@ -149,14 +150,16 @@ class FakeSwift(object):
|
|||||||
|
|
||||||
def _find_response(self, method, path):
|
def _find_response(self, method, path):
|
||||||
path = normalize_path(path)
|
path = normalize_path(path)
|
||||||
resp = self._responses[(method, path)]
|
resp_or_resps = self._responses[(method, path)]
|
||||||
if isinstance(resp, list):
|
if isinstance(resp_or_resps, list):
|
||||||
try:
|
resps = resp_or_resps
|
||||||
resp = resp.pop(0)
|
if len(resps) > 1:
|
||||||
except IndexError:
|
resp = resps.pop(0)
|
||||||
raise IndexError("Didn't find any more %r "
|
else:
|
||||||
"in allowed responses" % (
|
# we'll return the last registered response forever
|
||||||
(method, path),))
|
resp = resps[0]
|
||||||
|
else:
|
||||||
|
resp = resp_or_resps
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
def _select_response(self, env, method, path):
|
def _select_response(self, env, method, path):
|
||||||
@ -333,11 +336,20 @@ class FakeSwift(object):
|
|||||||
|
|
||||||
def register(self, method, path, response_class, headers, body=b''):
|
def register(self, method, path, response_class, headers, body=b''):
|
||||||
path = normalize_path(path)
|
path = normalize_path(path)
|
||||||
|
# many historical tests assume this is "private" attribute is a simple
|
||||||
|
# map of tuple => tuple that they can go grubbing around in
|
||||||
self._responses[(method, path)] = (response_class, headers, body)
|
self._responses[(method, path)] = (response_class, headers, body)
|
||||||
|
|
||||||
def register_responses(self, method, path, responses):
|
def register_next_response(self, method, path,
|
||||||
|
response_class, headers, body=b''):
|
||||||
path = normalize_path(path)
|
path = normalize_path(path)
|
||||||
self._responses[(method, path)] = list(responses)
|
resp_key = (method, path)
|
||||||
|
next_resp = (response_class, headers, body)
|
||||||
|
# setdefault is weird; I hope this makes sense
|
||||||
|
maybe_resp = self._responses.setdefault(resp_key, [])
|
||||||
|
if isinstance(maybe_resp, tuple):
|
||||||
|
self._responses[resp_key] = [maybe_resp]
|
||||||
|
self._responses[resp_key].append(next_resp)
|
||||||
|
|
||||||
|
|
||||||
class FakeAppThatExcepts(object):
|
class FakeAppThatExcepts(object):
|
||||||
|
@ -508,3 +508,145 @@ class TestFakeSwift(unittest.TestCase):
|
|||||||
self.assertEqual('bytes=0-2', req.headers.get('Range'))
|
self.assertEqual('bytes=0-2', req.headers.get('Range'))
|
||||||
self.assertEqual('bytes=0-2',
|
self.assertEqual('bytes=0-2',
|
||||||
swift.calls_with_headers[-1].headers.get('Range'))
|
swift.calls_with_headers[-1].headers.get('Range'))
|
||||||
|
|
||||||
|
|
||||||
|
class TestFakeSwiftMultipleResponses(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_register_response_is_forever(self):
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register('GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
# you can get this response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
def test_register_response_is_last_response_wins(self):
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register('GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
swift.register('GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Baz'}, b'other')
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
# you can get this new response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
def test_register_next_response_is_last_response_wins(self):
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
swift.register_next_response(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Baz'}, b'other')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
# you can get this new response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
def test_register_next_response_keeps_current_registered_response(self):
|
||||||
|
# we expect test authors will typically 'd register ALL their responses
|
||||||
|
# before you start calling FakeSwift
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
|
||||||
|
# we get the registered response, obviously
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
# because before calling register_next_response, no resp are consumed
|
||||||
|
swift.register_next_response(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Baz'}, b'other')
|
||||||
|
|
||||||
|
# so, this is the "current" response, not the *next* response
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
# the *next* response is the next response
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
def test_register_next_response_first(self):
|
||||||
|
# you can just use register_next_response
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register_next_response(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
swift.register_next_response(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Baz'}, b'other')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
|
||||||
|
# it works just like you'd called register
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
# you can get this new response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
def test_register_resets(self):
|
||||||
|
swift = FakeSwift()
|
||||||
|
swift.register_next_response(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Bar'}, b'stuff')
|
||||||
|
req = Request.blank('/v1/a/c/o')
|
||||||
|
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
# you can get this response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Bar', resp.headers['X-Foo'])
|
||||||
|
|
||||||
|
# if you call register mid test you immediately reset the resp
|
||||||
|
swift.register(
|
||||||
|
'GET', '/v1/a/c/o',
|
||||||
|
HTTPOk, {'X-Foo': 'Baz'}, b'other')
|
||||||
|
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
# you can get this new response as much as you want
|
||||||
|
for i in range(10):
|
||||||
|
resp = req.get_response(swift)
|
||||||
|
self.assertEqual(200, resp.status_int)
|
||||||
|
self.assertEqual('Baz', resp.headers['X-Foo'])
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -16,13 +16,15 @@
|
|||||||
"""Tests for swift.common.request_helpers"""
|
"""Tests for swift.common.request_helpers"""
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from swift.common.swob import Request, HTTPException, HeaderKeyDict
|
from swift.common.swob import Request, HTTPException, HeaderKeyDict, HTTPOk
|
||||||
from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
|
from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
|
||||||
from swift.common import request_helpers as rh
|
from swift.common import request_helpers as rh
|
||||||
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
|
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
|
||||||
|
|
||||||
|
from test.debug_logger import debug_logger
|
||||||
from test.unit import patch_policies
|
from test.unit import patch_policies
|
||||||
from test.unit.common.test_utils import FakeResponse
|
from test.unit.common.test_utils import FakeResponse
|
||||||
|
from test.unit.common.middleware.helpers import FakeSwift
|
||||||
|
|
||||||
|
|
||||||
server_types = ['account', 'container', 'object']
|
server_types = ['account', 'container', 'object']
|
||||||
@ -704,3 +706,60 @@ class TestHTTPResponseToDocumentIters(unittest.TestCase):
|
|||||||
'X-Object-Meta-Color': 'blue',
|
'X-Object-Meta-Color': 'blue',
|
||||||
})
|
})
|
||||||
self.assertIsNone(req.range)
|
self.assertIsNone(req.range)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSegmentedIterable(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.logger = debug_logger()
|
||||||
|
self.app = FakeSwift()
|
||||||
|
self.expected_unread_requests = {}
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.assertFalse(self.app.unclosed_requests)
|
||||||
|
self.assertEqual(self.app.unread_requests,
|
||||||
|
self.expected_unread_requests)
|
||||||
|
|
||||||
|
def test_simple_segments_app_iter(self):
|
||||||
|
self.app.register('GET', '/a/c/seg1', HTTPOk, {}, 'segment1')
|
||||||
|
self.app.register('GET', '/a/c/seg2', HTTPOk, {}, 'segment2')
|
||||||
|
req = Request.blank('/v1/a/c/mpu')
|
||||||
|
listing_iter = [
|
||||||
|
{'path': '/a/c/seg1', 'first_byte': None, 'last_byte': None},
|
||||||
|
{'path': '/a/c/seg2', 'first_byte': None, 'last_byte': None},
|
||||||
|
]
|
||||||
|
si = rh.SegmentedIterable(req, self.app, listing_iter, 60, self.logger,
|
||||||
|
'test-agent', 'test-source')
|
||||||
|
body = b''.join(si.app_iter)
|
||||||
|
self.assertEqual(b'segment1segment2', body)
|
||||||
|
|
||||||
|
def test_simple_segments_app_iter_ranges(self):
|
||||||
|
self.app.register('GET', '/a/c/seg1', HTTPOk, {}, 'segment1')
|
||||||
|
self.app.register('GET', '/a/c/seg2', HTTPOk, {}, 'segment2')
|
||||||
|
req = Request.blank('/v1/a/c/mpu')
|
||||||
|
listing_iter = [
|
||||||
|
{'path': '/a/c/seg1', 'first_byte': None, 'last_byte': None},
|
||||||
|
{'path': '/a/c/seg2', 'first_byte': None, 'last_byte': None},
|
||||||
|
]
|
||||||
|
si = rh.SegmentedIterable(req, self.app, listing_iter, 60, self.logger,
|
||||||
|
'test-agent', 'test-source')
|
||||||
|
body = b''.join(si.app_iter_ranges(
|
||||||
|
[(0, 8), (8, 16)], b'app/foo', b'bound', 16))
|
||||||
|
expected = b'\r\n'.join([
|
||||||
|
b'--bound',
|
||||||
|
b'Content-Type: app/foo',
|
||||||
|
b'Content-Range: bytes 0-7/16',
|
||||||
|
b'',
|
||||||
|
b'segment1',
|
||||||
|
b'--bound',
|
||||||
|
b'Content-Type: app/foo',
|
||||||
|
b'Content-Range: bytes 8-15/16',
|
||||||
|
b'',
|
||||||
|
b'segment2',
|
||||||
|
b'--bound--',
|
||||||
|
])
|
||||||
|
self.assertEqual(expected, body)
|
||||||
|
# XXX Spliterator stops SegementedIterable from asking to exhasut the
|
||||||
|
# segment response after it gets the last byte in app_iter_ranges
|
||||||
|
self.expected_unread_requests[
|
||||||
|
('GET', '/a/c/seg2?multipart-manifest=get')] = 1
|
||||||
|
Loading…
Reference in New Issue
Block a user