openstack
/
python-swiftclient
Code Issues Proposed changes

Check responses when retrying bodies 

Previously, if a Range request came back 200 OK (rather than 206 Partial
Content), we would mangle the response body. This could happen if there
was a middleware that would silently drop Range headers, for example.

Now, if the response does not include a Content-Range header, we will
log a warning and seek to our previous position in the stream. If the
Content-Range header has an unexpected value, we will raise an exception.

Change-Id: I94d4536cc1489968d45a2b6ba7edd70c85800275
This commit is contained in:
Tim Burke 2016-01-18 11:38:44 -08:00
parent 5050027610
commit fd5579a154
2 changed files with 85 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
swiftclient/client.py
View File

@ -249,6 +249,23 @@ class _RetryBody(_ObjectBody):
response_dict=self.response_dict,
headers=self.headers,
attempts=self.conn.attempts)
expected_range = 'bytes %d-%d/%d' % (
self.bytes_read,
self.expected_length - 1,
self.expected_length)
if 'content-range' not in hdrs:
# Server didn't respond with partial content; manually seek
logger.warning('Received 200 while retrying %s/%s; seeking...',
self.container, self.obj)
to_read = self.bytes_read
while to_read > 0:
buf = body.resp.read(min(to_read, self.chunk_size))
to_read -= len(buf)
elif hdrs['content-range'] != expected_range:
msg = ('Expected range "%s" while retrying %s/%s '
'but got "%s"' % (expected_range, self.container,
self.obj, hdrs['content-range']))
raise ClientException(msg)
self.resp = body.resp
buf = self.read(length)
return buf

70
tests/unit/test_swiftclient.py
View File

@ -855,9 +855,11 @@ class TestGetObject(MockHttpTest):
StubResponse(200, 'abcdef', {'etag': 'some etag',
'content-length': '6'}),
StubResponse(206, 'cdef', {'etag': 'some etag',
'content-length': '4'}),
'content-length': '4',
'content-range': 'bytes 2-5/6'}),
StubResponse(206, 'ef', {'etag': 'some etag',
'content-length': '2'}),
'content-length': '2',
'content-range': 'bytes 4-5/6'}),
)
__, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
self.assertEqual(next(resp), 'ab')
@ -887,6 +889,70 @@ class TestGetObject(MockHttpTest):
}),
])
def test_chunk_size_iter_retry_no_range_support(self):
conn = c.Connection('http://auth.url/', 'some_user', 'some_key')
with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth:
mock_get_auth.return_value = ('http://auth.url', 'tToken')
c.http_connection = self.fake_http_connection(*[
StubResponse(200, 'abcdef', {'etag': 'some etag',
'content-length': '6'})
] * 3)
__, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
self.assertEqual(next(resp), 'ab')
self.assertEqual(1, conn.attempts)
# simulate a dropped connection
resp.resp.read()
self.assertEqual(next(resp), 'cd')
self.assertEqual(2, conn.attempts)
# simulate a dropped connection
resp.resp.read()
self.assertEqual(next(resp), 'ef')
self.assertEqual(3, conn.attempts)
self.assertRaises(StopIteration, next, resp)
self.assertRequests([
('GET', '/asdf/asdf', '', {
'x-auth-token': 'tToken',
}),
('GET', '/asdf/asdf', '', {
'range': 'bytes=2-',
'if-match': 'some etag',
'x-auth-token': 'tToken',
}),
('GET', '/asdf/asdf', '', {
'range': 'bytes=4-',
'if-match': 'some etag',
'x-auth-token': 'tToken',
}),
])
def test_chunk_size_iter_retry_bad_range_response(self):
conn = c.Connection('http://auth.url/', 'some_user', 'some_key')
with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth:
mock_get_auth.return_value = ('http://auth.url', 'tToken')
c.http_connection = self.fake_http_connection(
StubResponse(200, 'abcdef', {'etag': 'some etag',
'content-length': '6'}),
StubResponse(206, 'abcdef', {'etag': 'some etag',
'content-length': '6',
'content-range': 'chunk 1-2/3'})
)
__, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
self.assertEqual(next(resp), 'ab')
self.assertEqual(1, conn.attempts)
# simulate a dropped connection
resp.resp.read()
self.assertRaises(c.ClientException, next, resp)
self.assertRequests([
('GET', '/asdf/asdf', '', {
'x-auth-token': 'tToken',
}),
('GET', '/asdf/asdf', '', {
'range': 'bytes=2-',
'if-match': 'some etag',
'x-auth-token': 'tToken',
}),
])
def test_get_object_with_resp_chunk_size_zero(self):
def get_connection(self):
def get_auth():