Swift object client: use urllib3 builtin support for chunked transfer
Urllib3 has native support for chunked encoding, so let's use this instead of rolling our own. Less code to maintain, additional logging and timing (thanks to our common RestClient). Yeah \O/. Change-Id: I4a253a5cec0fc35009af25872239363625d417e3
This commit is contained in:
parent
c808dc503b
commit
4408c4a5fe
releasenotes/notes
tempest
api/object_storage
lib
common
services
services/object_storage
tests/lib
@ -0,0 +1,9 @@
|
||||
---
|
||||
features:
|
||||
- The RestClient (in tempest.lib.common.rest_client) now supports POSTing
|
||||
and PUTing data with chunked transfer encoding. Just pass an `iterable`
|
||||
object as the `body` argument and set the `chunked` argument to `True`.
|
||||
- A new generator called `chunkify` is added in
|
||||
tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices)
|
||||
from a Python sequence.
|
||||
|
@ -20,7 +20,6 @@ import time
|
||||
import zlib
|
||||
|
||||
import six
|
||||
from six import moves
|
||||
|
||||
from tempest.api.object_storage import base
|
||||
from tempest.common import custom_matchers
|
||||
@ -201,8 +200,8 @@ class ObjectTest(base.BaseObjectTest):
|
||||
status, _, resp_headers = self.object_client.put_object_with_chunk(
|
||||
container=self.container_name,
|
||||
name=object_name,
|
||||
contents=moves.cStringIO(data),
|
||||
chunk_size=512)
|
||||
contents=data_utils.chunkify(data, 512)
|
||||
)
|
||||
self.assertHeaders(resp_headers, 'Object', 'PUT')
|
||||
|
||||
# check uploaded content
|
||||
|
@ -243,7 +243,8 @@ class RestClient(object):
|
||||
details = pattern.format(read_code, expected_code)
|
||||
raise exceptions.InvalidHttpSuccessCode(details)
|
||||
|
||||
def post(self, url, body, headers=None, extra_headers=False):
|
||||
def post(self, url, body, headers=None, extra_headers=False,
|
||||
chunked=False):
|
||||
"""Send a HTTP POST request using keystone auth
|
||||
|
||||
:param str url: the relative url to send the post request to
|
||||
@ -253,11 +254,12 @@ class RestClient(object):
|
||||
returned by the get_headers() method are to
|
||||
be used but additional headers are needed in
|
||||
the request pass them in as a dict.
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
:rtype: tuple
|
||||
"""
|
||||
return self.request('POST', url, extra_headers, headers, body)
|
||||
return self.request('POST', url, extra_headers, headers, body, chunked)
|
||||
|
||||
def get(self, url, headers=None, extra_headers=False):
|
||||
"""Send a HTTP GET request using keystone service catalog and auth
|
||||
@ -306,7 +308,7 @@ class RestClient(object):
|
||||
"""
|
||||
return self.request('PATCH', url, extra_headers, headers, body)
|
||||
|
||||
def put(self, url, body, headers=None, extra_headers=False):
|
||||
def put(self, url, body, headers=None, extra_headers=False, chunked=False):
|
||||
"""Send a HTTP PUT request using keystone service catalog and auth
|
||||
|
||||
:param str url: the relative url to send the post request to
|
||||
@ -316,11 +318,12 @@ class RestClient(object):
|
||||
returned by the get_headers() method are to
|
||||
be used but additional headers are needed in
|
||||
the request pass them in as a dict.
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
:rtype: tuple
|
||||
"""
|
||||
return self.request('PUT', url, extra_headers, headers, body)
|
||||
return self.request('PUT', url, extra_headers, headers, body, chunked)
|
||||
|
||||
def head(self, url, headers=None, extra_headers=False):
|
||||
"""Send a HTTP HEAD request using keystone service catalog and auth
|
||||
@ -520,7 +523,7 @@ class RestClient(object):
|
||||
if method != 'HEAD' and not resp_body and resp.status >= 400:
|
||||
self.LOG.warning("status >= 400 response with empty body")
|
||||
|
||||
def _request(self, method, url, headers=None, body=None):
|
||||
def _request(self, method, url, headers=None, body=None, chunked=False):
|
||||
"""A simple HTTP request interface."""
|
||||
# Authenticate the request with the auth provider
|
||||
req_url, req_headers, req_body = self.auth_provider.auth_request(
|
||||
@ -530,7 +533,9 @@ class RestClient(object):
|
||||
start = time.time()
|
||||
self._log_request_start(method, req_url)
|
||||
resp, resp_body = self.raw_request(
|
||||
req_url, method, headers=req_headers, body=req_body)
|
||||
req_url, method, headers=req_headers, body=req_body,
|
||||
chunked=chunked
|
||||
)
|
||||
end = time.time()
|
||||
self._log_request(method, req_url, resp, secs=(end - start),
|
||||
req_headers=req_headers, req_body=req_body,
|
||||
@ -541,7 +546,7 @@ class RestClient(object):
|
||||
|
||||
return resp, resp_body
|
||||
|
||||
def raw_request(self, url, method, headers=None, body=None):
|
||||
def raw_request(self, url, method, headers=None, body=None, chunked=False):
|
||||
"""Send a raw HTTP request without the keystone catalog or auth
|
||||
|
||||
This method sends a HTTP request in the same manner as the request()
|
||||
@ -554,17 +559,18 @@ class RestClient(object):
|
||||
:param str headers: Headers to use for the request if none are specifed
|
||||
the headers
|
||||
:param str body: Body to send with the request
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:rtype: tuple
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers()
|
||||
return self.http_obj.request(url, method,
|
||||
headers=headers, body=body)
|
||||
return self.http_obj.request(url, method, headers=headers,
|
||||
body=body, chunked=chunked)
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
body=None, chunked=False):
|
||||
"""Send a HTTP request with keystone auth and using the catalog
|
||||
|
||||
This method will send an HTTP request using keystone auth in the
|
||||
@ -590,6 +596,7 @@ class RestClient(object):
|
||||
get_headers() method are used. If the request
|
||||
explicitly requires no headers use an empty dict.
|
||||
:param str body: Body to send with the request
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:rtype: tuple
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
@ -629,8 +636,8 @@ class RestClient(object):
|
||||
except (ValueError, TypeError):
|
||||
headers = self.get_headers()
|
||||
|
||||
resp, resp_body = self._request(method, url,
|
||||
headers=headers, body=body)
|
||||
resp, resp_body = self._request(method, url, headers=headers,
|
||||
body=body, chunked=chunked)
|
||||
|
||||
while (resp.status == 413 and
|
||||
'retry-after' in resp and
|
||||
|
@ -19,6 +19,8 @@ import random
|
||||
import string
|
||||
import uuid
|
||||
|
||||
import six.moves
|
||||
|
||||
|
||||
def rand_uuid():
|
||||
"""Generate a random UUID string
|
||||
@ -196,3 +198,10 @@ def get_ipv6_addr_by_EUI64(cidr, mac):
|
||||
except TypeError:
|
||||
raise TypeError('Bad prefix type for generate IPv6 address by '
|
||||
'EUI-64: %s' % cidr)
|
||||
|
||||
|
||||
# Courtesy of http://stackoverflow.com/a/312464
|
||||
def chunkify(sequence, chunksize):
|
||||
"""Yield successive chunks from `sequence`."""
|
||||
for i in six.moves.xrange(0, len(sequence), chunksize):
|
||||
yield sequence[i:i + chunksize]
|
||||
|
@ -48,9 +48,9 @@ class BaseComputeClient(rest_client.RestClient):
|
||||
return headers
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
body=None, chunked=False):
|
||||
resp, resp_body = super(BaseComputeClient, self).request(
|
||||
method, url, extra_headers, headers, body)
|
||||
method, url, extra_headers, headers, body, chunked)
|
||||
if (COMPUTE_MICROVERSION and
|
||||
COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION):
|
||||
api_version_utils.assert_version_header_matches_request(
|
||||
|
@ -75,8 +75,12 @@ class TokenClient(rest_client.RestClient):
|
||||
return rest_client.ResponseBody(resp, body['access'])
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
"""A simple HTTP request interface."""
|
||||
body=None, chunked=False):
|
||||
"""A simple HTTP request interface.
|
||||
|
||||
Note: this overloads the `request` method from the parent class and
|
||||
thus must implement the same method signature.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers(accept_type="json")
|
||||
elif extra_headers:
|
||||
|
@ -122,8 +122,12 @@ class V3TokenClient(rest_client.RestClient):
|
||||
return rest_client.ResponseBody(resp, body)
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
"""A simple HTTP request interface."""
|
||||
body=None, chunked=False):
|
||||
"""A simple HTTP request interface.
|
||||
|
||||
Note: this overloads the `request` method from the parent class and
|
||||
thus must implement the same method signature.
|
||||
"""
|
||||
if headers is None:
|
||||
# Always accept 'json', for xml token client too.
|
||||
# Because XML response is not easily
|
||||
|
@ -149,25 +149,30 @@ class ObjectClient(rest_client.RestClient):
|
||||
self.expected_success(201, resp.status)
|
||||
return resp, body
|
||||
|
||||
def put_object_with_chunk(self, container, name, contents, chunk_size):
|
||||
"""Put an object with Transfer-Encoding header"""
|
||||
def put_object_with_chunk(self, container, name, contents):
|
||||
"""Put an object with Transfer-Encoding header
|
||||
|
||||
:param container: name of the container
|
||||
:type container: string
|
||||
:param name: name of the object
|
||||
:type name: string
|
||||
:param contents: object data
|
||||
:type contents: iterable
|
||||
"""
|
||||
headers = {'Transfer-Encoding': 'chunked'}
|
||||
if self.token:
|
||||
headers['X-Auth-Token'] = self.token
|
||||
|
||||
conn = put_object_connection(self.base_url, container, name, contents,
|
||||
chunk_size, headers)
|
||||
|
||||
resp = conn.getresponse()
|
||||
body = resp.read()
|
||||
|
||||
resp_headers = {}
|
||||
for header, value in resp.getheaders():
|
||||
resp_headers[header.lower()] = value
|
||||
url = "%s/%s" % (container, name)
|
||||
resp, body = self.put(
|
||||
url, headers=headers,
|
||||
body=contents,
|
||||
chunked=True
|
||||
)
|
||||
|
||||
self._error_checker('PUT', None, headers, contents, resp, body)
|
||||
self.expected_success(201, resp.status)
|
||||
return resp.status, resp.reason, resp_headers
|
||||
return resp.status, resp.reason, resp
|
||||
|
||||
def create_object_continue(self, container, object_name,
|
||||
data, metadata=None):
|
||||
@ -262,30 +267,7 @@ def put_object_connection(base_url, container, name, contents=None,
|
||||
headers = dict(headers)
|
||||
else:
|
||||
headers = {}
|
||||
if hasattr(contents, 'read'):
|
||||
conn.putrequest('PUT', path)
|
||||
for header, value in six.iteritems(headers):
|
||||
conn.putheader(header, value)
|
||||
if 'Content-Length' not in headers:
|
||||
if 'Transfer-Encoding' not in headers:
|
||||
conn.putheader('Transfer-Encoding', 'chunked')
|
||||
conn.endheaders()
|
||||
chunk = contents.read(chunk_size)
|
||||
while chunk:
|
||||
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
|
||||
chunk = contents.read(chunk_size)
|
||||
conn.send('0\r\n\r\n')
|
||||
else:
|
||||
conn.endheaders()
|
||||
left = headers['Content-Length']
|
||||
while left > 0:
|
||||
size = chunk_size
|
||||
if size > left:
|
||||
size = left
|
||||
chunk = contents.read(size)
|
||||
conn.send(chunk)
|
||||
left -= len(chunk)
|
||||
else:
|
||||
conn.request('PUT', path, contents, headers)
|
||||
|
||||
conn.request('PUT', path, contents, headers)
|
||||
|
||||
return conn
|
||||
|
@ -169,3 +169,10 @@ class TestDataUtils(base.TestCase):
|
||||
bad_mac = 99999999999999999999
|
||||
self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64,
|
||||
cidr, bad_mac)
|
||||
|
||||
def test_chunkify(self):
|
||||
data = "aaa"
|
||||
chunks = data_utils.chunkify(data, 2)
|
||||
self.assertEqual("aa", next(chunks))
|
||||
self.assertEqual("a", next(chunks))
|
||||
self.assertRaises(StopIteration, next, chunks)
|
||||
|
@ -21,7 +21,7 @@ class fake_httplib2(object):
|
||||
self.return_type = return_type
|
||||
|
||||
def request(self, uri, method="GET", body=None, headers=None,
|
||||
redirections=5, connection_type=None):
|
||||
redirections=5, connection_type=None, chunked=False):
|
||||
if not self.return_type:
|
||||
fake_headers = fake_http_response(headers)
|
||||
return_obj = {
|
||||
|
Loading…
x
Reference in New Issue
Block a user