openstack
/
tempest
Code Issues Proposed changes

Swift object client: use urllib3 builtin support for chunked transfer 

Urllib3 has native support for chunked encoding, so let's use this
instead of rolling our own. Less code to maintain, additional logging
and timing (thanks to our common RestClient). Yeah \O/.

Change-Id: I4a253a5cec0fc35009af25872239363625d417e3
This commit is contained in:
Jordan Pittier 2016-04-29 15:05:09 +02:00
parent c808dc503b
commit 4408c4a5fe
10 changed files with 80 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml Normal file
View File

@ -0,0 +1,9 @@
---
features:
- The RestClient (in tempest.lib.common.rest_client) now supports POSTing
and PUTing data with chunked transfer encoding. Just pass an `iterable`
object as the `body` argument and set the `chunked` argument to `True`.
- A new generator called `chunkify` is added in
tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices)
from a Python sequence.

5
tempest/api/object_storage/test_object_services.py
View File

@ -20,7 +20,6 @@ import time
import zlib
import six
from six import moves
from tempest.api.object_storage import base
from tempest.common import custom_matchers
@ -201,8 +200,8 @@ class ObjectTest(base.BaseObjectTest):
status, _, resp_headers = self.object_client.put_object_with_chunk(
container=self.container_name,
name=object_name,
contents=moves.cStringIO(data),
chunk_size=512)
contents=data_utils.chunkify(data, 512)
)
self.assertHeaders(resp_headers, 'Object', 'PUT')
# check uploaded content

31
tempest/lib/common/rest_client.py
View File

@ -243,7 +243,8 @@ class RestClient(object):
details = pattern.format(read_code, expected_code)
raise exceptions.InvalidHttpSuccessCode(details)
def post(self, url, body, headers=None, extra_headers=False):
def post(self, url, body, headers=None, extra_headers=False,
chunked=False):
"""Send a HTTP POST request using keystone auth
:param str url: the relative url to send the post request to
@ -253,11 +254,12 @@ class RestClient(object):
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
:param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
return self.request('POST', url, extra_headers, headers, body)
return self.request('POST', url, extra_headers, headers, body, chunked)
def get(self, url, headers=None, extra_headers=False):
"""Send a HTTP GET request using keystone service catalog and auth
@ -306,7 +308,7 @@ class RestClient(object):
"""
return self.request('PATCH', url, extra_headers, headers, body)
def put(self, url, body, headers=None, extra_headers=False):
def put(self, url, body, headers=None, extra_headers=False, chunked=False):
"""Send a HTTP PUT request using keystone service catalog and auth
:param str url: the relative url to send the post request to
@ -316,11 +318,12 @@ class RestClient(object):
returned by the get_headers() method are to
be used but additional headers are needed in
the request pass them in as a dict.
:param bool chunked: sends the body with chunked encoding
:return: a tuple with the first entry containing the response headers
and the second the response body
:rtype: tuple
"""
return self.request('PUT', url, extra_headers, headers, body)
return self.request('PUT', url, extra_headers, headers, body, chunked)
def head(self, url, headers=None, extra_headers=False):
"""Send a HTTP HEAD request using keystone service catalog and auth
@ -520,7 +523,7 @@ class RestClient(object):
if method != 'HEAD' and not resp_body and resp.status >= 400:
self.LOG.warning("status >= 400 response with empty body")
def _request(self, method, url, headers=None, body=None):
def _request(self, method, url, headers=None, body=None, chunked=False):
"""A simple HTTP request interface."""
# Authenticate the request with the auth provider
req_url, req_headers, req_body = self.auth_provider.auth_request(
@ -530,7 +533,9 @@ class RestClient(object):
start = time.time()
self._log_request_start(method, req_url)
resp, resp_body = self.raw_request(
req_url, method, headers=req_headers, body=req_body)
req_url, method, headers=req_headers, body=req_body,
chunked=chunked
)
end = time.time()
self._log_request(method, req_url, resp, secs=(end - start),
req_headers=req_headers, req_body=req_body,
@ -541,7 +546,7 @@ class RestClient(object):
return resp, resp_body
def raw_request(self, url, method, headers=None, body=None):
def raw_request(self, url, method, headers=None, body=None, chunked=False):
"""Send a raw HTTP request without the keystone catalog or auth
This method sends a HTTP request in the same manner as the request()
@ -554,17 +559,18 @@ class RestClient(object):
:param str headers: Headers to use for the request if none are specifed
the headers
:param str body: Body to send with the request
:param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
"""
if headers is None:
headers = self.get_headers()
return self.http_obj.request(url, method,
headers=headers, body=body)
return self.http_obj.request(url, method, headers=headers,
body=body, chunked=chunked)
def request(self, method, url, extra_headers=False, headers=None,
body=None):
body=None, chunked=False):
"""Send a HTTP request with keystone auth and using the catalog
This method will send an HTTP request using keystone auth in the
@ -590,6 +596,7 @@ class RestClient(object):
get_headers() method are used. If the request
explicitly requires no headers use an empty dict.
:param str body: Body to send with the request
:param bool chunked: sends the body with chunked encoding
:rtype: tuple
:return: a tuple with the first entry containing the response headers
and the second the response body
@ -629,8 +636,8 @@ class RestClient(object):
except (ValueError, TypeError):
headers = self.get_headers()
resp, resp_body = self._request(method, url,
headers=headers, body=body)
resp, resp_body = self._request(method, url, headers=headers,
body=body, chunked=chunked)
while (resp.status == 413 and
'retry-after' in resp and

9
tempest/lib/common/utils/data_utils.py
View File

@ -19,6 +19,8 @@ import random
import string
import uuid
import six.moves
def rand_uuid():
"""Generate a random UUID string
@ -196,3 +198,10 @@ def get_ipv6_addr_by_EUI64(cidr, mac):
except TypeError:
raise TypeError('Bad prefix type for generate IPv6 address by '
'EUI-64: %s' % cidr)
# Courtesy of http://stackoverflow.com/a/312464
def chunkify(sequence, chunksize):
"""Yield successive chunks from `sequence`."""
for i in six.moves.xrange(0, len(sequence), chunksize):
yield sequence[i:i + chunksize]

4
tempest/lib/services/compute/base_compute_client.py
View File

@ -48,9 +48,9 @@ class BaseComputeClient(rest_client.RestClient):
return headers
def request(self, method, url, extra_headers=False, headers=None,
body=None):
body=None, chunked=False):
resp, resp_body = super(BaseComputeClient, self).request(
method, url, extra_headers, headers, body)
method, url, extra_headers, headers, body, chunked)
if (COMPUTE_MICROVERSION and
COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION):
api_version_utils.assert_version_header_matches_request(

8
tempest/lib/services/identity/v2/token_client.py
View File

@ -75,8 +75,12 @@ class TokenClient(rest_client.RestClient):
return rest_client.ResponseBody(resp, body['access'])
def request(self, method, url, extra_headers=False, headers=None,
body=None):
"""A simple HTTP request interface."""
body=None, chunked=False):
"""A simple HTTP request interface.
Note: this overloads the `request` method from the parent class and
thus must implement the same method signature.
"""
if headers is None:
headers = self.get_headers(accept_type="json")
elif extra_headers:

8
tempest/lib/services/identity/v3/token_client.py
View File

@ -122,8 +122,12 @@ class V3TokenClient(rest_client.RestClient):
return rest_client.ResponseBody(resp, body)
def request(self, method, url, extra_headers=False, headers=None,
body=None):
"""A simple HTTP request interface."""
body=None, chunked=False):
"""A simple HTTP request interface.
Note: this overloads the `request` method from the parent class and
thus must implement the same method signature.
"""
if headers is None:
# Always accept 'json', for xml token client too.
# Because XML response is not easily

56
tempest/services/object_storage/object_client.py
View File

@ -149,25 +149,30 @@ class ObjectClient(rest_client.RestClient):
self.expected_success(201, resp.status)
return resp, body
def put_object_with_chunk(self, container, name, contents, chunk_size):
"""Put an object with Transfer-Encoding header"""
def put_object_with_chunk(self, container, name, contents):
"""Put an object with Transfer-Encoding header
:param container: name of the container
:type container: string
:param name: name of the object
:type name: string
:param contents: object data
:type contents: iterable
"""
headers = {'Transfer-Encoding': 'chunked'}
if self.token:
headers['X-Auth-Token'] = self.token
conn = put_object_connection(self.base_url, container, name, contents,
chunk_size, headers)
resp = conn.getresponse()
body = resp.read()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
url = "%s/%s" % (container, name)
resp, body = self.put(
url, headers=headers,
body=contents,
chunked=True
)
self._error_checker('PUT', None, headers, contents, resp, body)
self.expected_success(201, resp.status)
return resp.status, resp.reason, resp_headers
return resp.status, resp.reason, resp
def create_object_continue(self, container, object_name,
data, metadata=None):
@ -262,30 +267,7 @@ def put_object_connection(base_url, container, name, contents=None,
headers = dict(headers)
else:
headers = {}
if hasattr(contents, 'read'):
conn.putrequest('PUT', path)
for header, value in six.iteritems(headers):
conn.putheader(header, value)
if 'Content-Length' not in headers:
if 'Transfer-Encoding' not in headers:
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
chunk = contents.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents.read(chunk_size)
conn.send('0\r\n\r\n')
else:
conn.endheaders()
left = headers['Content-Length']
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents.read(size)
conn.send(chunk)
left -= len(chunk)
else:
conn.request('PUT', path, contents, headers)
conn.request('PUT', path, contents, headers)
return conn

7
tempest/tests/lib/common/utils/test_data_utils.py
View File

@ -169,3 +169,10 @@ class TestDataUtils(base.TestCase):
bad_mac = 99999999999999999999
self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64,
cidr, bad_mac)
def test_chunkify(self):
data = "aaa"
chunks = data_utils.chunkify(data, 2)
self.assertEqual("aa", next(chunks))
self.assertEqual("a", next(chunks))
self.assertRaises(StopIteration, next, chunks)

2
tempest/tests/lib/fake_http.py
View File

@ -21,7 +21,7 @@ class fake_httplib2(object):
self.return_type = return_type
def request(self, uri, method="GET", body=None, headers=None,
redirections=5, connection_type=None):
redirections=5, connection_type=None, chunked=False):
if not self.return_type:
fake_headers = fake_http_response(headers)
return_obj = {