diff --git a/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml new file mode 100644 index 0000000000..eb45523fee --- /dev/null +++ b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml @@ -0,0 +1,9 @@ +--- +features: + - The RestClient (in tempest.lib.common.rest_client) now supports POSTing + and PUTing data with chunked transfer encoding. Just pass an `iterable` + object as the `body` argument and set the `chunked` argument to `True`. + - A new generator called `chunkify` is added in + tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices) + from a Python sequence. + diff --git a/tempest/api/object_storage/test_object_services.py b/tempest/api/object_storage/test_object_services.py index dc926e0f41..a88e4f41a4 100644 --- a/tempest/api/object_storage/test_object_services.py +++ b/tempest/api/object_storage/test_object_services.py @@ -20,7 +20,6 @@ import time import zlib import six -from six import moves from tempest.api.object_storage import base from tempest.common import custom_matchers @@ -201,8 +200,8 @@ class ObjectTest(base.BaseObjectTest): status, _, resp_headers = self.object_client.put_object_with_chunk( container=self.container_name, name=object_name, - contents=moves.cStringIO(data), - chunk_size=512) + contents=data_utils.chunkify(data, 512) + ) self.assertHeaders(resp_headers, 'Object', 'PUT') # check uploaded content diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py index 30750dec17..179db17a58 100644 --- a/tempest/lib/common/rest_client.py +++ b/tempest/lib/common/rest_client.py @@ -243,7 +243,8 @@ class RestClient(object): details = pattern.format(read_code, expected_code) raise exceptions.InvalidHttpSuccessCode(details) - def post(self, url, body, headers=None, extra_headers=False): + def post(self, url, body, headers=None, extra_headers=False, + chunked=False): """Send a HTTP POST request using keystone auth :param str url: the relative url to send the post request to @@ -253,11 +254,12 @@ class RestClient(object): returned by the get_headers() method are to be used but additional headers are needed in the request pass them in as a dict. + :param bool chunked: sends the body with chunked encoding :return: a tuple with the first entry containing the response headers and the second the response body :rtype: tuple """ - return self.request('POST', url, extra_headers, headers, body) + return self.request('POST', url, extra_headers, headers, body, chunked) def get(self, url, headers=None, extra_headers=False): """Send a HTTP GET request using keystone service catalog and auth @@ -306,7 +308,7 @@ class RestClient(object): """ return self.request('PATCH', url, extra_headers, headers, body) - def put(self, url, body, headers=None, extra_headers=False): + def put(self, url, body, headers=None, extra_headers=False, chunked=False): """Send a HTTP PUT request using keystone service catalog and auth :param str url: the relative url to send the post request to @@ -316,11 +318,12 @@ class RestClient(object): returned by the get_headers() method are to be used but additional headers are needed in the request pass them in as a dict. + :param bool chunked: sends the body with chunked encoding :return: a tuple with the first entry containing the response headers and the second the response body :rtype: tuple """ - return self.request('PUT', url, extra_headers, headers, body) + return self.request('PUT', url, extra_headers, headers, body, chunked) def head(self, url, headers=None, extra_headers=False): """Send a HTTP HEAD request using keystone service catalog and auth @@ -520,7 +523,7 @@ class RestClient(object): if method != 'HEAD' and not resp_body and resp.status >= 400: self.LOG.warning("status >= 400 response with empty body") - def _request(self, method, url, headers=None, body=None): + def _request(self, method, url, headers=None, body=None, chunked=False): """A simple HTTP request interface.""" # Authenticate the request with the auth provider req_url, req_headers, req_body = self.auth_provider.auth_request( @@ -530,7 +533,9 @@ class RestClient(object): start = time.time() self._log_request_start(method, req_url) resp, resp_body = self.raw_request( - req_url, method, headers=req_headers, body=req_body) + req_url, method, headers=req_headers, body=req_body, + chunked=chunked + ) end = time.time() self._log_request(method, req_url, resp, secs=(end - start), req_headers=req_headers, req_body=req_body, @@ -541,7 +546,7 @@ class RestClient(object): return resp, resp_body - def raw_request(self, url, method, headers=None, body=None): + def raw_request(self, url, method, headers=None, body=None, chunked=False): """Send a raw HTTP request without the keystone catalog or auth This method sends a HTTP request in the same manner as the request() @@ -554,17 +559,18 @@ class RestClient(object): :param str headers: Headers to use for the request if none are specifed the headers :param str body: Body to send with the request + :param bool chunked: sends the body with chunked encoding :rtype: tuple :return: a tuple with the first entry containing the response headers and the second the response body """ if headers is None: headers = self.get_headers() - return self.http_obj.request(url, method, - headers=headers, body=body) + return self.http_obj.request(url, method, headers=headers, + body=body, chunked=chunked) def request(self, method, url, extra_headers=False, headers=None, - body=None): + body=None, chunked=False): """Send a HTTP request with keystone auth and using the catalog This method will send an HTTP request using keystone auth in the @@ -590,6 +596,7 @@ class RestClient(object): get_headers() method are used. If the request explicitly requires no headers use an empty dict. :param str body: Body to send with the request + :param bool chunked: sends the body with chunked encoding :rtype: tuple :return: a tuple with the first entry containing the response headers and the second the response body @@ -629,8 +636,8 @@ class RestClient(object): except (ValueError, TypeError): headers = self.get_headers() - resp, resp_body = self._request(method, url, - headers=headers, body=body) + resp, resp_body = self._request(method, url, headers=headers, + body=body, chunked=chunked) while (resp.status == 413 and 'retry-after' in resp and diff --git a/tempest/lib/common/utils/data_utils.py b/tempest/lib/common/utils/data_utils.py index 9605479623..45e50673d7 100644 --- a/tempest/lib/common/utils/data_utils.py +++ b/tempest/lib/common/utils/data_utils.py @@ -19,6 +19,8 @@ import random import string import uuid +import six.moves + def rand_uuid(): """Generate a random UUID string @@ -196,3 +198,10 @@ def get_ipv6_addr_by_EUI64(cidr, mac): except TypeError: raise TypeError('Bad prefix type for generate IPv6 address by ' 'EUI-64: %s' % cidr) + + +# Courtesy of http://stackoverflow.com/a/312464 +def chunkify(sequence, chunksize): + """Yield successive chunks from `sequence`.""" + for i in six.moves.xrange(0, len(sequence), chunksize): + yield sequence[i:i + chunksize] diff --git a/tempest/lib/services/compute/base_compute_client.py b/tempest/lib/services/compute/base_compute_client.py index 9161abbdec..a387b855f7 100644 --- a/tempest/lib/services/compute/base_compute_client.py +++ b/tempest/lib/services/compute/base_compute_client.py @@ -48,9 +48,9 @@ class BaseComputeClient(rest_client.RestClient): return headers def request(self, method, url, extra_headers=False, headers=None, - body=None): + body=None, chunked=False): resp, resp_body = super(BaseComputeClient, self).request( - method, url, extra_headers, headers, body) + method, url, extra_headers, headers, body, chunked) if (COMPUTE_MICROVERSION and COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION): api_version_utils.assert_version_header_matches_request( diff --git a/tempest/lib/services/identity/v2/token_client.py b/tempest/lib/services/identity/v2/token_client.py index 03501757a7..571602735d 100644 --- a/tempest/lib/services/identity/v2/token_client.py +++ b/tempest/lib/services/identity/v2/token_client.py @@ -75,8 +75,12 @@ class TokenClient(rest_client.RestClient): return rest_client.ResponseBody(resp, body['access']) def request(self, method, url, extra_headers=False, headers=None, - body=None): - """A simple HTTP request interface.""" + body=None, chunked=False): + """A simple HTTP request interface. + + Note: this overloads the `request` method from the parent class and + thus must implement the same method signature. + """ if headers is None: headers = self.get_headers(accept_type="json") elif extra_headers: diff --git a/tempest/lib/services/identity/v3/token_client.py b/tempest/lib/services/identity/v3/token_client.py index f342a497e9..964d43f0b7 100644 --- a/tempest/lib/services/identity/v3/token_client.py +++ b/tempest/lib/services/identity/v3/token_client.py @@ -122,8 +122,12 @@ class V3TokenClient(rest_client.RestClient): return rest_client.ResponseBody(resp, body) def request(self, method, url, extra_headers=False, headers=None, - body=None): - """A simple HTTP request interface.""" + body=None, chunked=False): + """A simple HTTP request interface. + + Note: this overloads the `request` method from the parent class and + thus must implement the same method signature. + """ if headers is None: # Always accept 'json', for xml token client too. # Because XML response is not easily diff --git a/tempest/services/object_storage/object_client.py b/tempest/services/object_storage/object_client.py index fa43d947cd..33dba6e412 100644 --- a/tempest/services/object_storage/object_client.py +++ b/tempest/services/object_storage/object_client.py @@ -149,25 +149,30 @@ class ObjectClient(rest_client.RestClient): self.expected_success(201, resp.status) return resp, body - def put_object_with_chunk(self, container, name, contents, chunk_size): - """Put an object with Transfer-Encoding header""" + def put_object_with_chunk(self, container, name, contents): + """Put an object with Transfer-Encoding header + + :param container: name of the container + :type container: string + :param name: name of the object + :type name: string + :param contents: object data + :type contents: iterable + """ headers = {'Transfer-Encoding': 'chunked'} if self.token: headers['X-Auth-Token'] = self.token - conn = put_object_connection(self.base_url, container, name, contents, - chunk_size, headers) - - resp = conn.getresponse() - body = resp.read() - - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + url = "%s/%s" % (container, name) + resp, body = self.put( + url, headers=headers, + body=contents, + chunked=True + ) self._error_checker('PUT', None, headers, contents, resp, body) self.expected_success(201, resp.status) - return resp.status, resp.reason, resp_headers + return resp.status, resp.reason, resp def create_object_continue(self, container, object_name, data, metadata=None): @@ -262,30 +267,7 @@ def put_object_connection(base_url, container, name, contents=None, headers = dict(headers) else: headers = {} - if hasattr(contents, 'read'): - conn.putrequest('PUT', path) - for header, value in six.iteritems(headers): - conn.putheader(header, value) - if 'Content-Length' not in headers: - if 'Transfer-Encoding' not in headers: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = contents.read(chunk_size) - conn.send('0\r\n\r\n') - else: - conn.endheaders() - left = headers['Content-Length'] - while left > 0: - size = chunk_size - if size > left: - size = left - chunk = contents.read(size) - conn.send(chunk) - left -= len(chunk) - else: - conn.request('PUT', path, contents, headers) + + conn.request('PUT', path, contents, headers) return conn diff --git a/tempest/tests/lib/common/utils/test_data_utils.py b/tempest/tests/lib/common/utils/test_data_utils.py index f9e1f44492..399c4afc1c 100644 --- a/tempest/tests/lib/common/utils/test_data_utils.py +++ b/tempest/tests/lib/common/utils/test_data_utils.py @@ -169,3 +169,10 @@ class TestDataUtils(base.TestCase): bad_mac = 99999999999999999999 self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64, cidr, bad_mac) + + def test_chunkify(self): + data = "aaa" + chunks = data_utils.chunkify(data, 2) + self.assertEqual("aa", next(chunks)) + self.assertEqual("a", next(chunks)) + self.assertRaises(StopIteration, next, chunks) diff --git a/tempest/tests/lib/fake_http.py b/tempest/tests/lib/fake_http.py index 397c856459..cfa4b93685 100644 --- a/tempest/tests/lib/fake_http.py +++ b/tempest/tests/lib/fake_http.py @@ -21,7 +21,7 @@ class fake_httplib2(object): self.return_type = return_type def request(self, uri, method="GET", body=None, headers=None, - redirections=5, connection_type=None): + redirections=5, connection_type=None, chunked=False): if not self.return_type: fake_headers = fake_http_response(headers) return_obj = {