Wrap raw iterators to ensure we send entire contents to server

Currently, if you attempt to stream an upload from an iterator, as in

    def data():
        yield 'foo'
        yield ''
        yield 'bar'
    conn.put_object('c', 'o', data())

... requests will faithfully emit a zero-length chunk, ending the
transfer. Swift will then close the connection, possibly (if Connection:
keep-alive was set) after attempting to parse the next chunk as a new
request.

Now, Swift will receive all of the bytes from the iterable, and any
zero-byte chunks will be ignored.

This will be fixed in requests [1], but not until an eventual 3.0.0
release.

[1] https://github.com/kennethreitz/requests/pull/2631

Change-Id: I19579ed7a0181ac3f488433e7c1839f7f7a040b8

swiftclient/client.py

@@ -32,7 +32,7 @@ import six
 from swiftclient import version as swiftclient_version
 from swiftclient.exceptions import ClientException
 from swiftclient.utils import (
-    LengthWrapper, ReadableToIterable, parse_api_response)
+    iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response)
 
 # Defautl is 100, increase to 256
 http_client._MAXHEADERS = 256
@@ -1126,6 +1126,10 @@ def put_object(url, token=None, container=None, name=None, contents=None,
             warn_msg = ('%s object has no "read" method, ignoring chunk_size'
                         % type(contents).__name__)
             warnings.warn(warn_msg, stacklevel=2)
+        # Match requests's is_stream test
+        if hasattr(contents, '__iter__') and not isinstance(contents, (
+                six.text_type, six.binary_type, list, tuple, dict)):
+            contents = iter_wrapper(contents)
         conn.request('PUT', path, contents, headers)
 
     resp = conn.getresponse()

swiftclient/utils.py

@@ -231,3 +231,12 @@ class LengthWrapper(object):
             self.md5sum.update(chunk.encode())
 
         return chunk
+
+
+def iter_wrapper(iterable):
+    for chunk in iterable:
+        if len(chunk) == 0:
+            # If we emit an empty chunk, requests will go ahead and send it,
+            # causing the server to close the connection
+            continue
+        yield chunk

tests/unit/test_swiftclient.py

@@ -984,6 +984,26 @@ class TestPutObject(MockHttpTest):
                 data += chunk
             self.assertEqual(data, raw_data)
 
+    def test_iter_upload(self):
+        def data():
+            for chunk in ('foo', '', 'bar'):
+                yield chunk
+        conn = c.http_connection(u'http://www.test.com/')
+        resp = MockHttpResponse(status=200)
+        conn[1].getresponse = resp.fake_response
+        conn[1]._request = resp._fake_request
+
+        c.put_object(url='http://www.test.com', http_conn=conn,
+                     contents=data())
+        req_headers = resp.requests_params['headers']
+        self.assertNotIn('Content-Length', req_headers)
+        req_data = resp.requests_params['data']
+        self.assertTrue(hasattr(req_data, '__iter__'))
+        # If we emit an empty chunk, requests will go ahead and send it,
+        # causing the server to close the connection. So make sure we don't
+        # do that.
+        self.assertEqual(['foo', 'bar'], list(req_data))
+
     def test_md5_mismatch(self):
         conn = c.http_connection('http://www.test.com')
         resp = MockHttpResponse(status=200, verify=True,