Add support for streaming object responses

When getting objects, sometimes you want a stream rather than reading
the whole value or passing to a file-like object. Add a stream_object
method that returns an iterator over the content to be returned.

Change-Id: I6f7ae89799e733657adc9989b3accfdf6e3f2f82

openstack/cloud/openstackcloud.py

@@ -7793,12 +7793,7 @@ class OpenStackCloud(_normalize.Normalizer):
         :raises: OpenStackCloudException on operation error.
         """
         endpoint = self._get_object_endpoint(container, obj, query_string)
-        try:
         return self._object_store_client.get(endpoint, stream=stream)
-        except exc.OpenStackCloudHTTPError as e:
-            if e.response.status_code == 404:
-                return None
-            raise
 
     def _get_object_endpoint(self, container, obj, query_string):
         endpoint = '{container}/{object}'.format(
@@ -7808,8 +7803,33 @@ class OpenStackCloud(_normalize.Normalizer):
                 endpoint=endpoint, query_string=query_string)
         return endpoint
 
+    def stream_object(
+            self, container, obj, query_string=None, resp_chunk_size=1024):
+        """Download the content via a streaming iterator.
+
+        :param string container: name of the container.
+        :param string obj: name of the object.
+        :param string query_string:
+            query args for uri. (delimiter, prefix, etc.)
+        :param int resp_chunk_size:
+            chunk size of data to read. Only used if the results are
+
+        :returns:
+            An iterator over the content or None if the object is not found.
+        :raises: OpenStackCloudException on operation error.
+        """
+        try:
+            with self.get_object_raw(
+                    container, obj, query_string=query_string) as response:
+                for ret in response.iter_content(chunk_size=resp_chunk_size):
+                    yield ret
+        except exc.OpenStackCloudHTTPError as e:
+            if e.response.status_code == 404:
+                return
+            raise
+
     def get_object(self, container, obj, query_string=None,
-                   resp_chunk_size=1024, outfile=None):
+                   resp_chunk_size=1024, outfile=None, stream=False):
         """Get the headers and body of an object
 
         :param string container: name of the container.

openstack/tests/unit/cloud/test_object.py

@@ -347,6 +347,45 @@ class TestObject(BaseTestObject):
 
         self.assertEqual((response_headers, text), resp)
 
+    def test_stream_object(self):
+        text = b'test body'
+        self.register_uris([
+            dict(method='GET', uri=self.object_endpoint,
+                 headers={
+                     'Content-Length': '20304400896',
+                     'Content-Type': 'application/octet-stream',
+                     'Accept-Ranges': 'bytes',
+                     'Last-Modified': 'Thu, 15 Dec 2016 13:34:14 GMT',
+                     'Etag': '"b5c454b44fbd5344793e3fb7e3850768"',
+                     'X-Timestamp': '1481808853.65009',
+                     'X-Trans-Id': 'tx68c2a2278f0c469bb6de1-005857ed80dfw1',
+                     'Date': 'Mon, 19 Dec 2016 14:24:00 GMT',
+                     'X-Static-Large-Object': 'True',
+                     'X-Object-Meta-Mtime': '1481513709.168512',
+                 },
+                 text='test body')])
+
+        response_text = b''
+        for data in self.cloud.stream_object(self.container, self.object):
+            response_text += data
+
+        self.assert_calls()
+
+        self.assertEqual(text, response_text)
+
+    def test_stream_object_not_found(self):
+        self.register_uris([
+            dict(method='GET', uri=self.object_endpoint, status_code=404),
+        ])
+
+        response_text = b''
+        for data in self.cloud.stream_object(self.container, self.object):
+            response_text += data
+
+        self.assert_calls()
+
+        self.assertEqual(b'', response_text)
+
     def test_get_object_not_found(self):
         self.register_uris([dict(method='GET',
                                  uri=self.object_endpoint, status_code=404)])

releasenotes/notes/stream-object-6ecd43511dca726b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Added ``stream_object`` method for getting object content in an iterator.