Add ability to stream object directly to file

For object downloads, allow the user to specify a file to write the
content to, rather than returning it all in memory.

Change-Id: Ic926fd63a9801049e8120d7597df9961a5f9e657

releasenotes/notes/stream-to-file-91f48d6dcea399c6.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - get_object now supports streaming output directly to a file.

shade/openstackcloud.py

@@ -6063,14 +6063,21 @@ class OpenStackCloud(_normalize.Normalizer):
             raise
 
     def get_object(self, container, obj, query_string=None,
-                   resp_chunk_size=None):
+                   resp_chunk_size=1024, outfile=None):
         """Get the headers and body of an object from swift
 
         :param string container: name of the container.
         :param string obj: name of the object.
         :param string query_string: query args for uri.
                                     (delimiter, prefix, etc.)
-        :param int resp_chunk_size: chunk size of data to read.
+        :param int resp_chunk_size: chunk size of data to read. Only used
+                                    if the results are being written to a
+                                    file. (optional, defaults to 1k)
+        :param outfile: Write the object to a file instead of
+                        returning the contents. If this option is
+                        given, body in the return tuple will be None. outfile
+                        can either be a file path given as a string, or a
+                        File like object.
 
         :returns: Tuple (headers, body) of the object, or None if the object
                   is not found (404)
@@ -6083,10 +6090,25 @@ class OpenStackCloud(_normalize.Normalizer):
             if query_string:
                 endpoint = '{endpoint}?{query_string}'.format(
                     endpoint=endpoint, query_string=query_string)
-            response = self._object_store_client.get(endpoint)
+            response = self._object_store_client.get(
+                endpoint, stream=True)
             response_headers = {
                 k.lower(): v for k, v in response.headers.items()}
-            return (response_headers, response.text)
+            if outfile:
+                if isinstance(outfile, six.string_types):
+                    outfile_handle = open(outfile, 'wb')
+                else:
+                    outfile_handle = outfile
+                for chunk in response.iter_content(
+                        resp_chunk_size, decode_unicode=False):
+                    outfile_handle.write(chunk)
+                if isinstance(outfile, six.string_types):
+                    outfile_handle.close()
+                else:
+                    outfile_handle.flush()
+                return (response_headers, None)
+            else:
+                return (response_headers, response.text)
         except OpenStackCloudHTTPError as e:
             if e.response.status_code == 404:
                 return None

shade/tests/functional/test_object.py

@@ -97,3 +97,72 @@ class TestObject(base.BaseFunctionalTestCase):
         self.assertEqual(container_name,
                          self.demo_cloud.list_containers()[0]['name'])
         self.demo_cloud.delete_container(container_name)
+
+    def test_download_object_to_file(self):
+        '''Test uploading small and large files.'''
+        container_name = self.getUniqueString('container')
+        self.addDetail('container', content.text_content(container_name))
+        self.addCleanup(self.demo_cloud.delete_container, container_name)
+        self.demo_cloud.create_container(container_name)
+        self.assertEqual(container_name,
+                         self.demo_cloud.list_containers()[0]['name'])
+        sizes = (
+            (64 * 1024, 1),  # 64K, one segment
+            (64 * 1024, 5)   # 64MB, 5 segments
+        )
+        for size, nseg in sizes:
+            fake_content = ''
+            segment_size = int(round(size / nseg))
+            with tempfile.NamedTemporaryFile() as fake_file:
+                fake_content = ''.join(random.SystemRandom().choice(
+                    string.ascii_uppercase + string.digits)
+                    for _ in range(size)).encode('latin-1')
+
+                fake_file.write(fake_content)
+                fake_file.flush()
+                name = 'test-%d' % size
+                self.addCleanup(
+                    self.demo_cloud.delete_object, container_name, name)
+                self.demo_cloud.create_object(
+                    container_name, name,
+                    fake_file.name,
+                    segment_size=segment_size,
+                    metadata={'foo': 'bar'})
+                self.assertFalse(self.demo_cloud.is_object_stale(
+                    container_name, name,
+                    fake_file.name
+                    )
+                )
+            self.assertEqual(
+                'bar', self.demo_cloud.get_object_metadata(
+                    container_name, name)['x-object-meta-foo']
+            )
+            self.demo_cloud.update_object(container=container_name, name=name,
+                                          metadata={'testk': 'testv'})
+            self.assertEqual(
+                'testv', self.demo_cloud.get_object_metadata(
+                    container_name, name)['x-object-meta-testk']
+            )
+            try:
+                with tempfile.NamedTemporaryFile() as fake_file:
+                    self.demo_cloud.get_object(
+                        container_name, name, outfile=fake_file.name)
+                    downloaded_content = open(fake_file.name, 'rb').read()
+                    self.assertEqual(fake_content, downloaded_content)
+            except exc.OpenStackCloudException as e:
+                self.addDetail(
+                    'failed_response',
+                    content.text_content(str(e.response.headers)))
+                self.addDetail(
+                    'failed_response',
+                    content.text_content(e.response.text))
+                raise
+            self.assertEqual(
+                name,
+                self.demo_cloud.list_objects(container_name)[0]['name'])
+            self.assertTrue(
+                self.demo_cloud.delete_object(container_name, name))
+        self.assertEqual([], self.demo_cloud.list_objects(container_name))
+        self.assertEqual(container_name,
+                         self.demo_cloud.list_containers()[0]['name'])
+        self.demo_cloud.delete_container(container_name)