Allow uploading big files to swift (5GB)

This patch enables the swift upload for big files (+5GB)
in TripleO.
Also, adds support to the undercloud backup CLI to upload
big backups to swift.

Change-Id: I80163c8df15377dd3b5a44b5439a23223d8cccee
Depends-On: Iff73833f1d470750862873f70a4a9aaba50bd164
Closes-Bug: 1761412

releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Allow uploading files bigger than 5GB to swift.
+    Currently we have support for uploading files
+    to swift using the swift client class, this class
+    does not allow to upload files bigger than 5GB.
+    This change enables the upload of files bigger than
+    5GB by using the swift service class and adjusting
+    the headers to allow this operations. This new helper
+    will be used for the Undercloud backup, to be able to
+    store files bigger than 5GB.

tripleo_common/actions/base.py

@@ -26,6 +26,7 @@ from mistralclient.api import client as mistral_client
 from novaclient.client import Client as nova_client
 from swiftclient import client as swift_client
 from swiftclient import exceptions as swiftexceptions
+from swiftclient import service as swift_service
 from zaqarclient.queues.v2 import client as zaqarclient
 
 from tripleo_common import constants
@@ -62,6 +63,22 @@ class TripleOAction(actions.Action):
         }
         return swift_client.Connection(**kwargs)
 
+    # This version returns the SwiftService API
+    def get_object_service(self, context):
+        swift_endpoint = keystone_utils.get_endpoint_for_project(
+            context, 'swift')
+
+        swift_opts = {
+            'os_storage_url': swift_endpoint.url % {
+                'tenant_id': context.project_id
+            },
+            'os_auth_token': context.auth_token,
+            'os_region_name': swift_endpoint.region,
+            'os_project_id': context.security.project_id,
+        }
+
+        return swift_service.SwiftService(options=swift_opts)
+
     def get_baremetal_client(self, context):
         ironic_endpoint = keystone_utils.get_endpoint_for_project(
             context, 'ironic')

tripleo_common/actions/logging_to_swift.py

@@ -158,6 +158,8 @@ class PrepareLogDownloadAction(base.TripleOAction):
 
     def run(self, context):
         swift = self.get_object_client(context)
+        swift_service = self.get_object_service(context)
+
         tmp_dir = tempfile.mkdtemp()
         tarball_name = 'logs-%s.tar.gz' % timeutils.timestamp()
 
@@ -165,7 +167,7 @@ class PrepareLogDownloadAction(base.TripleOAction):
             swiftutils.download_container(
                 swift, self.logging_container, tmp_dir)
             swiftutils.create_and_upload_tarball(
-                swift, tmp_dir, self.downloads_container,
+                swift_service, tmp_dir, self.downloads_container,
                 tarball_name, self.delete_after)
         except swiftexceptions.ClientException as err:
             msg = "Error attempting an operation on container: %s" % err

tripleo_common/actions/plan.py

@@ -188,13 +188,15 @@ class ExportPlanAction(base.TripleOAction):
 
     def run(self, context):
         swift = self.get_object_client(context)
+        swift_service = self.get_object_service(context)
+
         tmp_dir = tempfile.mkdtemp()
         tarball_name = '%s.tar.gz' % self.plan
 
         try:
             swiftutils.download_container(swift, self.plan, tmp_dir)
             swiftutils.create_and_upload_tarball(
-                swift, tmp_dir, self.exports_container, tarball_name,
+                swift_service, tmp_dir, self.exports_container, tarball_name,
                 delete_after=self.delete_after)
         except swiftexceptions.ClientException as err:
             msg = "Error attempting an operation on container: %s" % err

tripleo_common/actions/undercloud.py

@@ -198,11 +198,10 @@ class UploadUndercloudBackupToSwift(tripleobase.TripleOAction):
     def run(self, context):
         try:
             LOG.info('Uploading backup to swift')
-            swift = self.get_object_client(context)
+            swift_service = self.get_object_service(context)
-
             # Create tarball without gzip and store it 24h
             swiftutils.create_and_upload_tarball(
-                swift, self.backup_path, self.container,
+                swift_service, self.backup_path, self.container,
                 self.tarball_name, '-cf', self.expire)
 
             msg = 'Backup uploaded to undercloud-backups succesfully'

tripleo_common/tests/actions/test_logging_to_swift.py

@@ -122,8 +122,12 @@ class PrepareLogDownloadActionTest(base.TestCase):
 
         self.ctx = mock.MagicMock()
 
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
     @mock.patch('tripleo_common.utils.tarball.create_tarball')
-    def test_run_success(self, mock_create_tarball):
+    def test_run_success(self,
+                         mock_create_tarball,
+                         mock_get_obj_service):
+
         get_object_mock_calls = [
             mock.call('logging-container', lf) for lf in self.log_files
         ]
@@ -131,6 +135,12 @@ class PrepareLogDownloadActionTest(base.TestCase):
             mock.call('logging-container')
         ]
 
+        swift_service = mock.MagicMock()
+        swift_service.delete.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = logging_to_swift.PrepareLogDownloadAction(
             'logging-container', 'downloads-container', 3600
         )
@@ -142,10 +152,20 @@ class PrepareLogDownloadActionTest(base.TestCase):
             get_object_mock_calls, any_order=True)
         mock_create_tarball.assert_called_once()
 
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
     @mock.patch('tripleo_common.utils.tarball.create_tarball')
-    def test_run_error_creating_tarball(self, mock_create_tarball):
+    def test_run_error_creating_tarball(self,
+                                        mock_create_tarball,
+                                        mock_get_obj_service):
+
         mock_create_tarball.side_effect = processutils.ProcessExecutionError
 
+        swift_service = mock.MagicMock()
+        swift_service.delete.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = logging_to_swift.PrepareLogDownloadAction(
             'logging-container', 'downloads-container', 3600
         )

tripleo_common/tests/actions/test_plan.py

@@ -398,27 +398,45 @@ class ExportPlanActionTest(base.TestCase):
 
         self.ctx = mock.MagicMock()
 
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
     @mock.patch('tripleo_common.utils.tarball.create_tarball')
-    def test_run_success(self, mock_create_tarball):
+    def test_run_success(self,
+                         mock_create_tarball,
+                         mock_get_obj_service):
+
         get_object_mock_calls = [
             mock.call(self.plan, tf) for tf in self.template_files
         ]
 
+        swift_service = mock.MagicMock()
+        swift_service.upload.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = plan.ExportPlanAction(self.plan, self.delete_after,
                                        self.exports_container)
         action.run(self.ctx)
 
         self.swift.get_container.assert_called_once_with(self.plan)
-        self.swift.put_container.assert_called_once_with('plan-exports')
         self.swift.get_object.assert_has_calls(
             get_object_mock_calls, any_order=True)
-        self.swift.put_object.assert_called_once()
+        swift_service.upload.assert_called_once()
         mock_create_tarball.assert_called_once()
 
-    def test_run_container_does_not_exist(self):
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
+    def test_run_container_does_not_exist(self,
+                                          mock_get_obj_service):
+
         self.swift.get_container.side_effect = swiftexceptions.ClientException(
             self.plan)
 
+        swift_service = mock.MagicMock()
+        swift_service.delete.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = plan.ExportPlanAction(self.plan, self.delete_after,
                                        self.exports_container)
         result = action.run(self.ctx)
@@ -426,10 +444,20 @@ class ExportPlanActionTest(base.TestCase):
         error = "Error attempting an operation on container: %s" % self.plan
         self.assertIn(error, result.error)
 
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
     @mock.patch('tripleo_common.utils.tarball.create_tarball')
-    def test_run_error_creating_tarball(self, mock_create_tarball):
+    def test_run_error_creating_tarball(self,
+                                        mock_create_tarball,
+                                        mock_get_obj_service):
+
         mock_create_tarball.side_effect = processutils.ProcessExecutionError
 
+        swift_service = mock.MagicMock()
+        swift_service.delete.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = plan.ExportPlanAction(self.plan, self.delete_after,
                                        self.exports_container)
         result = action.run(self.ctx)

tripleo_common/tests/actions/test_undercloud.py

@@ -152,8 +152,12 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase):
         self.addCleanup(swift_patcher.stop)
         self.ctx = mock.MagicMock()
 
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
     @mock.patch('tripleo_common.utils.tarball.create_tarball')
-    def test_simple_success(self, mock_create_tarball):
+    def test_upload_to_swift_success(self,
+                                     mock_create_tarball,
+                                     mock_get_obj_service):
+
         self.swift.head_object.return_value = {
             'content-length': 1
         }
@@ -161,15 +165,33 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase):
             {}, []
         )
 
+        swift_service = mock.MagicMock()
+        swift_service.delete.return_value = ([
+            {'success': True},
+        ])
+        mock_get_obj_service.return_value = swift_service
+
         action = undercloud.UploadUndercloudBackupToSwift(
             self.backup_path, self.container)
         action.run(self.ctx)
 
-        self.swift.put_object.assert_called_once_with(
+        swift_service.upload.has_calls()
+
+        options = {'use_slo': True,
+                   'changed': None,
+                   'meta': [],
+                   'fail_fast': True,
+                   'leave_segments': False,
+                   'header': ['X-Delete-After: 86400'],
+                   'skip_identical': False,
+                   'segment_size': 1048576000,
+                   'segment_container': None,
+                   'dir_marker': False}
+
+        swift_service.upload.assert_called_once_with(
             self.container,
-            action.tarball_name,
             mock.ANY,
-            headers={'X-Delete-After': 86400}
+            options=options
             )
 
         mock_create_tarball.assert_called_once()

tripleo_common/utils/swift.py

@@ -21,6 +21,9 @@ import tempfile
 from tripleo_common import constants
 from tripleo_common.utils import tarball
 
+from swiftclient.service import SwiftError
+from swiftclient.service import SwiftUploadObject
+
 LOG = logging.getLogger(__name__)
 
 
@@ -81,18 +84,70 @@ def create_container(swiftclient, container):
     swiftclient.put_container(container)
 
 
-def create_and_upload_tarball(swiftclient,
+def create_and_upload_tarball(swiftservice,
                               tmp_dir,
                               container,
                               tarball_name,
                               tarball_options='-czf',
-                              delete_after=3600):
+                              delete_after=3600,
-    """Create a tarball containing the tmp_dir and upload it to Swift."""
+                              segment_size=1048576000,
-    headers = {'X-Delete-After': delete_after}
+                              use_slo=True,
+                              segment_container=None,
+                              leave_segments=False,
+                              changed=None,
+                              skip_identical=False,
+                              fail_fast=True,
+                              dir_marker=False):
+    """Create a tarball containing the tmp_dir and upload it to Swift.
 
-    create_container(swiftclient, container)
+       This method allows to upload files bigger than 5GB.
+       It will create 2 swift containers to store the segments and
+       one container to reference the manifest with the segment pointers
+    """
 
+    try:
         with tempfile.NamedTemporaryFile() as tmp_tarball:
-        tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
+            tarball.create_tarball(tmp_dir,
-        swiftclient.put_object(container, tarball_name, tmp_tarball,
+                                   tmp_tarball.name,
-                               headers=headers)
+                                   tarball_options)
+            objs = [SwiftUploadObject(tmp_tarball,
+                                      object_name=tarball_name)]
+            options = {'meta': [],
+                       'header': ['X-Delete-After: ' + str(delete_after)],
+                       'segment_size': segment_size,
+                       'use_slo': use_slo,
+                       'segment_container': segment_container,
+                       'leave_segments': leave_segments,
+                       'changed': changed,
+                       'skip_identical': skip_identical,
+                       'fail_fast': fail_fast,
+                       'dir_marker': dir_marker
+                       }
+
+            for r in swiftservice.upload(container,
+                                         objs,
+                                         options=options):
+                if r['success']:
+                    if 'object' in r:
+                        LOG.info(r['object'])
+                    elif 'for_object' in r:
+                        LOG.info(
+                            '%s segment %s' % (r['for_object'],
+                                               r['segment_index'])
+                            )
+                else:
+                    error = r['error']
+                    if r['action'] == "create_container":
+                        LOG.warning(
+                            'Warning: failed to create container '
+                            "'%s'%s", container, error
+                        )
+                    elif r['action'] == "upload_object":
+                        LOG.error(
+                            "Failed to upload object %s to container %s: %s" %
+                            (container, r['object'], error)
+                        )
+                    else:
+                        LOG.error("%s" % error)
+    except SwiftError as e:
+        LOG.error(e.value)