From ba013510b25eef75776cf453cd31d5a1da3a0f80 Mon Sep 17 00:00:00 2001 From: Carlos Camacho Date: Thu, 5 Apr 2018 08:42:39 +0000 Subject: [PATCH] Allow uploading big files to swift (5GB) This patch enables the swift upload for big files (+5GB) in TripleO. Also, adds support to the undercloud backup CLI to upload big backups to swift. Change-Id: I80163c8df15377dd3b5a44b5439a23223d8cccee Depends-On: Iff73833f1d470750862873f70a4a9aaba50bd164 Closes-Bug: 1761412 (cherry picked from commit: I80163c8df15377dd3b5a44b5439a23223d8cccee) --- ...low-upload-big-files-f67ff35fcd166612.yaml | 12 +++ tripleo_common/actions/base.py | 17 +++++ tripleo_common/actions/logging_to_swift.py | 4 +- tripleo_common/actions/plan.py | 4 +- tripleo_common/actions/undercloud.py | 5 +- .../tests/actions/test_logging_to_swift.py | 24 +++++- tripleo_common/tests/actions/test_plan.py | 43 ++++++++--- .../tests/actions/test_undercloud.py | 30 +++++++- tripleo_common/utils/swift.py | 73 ++++++++++++++++--- 9 files changed, 183 insertions(+), 29 deletions(-) create mode 100644 releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml diff --git a/releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml b/releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml new file mode 100644 index 000000000..296a92b34 --- /dev/null +++ b/releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Allow uploading files bigger than 5GB to swift. + Currently we have support for uploading files + to swift using the swift client class, this class + does not allow to upload files bigger than 5GB. + This change enables the upload of files bigger than + 5GB by using the swift service class and adjusting + the headers to allow this operations. This new helper + will be used for the Undercloud backup, to be able to + store files bigger than 5GB. diff --git a/tripleo_common/actions/base.py b/tripleo_common/actions/base.py index 336960a82..8ab1514b2 100644 --- a/tripleo_common/actions/base.py +++ b/tripleo_common/actions/base.py @@ -26,6 +26,7 @@ from mistralclient.api import client as mistral_client from novaclient.client import Client as nova_client from swiftclient import client as swift_client from swiftclient import exceptions as swiftexceptions +from swiftclient import service as swift_service from zaqarclient.queues.v2 import client as zaqarclient from tripleo_common import constants @@ -62,6 +63,22 @@ class TripleOAction(actions.Action): } return swift_client.Connection(**kwargs) + # This version returns the SwiftService API + def get_object_service(self, context): + swift_endpoint = keystone_utils.get_endpoint_for_project( + context, 'swift') + + swift_opts = { + 'os_storage_url': swift_endpoint.url % { + 'tenant_id': context.project_id + }, + 'os_auth_token': context.auth_token, + 'os_region_name': swift_endpoint.region, + 'os_project_id': context.security.project_id, + } + + return swift_service.SwiftService(options=swift_opts) + def get_baremetal_client(self, context): ironic_endpoint = keystone_utils.get_endpoint_for_project( context, 'ironic') diff --git a/tripleo_common/actions/logging_to_swift.py b/tripleo_common/actions/logging_to_swift.py index 46a46daef..5da994da1 100644 --- a/tripleo_common/actions/logging_to_swift.py +++ b/tripleo_common/actions/logging_to_swift.py @@ -157,6 +157,8 @@ class PrepareLogDownloadAction(base.TripleOAction): def run(self, context): swift = self.get_object_client(context) + swift_service = self.get_object_service(context) + tmp_dir = tempfile.mkdtemp() tarball_name = 'logs-%s.tar.gz' % timeutils.timestamp() @@ -164,7 +166,7 @@ class PrepareLogDownloadAction(base.TripleOAction): swiftutils.download_container( swift, self.logging_container, tmp_dir) swiftutils.create_and_upload_tarball( - swift, tmp_dir, self.downloads_container, + swift_service, tmp_dir, self.downloads_container, tarball_name, self.delete_after) except swiftexceptions.ClientException as err: msg = "Error attempting an operation on container: %s" % err diff --git a/tripleo_common/actions/plan.py b/tripleo_common/actions/plan.py index 8fd85e1a4..54b75a4e0 100644 --- a/tripleo_common/actions/plan.py +++ b/tripleo_common/actions/plan.py @@ -186,13 +186,15 @@ class ExportPlanAction(base.TripleOAction): def run(self, context): swift = self.get_object_client(context) + swift_service = self.get_object_service(context) + tmp_dir = tempfile.mkdtemp() tarball_name = '%s.tar.gz' % self.plan try: swiftutils.download_container(swift, self.plan, tmp_dir) swiftutils.create_and_upload_tarball( - swift, tmp_dir, self.exports_container, tarball_name, + swift_service, tmp_dir, self.exports_container, tarball_name, delete_after=self.delete_after) except swiftexceptions.ClientException as err: msg = "Error attempting an operation on container: %s" % err diff --git a/tripleo_common/actions/undercloud.py b/tripleo_common/actions/undercloud.py index cd9e76d5e..745ceb36e 100644 --- a/tripleo_common/actions/undercloud.py +++ b/tripleo_common/actions/undercloud.py @@ -198,11 +198,10 @@ class UploadUndercloudBackupToSwift(tripleobase.TripleOAction): def run(self, context): try: LOG.info('Uploading backup to swift') - swift = self.get_object_client(context) - + swift_service = self.get_object_service(context) # Create tarball without gzip and store it 24h swiftutils.create_and_upload_tarball( - swift, self.backup_path, self.container, + swift_service, self.backup_path, self.container, self.tarball_name, '-cf', self.expire) msg = 'Backup uploaded to undercloud-backups succesfully' diff --git a/tripleo_common/tests/actions/test_logging_to_swift.py b/tripleo_common/tests/actions/test_logging_to_swift.py index d1f489afb..42a4f65e5 100644 --- a/tripleo_common/tests/actions/test_logging_to_swift.py +++ b/tripleo_common/tests/actions/test_logging_to_swift.py @@ -125,8 +125,12 @@ class PrepareLogDownloadActionTest(base.TestCase): self.ctx = mock.MagicMock() + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') @mock.patch('tripleo_common.utils.tarball.create_tarball') - def test_run_success(self, mock_create_tarball): + def test_run_success(self, + mock_create_tarball, + mock_get_obj_service): + get_object_mock_calls = [ mock.call('logging-container', lf) for lf in self.log_files ] @@ -134,6 +138,12 @@ class PrepareLogDownloadActionTest(base.TestCase): mock.call('logging-container') ] + swift_service = mock.MagicMock() + swift_service.delete.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service + action = logging_to_swift.PrepareLogDownloadAction( 'logging-container', 'downloads-container', 3600 ) @@ -145,10 +155,20 @@ class PrepareLogDownloadActionTest(base.TestCase): get_object_mock_calls, any_order=True) mock_create_tarball.assert_called_once() + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') @mock.patch('tripleo_common.utils.tarball.create_tarball') - def test_run_error_creating_tarball(self, mock_create_tarball): + def test_run_error_creating_tarball(self, + mock_create_tarball, + mock_get_obj_service): + mock_create_tarball.side_effect = processutils.ProcessExecutionError + swift_service = mock.MagicMock() + swift_service.delete.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service + action = logging_to_swift.PrepareLogDownloadAction( 'logging-container', 'downloads-container', 3600 ) diff --git a/tripleo_common/tests/actions/test_plan.py b/tripleo_common/tests/actions/test_plan.py index cce40419d..9dd68431d 100644 --- a/tripleo_common/tests/actions/test_plan.py +++ b/tripleo_common/tests/actions/test_plan.py @@ -398,30 +398,45 @@ class ExportPlanActionTest(base.TestCase): self.ctx = mock.MagicMock() + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') @mock.patch('tripleo_common.utils.tarball.create_tarball') - def test_run_success(self, mock_create_tarball): + def test_run_success(self, + mock_create_tarball, + mock_get_obj_service): + get_object_mock_calls = [ mock.call(self.plan, tf) for tf in self.template_files ] - get_container_mock_calls = [ - mock.call(self.plan), - mock.call('plan-exports') - ] + + swift_service = mock.MagicMock() + swift_service.upload.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service action = plan.ExportPlanAction(self.plan, self.delete_after, self.exports_container) action.run(self.ctx) - self.swift.get_container.assert_has_calls(get_container_mock_calls) + self.swift.get_container.assert_called_once_with(self.plan) self.swift.get_object.assert_has_calls( get_object_mock_calls, any_order=True) - self.swift.put_object.assert_called_once() + swift_service.upload.assert_called_once() mock_create_tarball.assert_called_once() - def test_run_container_does_not_exist(self): + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') + def test_run_container_does_not_exist(self, + mock_get_obj_service): + self.swift.get_container.side_effect = swiftexceptions.ClientException( self.plan) + swift_service = mock.MagicMock() + swift_service.delete.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service + action = plan.ExportPlanAction(self.plan, self.delete_after, self.exports_container) result = action.run(self.ctx) @@ -429,10 +444,20 @@ class ExportPlanActionTest(base.TestCase): error = "Error attempting an operation on container: %s" % self.plan self.assertIn(error, result.error) + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') @mock.patch('tripleo_common.utils.tarball.create_tarball') - def test_run_error_creating_tarball(self, mock_create_tarball): + def test_run_error_creating_tarball(self, + mock_create_tarball, + mock_get_obj_service): + mock_create_tarball.side_effect = processutils.ProcessExecutionError + swift_service = mock.MagicMock() + swift_service.delete.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service + action = plan.ExportPlanAction(self.plan, self.delete_after, self.exports_container) result = action.run(self.ctx) diff --git a/tripleo_common/tests/actions/test_undercloud.py b/tripleo_common/tests/actions/test_undercloud.py index ea6585666..2fccb4cf1 100644 --- a/tripleo_common/tests/actions/test_undercloud.py +++ b/tripleo_common/tests/actions/test_undercloud.py @@ -152,8 +152,12 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase): self.addCleanup(swift_patcher.stop) self.ctx = mock.MagicMock() + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service') @mock.patch('tripleo_common.utils.tarball.create_tarball') - def test_simple_success(self, mock_create_tarball): + def test_upload_to_swift_success(self, + mock_create_tarball, + mock_get_obj_service): + self.swift.head_object.return_value = { 'content-length': 1 } @@ -161,15 +165,33 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase): {}, [] ) + swift_service = mock.MagicMock() + swift_service.delete.return_value = ([ + {'success': True}, + ]) + mock_get_obj_service.return_value = swift_service + action = undercloud.UploadUndercloudBackupToSwift( self.backup_path, self.container) action.run(self.ctx) - self.swift.put_object.assert_called_once_with( + swift_service.upload.has_calls() + + options = {'use_slo': True, + 'changed': None, + 'meta': [], + 'fail_fast': True, + 'leave_segments': False, + 'header': ['X-Delete-After: 86400'], + 'skip_identical': False, + 'segment_size': 1048576000, + 'segment_container': None, + 'dir_marker': False} + + swift_service.upload.assert_called_once_with( self.container, - action.tarball_name, mock.ANY, - headers={'X-Delete-After': 86400} + options=options ) mock_create_tarball.assert_called_once() diff --git a/tripleo_common/utils/swift.py b/tripleo_common/utils/swift.py index f03f4b6ab..f353ef0a7 100644 --- a/tripleo_common/utils/swift.py +++ b/tripleo_common/utils/swift.py @@ -23,6 +23,9 @@ from swiftclient import exceptions as swiftexceptions from tripleo_common import constants from tripleo_common.utils import tarball +from swiftclient.service import SwiftError +from swiftclient.service import SwiftUploadObject + LOG = logging.getLogger(__name__) @@ -85,18 +88,70 @@ def get_or_create_container(swiftclient, container): return swiftclient.put_container(container) -def create_and_upload_tarball(swiftclient, +def create_and_upload_tarball(swiftservice, tmp_dir, container, tarball_name, tarball_options='-czf', - delete_after=3600): - """Create a tarball containing the tmp_dir and upload it to Swift.""" - headers = {'X-Delete-After': delete_after} + delete_after=3600, + segment_size=1048576000, + use_slo=True, + segment_container=None, + leave_segments=False, + changed=None, + skip_identical=False, + fail_fast=True, + dir_marker=False): + """Create a tarball containing the tmp_dir and upload it to Swift. - get_or_create_container(swiftclient, container) + This method allows to upload files bigger than 5GB. + It will create 2 swift containers to store the segments and + one container to reference the manifest with the segment pointers + """ - with tempfile.NamedTemporaryFile() as tmp_tarball: - tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options) - swiftclient.put_object(container, tarball_name, tmp_tarball, - headers=headers) + try: + with tempfile.NamedTemporaryFile() as tmp_tarball: + tarball.create_tarball(tmp_dir, + tmp_tarball.name, + tarball_options) + objs = [SwiftUploadObject(tmp_tarball, + object_name=tarball_name)] + options = {'meta': [], + 'header': ['X-Delete-After: ' + str(delete_after)], + 'segment_size': segment_size, + 'use_slo': use_slo, + 'segment_container': segment_container, + 'leave_segments': leave_segments, + 'changed': changed, + 'skip_identical': skip_identical, + 'fail_fast': fail_fast, + 'dir_marker': dir_marker + } + + for r in swiftservice.upload(container, + objs, + options=options): + if r['success']: + if 'object' in r: + LOG.info(r['object']) + elif 'for_object' in r: + LOG.info( + '%s segment %s' % (r['for_object'], + r['segment_index']) + ) + else: + error = r['error'] + if r['action'] == "create_container": + LOG.warning( + 'Warning: failed to create container ' + "'%s'%s", container, error + ) + elif r['action'] == "upload_object": + LOG.error( + "Failed to upload object %s to container %s: %s" % + (container, r['object'], error) + ) + else: + LOG.error("%s" % error) + except SwiftError as e: + LOG.error(e.value)