openstack/tripleo-common
Code Issues Proposed changes

Allow uploading big files to swift (5GB)

Browse Source 
This patch enables the swift upload for big files (+5GB)
in TripleO.
Also, adds support to the undercloud backup CLI to upload
big backups to swift.

Change-Id: I80163c8df15377dd3b5a44b5439a23223d8cccee
Depends-On: Iff73833f1d470750862873f70a4a9aaba50bd164
Closes-Bug: 1761412
This commit is contained in:
Carlos Camacho 2018-04-05 08:42:39 +00:00
parent 2cf8cbddae
commit 13f2704c8e
9 changed files with 182 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
releasenotes/notes/allow-upload-big-files-f67ff35fcd166612.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- |
Allow uploading files bigger than 5GB to swift.
Currently we have support for uploading files
to swift using the swift client class, this class
does not allow to upload files bigger than 5GB.
This change enables the upload of files bigger than
5GB by using the swift service class and adjusting
the headers to allow this operations. This new helper
will be used for the Undercloud backup, to be able to
store files bigger than 5GB.

17
tripleo_common/actions/base.py
View File

@ -26,6 +26,7 @@ from mistralclient.api import client as mistral_client
from novaclient.client import Client as nova_client
from swiftclient import client as swift_client
from swiftclient import exceptions as swiftexceptions
from swiftclient import service as swift_service
from zaqarclient.queues.v2 import client as zaqarclient
from tripleo_common import constants
@ -62,6 +63,22 @@ class TripleOAction(actions.Action):
}
return swift_client.Connection(**kwargs)
# This version returns the SwiftService API
def get_object_service(self, context):
swift_endpoint = keystone_utils.get_endpoint_for_project(
context, 'swift')
swift_opts = {
'os_storage_url': swift_endpoint.url % {
'tenant_id': context.project_id
},
'os_auth_token': context.auth_token,
'os_region_name': swift_endpoint.region,
'os_project_id': context.security.project_id,
}
return swift_service.SwiftService(options=swift_opts)
def get_baremetal_client(self, context):
ironic_endpoint = keystone_utils.get_endpoint_for_project(
context, 'ironic')

4
tripleo_common/actions/logging_to_swift.py
View File

@ -158,6 +158,8 @@ class PrepareLogDownloadAction(base.TripleOAction):
def run(self, context):
swift = self.get_object_client(context)
swift_service = self.get_object_service(context)
tmp_dir = tempfile.mkdtemp()
tarball_name = 'logs-%s.tar.gz' % timeutils.timestamp()
@ -165,7 +167,7 @@ class PrepareLogDownloadAction(base.TripleOAction):
swiftutils.download_container(
swift, self.logging_container, tmp_dir)
swiftutils.create_and_upload_tarball(
swift, tmp_dir, self.downloads_container,
swift_service, tmp_dir, self.downloads_container,
tarball_name, self.delete_after)
except swiftexceptions.ClientException as err:
msg = "Error attempting an operation on container: %s" % err

4
tripleo_common/actions/plan.py
View File

@ -188,13 +188,15 @@ class ExportPlanAction(base.TripleOAction):
def run(self, context):
swift = self.get_object_client(context)
swift_service = self.get_object_service(context)
tmp_dir = tempfile.mkdtemp()
tarball_name = '%s.tar.gz' % self.plan
try:
swiftutils.download_container(swift, self.plan, tmp_dir)
swiftutils.create_and_upload_tarball(
swift, tmp_dir, self.exports_container, tarball_name,
swift_service, tmp_dir, self.exports_container, tarball_name,
delete_after=self.delete_after)
except swiftexceptions.ClientException as err:
msg = "Error attempting an operation on container: %s" % err

5
tripleo_common/actions/undercloud.py
View File

@ -198,11 +198,10 @@ class UploadUndercloudBackupToSwift(tripleobase.TripleOAction):
def run(self, context):
try:
LOG.info('Uploading backup to swift')
swift = self.get_object_client(context)
swift_service = self.get_object_service(context)
# Create tarball without gzip and store it 24h
swiftutils.create_and_upload_tarball(
swift, self.backup_path, self.container,
swift_service, self.backup_path, self.container,
self.tarball_name, '-cf', self.expire)
msg = 'Backup uploaded to undercloud-backups succesfully'

24
tripleo_common/tests/actions/test_logging_to_swift.py
View File

@ -122,8 +122,12 @@ class PrepareLogDownloadActionTest(base.TestCase):
self.ctx = mock.MagicMock()
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_run_success(self, mock_create_tarball):
def test_run_success(self,
mock_create_tarball,
mock_get_obj_service):
get_object_mock_calls = [
mock.call('logging-container', lf) for lf in self.log_files
]
@ -131,6 +135,12 @@ class PrepareLogDownloadActionTest(base.TestCase):
mock.call('logging-container')
]
swift_service = mock.MagicMock()
swift_service.delete.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = logging_to_swift.PrepareLogDownloadAction(
'logging-container', 'downloads-container', 3600
)
@ -142,10 +152,20 @@ class PrepareLogDownloadActionTest(base.TestCase):
get_object_mock_calls, any_order=True)
mock_create_tarball.assert_called_once()
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_run_error_creating_tarball(self, mock_create_tarball):
def test_run_error_creating_tarball(self,
mock_create_tarball,
mock_get_obj_service):
mock_create_tarball.side_effect = processutils.ProcessExecutionError
swift_service = mock.MagicMock()
swift_service.delete.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = logging_to_swift.PrepareLogDownloadAction(
'logging-container', 'downloads-container', 3600
)

38
tripleo_common/tests/actions/test_plan.py
View File

@ -398,27 +398,45 @@ class ExportPlanActionTest(base.TestCase):
self.ctx = mock.MagicMock()
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_run_success(self, mock_create_tarball):
def test_run_success(self,
mock_create_tarball,
mock_get_obj_service):
get_object_mock_calls = [
mock.call(self.plan, tf) for tf in self.template_files
]
swift_service = mock.MagicMock()
swift_service.upload.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = plan.ExportPlanAction(self.plan, self.delete_after,
self.exports_container)
action.run(self.ctx)
self.swift.get_container.assert_called_once_with(self.plan)
self.swift.put_container.assert_called_once_with('plan-exports')
self.swift.get_object.assert_has_calls(
get_object_mock_calls, any_order=True)
self.swift.put_object.assert_called_once()
swift_service.upload.assert_called_once()
mock_create_tarball.assert_called_once()
def test_run_container_does_not_exist(self):
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
def test_run_container_does_not_exist(self,
mock_get_obj_service):
self.swift.get_container.side_effect = swiftexceptions.ClientException(
self.plan)
swift_service = mock.MagicMock()
swift_service.delete.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = plan.ExportPlanAction(self.plan, self.delete_after,
self.exports_container)
result = action.run(self.ctx)
@ -426,10 +444,20 @@ class ExportPlanActionTest(base.TestCase):
error = "Error attempting an operation on container: %s" % self.plan
self.assertIn(error, result.error)
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_run_error_creating_tarball(self, mock_create_tarball):
def test_run_error_creating_tarball(self,
mock_create_tarball,
mock_get_obj_service):
mock_create_tarball.side_effect = processutils.ProcessExecutionError
swift_service = mock.MagicMock()
swift_service.delete.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = plan.ExportPlanAction(self.plan, self.delete_after,
self.exports_container)
result = action.run(self.ctx)

30
tripleo_common/tests/actions/test_undercloud.py
View File

@ -152,8 +152,12 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase):
self.addCleanup(swift_patcher.stop)
self.ctx = mock.MagicMock()
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_service')
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_simple_success(self, mock_create_tarball):
def test_upload_to_swift_success(self,
mock_create_tarball,
mock_get_obj_service):
self.swift.head_object.return_value = {
'content-length': 1
}
@ -161,15 +165,33 @@ class UploadUndercloudBackupToSwiftTest(base.TestCase):
{}, []
)
swift_service = mock.MagicMock()
swift_service.delete.return_value = ([
{'success': True},
])
mock_get_obj_service.return_value = swift_service
action = undercloud.UploadUndercloudBackupToSwift(
self.backup_path, self.container)
action.run(self.ctx)
self.swift.put_object.assert_called_once_with(
swift_service.upload.has_calls()
options = {'use_slo': True,
'changed': None,
'meta': [],
'fail_fast': True,
'leave_segments': False,
'header': ['X-Delete-After: 86400'],
'skip_identical': False,
'segment_size': 1048576000,
'segment_container': None,
'dir_marker': False}
swift_service.upload.assert_called_once_with(
self.container,
action.tarball_name,
mock.ANY,
headers={'X-Delete-After': 86400}
options=options
)
mock_create_tarball.assert_called_once()

73
tripleo_common/utils/swift.py
View File

@ -21,6 +21,9 @@ import tempfile
from tripleo_common import constants
from tripleo_common.utils import tarball
from swiftclient.service import SwiftError
from swiftclient.service import SwiftUploadObject
LOG = logging.getLogger(__name__)
@ -81,18 +84,70 @@ def create_container(swiftclient, container):
swiftclient.put_container(container)
def create_and_upload_tarball(swiftclient,
def create_and_upload_tarball(swiftservice,
tmp_dir,
container,
tarball_name,
tarball_options='-czf',
delete_after=3600):
"""Create a tarball containing the tmp_dir and upload it to Swift."""
headers = {'X-Delete-After': delete_after}
delete_after=3600,
segment_size=1048576000,
use_slo=True,
segment_container=None,
leave_segments=False,
changed=None,
skip_identical=False,
fail_fast=True,
dir_marker=False):
"""Create a tarball containing the tmp_dir and upload it to Swift.
create_container(swiftclient, container)
This method allows to upload files bigger than 5GB.
It will create 2 swift containers to store the segments and
one container to reference the manifest with the segment pointers
"""
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
swiftclient.put_object(container, tarball_name, tmp_tarball,
headers=headers)
try:
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir,
tmp_tarball.name,
tarball_options)
objs = [SwiftUploadObject(tmp_tarball,
object_name=tarball_name)]
options = {'meta': [],
'header': ['X-Delete-After: ' + str(delete_after)],
'segment_size': segment_size,
'use_slo': use_slo,
'segment_container': segment_container,
'leave_segments': leave_segments,
'changed': changed,
'skip_identical': skip_identical,
'fail_fast': fail_fast,
'dir_marker': dir_marker
}
for r in swiftservice.upload(container,
objs,
options=options):
if r['success']:
if 'object' in r:
LOG.info(r['object'])
elif 'for_object' in r:
LOG.info(
'%s segment %s' % (r['for_object'],
r['segment_index'])
)
else:
error = r['error']
if r['action'] == "create_container":
LOG.warning(
'Warning: failed to create container '
"'%s'%s", container, error
)
elif r['action'] == "upload_object":
LOG.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error)
)
else:
LOG.error("%s" % error)
except SwiftError as e:
LOG.error(e.value)