diff --git a/releasenotes/notes/undercloud-backup-actions-and-workflow-1d661bba3fb2f974.yaml b/releasenotes/notes/undercloud-backup-actions-and-workflow-1d661bba3fb2f974.yaml new file mode 100644 index 000000000..e498a26ca --- /dev/null +++ b/releasenotes/notes/undercloud-backup-actions-and-workflow-1d661bba3fb2f974.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Introduce Undercloud Backup workflow as well as set of Mistral actions to + perform Undercloud Backup diff --git a/setup.cfg b/setup.cfg index c82c32e61..9b2a8fab3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -140,6 +140,12 @@ mistral.actions = tripleo.files.make_temp_dir = tripleo_common.actions.files:MakeTempDir tripleo.files.remove_temp_dir = tripleo_common.actions.files:RemoveTempDir tripleo.ansible-generate-inventory = tripleo_common.actions.ansible:AnsibleGenerateInventoryAction + tripleo.undercloud.get_free_space = tripleo_common.actions.undercloud:GetFreeSpace + tripleo.undercloud.create_backup_dir = tripleo_common.actions.undercloud:CreateBackupDir + tripleo.undercloud.create_database_backup = tripleo_common.actions.undercloud:CreateDatabaseBackup + tripleo.undercloud.create_file_system_backup = tripleo_common.actions.undercloud:CreateFileSystemBackup + tripleo.undercloud.upload_backup_to_swift = tripleo_common.actions.undercloud:UploadUndercloudBackupToSwift + tripleo.undercloud.remove_temp_dir = tripleo_common.actions.undercloud:RemoveTempDir # deprecated for pike release, will be removed in queens tripleo.ansible = tripleo_common.actions.ansible:AnsibleAction tripleo.ansible-playbook = tripleo_common.actions.ansible:AnsiblePlaybookAction diff --git a/sudoers b/sudoers index db590fd48..9e89bf3f4 100644 --- a/sudoers +++ b/sudoers @@ -8,4 +8,6 @@ mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_identit mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \ !/usr/bin/rm /tmp/validations_identity_* *, !/usr/bin/rm /tmp/validations_identity_*..* mistral ALL = NOPASSWD: /bin/nova-manage cell_v2 discover_hosts * +mistral ALL = NOPASSWD: /usr/bin/tar --ignore-failed-read -C / -cf /var/tmp/undercloud-backup-*.tar * +mistral ALL = NOPASSWD: /usr/bin/chown mistral. /var/tmp/undercloud-backup-*/filesystem-*.tar validations ALL = NOPASSWD: ALL diff --git a/tripleo_common/actions/undercloud.py b/tripleo_common/actions/undercloud.py new file mode 100644 index 000000000..cd9e76d5e --- /dev/null +++ b/tripleo_common/actions/undercloud.py @@ -0,0 +1,236 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import os +import re +import shutil +import six +import subprocess +import tempfile +import time + +from mistral_lib import actions +from mistral_lib.actions import base + +from tripleo_common.actions import base as tripleobase +from tripleo_common.utils import swift as swiftutils + +LOG = logging.getLogger(__name__) + + +class GetFreeSpace(base.Action): + """Get the Undercloud free space for the backup. + + The default path to check will be /var/tmp and the + default minimum size will be 10240 MB (10GB). + """ + + def __init__(self, min_space=10240, temp_dir="/var/tmp/"): + self.min_space = min_space + self.temp_dir = temp_dir + + def run(self, context): + temp_path = self.temp_dir + min_space = self.min_space + while not os.path.isdir(temp_path): + head, tail = os.path.split(temp_path) + temp_path = head + available_space = ( + (os.statvfs(temp_path).f_frsize * os.statvfs(temp_path).f_bavail) / + (1024 * 1024)) + if (available_space < min_space): + msg = "There is not enough space, avail. - %s MB" \ + % str(int(available_space)) + return actions.Result(error={'msg': msg}) + else: + msg = "There is enough space, avail. - %s MB" \ + % str(int(available_space)) + return actions.Result(data={'msg': msg}) + + +class CreateBackupDir(base.Action): + """Creates the Backup temporary directory. + + We will run the backup locally, so we need to create a temporary + directory. The directory created will match the regular expression + ^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$ + """ + + def __init__(self): + pass + + def run(self, context): + try: + _path = tempfile.mkdtemp(prefix='undercloud-backup-', + dir='/var/tmp/') + return actions.Result(data={"path": _path}) + except Exception as msg: + return actions.Result(error={"msg": six.text_type(msg)}) + + +class CreateDatabaseBackup(base.Action): + """Creates a database full backup. + + This action will run the DB dump using a single transaction and storing + the result in a given folder. + """ + + def __init__(self, path, dbuser, dbpassword): + self.path = path + self.dbuser = dbuser + self.dbpassword = dbpassword + self.backup_name = os.path.join(self.path, + 'all-databases-' + + time.strftime("%Y%m%d%H%M%S") + + '.sql.gz') + + def run(self, context): + pid_file = tempfile.gettempdir() + os.sep + "mysqldump.pid" + if os.path.exists(pid_file): + msg = 'Another Backup process is running' + return actions.Result(error={"msg": six.text_type(msg)}) + lockfile = open(pid_file, 'w') + lockfile.write("%s\n" % os.getpid()) + lockfile.close + + # Backup all databases with nice and ionice just not to create + # a huge load on undercloud. Output will be redirected to mysqldump + # variable and will be gzipped. + script = """ + #!/bin/bash + nice -n 19 ionice -c2 -n7 \ + mysqldump -u%s -p%s --opt --all-databases | gzip > %s + """ % (self.dbuser, self.dbpassword, self.backup_name) + + proc_failed = False + + try: + subprocess.check_call(script, shell=True) + except subprocess.CalledProcessError: + proc_failed = True + msg = 'Database dump failed. Deleting temporary directory' + os.remove(self.backup_name) + else: + msg = 'Database dump created succesfully' + finally: + os.remove(pid_file) + + if proc_failed: + return actions.Result(error={'msg': six.text_type(msg)}) + else: + return actions.Result(data={'msg': six.text_type(msg)}) + + +class CreateFileSystemBackup(base.Action): + """Creates a File System backup. + + This action will run a filesystem backup based on an array of resources + to be backed up. This method gets the sources paths and the destination + folder as parameters. + """ + + def __init__(self, sources_path, path): + self.sources_path = sources_path + self.path = path + self.outfile = os.path.join(self.path, + 'filesystem-' + + time.strftime("%Y%m%d%H%M%S") + + '.tar') + + def run(self, context): + + backup_sources = self.sources_path.split(',') + separated_string = ' '.join(backup_sources) + + script = """ + #!/bin/bash + sudo tar --ignore-failed-read -C / -cf %s %s + sudo chown mistral. %s + """ % (self.outfile, separated_string, self.outfile) + + proc_failed = False + try: + subprocess.check_call(script, shell=True) + except subprocess.CalledProcessError: + proc_failed = True + msg = 'File system backup failed' + os.remove(self.outfile) + else: + msg = 'File system backup created succesfully at: ' + self.outfile + + if proc_failed: + # Delete failed backup here + return actions.Result(error={'msg': six.text_type(msg)}) + else: + return actions.Result(data={'msg': msg}) + + +class UploadUndercloudBackupToSwift(tripleobase.TripleOAction): + """Push the Undercloud backup to a swift container. + + This action will push the files in the temporary folder to the swift + container storing the Undercloud backups as uncompressed tarball file. + The backup will be stored 1 day (86400 s) + """ + + def __init__(self, + backup_path, + container='undercloud-backups', + expire=86400): + self.backup_path = backup_path + self.container = container + self.expire = expire + self.tarball_name = 'UC-backup-%s.tar' % time.strftime( + "%Y%m%d%H%M%S") + + def run(self, context): + try: + LOG.info('Uploading backup to swift') + swift = self.get_object_client(context) + + # Create tarball without gzip and store it 24h + swiftutils.create_and_upload_tarball( + swift, self.backup_path, self.container, + self.tarball_name, '-cf', self.expire) + + msg = 'Backup uploaded to undercloud-backups succesfully' + return actions.Result(data={'msg': msg}) + except Exception as msg: + return actions.Result(error={'msg': six.text_type(msg)}) + + +class RemoveTempDir(base.Action): + """Removes temporary directory on localhost by path. + + The path must match the regular expression + ^/var/tmp/undercloud-backup-[A-Za-z0-9_]+$ + """ + + def __init__(self, path): + self.path = path + + def run(self, context): + # regex from tempfile's _RandomNameSequence characters + _regex = '^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$' + if (not isinstance(self.path, six.string_types) or + not re.match(_regex, self.path)): + msg = "Path does not match %s" % _regex + return actions.Result(error={"msg": msg}) + try: + shutil.rmtree(self.path) + msg = "Deleted directory %s" % self.path + return actions.Result(data={"msg": msg}) + except Exception as msg: + return actions.Result(error={"msg": six.text_type(msg)}) diff --git a/tripleo_common/tests/actions/test_undercloud.py b/tripleo_common/tests/actions/test_undercloud.py new file mode 100644 index 000000000..ea6585666 --- /dev/null +++ b/tripleo_common/tests/actions/test_undercloud.py @@ -0,0 +1,175 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import mock + +from tripleo_common.actions import undercloud +from tripleo_common.tests import base + + +class GetFreeSpaceTest(base.TestCase): + def setUp(self): + super(GetFreeSpaceTest, self).setUp() + self.temp_dir = "/var/tmp" + + @mock.patch("os.path.isdir") + @mock.patch("os.statvfs") + def test_run_false(self, mock_statvfs, mock_isdir): + mock_isdir.return_value = True + mock_statvfs.return_value = mock.MagicMock( + spec_set=['f_frsize', 'f_bavail'], + f_frsize=4096, f_bavail=1024) + action = undercloud.GetFreeSpace() + action_result = action.run(context={}) + mock_isdir.assert_called() + mock_statvfs.assert_called() + self.assertEqual("There is not enough space, avail. - 4 MB", + action_result.error['msg']) + + @mock.patch("os.path.isdir") + @mock.patch("os.statvfs") + def test_run_true(self, mock_statvfs, mock_isdir): + mock_isdir.return_value = True + mock_statvfs.return_value = mock.MagicMock( + spec_set=['f_frsize', 'f_bavail'], + f_frsize=4096, f_bavail=10240000) + action = undercloud.GetFreeSpace() + action_result = action.run(context={}) + mock_isdir.assert_called() + mock_statvfs.assert_called() + self.assertEqual("There is enough space, avail. - 40000 MB", + action_result.data['msg']) + + +class RemoveTempDirTest(base.TestCase): + + def setUp(self): + super(RemoveTempDirTest, self).setUp() + self.path = "/var/tmp/undercloud-backup-dG6hr_" + + @mock.patch("shutil.rmtree") + def test_sucess_remove_temp_dir(self, mock_rmtree): + mock_rmtree.return_value = None # rmtree has no return value + action = undercloud.RemoveTempDir(self.path) + action_result = action.run(context={}) + mock_rmtree.assert_called() + self.assertFalse(action_result.cancel) + self.assertIsNone(action_result.error) + self.assertEqual('Deleted directory /var/tmp/undercloud-backup-dG6hr_', + action_result.data['msg']) + + +class CreateDatabaseBackupTest(base.TestCase): + + def setUp(self): + super(CreateDatabaseBackupTest, self).setUp() + self.dbback = undercloud.CreateDatabaseBackup( + '/var/tmp/undercloud-backup-dG6hr_', + 'root', 'dbpassword') + + @mock.patch( + 'tripleo_common.actions.base.TripleOAction.get_object_client') + @mock.patch('subprocess.check_call') + def test_create_database_backup( + self, mock_check_call, mock_get_object_client): + self.dbback.logger = mock.Mock() + self.dbback.run(mock_get_object_client) + assert_string = ('\n #!/bin/bash\n ' + 'nice -n 19 ionice -c2 -n7 ' + 'mysqldump -uroot -pdbpassword --opt ' + '--all-databases | gzip > ' + + self.dbback.backup_name + + '\n ') + mock_check_call.assert_called_once_with(assert_string, shell=True) + + +class CreateFileSystemBackupTest(base.TestCase): + + def setUp(self): + super(CreateFileSystemBackupTest, self).setUp() + self.fsback = undercloud.CreateFileSystemBackup( + '/home/stack/,/etc/hosts', + '/var/tmp/undercloud-backup-ef9b_H') + + @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') + @mock.patch('subprocess.check_call') + def test_create_file_system_backup( + self, + mock_check_call, + mock_get_object_client): + self.fsback.logger = mock.Mock() + self.fsback.run(mock_get_object_client) + assert_string = ('\n #!/bin/bash\n ' + 'sudo tar --ignore-failed-read -C / ' + '-cf ' + + self.fsback.outfile + + ' /home/stack/ /etc/hosts\n ' + 'sudo chown mistral. ' + + self.fsback.outfile + + '\n ') + mock_check_call.assert_called_once_with(assert_string, shell=True) + + +class CreateBackupDirTest(base.TestCase): + + def setUp(self): + super(CreateBackupDirTest, self).setUp() + self.temp_dir = '/var/tmp/undercloud-backup-XXXXXX' + + @mock.patch('tempfile.mkdtemp') + def test_run(self, mock_mkdtemp): + mock_mkdtemp.return_value = self.temp_dir + action = undercloud.CreateBackupDir() + action_result = action.run(context={}) + mock_mkdtemp.assert_called() + self.assertEqual(self.temp_dir, + action_result.data['path']) + + +class UploadUndercloudBackupToSwiftTest(base.TestCase): + + def setUp(self,): + super(UploadUndercloudBackupToSwiftTest, self).setUp() + self.container = 'undercloud-backups' + self.backup_path = '/var/tmp/undercloud-backups-sdf_45' + self.tarball_name = 'UC-backup-20180112124502.tar' + self.swift = mock.MagicMock() + swift_patcher = mock.patch( + 'tripleo_common.actions.base.TripleOAction.get_object_client', + return_value=self.swift) + swift_patcher.start() + self.addCleanup(swift_patcher.stop) + self.ctx = mock.MagicMock() + + @mock.patch('tripleo_common.utils.tarball.create_tarball') + def test_simple_success(self, mock_create_tarball): + self.swift.head_object.return_value = { + 'content-length': 1 + } + self.swift.get_container.return_value = ( + {}, [] + ) + + action = undercloud.UploadUndercloudBackupToSwift( + self.backup_path, self.container) + action.run(self.ctx) + + self.swift.put_object.assert_called_once_with( + self.container, + action.tarball_name, + mock.ANY, + headers={'X-Delete-After': 86400} + ) + + mock_create_tarball.assert_called_once() diff --git a/tripleo_common/utils/swift.py b/tripleo_common/utils/swift.py index 1d8c622f7..f03f4b6ab 100644 --- a/tripleo_common/utils/swift.py +++ b/tripleo_common/utils/swift.py @@ -89,6 +89,7 @@ def create_and_upload_tarball(swiftclient, tmp_dir, container, tarball_name, + tarball_options='-czf', delete_after=3600): """Create a tarball containing the tmp_dir and upload it to Swift.""" headers = {'X-Delete-After': delete_after} @@ -96,6 +97,6 @@ def create_and_upload_tarball(swiftclient, get_or_create_container(swiftclient, container) with tempfile.NamedTemporaryFile() as tmp_tarball: - tarball.create_tarball(tmp_dir, tmp_tarball.name) + tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options) swiftclient.put_object(container, tarball_name, tmp_tarball, headers=headers) diff --git a/workbooks/undercloud_backup.yaml b/workbooks/undercloud_backup.yaml new file mode 100644 index 000000000..be11ba34e --- /dev/null +++ b/workbooks/undercloud_backup.yaml @@ -0,0 +1,133 @@ +--- +version: '2.0' +name: tripleo.undercloud_backup.v1 +description: TripleO Undercloud backup workflows + +workflows: + + backup: + description: This workflow will launch the Undercloud backup + tags: + - tripleo-common-managed + input: + - sources_path: '/home/stack/' + - queue_name: tripleo + tasks: + # Action to know if there is enough available space + # to run the Undercloud backup + get_free_space: + action: tripleo.undercloud.get_free_space + publish: + status: SUCCESS + message: <% task().result %> + free_space: <% task().result %> + on-success: create_backup_dir + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # We create a temp directory to store the Undercloud + # backup + create_backup_dir: + action: tripleo.undercloud.create_backup_dir + publish: + status: SUCCESS + message: <% task().result %> + backup_path: <% task().result %> + on-success: get_database_credentials + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # The Undercloud database password for the root + # user is stored in a Mistral environment, we + # need the password in order to run the database dump + get_database_credentials: + action: mistral.environments_get name='tripleo.undercloud-config' + publish: + status: SUCCESS + message: <% task().result %> + undercloud_db_password: <% task(get_database_credentials).result.variables.undercloud_db_password %> + on-success: create_database_backup + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # Run the DB dump of all the databases and store the result + # in the temporary folder + create_database_backup: + input: + path: <% $.backup_path.path %> + dbuser: root + dbpassword: <% $.undercloud_db_password %> + action: tripleo.undercloud.create_database_backup + publish: + status: SUCCESS + message: <% task().result %> + on-success: create_fs_backup + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # This action will run the fs backup + create_fs_backup: + input: + sources_path: <% $.sources_path %> + path: <% $.backup_path.path %> + action: tripleo.undercloud.create_file_system_backup + publish: + status: SUCCESS + message: <% task().result %> + on-success: upload_backup + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # This action will push the backup to swift + upload_backup: + input: + backup_path: <% $.backup_path.path %> + action: tripleo.undercloud.upload_backup_to_swift + publish: + status: SUCCESS + message: <% task().result %> + on-success: cleanup_backup + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # This action will remove the backup temp folder + cleanup_backup: + input: + path: <% $.backup_path.path %> + action: tripleo.undercloud.remove_temp_dir + publish: + status: SUCCESS + message: <% task().result %> + on-success: send_message + on-error: send_message + publish-on-error: + status: FAILED + message: <% task().result %> + + # Sending a message to show that the backup finished + send_message: + action: zaqar.queue_post + retry: count=5 delay=1 + input: + queue_name: <% $.queue_name %> + messages: + body: + type: tripleo.undercloud_backup.v1.launch + payload: + status: <% $.get('status', 'SUCCESS') %> + execution: <% execution() %> + message: <% $.get('message', '') %> + on-success: + - fail: <% $.get('status') = "FAILED" %>