Add UC Backup Mistral workflow definition and actions

This patch adds the workflow definition and actions
to perform an Undercloud backup.

Workflow can be executed as:
  mistral execution-create tripleo.undercloud_backup.v1.launch

Actions can be invoked as:
  mistral run-action tripleo.undercloud.get_free_space
  mistral run-action tripleo.undercloud.create_backup_dir
  mistral run-action tripleo.undercloud.create_database_backup \
    '{"path": "/tmp", "dbpassword": "", "dbuser":"root"}'
  mistral run-action tripleo.undercloud.create_file_system_backup \
    '{ "path": "/tmp/undercloud-backup-dCxQSn", "sources_path": "/home/stack/"}'
  mistral run-action tripleo.undercloud.upload_backup_to_swift
  mistral run-action tripleo.undercloud.remove_temp_dir

Related bp undercloud-backup-restore

Co-Authored-By: Sergii Golovatiuk <sgolovat@redhat.com>

Change-Id: Iebd67568d5e72967e694b88fc8c73361543929a1

releasenotes/notes/undercloud-backup-actions-and-workflow-1d661bba3fb2f974.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Introduce Undercloud Backup workflow as well as set of Mistral actions to
+    perform Undercloud Backup

setup.cfg

@@ -140,6 +140,12 @@ mistral.actions =
     tripleo.files.make_temp_dir = tripleo_common.actions.files:MakeTempDir
     tripleo.files.remove_temp_dir = tripleo_common.actions.files:RemoveTempDir
     tripleo.ansible-generate-inventory = tripleo_common.actions.ansible:AnsibleGenerateInventoryAction
+    tripleo.undercloud.get_free_space = tripleo_common.actions.undercloud:GetFreeSpace
+    tripleo.undercloud.create_backup_dir = tripleo_common.actions.undercloud:CreateBackupDir
+    tripleo.undercloud.create_database_backup = tripleo_common.actions.undercloud:CreateDatabaseBackup
+    tripleo.undercloud.create_file_system_backup = tripleo_common.actions.undercloud:CreateFileSystemBackup
+    tripleo.undercloud.upload_backup_to_swift = tripleo_common.actions.undercloud:UploadUndercloudBackupToSwift
+    tripleo.undercloud.remove_temp_dir = tripleo_common.actions.undercloud:RemoveTempDir
     # deprecated for pike release, will be removed in queens
     tripleo.ansible = tripleo_common.actions.ansible:AnsibleAction
     tripleo.ansible-playbook = tripleo_common.actions.ansible:AnsiblePlaybookAction

sudoers

@@ -8,4 +8,6 @@ mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_identit
 mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
         !/usr/bin/rm /tmp/validations_identity_* *, !/usr/bin/rm /tmp/validations_identity_*..*
 mistral ALL = NOPASSWD: /bin/nova-manage cell_v2 discover_hosts *
+mistral ALL = NOPASSWD: /usr/bin/tar --ignore-failed-read -C / -cf /var/tmp/undercloud-backup-*.tar *
+mistral ALL = NOPASSWD: /usr/bin/chown mistral. /var/tmp/undercloud-backup-*/filesystem-*.tar
 validations ALL = NOPASSWD: ALL

tripleo_common/actions/undercloud.py

@@ -0,0 +1,236 @@
+# Copyright 2017 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+import os
+import re
+import shutil
+import six
+import subprocess
+import tempfile
+import time
+
+from mistral_lib import actions
+from mistral_lib.actions import base
+
+from tripleo_common.actions import base as tripleobase
+from tripleo_common.utils import swift as swiftutils
+
+LOG = logging.getLogger(__name__)
+
+
+class GetFreeSpace(base.Action):
+    """Get the Undercloud free space for the backup.
+
+       The default path to check will be /var/tmp and the
+       default minimum size will be 10240 MB (10GB).
+    """
+
+    def __init__(self, min_space=10240, temp_dir="/var/tmp/"):
+        self.min_space = min_space
+        self.temp_dir = temp_dir
+
+    def run(self, context):
+        temp_path = self.temp_dir
+        min_space = self.min_space
+        while not os.path.isdir(temp_path):
+            head, tail = os.path.split(temp_path)
+            temp_path = head
+        available_space = (
+            (os.statvfs(temp_path).f_frsize * os.statvfs(temp_path).f_bavail) /
+            (1024 * 1024))
+        if (available_space < min_space):
+            msg = "There is not enough space, avail. - %s MB" \
+                  % str(int(available_space))
+            return actions.Result(error={'msg': msg})
+        else:
+            msg = "There is enough space, avail. - %s MB" \
+                  % str(int(available_space))
+            return actions.Result(data={'msg': msg})
+
+
+class CreateBackupDir(base.Action):
+    """Creates the Backup temporary directory.
+
+       We will run the backup locally, so we need to create a temporary
+       directory.  The directory created will match the regular expression
+       ^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$
+    """
+
+    def __init__(self):
+        pass
+
+    def run(self, context):
+        try:
+            _path = tempfile.mkdtemp(prefix='undercloud-backup-',
+                                     dir='/var/tmp/')
+            return actions.Result(data={"path": _path})
+        except Exception as msg:
+            return actions.Result(error={"msg": six.text_type(msg)})
+
+
+class CreateDatabaseBackup(base.Action):
+    """Creates a database full backup.
+
+       This action will run the DB dump using a single transaction and storing
+       the result in a given folder.
+    """
+
+    def __init__(self, path, dbuser, dbpassword):
+        self.path = path
+        self.dbuser = dbuser
+        self.dbpassword = dbpassword
+        self.backup_name = os.path.join(self.path,
+                                        'all-databases-' +
+                                        time.strftime("%Y%m%d%H%M%S") +
+                                        '.sql.gz')
+
+    def run(self, context):
+        pid_file = tempfile.gettempdir() + os.sep + "mysqldump.pid"
+        if os.path.exists(pid_file):
+            msg = 'Another Backup process is running'
+            return actions.Result(error={"msg": six.text_type(msg)})
+        lockfile = open(pid_file, 'w')
+        lockfile.write("%s\n" % os.getpid())
+        lockfile.close
+
+        # Backup all databases with nice and ionice just not to create
+        # a huge load on undercloud. Output will be redirected to mysqldump
+        # variable and will be gzipped.
+        script = """
+        #!/bin/bash
+        nice -n 19 ionice -c2 -n7 \
+            mysqldump -u%s -p%s --opt --all-databases | gzip > %s
+        """ % (self.dbuser, self.dbpassword, self.backup_name)
+
+        proc_failed = False
+
+        try:
+            subprocess.check_call(script, shell=True)
+        except subprocess.CalledProcessError:
+            proc_failed = True
+            msg = 'Database dump failed. Deleting temporary directory'
+            os.remove(self.backup_name)
+        else:
+            msg = 'Database dump created succesfully'
+        finally:
+            os.remove(pid_file)
+
+        if proc_failed:
+            return actions.Result(error={'msg': six.text_type(msg)})
+        else:
+            return actions.Result(data={'msg': six.text_type(msg)})
+
+
+class CreateFileSystemBackup(base.Action):
+    """Creates a File System backup.
+
+       This action will run a filesystem backup based on an array of resources
+       to be backed up.  This method gets the sources paths and the destination
+       folder as parameters.
+    """
+
+    def __init__(self, sources_path, path):
+        self.sources_path = sources_path
+        self.path = path
+        self.outfile = os.path.join(self.path,
+                                    'filesystem-' +
+                                    time.strftime("%Y%m%d%H%M%S") +
+                                    '.tar')
+
+    def run(self, context):
+
+        backup_sources = self.sources_path.split(',')
+        separated_string = ' '.join(backup_sources)
+
+        script = """
+        #!/bin/bash
+        sudo tar --ignore-failed-read -C / -cf %s %s
+        sudo chown mistral. %s
+        """ % (self.outfile, separated_string, self.outfile)
+
+        proc_failed = False
+        try:
+            subprocess.check_call(script, shell=True)
+        except subprocess.CalledProcessError:
+            proc_failed = True
+            msg = 'File system backup failed'
+            os.remove(self.outfile)
+        else:
+            msg = 'File system backup created succesfully at: ' + self.outfile
+
+        if proc_failed:
+            # Delete failed backup here
+            return actions.Result(error={'msg': six.text_type(msg)})
+        else:
+            return actions.Result(data={'msg': msg})
+
+
+class UploadUndercloudBackupToSwift(tripleobase.TripleOAction):
+    """Push the Undercloud backup to a swift container.
+
+       This action will push the files in the temporary folder to the swift
+       container storing the Undercloud backups as uncompressed tarball file.
+       The backup will be stored 1 day (86400 s)
+    """
+
+    def __init__(self,
+                 backup_path,
+                 container='undercloud-backups',
+                 expire=86400):
+        self.backup_path = backup_path
+        self.container = container
+        self.expire = expire
+        self.tarball_name = 'UC-backup-%s.tar' % time.strftime(
+                            "%Y%m%d%H%M%S")
+
+    def run(self, context):
+        try:
+            LOG.info('Uploading backup to swift')
+            swift = self.get_object_client(context)
+
+            # Create tarball without gzip and store it 24h
+            swiftutils.create_and_upload_tarball(
+                swift, self.backup_path, self.container,
+                self.tarball_name, '-cf', self.expire)
+
+            msg = 'Backup uploaded to undercloud-backups succesfully'
+            return actions.Result(data={'msg': msg})
+        except Exception as msg:
+            return actions.Result(error={'msg': six.text_type(msg)})
+
+
+class RemoveTempDir(base.Action):
+    """Removes temporary directory on localhost by path.
+
+    The path must match the regular expression
+    ^/var/tmp/undercloud-backup-[A-Za-z0-9_]+$
+    """
+
+    def __init__(self, path):
+        self.path = path
+
+    def run(self, context):
+        # regex from tempfile's _RandomNameSequence characters
+        _regex = '^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$'
+        if (not isinstance(self.path, six.string_types) or
+                not re.match(_regex, self.path)):
+            msg = "Path does not match %s" % _regex
+            return actions.Result(error={"msg": msg})
+        try:
+            shutil.rmtree(self.path)
+            msg = "Deleted directory %s" % self.path
+            return actions.Result(data={"msg": msg})
+        except Exception as msg:
+            return actions.Result(error={"msg": six.text_type(msg)})

tripleo_common/tests/actions/test_undercloud.py

@@ -0,0 +1,175 @@
+# Copyright 2017 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+
+from tripleo_common.actions import undercloud
+from tripleo_common.tests import base
+
+
+class GetFreeSpaceTest(base.TestCase):
+    def setUp(self):
+        super(GetFreeSpaceTest, self).setUp()
+        self.temp_dir = "/var/tmp"
+
+    @mock.patch("os.path.isdir")
+    @mock.patch("os.statvfs")
+    def test_run_false(self, mock_statvfs, mock_isdir):
+        mock_isdir.return_value = True
+        mock_statvfs.return_value = mock.MagicMock(
+            spec_set=['f_frsize', 'f_bavail'],
+            f_frsize=4096, f_bavail=1024)
+        action = undercloud.GetFreeSpace()
+        action_result = action.run(context={})
+        mock_isdir.assert_called()
+        mock_statvfs.assert_called()
+        self.assertEqual("There is not enough space, avail. - 4 MB",
+                         action_result.error['msg'])
+
+    @mock.patch("os.path.isdir")
+    @mock.patch("os.statvfs")
+    def test_run_true(self, mock_statvfs, mock_isdir):
+        mock_isdir.return_value = True
+        mock_statvfs.return_value = mock.MagicMock(
+            spec_set=['f_frsize', 'f_bavail'],
+            f_frsize=4096, f_bavail=10240000)
+        action = undercloud.GetFreeSpace()
+        action_result = action.run(context={})
+        mock_isdir.assert_called()
+        mock_statvfs.assert_called()
+        self.assertEqual("There is enough space, avail. - 40000 MB",
+                         action_result.data['msg'])
+
+
+class RemoveTempDirTest(base.TestCase):
+
+    def setUp(self):
+        super(RemoveTempDirTest, self).setUp()
+        self.path = "/var/tmp/undercloud-backup-dG6hr_"
+
+    @mock.patch("shutil.rmtree")
+    def test_sucess_remove_temp_dir(self, mock_rmtree):
+        mock_rmtree.return_value = None  # rmtree has no return value
+        action = undercloud.RemoveTempDir(self.path)
+        action_result = action.run(context={})
+        mock_rmtree.assert_called()
+        self.assertFalse(action_result.cancel)
+        self.assertIsNone(action_result.error)
+        self.assertEqual('Deleted directory /var/tmp/undercloud-backup-dG6hr_',
+                         action_result.data['msg'])
+
+
+class CreateDatabaseBackupTest(base.TestCase):
+
+    def setUp(self):
+        super(CreateDatabaseBackupTest, self).setUp()
+        self.dbback = undercloud.CreateDatabaseBackup(
+            '/var/tmp/undercloud-backup-dG6hr_',
+            'root', 'dbpassword')
+
+    @mock.patch(
+        'tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('subprocess.check_call')
+    def test_create_database_backup(
+            self, mock_check_call, mock_get_object_client):
+        self.dbback.logger = mock.Mock()
+        self.dbback.run(mock_get_object_client)
+        assert_string = ('\n        #!/bin/bash\n        '
+                         'nice -n 19 ionice -c2 -n7             '
+                         'mysqldump -uroot -pdbpassword --opt '
+                         '--all-databases | gzip > ' +
+                         self.dbback.backup_name +
+                         '\n        ')
+        mock_check_call.assert_called_once_with(assert_string, shell=True)
+
+
+class CreateFileSystemBackupTest(base.TestCase):
+
+    def setUp(self):
+        super(CreateFileSystemBackupTest, self).setUp()
+        self.fsback = undercloud.CreateFileSystemBackup(
+            '/home/stack/,/etc/hosts',
+            '/var/tmp/undercloud-backup-ef9b_H')
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
+    @mock.patch('subprocess.check_call')
+    def test_create_file_system_backup(
+            self,
+            mock_check_call,
+            mock_get_object_client):
+        self.fsback.logger = mock.Mock()
+        self.fsback.run(mock_get_object_client)
+        assert_string = ('\n        #!/bin/bash\n        '
+                         'sudo tar --ignore-failed-read -C / '
+                         '-cf ' +
+                         self.fsback.outfile +
+                         ' /home/stack/ /etc/hosts\n        '
+                         'sudo chown mistral. ' +
+                         self.fsback.outfile +
+                         '\n        ')
+        mock_check_call.assert_called_once_with(assert_string, shell=True)
+
+
+class CreateBackupDirTest(base.TestCase):
+
+    def setUp(self):
+        super(CreateBackupDirTest, self).setUp()
+        self.temp_dir = '/var/tmp/undercloud-backup-XXXXXX'
+
+    @mock.patch('tempfile.mkdtemp')
+    def test_run(self, mock_mkdtemp):
+        mock_mkdtemp.return_value = self.temp_dir
+        action = undercloud.CreateBackupDir()
+        action_result = action.run(context={})
+        mock_mkdtemp.assert_called()
+        self.assertEqual(self.temp_dir,
+                         action_result.data['path'])
+
+
+class UploadUndercloudBackupToSwiftTest(base.TestCase):
+
+    def setUp(self,):
+        super(UploadUndercloudBackupToSwiftTest, self).setUp()
+        self.container = 'undercloud-backups'
+        self.backup_path = '/var/tmp/undercloud-backups-sdf_45'
+        self.tarball_name = 'UC-backup-20180112124502.tar'
+        self.swift = mock.MagicMock()
+        swift_patcher = mock.patch(
+            'tripleo_common.actions.base.TripleOAction.get_object_client',
+            return_value=self.swift)
+        swift_patcher.start()
+        self.addCleanup(swift_patcher.stop)
+        self.ctx = mock.MagicMock()
+
+    @mock.patch('tripleo_common.utils.tarball.create_tarball')
+    def test_simple_success(self, mock_create_tarball):
+        self.swift.head_object.return_value = {
+            'content-length': 1
+        }
+        self.swift.get_container.return_value = (
+            {}, []
+        )
+
+        action = undercloud.UploadUndercloudBackupToSwift(
+            self.backup_path, self.container)
+        action.run(self.ctx)
+
+        self.swift.put_object.assert_called_once_with(
+            self.container,
+            action.tarball_name,
+            mock.ANY,
+            headers={'X-Delete-After': 86400}
+            )
+
+        mock_create_tarball.assert_called_once()

tripleo_common/utils/swift.py

@@ -89,6 +89,7 @@ def create_and_upload_tarball(swiftclient,
                               tmp_dir,
                               container,
                               tarball_name,
+                              tarball_options='-czf',
                               delete_after=3600):
     """Create a tarball containing the tmp_dir and upload it to Swift."""
     headers = {'X-Delete-After': delete_after}
@@ -96,6 +97,6 @@ def create_and_upload_tarball(swiftclient,
     get_or_create_container(swiftclient, container)
 
     with tempfile.NamedTemporaryFile() as tmp_tarball:
-        tarball.create_tarball(tmp_dir, tmp_tarball.name)
+        tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
         swiftclient.put_object(container, tarball_name, tmp_tarball,
                                headers=headers)

workbooks/undercloud_backup.yaml

@@ -0,0 +1,133 @@
+---
+version: '2.0'
+name: tripleo.undercloud_backup.v1
+description: TripleO Undercloud backup workflows
+
+workflows:
+
+  backup:
+    description: This workflow will launch the Undercloud backup
+    tags:
+      - tripleo-common-managed
+    input:
+      - sources_path: '/home/stack/'
+      - queue_name: tripleo
+    tasks:
+      # Action to know if there is enough available space
+      # to run the Undercloud backup
+      get_free_space:
+        action: tripleo.undercloud.get_free_space
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+            free_space: <% task().result %>
+        on-success: create_backup_dir
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # We create a temp directory to store the Undercloud
+      # backup
+      create_backup_dir:
+        action: tripleo.undercloud.create_backup_dir
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+            backup_path: <% task().result %>
+        on-success: get_database_credentials
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # The Undercloud database password for the root
+      # user is stored in a Mistral environment, we
+      # need the password in order to run the database dump
+      get_database_credentials:
+        action: mistral.environments_get name='tripleo.undercloud-config'
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+            undercloud_db_password: <% task(get_database_credentials).result.variables.undercloud_db_password %>
+        on-success: create_database_backup
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # Run the DB dump of all the databases and store the result
+      # in the temporary folder
+      create_database_backup:
+        input:
+            path: <% $.backup_path.path %>
+            dbuser: root
+            dbpassword: <% $.undercloud_db_password %>
+        action: tripleo.undercloud.create_database_backup
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+        on-success: create_fs_backup
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # This action will run the fs backup
+      create_fs_backup:
+        input:
+            sources_path: <% $.sources_path %>
+            path: <% $.backup_path.path %>
+        action: tripleo.undercloud.create_file_system_backup
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+        on-success: upload_backup
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # This action will push the backup to swift
+      upload_backup:
+        input:
+            backup_path: <% $.backup_path.path %>
+        action: tripleo.undercloud.upload_backup_to_swift
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+        on-success: cleanup_backup
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # This action will remove the backup temp folder
+      cleanup_backup:
+        input:
+            path: <% $.backup_path.path %>
+        action: tripleo.undercloud.remove_temp_dir
+        publish:
+            status: SUCCESS
+            message: <% task().result %>
+        on-success: send_message
+        on-error: send_message
+        publish-on-error:
+          status: FAILED
+          message: <% task().result %>
+
+      # Sending a message to show that the backup finished
+      send_message:
+        action: zaqar.queue_post
+        retry: count=5 delay=1
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.undercloud_backup.v1.launch
+              payload:
+                status: <% $.get('status', 'SUCCESS') %>
+                execution: <% execution() %>
+                message: <% $.get('message', '') %>
+        on-success:
+          - fail: <% $.get('status') = "FAILED" %>