Add UC Backup Mistral workflow definition and actions

This patch adds the workflow definition and actions
to perform an Undercloud backup.

Workflow can be executed as:
  mistral execution-create tripleo.undercloud_backup.v1.launch

Actions can be invoked as:
  mistral run-action tripleo.undercloud.get_free_space
  mistral run-action tripleo.undercloud.create_backup_dir
  mistral run-action tripleo.undercloud.create_database_backup \
    '{"path": "/tmp", "dbpassword": "", "dbuser":"root"}'
  mistral run-action tripleo.undercloud.create_file_system_backup \
    '{ "path": "/tmp/undercloud-backup-dCxQSn", "sources_path": "/home/stack/"}'
  mistral run-action tripleo.undercloud.upload_backup_to_swift
  mistral run-action tripleo.undercloud.remove_temp_dir

Related bp undercloud-backup-restore

Co-Authored-By: Sergii Golovatiuk <sgolovat@redhat.com>

Change-Id: Iebd67568d5e72967e694b88fc8c73361543929a1
This commit is contained in:
Carlos Camacho 2017-11-03 11:53:39 +00:00
parent 1c760d570a
commit bfc687b4d3
7 changed files with 559 additions and 1 deletions

View File

@ -0,0 +1,5 @@
---
features:
- |
Introduce Undercloud Backup workflow as well as set of Mistral actions to
perform Undercloud Backup

View File

@ -140,6 +140,12 @@ mistral.actions =
tripleo.files.make_temp_dir = tripleo_common.actions.files:MakeTempDir
tripleo.files.remove_temp_dir = tripleo_common.actions.files:RemoveTempDir
tripleo.ansible-generate-inventory = tripleo_common.actions.ansible:AnsibleGenerateInventoryAction
tripleo.undercloud.get_free_space = tripleo_common.actions.undercloud:GetFreeSpace
tripleo.undercloud.create_backup_dir = tripleo_common.actions.undercloud:CreateBackupDir
tripleo.undercloud.create_database_backup = tripleo_common.actions.undercloud:CreateDatabaseBackup
tripleo.undercloud.create_file_system_backup = tripleo_common.actions.undercloud:CreateFileSystemBackup
tripleo.undercloud.upload_backup_to_swift = tripleo_common.actions.undercloud:UploadUndercloudBackupToSwift
tripleo.undercloud.remove_temp_dir = tripleo_common.actions.undercloud:RemoveTempDir
# deprecated for pike release, will be removed in queens
tripleo.ansible = tripleo_common.actions.ansible:AnsibleAction
tripleo.ansible-playbook = tripleo_common.actions.ansible:AnsiblePlaybookAction

View File

@ -8,4 +8,6 @@ mistral ALL = NOPASSWD: /usr/bin/chown -h validations\: /tmp/validations_identit
mistral ALL = NOPASSWD: /usr/bin/rm -f /tmp/validations_identity_[A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_][A-Za-z0-9_], \
!/usr/bin/rm /tmp/validations_identity_* *, !/usr/bin/rm /tmp/validations_identity_*..*
mistral ALL = NOPASSWD: /bin/nova-manage cell_v2 discover_hosts *
mistral ALL = NOPASSWD: /usr/bin/tar --ignore-failed-read -C / -cf /var/tmp/undercloud-backup-*.tar *
mistral ALL = NOPASSWD: /usr/bin/chown mistral. /var/tmp/undercloud-backup-*/filesystem-*.tar
validations ALL = NOPASSWD: ALL

View File

@ -0,0 +1,236 @@
# Copyright 2017 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import re
import shutil
import six
import subprocess
import tempfile
import time
from mistral_lib import actions
from mistral_lib.actions import base
from tripleo_common.actions import base as tripleobase
from tripleo_common.utils import swift as swiftutils
LOG = logging.getLogger(__name__)
class GetFreeSpace(base.Action):
"""Get the Undercloud free space for the backup.
The default path to check will be /var/tmp and the
default minimum size will be 10240 MB (10GB).
"""
def __init__(self, min_space=10240, temp_dir="/var/tmp/"):
self.min_space = min_space
self.temp_dir = temp_dir
def run(self, context):
temp_path = self.temp_dir
min_space = self.min_space
while not os.path.isdir(temp_path):
head, tail = os.path.split(temp_path)
temp_path = head
available_space = (
(os.statvfs(temp_path).f_frsize * os.statvfs(temp_path).f_bavail) /
(1024 * 1024))
if (available_space < min_space):
msg = "There is not enough space, avail. - %s MB" \
% str(int(available_space))
return actions.Result(error={'msg': msg})
else:
msg = "There is enough space, avail. - %s MB" \
% str(int(available_space))
return actions.Result(data={'msg': msg})
class CreateBackupDir(base.Action):
"""Creates the Backup temporary directory.
We will run the backup locally, so we need to create a temporary
directory. The directory created will match the regular expression
^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$
"""
def __init__(self):
pass
def run(self, context):
try:
_path = tempfile.mkdtemp(prefix='undercloud-backup-',
dir='/var/tmp/')
return actions.Result(data={"path": _path})
except Exception as msg:
return actions.Result(error={"msg": six.text_type(msg)})
class CreateDatabaseBackup(base.Action):
"""Creates a database full backup.
This action will run the DB dump using a single transaction and storing
the result in a given folder.
"""
def __init__(self, path, dbuser, dbpassword):
self.path = path
self.dbuser = dbuser
self.dbpassword = dbpassword
self.backup_name = os.path.join(self.path,
'all-databases-' +
time.strftime("%Y%m%d%H%M%S") +
'.sql.gz')
def run(self, context):
pid_file = tempfile.gettempdir() + os.sep + "mysqldump.pid"
if os.path.exists(pid_file):
msg = 'Another Backup process is running'
return actions.Result(error={"msg": six.text_type(msg)})
lockfile = open(pid_file, 'w')
lockfile.write("%s\n" % os.getpid())
lockfile.close
# Backup all databases with nice and ionice just not to create
# a huge load on undercloud. Output will be redirected to mysqldump
# variable and will be gzipped.
script = """
#!/bin/bash
nice -n 19 ionice -c2 -n7 \
mysqldump -u%s -p%s --opt --all-databases | gzip > %s
""" % (self.dbuser, self.dbpassword, self.backup_name)
proc_failed = False
try:
subprocess.check_call(script, shell=True)
except subprocess.CalledProcessError:
proc_failed = True
msg = 'Database dump failed. Deleting temporary directory'
os.remove(self.backup_name)
else:
msg = 'Database dump created succesfully'
finally:
os.remove(pid_file)
if proc_failed:
return actions.Result(error={'msg': six.text_type(msg)})
else:
return actions.Result(data={'msg': six.text_type(msg)})
class CreateFileSystemBackup(base.Action):
"""Creates a File System backup.
This action will run a filesystem backup based on an array of resources
to be backed up. This method gets the sources paths and the destination
folder as parameters.
"""
def __init__(self, sources_path, path):
self.sources_path = sources_path
self.path = path
self.outfile = os.path.join(self.path,
'filesystem-' +
time.strftime("%Y%m%d%H%M%S") +
'.tar')
def run(self, context):
backup_sources = self.sources_path.split(',')
separated_string = ' '.join(backup_sources)
script = """
#!/bin/bash
sudo tar --ignore-failed-read -C / -cf %s %s
sudo chown mistral. %s
""" % (self.outfile, separated_string, self.outfile)
proc_failed = False
try:
subprocess.check_call(script, shell=True)
except subprocess.CalledProcessError:
proc_failed = True
msg = 'File system backup failed'
os.remove(self.outfile)
else:
msg = 'File system backup created succesfully at: ' + self.outfile
if proc_failed:
# Delete failed backup here
return actions.Result(error={'msg': six.text_type(msg)})
else:
return actions.Result(data={'msg': msg})
class UploadUndercloudBackupToSwift(tripleobase.TripleOAction):
"""Push the Undercloud backup to a swift container.
This action will push the files in the temporary folder to the swift
container storing the Undercloud backups as uncompressed tarball file.
The backup will be stored 1 day (86400 s)
"""
def __init__(self,
backup_path,
container='undercloud-backups',
expire=86400):
self.backup_path = backup_path
self.container = container
self.expire = expire
self.tarball_name = 'UC-backup-%s.tar' % time.strftime(
"%Y%m%d%H%M%S")
def run(self, context):
try:
LOG.info('Uploading backup to swift')
swift = self.get_object_client(context)
# Create tarball without gzip and store it 24h
swiftutils.create_and_upload_tarball(
swift, self.backup_path, self.container,
self.tarball_name, '-cf', self.expire)
msg = 'Backup uploaded to undercloud-backups succesfully'
return actions.Result(data={'msg': msg})
except Exception as msg:
return actions.Result(error={'msg': six.text_type(msg)})
class RemoveTempDir(base.Action):
"""Removes temporary directory on localhost by path.
The path must match the regular expression
^/var/tmp/undercloud-backup-[A-Za-z0-9_]+$
"""
def __init__(self, path):
self.path = path
def run(self, context):
# regex from tempfile's _RandomNameSequence characters
_regex = '^/var/tmp/undercloud-backup-[A-Za-z0-9_]{6}$'
if (not isinstance(self.path, six.string_types) or
not re.match(_regex, self.path)):
msg = "Path does not match %s" % _regex
return actions.Result(error={"msg": msg})
try:
shutil.rmtree(self.path)
msg = "Deleted directory %s" % self.path
return actions.Result(data={"msg": msg})
except Exception as msg:
return actions.Result(error={"msg": six.text_type(msg)})

View File

@ -0,0 +1,175 @@
# Copyright 2017 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from tripleo_common.actions import undercloud
from tripleo_common.tests import base
class GetFreeSpaceTest(base.TestCase):
def setUp(self):
super(GetFreeSpaceTest, self).setUp()
self.temp_dir = "/var/tmp"
@mock.patch("os.path.isdir")
@mock.patch("os.statvfs")
def test_run_false(self, mock_statvfs, mock_isdir):
mock_isdir.return_value = True
mock_statvfs.return_value = mock.MagicMock(
spec_set=['f_frsize', 'f_bavail'],
f_frsize=4096, f_bavail=1024)
action = undercloud.GetFreeSpace()
action_result = action.run(context={})
mock_isdir.assert_called()
mock_statvfs.assert_called()
self.assertEqual("There is not enough space, avail. - 4 MB",
action_result.error['msg'])
@mock.patch("os.path.isdir")
@mock.patch("os.statvfs")
def test_run_true(self, mock_statvfs, mock_isdir):
mock_isdir.return_value = True
mock_statvfs.return_value = mock.MagicMock(
spec_set=['f_frsize', 'f_bavail'],
f_frsize=4096, f_bavail=10240000)
action = undercloud.GetFreeSpace()
action_result = action.run(context={})
mock_isdir.assert_called()
mock_statvfs.assert_called()
self.assertEqual("There is enough space, avail. - 40000 MB",
action_result.data['msg'])
class RemoveTempDirTest(base.TestCase):
def setUp(self):
super(RemoveTempDirTest, self).setUp()
self.path = "/var/tmp/undercloud-backup-dG6hr_"
@mock.patch("shutil.rmtree")
def test_sucess_remove_temp_dir(self, mock_rmtree):
mock_rmtree.return_value = None # rmtree has no return value
action = undercloud.RemoveTempDir(self.path)
action_result = action.run(context={})
mock_rmtree.assert_called()
self.assertFalse(action_result.cancel)
self.assertIsNone(action_result.error)
self.assertEqual('Deleted directory /var/tmp/undercloud-backup-dG6hr_',
action_result.data['msg'])
class CreateDatabaseBackupTest(base.TestCase):
def setUp(self):
super(CreateDatabaseBackupTest, self).setUp()
self.dbback = undercloud.CreateDatabaseBackup(
'/var/tmp/undercloud-backup-dG6hr_',
'root', 'dbpassword')
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('subprocess.check_call')
def test_create_database_backup(
self, mock_check_call, mock_get_object_client):
self.dbback.logger = mock.Mock()
self.dbback.run(mock_get_object_client)
assert_string = ('\n #!/bin/bash\n '
'nice -n 19 ionice -c2 -n7 '
'mysqldump -uroot -pdbpassword --opt '
'--all-databases | gzip > ' +
self.dbback.backup_name +
'\n ')
mock_check_call.assert_called_once_with(assert_string, shell=True)
class CreateFileSystemBackupTest(base.TestCase):
def setUp(self):
super(CreateFileSystemBackupTest, self).setUp()
self.fsback = undercloud.CreateFileSystemBackup(
'/home/stack/,/etc/hosts',
'/var/tmp/undercloud-backup-ef9b_H')
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('subprocess.check_call')
def test_create_file_system_backup(
self,
mock_check_call,
mock_get_object_client):
self.fsback.logger = mock.Mock()
self.fsback.run(mock_get_object_client)
assert_string = ('\n #!/bin/bash\n '
'sudo tar --ignore-failed-read -C / '
'-cf ' +
self.fsback.outfile +
' /home/stack/ /etc/hosts\n '
'sudo chown mistral. ' +
self.fsback.outfile +
'\n ')
mock_check_call.assert_called_once_with(assert_string, shell=True)
class CreateBackupDirTest(base.TestCase):
def setUp(self):
super(CreateBackupDirTest, self).setUp()
self.temp_dir = '/var/tmp/undercloud-backup-XXXXXX'
@mock.patch('tempfile.mkdtemp')
def test_run(self, mock_mkdtemp):
mock_mkdtemp.return_value = self.temp_dir
action = undercloud.CreateBackupDir()
action_result = action.run(context={})
mock_mkdtemp.assert_called()
self.assertEqual(self.temp_dir,
action_result.data['path'])
class UploadUndercloudBackupToSwiftTest(base.TestCase):
def setUp(self,):
super(UploadUndercloudBackupToSwiftTest, self).setUp()
self.container = 'undercloud-backups'
self.backup_path = '/var/tmp/undercloud-backups-sdf_45'
self.tarball_name = 'UC-backup-20180112124502.tar'
self.swift = mock.MagicMock()
swift_patcher = mock.patch(
'tripleo_common.actions.base.TripleOAction.get_object_client',
return_value=self.swift)
swift_patcher.start()
self.addCleanup(swift_patcher.stop)
self.ctx = mock.MagicMock()
@mock.patch('tripleo_common.utils.tarball.create_tarball')
def test_simple_success(self, mock_create_tarball):
self.swift.head_object.return_value = {
'content-length': 1
}
self.swift.get_container.return_value = (
{}, []
)
action = undercloud.UploadUndercloudBackupToSwift(
self.backup_path, self.container)
action.run(self.ctx)
self.swift.put_object.assert_called_once_with(
self.container,
action.tarball_name,
mock.ANY,
headers={'X-Delete-After': 86400}
)
mock_create_tarball.assert_called_once()

View File

@ -89,6 +89,7 @@ def create_and_upload_tarball(swiftclient,
tmp_dir,
container,
tarball_name,
tarball_options='-czf',
delete_after=3600):
"""Create a tarball containing the tmp_dir and upload it to Swift."""
headers = {'X-Delete-After': delete_after}
@ -96,6 +97,6 @@ def create_and_upload_tarball(swiftclient,
get_or_create_container(swiftclient, container)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name)
tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
swiftclient.put_object(container, tarball_name, tmp_tarball,
headers=headers)

View File

@ -0,0 +1,133 @@
---
version: '2.0'
name: tripleo.undercloud_backup.v1
description: TripleO Undercloud backup workflows
workflows:
backup:
description: This workflow will launch the Undercloud backup
tags:
- tripleo-common-managed
input:
- sources_path: '/home/stack/'
- queue_name: tripleo
tasks:
# Action to know if there is enough available space
# to run the Undercloud backup
get_free_space:
action: tripleo.undercloud.get_free_space
publish:
status: SUCCESS
message: <% task().result %>
free_space: <% task().result %>
on-success: create_backup_dir
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# We create a temp directory to store the Undercloud
# backup
create_backup_dir:
action: tripleo.undercloud.create_backup_dir
publish:
status: SUCCESS
message: <% task().result %>
backup_path: <% task().result %>
on-success: get_database_credentials
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# The Undercloud database password for the root
# user is stored in a Mistral environment, we
# need the password in order to run the database dump
get_database_credentials:
action: mistral.environments_get name='tripleo.undercloud-config'
publish:
status: SUCCESS
message: <% task().result %>
undercloud_db_password: <% task(get_database_credentials).result.variables.undercloud_db_password %>
on-success: create_database_backup
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# Run the DB dump of all the databases and store the result
# in the temporary folder
create_database_backup:
input:
path: <% $.backup_path.path %>
dbuser: root
dbpassword: <% $.undercloud_db_password %>
action: tripleo.undercloud.create_database_backup
publish:
status: SUCCESS
message: <% task().result %>
on-success: create_fs_backup
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# This action will run the fs backup
create_fs_backup:
input:
sources_path: <% $.sources_path %>
path: <% $.backup_path.path %>
action: tripleo.undercloud.create_file_system_backup
publish:
status: SUCCESS
message: <% task().result %>
on-success: upload_backup
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# This action will push the backup to swift
upload_backup:
input:
backup_path: <% $.backup_path.path %>
action: tripleo.undercloud.upload_backup_to_swift
publish:
status: SUCCESS
message: <% task().result %>
on-success: cleanup_backup
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# This action will remove the backup temp folder
cleanup_backup:
input:
path: <% $.backup_path.path %>
action: tripleo.undercloud.remove_temp_dir
publish:
status: SUCCESS
message: <% task().result %>
on-success: send_message
on-error: send_message
publish-on-error:
status: FAILED
message: <% task().result %>
# Sending a message to show that the backup finished
send_message:
action: zaqar.queue_post
retry: count=5 delay=1
input:
queue_name: <% $.queue_name %>
messages:
body:
type: tripleo.undercloud_backup.v1.launch
payload:
status: <% $.get('status', 'SUCCESS') %>
execution: <% execution() %>
message: <% $.get('message', '') %>
on-success:
- fail: <% $.get('status') = "FAILED" %>