openstack/trove
Code Issues Proposed changes

Merge "Adding support for encrypted backups."

Browse Source
This commit is contained in:
Jenkins 2013-06-18 20:58:55 +00:00 committed by Gerrit Code Review
commit 0bdf7b3928
8 changed files with 173 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/reddwarf/reddwarf-guestagent.conf.sample
View File

@ -72,6 +72,8 @@ storage_strategy = SwiftStorage
storage_namespace = reddwarf.guestagent.strategies.storage.swift
backup_swift_container = database_backups
backup_use_gzip_compression = True
backup_use_openssl_encryption = True
backup_aes_cbc_key = "default_aes_cbc_key"
backup_use_snet = False
backup_chunk_size = 65536
backup_segment_max_size = 2147483648

4
reddwarf/common/cfg.py
View File

@ -140,6 +140,10 @@ common_opts = [
cfg.StrOpt('backup_swift_container', default='database_backups'),
cfg.BoolOpt('backup_use_gzip_compression', default=True,
help='Compress backups using gzip.'),
cfg.BoolOpt('backup_use_openssl_encryption', default=True,
help='Encrypt backups using openssl.'),
cfg.StrOpt('backup_aes_cbc_key', default='default_aes_cbc_key',
help='default openssl aes_cbc key.'),
cfg.BoolOpt('backup_use_snet', default=False,
help='Send backup files over snet.'),
cfg.IntOpt('backup_chunk_size', default=2 ** 16,

14
reddwarf/guestagent/strategies/backup/base.py
View File

@ -30,6 +30,9 @@ CHUNK_SIZE = CONF.backup_chunk_size
MAX_FILE_SIZE = CONF.backup_segment_max_size
BACKUP_CONTAINER = CONF.backup_swift_container
BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
BACKUP_ENCRYPT_KEY = CONF.backup_aes_cbc_key
LOG = logging.getLogger(__name__)
@ -49,6 +52,8 @@ class BackupRunner(Strategy):
# The actual system call to run the backup
cmd = None
is_zipped = BACKUP_USE_GZIP
is_encrypted = BACKUP_USE_OPENSSL
encrypt_key = BACKUP_ENCRYPT_KEY
def __init__(self, filename, **kwargs):
self.filename = filename
@ -119,6 +124,15 @@ class BackupRunner(Strategy):
def zip_manifest(self):
return '.gz' if self.is_zipped else ''
@property
def encrypt_cmd(self):
return (' | openssl enc -aes-256-cbc -salt -pass pass:%s' %
self.encrypt_key) if self.is_encrypted else ''
@property
def encrypt_manifest(self):
return '.enc' if self.is_encrypted else ''
def read(self, chunk_size):
"""Wrap self.process.stdout.read to allow for segmentation."""
if self.end_of_segment:

12
reddwarf/guestagent/strategies/backup/impl.py
View File

@ -32,12 +32,12 @@ class MySQLDump(base.BackupRunner):
' --opt'\
' --password=%(password)s'\
' -u %(user)s'
return cmd + self.zip_cmd
return cmd + self.zip_cmd + self.encrypt_cmd
@property
def manifest(self):
manifest = '%s' + self.zip_manifest
return manifest % self.filename
manifest = '%s' % self.filename
return manifest + self.zip_manifest + self.encrypt_manifest
class InnoBackupEx(base.BackupRunner):
@ -49,9 +49,9 @@ class InnoBackupEx(base.BackupRunner):
cmd = 'sudo innobackupex'\
' --stream=xbstream'\
' /var/lib/mysql 2>/tmp/innobackupex.log'
return cmd + self.zip_cmd
return cmd + self.zip_cmd + self.encrypt_cmd
@property
def manifest(self):
manifest = '%s.xbstream' + self.zip_manifest
return manifest % self.filename
manifest = '%s.xbstream' % self.filename
return manifest + self.zip_manifest + self.encrypt_manifest

28
reddwarf/guestagent/strategies/restore/base.py
View File

@ -27,6 +27,9 @@ import glob
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CHUNK_SIZE = CONF.backup_chunk_size
BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key
RESET_ROOT_RETRY_TIMEOUT = 100
RESET_ROOT_SLEEP_INTERVAL = 10
RESET_ROOT_MYSQL_COMMAND = """
@ -78,13 +81,20 @@ class RestoreRunner(Strategy):
# The backup format type
restore_type = None
# Decryption Parameters
is_zipped = BACKUP_USE_GZIP
is_encrypted = BACKUP_USE_OPENSSL
decrypt_key = BACKUP_DECRYPT_KEY
def __init__(self, restore_stream, **kwargs):
self.restore_stream = restore_stream
self.restore_location = kwargs.get('restore_location',
'/var/lib/mysql')
self.restore_cmd = self.restore_cmd % kwargs
self.prepare_cmd = self.prepare_cmd % kwargs \
if hasattr(self, 'prepare_cmd') else None
self.restore_cmd = (self.decrypt_cmd +
self.unzip_cmd +
(self.base_restore_cmd % kwargs))
self.prepare_cmd = self.base_prepare_cmd % kwargs \
if hasattr(self, 'base_prepare_cmd') else None
super(RestoreRunner, self).__init__()
def __enter__(self):
@ -175,3 +185,15 @@ class RestoreRunner(Strategy):
filelist = glob.glob(self.restore_location + "/ib_logfile*")
for f in filelist:
os.unlink(f)
@property
def decrypt_cmd(self):
if self.is_encrypted:
return ('openssl enc -d -aes-256-cbc -salt -pass pass:%s | '
% self.decrypt_key)
else:
return ''
@property
def unzip_cmd(self):
return 'gzip -d -c | ' if self.is_zipped else ''

16
reddwarf/guestagent/strategies/restore/impl.py
View File

@ -25,10 +25,9 @@ LOG = logging.getLogger(__name__)
class MySQLDump(base.RestoreRunner):
""" Implementation of Restore Strategy for MySQLDump """
__strategy_name__ = 'mysqldump'
is_zipped = True
restore_cmd = ('mysql '
'--password=%(password)s '
'-u %(user)s')
base_restore_cmd = ('mysql '
'--password=%(password)s '
'-u %(user)s')
def _pre_restore(self):
pass
@ -40,11 +39,10 @@ class MySQLDump(base.RestoreRunner):
class InnoBackupEx(base.RestoreRunner):
""" Implementation of Restore Strategy for InnoBackupEx """
__strategy_name__ = 'innobackupex'
is_zipped = True
restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s '
'--defaults-file=%(restore_location)s/backup-my.cnf '
'--ibbackup xtrabackup 2>/tmp/innoprepare.log')
base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s'
' --defaults-file=%(restore_location)s/backup-my.cnf'
' --ibbackup xtrabackup 2>/tmp/innoprepare.log')
def _pre_restore(self):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())

6
reddwarf/guestagent/strategies/storage/swift.py
View File

@ -21,7 +21,6 @@ from reddwarf.common import utils
from eventlet.green import subprocess
import zlib
UNZIPPER = zlib.decompressobj(16 + zlib.MAX_WBITS)
LOG = logging.getLogger(__name__)
@ -107,7 +106,6 @@ class SwiftDownloadStream(object):
def __init__(self, **kwargs):
self.process = None
self.pid = None
self.is_zipped = kwargs.get('is_zipped', False)
self.cmd = self.cmd % kwargs
def __enter__(self):
@ -128,9 +126,7 @@ class SwiftDownloadStream(object):
pass
def read(self, *args, **kwargs):
if not self.is_zipped:
return self.process.stdout.read(*args, **kwargs)
return UNZIPPER.decompress(self.process.stdout.read(*args, **kwargs))
return self.process.stdout.read(*args, **kwargs)
def run(self):
self.process = subprocess.Popen(self.cmd, shell=True,

114
reddwarf/tests/unittests/guestagent/test_backups.py Normal file
View File

@ -0,0 +1,114 @@
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License
import reddwarf.guestagent.strategies.backup.base as backupBase
import reddwarf.guestagent.strategies.restore.base as restoreBase
import testtools
from reddwarf.common import utils
BACKUP_XTRA_CLS = "reddwarf.guestagent.strategies.backup.impl.InnoBackupEx"
RESTORE_XTRA_CLS = "reddwarf.guestagent.strategies.restore.impl.InnoBackupEx"
BACKUP_SQLDUMP_CLS = "reddwarf.guestagent.strategies.backup.impl.MySQLDump"
RESTORE_SQLDUMP_CLS = "reddwarf.guestagent.strategies.restore.impl.MySQLDump"
PIPE = " | "
ZIP = "gzip"
UNZIP = "gzip -d -c"
ENCRYPT = "openssl enc -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
DECRYPT = "openssl enc -d -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
XTRA_BACKUP = "sudo innobackupex --stream=xbstream /var/lib/mysql 2>/" \
"tmp/innobackupex.log"
SQLDUMP_BACKUP = "/usr/bin/mysqldump --all-databases --opt " \
"--password=password -u user"
XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql"
SQLDUMP_RESTORE = "mysql --password=password -u user"
PREPARE = "sudo innobackupex --apply-log /var/lib/mysql " \
"--defaults-file=/var/lib/mysql/backup-my.cnf " \
"--ibbackup xtrabackup 2>/tmp/innoprepare.log"
CRYPTO_KEY = "default_aes_cbc_key"
class GuestAgentBackupTest(testtools.TestCase):
def test_backup_decrypted_xtrabackup_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = False
RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command, XTRA_BACKUP + PIPE + ZIP)
self.assertEqual(bkup.manifest, "12345.xbstream.gz")
def test_backup_encrypted_xtrabackup_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command,
XTRA_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
self.assertEqual(bkup.manifest, "12345.xbstream.gz.enc")
def test_backup_decrypted_mysqldump_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = False
RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command, SQLDUMP_BACKUP + PIPE + ZIP)
self.assertEqual(bkup.manifest, "12345.gz")
def test_backup_encrypted_mysqldump_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command,
SQLDUMP_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
self.assertEqual(bkup.manifest, "12345.gz.enc")
def test_restore_decrypted_xtrabackup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + XTRA_RESTORE)
self.assertEqual(restr.prepare_cmd, PREPARE)
def test_restore_encrypted_xtrabackup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql")
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + XTRA_RESTORE)
self.assertEqual(restr.prepare_cmd, PREPARE)
def test_restore_decrypted_mysqldump_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql",
user="user", password="password")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE)
self.assertIsNone(restr.prepare_cmd)
def test_restore_encrypted_mysqldump_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql",
user="user", password="password")
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE)
self.assertIsNone(restr.prepare_cmd)